diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index bb2c4e49fb..0ee739e6ec 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -266,13 +266,22 @@ fails closed. Shared supervisors receive final access tokens through their authenticated sessions. No SPIFFE socket is mounted in a shared supervisor or workload Pod. -The topology does not support upstream proxies. +- **Upstream proxy.** The gateway snapshots the operator's proxy settings, + credential Secret from the sandbox namespace, and CA bundle in the + generation's private launch record. Pool members stage the material in the + instance's private directory and use the same CONNECT path as dedicated + supervisors. Recovery and handoff reuse the record. Restart the sandbox + generation to pick up proxy changes. Upgrade the gateway and every pool + member before enabling these settings. The Helm chart grants the gateway + `get` on the configured credential Secret name in the sandbox namespace. + With a live replacement supervisor and a takeover-capable runtime, the gateway can move a paused sandbox generation to the replacement using a signed takeover grant. Recovery preserves workload processes and memory, but proxied connections and interactive sessions must reconnect. If recovery exceeds the configured deadline, the runtime terminates the workload. It cannot recover a lost workload Pod or node. See [supervisor recovery](../../docs/kubernetes/high-availability.mdx#recover-from-a-supervisor-failure). + Shared sandbox instances export no supervisor OTLP traces or OCSF events and do not relay agent traces; the driver warns at startup when the gateway configures any of them. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index adf41521aa..51d88d1044 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -586,9 +586,6 @@ impl KubernetesComputeConfig { Ok(()) } - /// The shared supervisor serves only sandbox instances it can isolate: it - /// holds no per-sandbox workload identity, upstream proxy credential, or - /// proxy CA. pub fn validate_supervisor_topology(&self) -> Result<(), String> { match self.supervisor_topology { None => { @@ -610,15 +607,6 @@ impl KubernetesComputeConfig { { return Err("supervisor.app_listener.port conflicts with shared_supervisor.runtime_port".into()); } - if self.https_proxy.is_some() - || self.proxy_auth_secret_name.is_some() - || self.proxy_ca_bundle.is_some() - { - return Err( - "supervisor_topology=gateway does not support upstream proxy settings" - .into(), - ); - } Ok(()) } } @@ -1232,17 +1220,27 @@ mod tests { .is_err() ); } - // A shared supervisor holds no per-sandbox workload identity or proxy credential. let with_proxy = KubernetesComputeConfig { https_proxy: Some("http://proxy.example:3128".to_string()), ..gateway(Some(shared())) }; - assert!(with_proxy.validate_supervisor_topology().is_err()); + assert!(with_proxy.validate_supervisor_topology().is_ok()); + assert!(with_proxy.validate_upstream_proxy_config().is_ok()); + let with_credentials = KubernetesComputeConfig { + proxy_auth_secret_name: Some("egress-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + proxy_auth_allow_insecure: Some(true), + ..with_proxy + }; + assert!(with_credentials.validate_supervisor_topology().is_ok()); + assert!(with_credentials.validate_upstream_proxy_config().is_ok()); let with_proxy_ca = KubernetesComputeConfig { + https_proxy: Some("https://proxy.example:3130".to_string()), proxy_ca_bundle: Some("/etc/proxy/ca.pem".to_string()), ..gateway(Some(shared())) }; - assert!(with_proxy_ca.validate_supervisor_topology().is_err()); + assert!(with_proxy_ca.validate_supervisor_topology().is_ok()); + assert!(with_proxy_ca.validate_upstream_proxy_config().is_ok()); let workspace = KubernetesComputeConfig { supervisor_topology: Some(SupervisorTopology::Workspace), ..KubernetesComputeConfig::default() diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f7eaf4bd6b..672823f195 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3518,6 +3518,65 @@ impl KubernetesComputeDriver { }) } + /// Snapshot proxy credentials from the workload namespace into the same + /// private generation record as its route and trust, including on takeover. + async fn shared_upstream_proxy_launch( + &self, + namespace: &str, + ca_bundle: Option>, + ) -> Result, KubernetesDriverError> + { + let Some(url) = &self.config.https_proxy else { + return Ok(None); + }; + let credential = match ( + self.config.proxy_auth_secret_name.as_deref(), + self.config.proxy_auth_secret_key.as_deref(), + ) { + (None, None) => None, + (Some(name), Some(key)) => { + let secret = Api::::namespaced(self.client.clone(), namespace) + .get(name) + .await + .map_err(KubernetesDriverError::from_kube)?; + let value = secret + .data + .and_then(|mut data| data.remove(key)) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "proxy credential Secret is missing its configured key".to_string(), + ) + })?; + Some(String::from_utf8(value.0).map_err(|_| { + KubernetesDriverError::Precondition( + "proxy credential Secret must contain UTF-8 user:pass".to_string(), + ) + })?) + } + _ => { + return Err(KubernetesDriverError::Precondition( + "proxy_auth_secret_name and proxy_auth_secret_key must be set together".into(), + )); + } + }; + let proxy = openshell_supervisor_control::UpstreamProxyLaunch { + url: url.clone(), + no_proxy: self.config.no_proxy.clone(), + credential, + auth_allow_insecure: self.config.proxy_auth_allow_insecure.unwrap_or(false), + connect_by_hostname: self.config.proxy_connect_by_hostname.unwrap_or(false), + ca_bundle_pem: ca_bundle.map(String::from_utf8).transpose().map_err(|_| { + KubernetesDriverError::Precondition( + "proxy_ca_bundle must contain UTF-8 PEM".to_string(), + ) + })?, + }; + proxy + .validate() + .map_err(|error| KubernetesDriverError::Precondition(error.to_string()))?; + Ok(Some(proxy)) + } + /// Create one generation's gated supervisor Pod and return its UID. #[allow(clippy::too_many_arguments, clippy::similar_names)] async fn create_sandbox_runtime_supervisor( @@ -4015,6 +4074,9 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::from_kube)?; } GenerationLaunchTarget::Shared { shared, route, .. } => { + let upstream_proxy = self + .shared_upstream_proxy_launch(namespace, staged.upstream_proxy_ca_bundle) + .await?; self.create_generation_image_pull_secrets( namespace, names, @@ -4029,6 +4091,7 @@ impl KubernetesComputeDriver { provisioned.runtime_descriptor, launch_authentication, proxy_ca, + upstream_proxy, )?; // The coordinator persists the launch material with the // generation before it assigns the instance, so a restarted @@ -5889,6 +5952,7 @@ fn shared_instance_registration( runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, + upstream_proxy: Option, ) -> Result { Ok(SandboxInstanceRegistration { route: route.to_string(), @@ -5922,6 +5986,7 @@ fn shared_instance_registration( certificate_pem: proxy_ca.certificate_pem, private_key_pem: openshell_supervisor_control::SecretPem::new(proxy_ca.private_key_pem), }, + upstream_proxy, trace_context: RegistrationTraceContext::current(), }) } @@ -9134,6 +9199,7 @@ fn spawn_namespace_file_watcher( #[cfg(test)] mod tests { use super::*; + use k8s_openapi::ByteString; /// Serve Pod GETs over one reusable HTTP/1 connection per client, /// delaying a request for Pod `slow-` by that many milliseconds. @@ -13587,6 +13653,90 @@ mod tests { (driver, steps, bodies) } + #[tokio::test] + async fn shared_proxy_launch_snapshots_credentials_from_each_sandbox_namespace() { + let ca = generate_proxy_ca_material().unwrap().certificate_pem; + for (namespace, path) in [ + ( + "workspace-a", + "/api/v1/namespaces/workspace-a/secrets/egress-auth", + ), + ( + "workspace-b", + "/api/v1/namespaces/workspace-b/secrets/egress-auth", + ), + ] { + let credential = format!("{namespace}:private-password"); + let secret = Secret { + data: Some(BTreeMap::from([( + "credentials".to_string(), + ByteString(credential.as_bytes().to_vec()), + )])), + ..Secret::default() + }; + let config = KubernetesComputeConfig { + https_proxy: Some("https://proxy.example:3130".to_string()), + proxy_auth_secret_name: Some("egress-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + let (driver, pending, _) = scripted_driver( + config, + vec![( + http::Method::GET, + path, + kube_test_response(http::StatusCode::OK, serde_json::to_value(secret).unwrap()), + )], + ); + let proxy = driver + .shared_upstream_proxy_launch(namespace, Some(ca.as_bytes().to_vec())) + .await + .unwrap() + .unwrap(); + assert_eq!(proxy.credential.as_deref(), Some(credential.as_str())); + assert_eq!(proxy.ca_bundle_pem.as_deref(), Some(ca.as_str())); + assert!(pending.lock().unwrap().is_empty()); + } + } + + #[tokio::test] + async fn shared_proxy_launch_refuses_missing_or_invalid_credentials() { + let path = "/api/v1/namespaces/workspace-a/secrets/egress-auth"; + let invalid = [ + None, + Some(vec![0xff]), + Some(b"private-password".to_vec()), + Some(vec![b'x'; 4097]), + ]; + for credential in invalid { + let secret = Secret { + data: credential + .map(|value| BTreeMap::from([("credentials".to_string(), ByteString(value))])), + ..Secret::default() + }; + let config = KubernetesComputeConfig { + https_proxy: Some("https://proxy.example:3130".to_string()), + proxy_auth_secret_name: Some("egress-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + let (driver, _, _) = scripted_driver( + config, + vec![( + http::Method::GET, + path, + kube_test_response(http::StatusCode::OK, serde_json::to_value(secret).unwrap()), + )], + ); + let error = driver + .shared_upstream_proxy_launch("workspace-a", None) + .await + .unwrap_err() + .to_string(); + assert!(!error.contains("private-password"), "{error}"); + } + } + fn pod_owner(name: &str, uid: &str) -> OwnerReference { OwnerReference { api_version: "v1".to_string(), diff --git a/crates/openshell-supervisor-control/src/lib.rs b/crates/openshell-supervisor-control/src/lib.rs index f7ddb4284d..2bb9ff7a11 100644 --- a/crates/openshell-supervisor-control/src/lib.rs +++ b/crates/openshell-supervisor-control/src/lib.rs @@ -58,6 +58,6 @@ pub use pool_limits::{ }; pub use registration::{ INSTANCE_REMOVAL_GRACE, MAX_LAUNCH_MATERIAL_BYTES, ProxyCaMaterial, RegistrationError, - RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, + RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, UpstreamProxyLaunch, }; pub use status::{SandboxInstancePhaseReport, SandboxInstanceStatusReport}; diff --git a/crates/openshell-supervisor-control/src/registration.rs b/crates/openshell-supervisor-control/src/registration.rs index dfd80910c3..bff73d8da9 100644 --- a/crates/openshell-supervisor-control/src/registration.rs +++ b/crates/openshell-supervisor-control/src/registration.rs @@ -69,6 +69,66 @@ pub struct ProxyCaMaterial { pub private_key_pem: SecretPem, } +/// Operator-owned proxy settings retained with one sandbox generation. +/// +/// Credentials and trust travel only over the private supervisor assignment +/// stream and are staged into the hosted instance's private directory. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpstreamProxyLaunch { + pub url: String, + pub no_proxy: Option, + pub credential: Option, + pub auth_allow_insecure: bool, + pub connect_by_hostname: bool, + pub ca_bundle_pem: Option, +} + +impl std::fmt::Debug for UpstreamProxyLaunch { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("UpstreamProxyLaunch([REDACTED])") + } +} + +impl UpstreamProxyLaunch { + /// Validate the same proxy boundary as dedicated supervisors, without + /// interpreting paths in another process's filesystem. + pub fn validate(&self) -> Result<(), RegistrationError> { + use openshell_core::driver_utils::{ + MAX_UPSTREAM_PROXY_CREDENTIAL_BYTES, UpstreamProxySettings, + parse_upstream_proxy_credential, validate_upstream_proxy_ca_bundle_pem, + validate_upstream_proxy_settings, + }; + + validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some(&self.url), + no_proxy: self.no_proxy.as_deref(), + auth_file: self.credential.as_ref().map(|_| "upstream-proxy-auth"), + auth_allow_insecure: self.auth_allow_insecure.then_some(true), + connect_by_hostname: self.connect_by_hostname.then_some(true), + ca_bundle: self.ca_bundle_pem.as_ref().map(|_| "upstream-proxy-ca.pem"), + auth_setting_label: Some("proxy credential Secret"), + }) + .map_err(RegistrationError::new)?; + if let Some(credential) = &self.credential { + if u64::try_from(credential.len()).unwrap_or(u64::MAX) + > MAX_UPSTREAM_PROXY_CREDENTIAL_BYTES + { + return Err(RegistrationError::new( + "proxy credential exceeds its size limit", + )); + } + parse_upstream_proxy_credential(credential) + .map_err(|error| RegistrationError::new(format!("proxy credential {error}")))?; + } + if let Some(pem) = &self.ca_bundle_pem { + validate_upstream_proxy_ca_bundle_pem(pem, "shared registration", "proxy_ca_bundle") + .map_err(RegistrationError::new)?; + } + Ok(()) + } +} + /// W3C trace context of the operation that created the generation. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -118,6 +178,8 @@ pub struct SandboxInstanceRegistration { pub container_image: String, pub proxy_ca: ProxyCaMaterial, #[serde(default, skip_serializing_if = "Option::is_none")] + pub upstream_proxy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub trace_context: Option, } @@ -175,6 +237,9 @@ impl SandboxInstanceRegistration { { return Err(RegistrationError::new("proxy CA material is required")); } + if let Some(proxy) = &self.upstream_proxy { + proxy.validate()?; + } Ok(()) } @@ -234,6 +299,67 @@ impl SandboxInstanceRegistration { mod tests { use super::*; + fn upstream_proxy() -> UpstreamProxyLaunch { + UpstreamProxyLaunch { + url: "https://proxy.example:3130".to_string(), + no_proxy: Some("internal.example".to_string()), + credential: Some("operator:private-proxy-password".to_string()), + auth_allow_insecure: false, + connect_by_hostname: true, + ca_bundle_pem: None, + } + } + + #[test] + fn upstream_proxy_launch_roundtrips_without_debugging_credentials() { + let proxy = upstream_proxy(); + proxy.validate().unwrap(); + let decoded: UpstreamProxyLaunch = + serde_json::from_slice(&serde_json::to_vec(&proxy).unwrap()).unwrap(); + decoded.validate().unwrap(); + assert_eq!(decoded.url, proxy.url); + assert_eq!(decoded.credential, proxy.credential); + assert_eq!(decoded.no_proxy, proxy.no_proxy); + assert!(decoded.connect_by_hostname); + assert!(!format!("{decoded:?}").contains("private-proxy-password")); + } + + #[test] + fn upstream_proxy_launch_refuses_invalid_boundaries_without_quoting_secrets() { + let cases = [ + UpstreamProxyLaunch { + url: "http://proxy.example:3128".to_string(), + ..upstream_proxy() + }, + UpstreamProxyLaunch { + url: "http://operator:private-proxy-password@proxy.example:3128".to_string(), + ..upstream_proxy() + }, + UpstreamProxyLaunch { + credential: Some("private-proxy-password".to_string()), + ..upstream_proxy() + }, + UpstreamProxyLaunch { + credential: Some(format!("operator:{}", "x".repeat(4096))), + ..upstream_proxy() + }, + UpstreamProxyLaunch { + ca_bundle_pem: Some("private-proxy-password".to_string()), + ..upstream_proxy() + }, + ]; + for proxy in cases { + let error = proxy.validate().unwrap_err().to_string(); + assert!(!error.contains("private-proxy-password"), "{error}"); + } + let acknowledged = UpstreamProxyLaunch { + url: "http://proxy.example:3128".to_string(), + auth_allow_insecure: true, + ..upstream_proxy() + }; + acknowledged.validate().unwrap(); + } + #[test] fn assignments_must_agree_with_their_launch_material() { let mismatched = SandboxInstanceAssignment { diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index f8be6d1439..0f1bbe48e1 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -789,6 +789,8 @@ impl SharedHost { let trust_dir = dir.join("trust"); crate::sandbox_instance_host::prepare_private_dir(&trust_dir).map_err(failure)?; let ssh_socket = dir.join("ssh.sock"); + let upstream_proxy_args = + stage_upstream_proxy(registration.upstream_proxy.as_ref(), dir).map_err(failure)?; let config = crate::SandboxInstanceConfig { app_router: self .app_listener @@ -809,11 +811,7 @@ impl SharedHost { policy_data: None, ssh_socket_path: Some(ssh_socket.to_string_lossy().into_owned()), shared_ssh_socket: false, - // The Kubernetes driver refuses the gateway topology when an - // upstream proxy is configured, - // and it configures no extra CA for dedicated supervisors. - upstream_proxy_args: - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + upstream_proxy_args, additional_ca_bundle: None, backend_descriptor: descriptor, auth_bundle: registration.auth, @@ -924,6 +922,40 @@ fn takeover_term( }) } +/// Materialize only this generation's operator settings. No host environment +/// or workload-supplied filesystem path participates in proxy selection. +fn stage_upstream_proxy( + proxy: Option<&openshell_supervisor_control::UpstreamProxyLaunch>, + dir: &Path, +) -> Result { + use openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs; + + let Some(proxy) = proxy else { + return Ok(UpstreamProxyArgs::default()); + }; + proxy + .validate() + .map_err(|error| miette::miette!("{error}"))?; + let stage = |name: &str, contents: Option<&str>| -> Result> { + contents + .map(|contents| { + let path = dir.join(name); + write_private_file(&path, contents.as_bytes())?; + Ok(path.to_string_lossy().into_owned()) + }) + .transpose() + }; + Ok(UpstreamProxyArgs { + https_proxy: Some(proxy.url.clone()), + no_proxy: proxy.no_proxy.clone(), + proxy_auth_file: stage("upstream-proxy-auth", proxy.credential.as_deref())?, + proxy_auth_allow_insecure: proxy.auth_allow_insecure, + proxy_connect_by_hostname: proxy.connect_by_hostname, + proxy_ca_bundle: stage("upstream-proxy-ca.pem", proxy.ca_bundle_pem.as_deref())?, + ..UpstreamProxyArgs::default() + }) +} + #[cfg(unix)] fn write_private_file(path: &Path, contents: &[u8]) -> Result<()> { use miette::{IntoDiagnostic as _, WrapErr as _}; @@ -1659,6 +1691,75 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn upstream_proxy_files_and_settings_are_private_to_each_instance() { + use openshell_supervisor_control::UpstreamProxyLaunch; + use std::os::unix::fs::PermissionsExt as _; + + let root = tempfile::tempdir().unwrap(); + for name in ["first", "second"] { + let dir = root.path().join(name); + crate::sandbox_instance_host::prepare_private_dir(&dir).unwrap(); + let proxy = UpstreamProxyLaunch { + url: format!("https://{name}.example:3130"), + no_proxy: Some(format!("{name}.internal")), + credential: Some(format!("{name}:private-password")), + auth_allow_insecure: false, + connect_by_hostname: true, + ca_bundle_pem: None, + }; + let args = stage_upstream_proxy(Some(&proxy), &dir).unwrap(); + assert_eq!(args.https_proxy.as_deref(), Some(proxy.url.as_str())); + assert_eq!(args.no_proxy, proxy.no_proxy); + assert!(args.proxy_connect_by_hostname); + let credential_path = Path::new(args.proxy_auth_file.as_ref().unwrap()); + assert_eq!(credential_path.parent(), Some(dir.as_path())); + assert_eq!( + std::fs::read_to_string(credential_path).unwrap(), + proxy.credential.unwrap() + ); + assert_eq!( + std::fs::metadata(credential_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert!( + stage_upstream_proxy( + Some(&UpstreamProxyLaunch { + credential: Some("replacement:credential".to_string()), + ..proxy + }), + &dir + ) + .is_err() + ); + } + let direct = stage_upstream_proxy(None, root.path()).unwrap(); + assert!(direct.https_proxy.is_none()); + assert!(direct.proxy_auth_file.is_none()); + assert!(direct.proxy_ca_bundle.is_none()); + } + + #[test] + fn invalid_upstream_proxy_fails_before_staging_files() { + let root = tempfile::tempdir().unwrap(); + let proxy = openshell_supervisor_control::UpstreamProxyLaunch { + url: "http://proxy.example:3128".to_string(), + no_proxy: None, + credential: Some("operator:private-password".to_string()), + auth_allow_insecure: false, + connect_by_hostname: false, + ca_bundle_pem: None, + }; + let error = stage_upstream_proxy(Some(&proxy), root.path()).unwrap_err(); + assert!(!error.to_string().contains("private-password")); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } + /// A hosted instance whose stop path reports on `stopped`. fn admit_supervising(host: &SharedHost, route: &str, stopped: tokio::sync::mpsc::Sender<()>) { let bindings = crate::SandboxInstanceBindings { diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 92fc803b1d..3c4b66933b 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -498,6 +498,11 @@ Disabling autoscaling renders `spec.replicas` from `replicaCount` again, which defaults to 1, so set `replicaCount` to the replica count you want in that upgrade. +The shared topology supports `upstreamProxy` for approved TLS egress through an +operator-managed HTTP CONNECT proxy. See +[Keep an Upstream Egress Proxy](https://docs.nvidia.com/openshell/latest/kubernetes/high-availability#keep-an-upstream-egress-proxy) +for configuration and upgrade requirements. + Set `supervisor.autoscaling.enabled=true` with the shared supervisor topology to autoscale its Deployment independently of the gateway. It uses average memory per Pod (`supervisor.autoscaling.targetMemoryAverageValue`, default diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 24f531a74a..47ed236293 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -499,6 +499,11 @@ Disabling autoscaling renders `spec.replicas` from `replicaCount` again, which defaults to 1, so set `replicaCount` to the replica count you want in that upgrade. +The shared topology supports `upstreamProxy` for approved TLS egress through an +operator-managed HTTP CONNECT proxy. See +[Keep an Upstream Egress Proxy](https://docs.nvidia.com/openshell/latest/kubernetes/high-availability#keep-an-upstream-egress-proxy) +for configuration and upgrade requirements. + Set `supervisor.autoscaling.enabled=true` with the shared supervisor topology to autoscale its Deployment independently of the gateway. It uses average memory per Pod (`supervisor.autoscaling.targetMemoryAverageValue`, default diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 975213633b..c7edc2c715 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -855,9 +855,6 @@ Validate chart values that Helm would otherwise accept silently. {{- if ne (get $gateway "compute_driver" | default "kubernetes") "kubernetes" -}} {{- fail "supervisor_topology requires the kubernetes compute driver" -}} {{- end -}} -{{- if or (get $kubernetes "https_proxy") (get $kubernetes "proxy_ca_bundle") (get $kubernetes "proxy_auth_secret_name") -}} -{{- fail "supervisor_topology=gateway does not support upstream proxy settings" -}} -{{- end -}} {{- if .Values.server.disableTls -}} {{- fail "supervisor_topology=gateway requires gateway TLS; the supervisor token travels only over TLS" -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 8af91e1901..8788f35957 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -124,6 +124,15 @@ rules: resources: ["networkpolicies"] verbs: ["create", "get"] {{- end }} + {{- $proxyAuthSecret := get $kubernetesConfig "proxy_auth_secret_name" | default "" }} + {{- if and (ne $workspaceMode "shared") (include "openshell.supervisorEnabled" .) $proxyAuthSecret }} + # The configured proxy Secret is read from each sandbox's namespace. + # resourceNames limits this cluster-wide permission to that exact name. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: [{{ $proxyAuthSecret | quote }}] + verbs: ["get"] + {{- end }} {{- if eq $workspaceMode "managed" }} # Operator-mode namespaces receive Secret permissions from the # openshell-workspace chart Role instead. diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 85d2e954d4..0a2203dae4 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -67,6 +67,14 @@ rules: # Bootstrap Secrets are generation-scoped. Recovery deletes them by exact # name. verbs: ["create", "delete"] + {{- $proxyAuthSecret := get $kubernetesConfig "proxy_auth_secret_name" | default "" }} + {{- if and (include "openshell.supervisorEnabled" .) $proxyAuthSecret }} + # Shared instances snapshot the operator's proxy credential at creation. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: [{{ $proxyAuthSecret | quote }}] + verbs: ["get"] + {{- end }} - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] diff --git a/deploy/helm/openshell/tests/shared_proxy_rbac_test.yaml b/deploy/helm/openshell/tests/shared_proxy_rbac_test.yaml new file mode 100644 index 0000000000..b622b96a74 --- /dev/null +++ b/deploy/helm/openshell/tests/shared_proxy_rbac_test.yaml @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: Shared supervisor proxy Secret reads +templates: + - templates/role.yaml + - templates/clusterrole.yaml +release: + name: openshell + namespace: gateway + +tests: + - it: grants the configured proxy Secret read in the shared sandbox namespace + template: templates/role.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + upstreamProxy: + url: https://proxy.example:3128 + authSecret: + name: proxy-auth + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["proxy-auth"] + verbs: ["get"] + + - it: keeps shared workspace proxy reads out of the ClusterRole + template: templates/clusterrole.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + upstreamProxy: + url: https://proxy.example:3128 + authSecret: + name: proxy-auth + asserts: + - notExists: + path: rules[*].resources[?(@ == "secrets")] + + - it: keeps dedicated supervisor proxy credentials unreadable by the gateway + template: templates/role.yaml + set: + upstreamProxy: + url: https://proxy.example:3128 + authSecret: + name: proxy-auth + asserts: + - notExists: + path: rules[*].resourceNames + + - it: uses the effective proxy Secret name for managed namespaces + template: templates/clusterrole.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + supervisor_topology: gateway + proxy_auth_secret_name: schema-proxy-auth + upstreamProxy: + url: https://proxy.example:3128 + authSecret: + name: legacy-proxy-auth + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["schema-proxy-auth"] + verbs: ["get"] + + - it: grants operator mode only the named proxy Secret read + template: templates/clusterrole.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator + operator_namespace_label: workspaces.example/team=agents + supervisor_topology: gateway + https_proxy: https://proxy.example:3128 + proxy_auth_secret_name: proxy-auth + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["proxy-auth"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index 4dbba473ff..fd915d95dd 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -937,17 +937,34 @@ tests: asserts: - failedTemplate: errorMessage: shared_supervisor is rendered by the chart; set only supervisor_topology - - it: rejects settings the shared supervisor cannot isolate - template: templates/supervisor.yaml + - it: accepts upstream proxy routing with shared supervisors + template: templates/gateway-config.yaml set: upstreamProxy: - url: http://proxy.example:3128 + url: https://proxy.example:3130 + noProxy: internal.example + connectByHostname: true + authSecret: + name: egress-auth + key: credentials + caBundle: + configMapName: egress-ca gatewayConfig: openshell.drivers.kubernetes: supervisor_topology: gateway asserts: - - failedTemplate: - errorMessage: supervisor_topology=gateway does not support upstream proxy settings + - matchRegex: + path: data["gateway.toml"] + pattern: 'https_proxy = "https://proxy.example:3130"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'proxy_auth_secret_name = "egress-auth"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'proxy_ca_bundle = ".*/ca.crt"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_topology = "gateway"' - it: rejects workspace until implemented template: templates/supervisor.yaml set: diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 57c4da5c0b..addd351819 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1600,17 +1600,20 @@ supervisor_image_pull_policy = "if_not_present" # https_proxy = "http://proxy.corp.example:8080" # no_proxy = ".svc,.svc.cluster.local,10.96.0.0/12,10.244.0.0/16" # Proxy credentials must be an existing Secret in the sandbox namespace. The -# key contains a `user:pass` value and is mounted only in the network -# supervisor container, never in workload environment or command arguments. +# key contains a `user:pass` value. The supervisor receives the credential in +# a private file outside the workload's environment and command arguments. # proxy_auth_secret_name = "corporate-proxy-auth" # proxy_auth_secret_key = "credentials" # The gateway validates the Secret name/key syntax and their configuration -# relationship at startup; it does not read the Secret from the Kubernetes API. -# Kubernetes resolves the Secret when the Sandbox Pod starts. A missing key or -# Secret prevents that Pod from starting; unreadable or malformed `user:pass` -# content is validated fail-closed by the supervisor at startup and never -# falls back to direct egress. -# Proxy credentials mount only in the separately scheduled supervisor Pod. +# relationship at startup. Dedicated supervisors mount the Secret; a missing +# Secret/key blocks their Pod, and malformed contents fail supervisor startup. +# With supervisor_topology = "gateway", the gateway reads and validates the +# credential from the sandbox namespace and stores it with the generation's +# private launch material. Pool members restore it in each instance's private +# directory, including after a handoff. Restart the sandbox generation after +# changing proxy settings, credentials, or trust. Upgrade the gateway and all +# pool members before enabling proxy settings. Both topologies refuse invalid +# credentials without a direct fallback. # Required with a credential Secret when the proxy URL is http://, because # Basic authentication is then cleartext on the connection to that proxy. An # https:// proxy carries the credential inside the verified TLS session and diff --git a/docs/kubernetes/high-availability.mdx b/docs/kubernetes/high-availability.mdx index 12c26c6fb9..55239406d1 100644 --- a/docs/kubernetes/high-availability.mdx +++ b/docs/kubernetes/high-availability.mdx @@ -612,6 +612,46 @@ more supervisors at once, set `supervisor.podDisruptionBudget.enabled` to helm upgrade openshell -n openshell --reuse-values --set supervisor.podDisruptionBudget.enabled=false ``` +### Keep an Upstream Egress Proxy + +Shared supervisors accept the same `upstreamProxy` settings as dedicated +supervisors. Use this when approved TLS traffic must pass through an HTTP +CONNECT proxy while applications keep their destination URLs and sandbox +policies. + +Add the proxy settings to the topology values above: + +```yaml +upstreamProxy: + url: https://proxy.example.test:3128 + noProxy: "" + caBundle: + configMapName: egress-proxy-ca +``` + +Replace the URL with your proxy listener. The CA ConfigMap belongs in the +gateway release namespace; include it when the proxy uses a private CA or +re-signs destination certificates. The CA bundle counts toward the 256 KiB +limit for each shared sandbox's serialized launch material. Include only the +proxy's CA certificates. + +If proxy authentication is required, set `authSecret.name` and +`authSecret.key` for an existing credential Secret in each sandbox namespace. +For an authenticated `http://` proxy, also set `authAllowInsecure: true`. + +The gateway needs `get` access to the configured credential Secret in each +sandbox namespace. The chart grants that exact Secret name in the shared +namespace Role. Managed and operator workspace modes grant it through the +ClusterRole across namespaces. If you manage RBAC separately, grant the read +only in the sandbox namespaces the gateway owns. + +Upgrade the gateway and every pool member before enabling proxy settings. +Each sandbox generation retains its proxy credentials and trust through a +handoff. Restart the generation after changing proxy settings, credentials, or +trust. Requests selected for proxying fail if the proxy is unavailable. These +settings route TLS traffic. +`noProxy` selects direct exceptions, and plain HTTP egress connects directly. + ### Recover from a Supervisor Failure When a supervisor fails, the gateway moves its sandboxes to a live supervisor. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index c9f936818e..57d38b5334 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -379,10 +379,11 @@ upstreamProxy: ``` -Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. +Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. Use an `https://` URL to encrypt the connection to the proxy. Set `upstreamProxy.caBundle.configMapName` when the proxy uses a private CA or re-signs destination certificates. Plain HTTP egress connects directly. -The credential mounts only in the separately scheduled supervisor Pod. The -sandbox workload cannot read it through its environment or volumes. +Dedicated supervisor Pods mount the credential in a private volume. For shared +supervisors, the gateway reads the Secret and delivers it in the sandbox +generation's private launch material. See [Keep an Upstream Egress Proxy](high-availability.mdx#keep-an-upstream-egress-proxy). ## RBAC diff --git a/e2e/rust/tests/kubernetes_corporate_proxy.rs b/e2e/rust/tests/kubernetes_corporate_proxy.rs index 0dc586c53c..ca63e45aba 100644 --- a/e2e/rust/tests/kubernetes_corporate_proxy.rs +++ b/e2e/rust/tests/kubernetes_corporate_proxy.rs @@ -17,6 +17,10 @@ //! gateway read, bootstrap-Secret key, supervisor mount, `--upstream-proxy-ca-bundle` //! argument — because the CONNECT handshake cannot succeed unless every link //! works. +//! +//! `mise run e2e:kubernetes:shared-supervisor-proxy` runs that same authenticated, +//! private-CA test with gateway topology, covering generation launch material +//! and private per-instance staging instead of a dedicated Pod's mounts. use std::io::Write as _; @@ -53,7 +57,7 @@ def wrap(sock): }; format!( r#"{tls_setup} -import base64, select, socket, threading +import base64, select, socket, ssl, threading expected = 'Basic ' + base64.b64encode(b'{USER}:{PASS}').decode() def head(c): data=b'' @@ -98,7 +102,10 @@ while True: c,_=s.accept() try: c=wrap(c) except OSError as e: - print('handshake-failed %s' % e, flush=True); c.close(); continue + # The host fixture's readiness probe opens and closes a plain TCP socket. + if not isinstance(e, ssl.SSLEOFError): + print('handshake-failed %s' % e, flush=True) + c.close(); continue threading.Thread(target=handle,args=(c,),daemon=True).start() "# ) @@ -213,6 +220,11 @@ async fn kubernetes_corporate_proxy_uses_secret_and_never_falls_back() { .await .expect("start TLS upstream") }; + // A live denied destination detects both proxy forwarding and direct + // fallback; a closed port would hide an unauthorized direct dial. + let denied_upstream = HostSupportContainer::start_python(&tls_upstream_script(), FIXTURE_PORT) + .await + .expect("start policy-denied TLS upstream"); let mut policy = NamedTempFile::new().expect("create policy file"); policy @@ -225,7 +237,7 @@ async fn kubernetes_corporate_proxy_uses_secret_and_never_falls_back() { .expect("UTF-8 policy path") .to_string(); let allowed = format!("https://{HOST_ALIAS}:{}/", upstream.port); - let denied = format!("https://{HOST_ALIAS}:9/"); + let denied = format!("https://{HOST_ALIAS}:{}/", denied_upstream.port); let script = format!( "import ssl, urllib.request\nctx=ssl._create_unverified_context()\n\n\ndef fetch(url):\n try:\n with urllib.request.urlopen(url,timeout=30,context=ctx) as r: return 'ok:'+r.read().decode()\n except Exception as e: return 'err:'+str(e)\nprint('ALLOWED '+fetch({allowed:?}),flush=True)\nprint('DENIED '+fetch({denied:?}),flush=True)" ); @@ -233,15 +245,15 @@ async fn kubernetes_corporate_proxy_uses_secret_and_never_falls_back() { .await .expect("create sandbox through corporate proxy"); let output = &sandbox.create_output; + let logs = proxy.logs().expect("read proxy logs"); assert!( output.contains(MARKER), - "approved HTTPS request must reach upstream through the proxy:\n{output}" + "approved HTTPS request must reach upstream through the proxy:\n{output}\nProxy logs:\n{logs}" ); assert!( - output.contains("DENIED err:") && output.contains("403"), - "policy-denied request must fail closed with 403:\n{output}" + output.contains("DENIED err:"), + "request to the live policy-denied destination must fail closed:\n{output}" ); - let logs = proxy.logs().expect("read proxy logs"); if mode == "no-proxy" { assert!( !logs.contains("CONNECT"), @@ -267,7 +279,7 @@ async fn kubernetes_corporate_proxy_uses_secret_and_never_falls_back() { ); } assert!( - !logs.contains(":9 ") && !logs.contains(":9\n"), + !logs.contains(&format!(":{} ", denied_upstream.port)), "denied request must never reach proxy:\n{logs}" ); } diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 4611fd137d..5a4e7029c3 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1631,14 +1631,23 @@ if [ "${OPENSHELL_E2E_KUBE_CORPORATE_PROXY:-0}" = "1" ]; then if [ "${CORPORATE_PROXY_MODE}" = "https-ca" ]; then CORPORATE_PROXY_SCHEME="https" fi + # Shared supervisors don't inherit the workload's host aliases. Use the + # already detected host IP for the operator's proxy connection. + CORPORATE_PROXY_HOST="${HOST_GATEWAY_IP}" + if [[ "${CORPORATE_PROXY_HOST}" == *:* ]]; then + CORPORATE_PROXY_HOST="[${CORPORATE_PROXY_HOST}]" + fi + # The base fixture uses canonical fields, which take precedence over the + # legacy upstreamProxy aliases. Override those same fields for each run. cat >"${CORPORATE_PROXY_VALUES}" <>"${CORPORATE_PROXY_VALUES}" <>"${CORPORATE_PROXY_VALUES}" < get secret -o jsonpath='{.data.upstream-proxy-ca\.pem}' | head -c 20 ``` -An empty last command with `proxy_ca_bundle` set in the rendered TOML means the -bundle never reached the sandbox; check the gateway logs for a `proxy_ca_bundle` -error, which fails closed at startup rather than falling back to direct egress. - -The proxy arguments and credential mount belong only to the separate supervisor -Pod. The workload Pod must never receive them. The credential is projected +For a dedicated supervisor, an empty last command with `proxy_ca_bundle` set in +the rendered TOML means the bundle never reached the supervisor. Check the +gateway logs for a `proxy_ca_bundle` error. + +With `supervisor_topology=gateway`, the gateway reads the credential Secret +from the sandbox namespace and retains its value and the proxy CA in that +generation's private launch material. Pool members stage files in each +instance's private directory. A handoff restores the same values. Restart the +sandbox generation after changing proxy settings, credentials, or trust. +Upgrade the gateway and every pool member before enabling proxy settings. +Missing or invalid credentials fail sandbox creation. Inspect gateway and +shared-supervisor logs for the error. Avoid dumping launch material because it +contains credentials. + +For dedicated supervisors, proxy arguments and the credential mount belong only +to the separate supervisor Pod. The workload Pod must never receive credentials +or bootstrap secrets. The credential is projected read-only as `openshell-upstream-proxy-auth` at `/run/openshell/upstream-proxy-auth` and passed by file path; it must never appear in environment variables, annotations, or command arguments. diff --git a/tasks/test.toml b/tasks/test.toml index 07f4986002..70d9d61598 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -376,6 +376,11 @@ run = """ OPENSHELL_E2E_KUBE_EXTRA_VALUES="${extra}${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh """ +["e2e:kubernetes:shared-supervisor-proxy"] +description = "Verify shared-supervisor CONNECT proxy credentials, private CA trust and policy denial" +env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_corporate_proxy", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBE_CORPORATE_PROXY = "1", OPENSHELL_E2E_KUBE_CORPORATE_PROXY_MODE = "https-ca", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes" } +run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor.yaml:deploy/helm/openshell/ci/values-tls-port-forward.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' + ["e2e:kubernetes:shared-supervisor-scaling"] description = "Run Kubernetes e2e that scales a one-member shared supervisor pool to two members and back to one with Helm, checking sandboxes are handed off, not restarted" env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor_scaling", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor-scaling" }