diff --git a/Cargo.lock b/Cargo.lock index 87021d5eea..9bb6846e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4040,6 +4040,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -4048,6 +4049,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_json", + "tar", "temp-env", "thiserror 2.0.18", "tokio", @@ -4058,6 +4060,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "uuid", ] [[package]] diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0396f51bc0..a502589dde 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -5,6 +5,15 @@ gateway. A supported runtime provisions `openshell-sandbox` inside the workload, `openshell-supervisor` outside it, a protected channel between them, and an independent outer network fence. Drivers do not implement policy evaluation. +Podman provisions a paired workload and supervisor container using its native +libpod API. The workload uses `network=none`; the external supervisor alone joins +the configured network. A per-sandbox named volume carries their mutually +authenticated gRPC Unix socket, with supervisor credentials kept in its separate +filesystem. Both containers run as the resolved non-root identity with all +capabilities dropped. They share only a user namespace for volume ownership, +not PID, mount, or network namespaces. Podman owns paired lifecycle and health; +the common protocol owns process, identity, TCP, DNS, and forwarding semantics. + ## Driver Contract Each runtime receives a sandbox spec and canonical policy from the gateway and diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 8b3e014e8c..cd463b86fe 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -17,6 +17,9 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } openshell-otel = { path = "../openshell-otel" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +tar = "0.4" +uuid = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 567abcbfcd..9a0f96475f 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -1,463 +1,61 @@ -# Rootless Podman Networking +# Podman Networking -Deep-dive into how networking works in the Podman compute driver when running -rootless with pasta as the network backend. Covers the external tooling -(Podman, Netavark, pasta, aardvark-dns), the three nested namespace layers, and -the complete data paths for SSH, outbound traffic, and supervisor-to-gateway -communication. - -For the general Podman driver architecture, lifecycle, API surface, and driver -comparison, see [README.md](README.md). - -## Component Stack - -Podman's networking is composed of four independent projects: - -| Component | Language | Role | -|---|---|---| -| Podman | Go | Container runtime; orchestrates network lifecycle. | -| Netavark | Rust | Network backend; creates interfaces, bridges, firewall rules. | -| aardvark-dns | Rust | Authoritative DNS server for container name resolution. | -| pasta, part of passt | C | User-mode networking; L2-to-L4 socket translation for rootless containers. | - -The key split: rootful containers default to Netavark bridge networking with -real kernel interfaces, while rootless containers commonly use pasta user-mode -networking without needing host privileges. - -## How Netavark Works - -Netavark is invoked by Podman as an external binary. It reads a JSON network -configuration from STDIN and executes one of three commands: - -- `netavark setup ` creates interfaces, assigns IPs, and sets up - firewall rules for NAT and port-forwarding. -- `netavark teardown ` reverses setup and removes interfaces and - firewall rules. -- `netavark create` takes a partial network config and completes it by - assigning subnets and gateways. - -For rootful bridge networking: - -1. Podman creates a network namespace for the container. -2. Podman invokes `netavark setup` with the network config JSON. -3. Netavark creates a bridge, such as `podman0`, if it does not exist. The - default subnet is `10.88.0.0/16`. -4. Netavark creates a veth pair. One end goes into the container's netns and - the other attaches to the bridge. -5. Netavark assigns an IP from the subnet to the container's veth interface. -6. Netavark configures iptables or nftables rules for masquerade and port - mappings. -7. Netavark starts aardvark-dns when DNS is enabled, listening on the bridge - gateway address. - -```text -Host Kernel - | - +-- Bridge interface, such as "podman0" - | | - | +-- veth pair endpoint, host side, container 1 - | +-- veth pair endpoint, host side, container 2 - | - +-- Host physical interface, such as eth0 - | - +-- NAT, iptables or nftables rules managed by Netavark -``` - -Netavark also supports macvlan networks, where the container gets a -sub-interface of a physical host NIC with its own MAC address, and external -plugins via a documented JSON API. - -## How Pasta Works - -Unprivileged users cannot create network interfaces on the host. They cannot -create veth pairs, bridges, or iptables rules. Netavark's bridge approach -cannot work directly for rootless containers without an additional rootless -networking layer. - -Pasta, part of the `passt` project, operates in userspace and translates -between the container's L2 TAP interface and the host's L4 sockets. It requires -no capabilities or privileges. - -```text -Container Network Namespace - | - +-- TAP device, such as "eth0" - | ^ - | | L2 frames, Ethernet - | v - +-- pasta process, userspace - | - | Translation: L2 frames <-> L4 sockets - | - v - Host Network Stack, native TCP/UDP/ICMP sockets -``` - -For an outbound TCP connection from a container: - -1. The application calls `connect()` to an external address. -2. The kernel routes the packet through the default gateway to the TAP device. -3. Pasta reads the raw Ethernet frame from the TAP file descriptor. -4. Pasta parses L2/L3/L4 headers and identifies the TCP SYN. -5. Pasta opens a native TCP socket on the host and calls `connect()` to the - same destination. -6. When the host socket connects, pasta reflects the SYN-ACK back through the - TAP as an L2 frame. -7. For ongoing data transfer, pasta translates between TAP frames and the host - socket, coordinating TCP windows and acknowledgments between the two sides. - -Pasta does not maintain per-connection packet buffers. It reflects observed -sending windows and ACKs directly between peers. This is a thinner translation -layer than a full TCP/IP stack. - -### Built-in Services - -Pasta includes minimal network services so the container stack can -auto-configure: - -| Service | Purpose | -|---|---| -| ARP proxy | Resolves the gateway address to the host's MAC address. | -| DHCP server | Hands out a single IPv4 address, usually matching the host's upstream interface. | -| NDP proxy | Handles IPv6 neighbor discovery and SLAAC prefix advertisement. | -| DHCPv6 server | Hands out a single IPv6 address, usually matching the host's upstream interface. | - -By default there is no NAT. Pasta copies the host's IP addresses into the -container namespace. - -### Local Connection Bypass - -For connections between the container and the host, pasta implements a local -bypass path: - -- Packets with a local destination skip L2 translation. -- TCP uses `splice(2)`. -- UDP uses `recvmmsg(2)` and `sendmmsg(2)`. - -### Port Forwarding - -By default, pasta uses auto-detection. It scans `/proc/net/tcp` and -`/proc/net/tcp6` periodically and automatically forwards ports that are bound -and listening. Port forwarding is configurable through pasta options. - -### Security Properties - -Pasta is designed for rootless use: - -- No dynamic memory allocation after startup. -- All capabilities dropped, except `CAP_NET_BIND_SERVICE` when granted. -- Restrictive seccomp profile. -- Detaches into its own user, mount, IPC, UTS, and PID namespaces. -- No external dependencies beyond libc. - -### Inter-Container Limitation - -Unlike bridge networking, pasta containers are isolated from each other by -default. No virtual bridge connects them. Communication requires port mappings -through the host, pods with a shared network namespace, or opting into rootless -Netavark bridge networking with `podman network create`. - -## Three Nested Namespaces - -The Podman compute driver creates three layers of network isolation: +Only the external supervisor has external network connectivity. The workload +container uses `network=none`; its loopback DNS relay and TCP socket mediation +reach the supervisor through a protected Unix socket, not a veth or proxy +environment variable. ```text -Namespace 1: Host - | - pasta manages port forwarding, such as 127.0.0.1: - gateway listens on its configured bind address and port - | -Namespace 2: Rootless Podman network namespace, managed by pasta - | - Bridge "openshell", often 10.89.x.0/24 - aardvark-dns for container name resolution - | - Container netns - supervisor, proxy, and relay client run here - | -Namespace 3: Inner sandbox netns, created by supervisor - | - veth pair, such as 10.200.0.1 <-> 10.200.0.2 - nftables forces ordinary traffic through proxy - user workload runs here +workload container supervisor container +agent -> sandbox -- private UDS / gRPC -> policy proxy -> Podman network -> destination + | + +-- authenticated gateway callback ``` -Pasta bridges namespace 1 and 2. The veth pair bridges namespace 2 and 3. The -proxy at the boundary of namespace 2 and 3 enforces network policy. +## Outer network fence -### Layer 1 Pasta +The driver creates the workload without networks, host aliases, published +ports, or added capabilities. It checks the Podman inspect response before +launch and restart. The sandbox installs seccomp mediation and Landlock before +executing the agent. It does not create network namespaces, configure nftables, +or require `CAP_NET_ADMIN`. -At driver startup, the driver ensures a Podman bridge network exists: +TCP opens, TCP byte streams, DNS requests/replies, and lifecycle operations +share the authenticated gRPC channel. DNS is resolved and authorized by the +supervisor. General UDP is unsupported. -```rust -client.ensure_network(&config.network_name).await?; -``` +## Supervisor callback network -This creates a bridge network named `openshell` by default, with DNS enabled. -In rootless mode, this bridge can exist inside a user namespace managed by -pasta. The bridge IP range is not reliably routable from the host. - -```text -Host - | - 127.0.0.1:, pasta binds this on the host - | - pasta process, translates L4 sockets <-> L2 TAP frames - | - rootless network namespace - | - Bridge "openshell", such as 10.89.1.0/24 - | - +-- 10.89.1.1, bridge gateway and aardvark-dns - | - +-- veth to container netns - | - 10.89.1.2, container IP -``` - -### Layer 2 Container Networking - -The container spec configures: - -- `nsmode: "bridge"` to use the Podman bridge network. -- `networks` to attach to the configured bridge, `openshell` by default. -- `portmappings` with `host_port: 0`, `container_port: 2222`, and `protocol: - "tcp"` to publish the SSH compatibility port on an ephemeral host port. -- `hostadd` entries for `host.containers.internal` and - `host.openshell.internal`, using Podman's `host-gateway` resolver or the - configured `host_gateway_ip`. - -Pasta is not explicitly configured by the driver. The driver requests bridge -mode and logs the network backend that Podman reports at startup. - -The `host.containers.internal` hostname is injected into `/etc/hosts` so the -supervisor can reach the gateway on the host. Linux defaults to -`host-gateway`; macOS Podman machine defaults to `192.168.127.254`, gvproxy's -host-loopback IP, because older Podman machine images can fail to resolve -`host-gateway`. Override this with `host_gateway_ip` or -`OPENSHELL_PODMAN_HOST_GATEWAY_IP` when a Podman machine uses a non-standard -host-loopback address. - -If `OPENSHELL_GRPC_ENDPOINT` is empty, the driver auto-detects: - -```rust -if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = - format!("{scheme}://host.containers.internal:{}", config.gateway_port); -} -``` - -The bridge gateway IP is not a stable substitute in rootless mode because it -can live inside the user namespace rather than on the host. - -Before the gateway binds its serving sockets, the driver reports the callback -listener required by the selected topology: - -- Rootful Linux Podman reports the configured bridge's gateway address exactly. -- Rootless Linux Podman explicitly reporting pasta requests the private IPv4 - source address selected by the host's default route. This avoids guessing - among private interfaces on a multihomed host. -- Rootless Linux Podman reporting slirp4netns, another named helper, or no - helper cannot use a direct local callback listener. The driver fails startup - unless `grpc_endpoint` names an explicitly remote endpoint. Supporting - slirp4netns requires a relay inside Podman's rootless network namespace. -- Podman Machine requests IPv4 loopback because gvproxy terminates the host - forwarding path there. -- An explicitly remote callback endpoint requests no additional local listener. - -On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman -and rootless pasta because the driver maps both local callback aliases to that -literal. Other rootless helpers still fail closed. Podman Machine requests -gateway loopback because its configured address is guest-visible and gvproxy -terminates that route on host loopback. The gateway validates and binds every -accepted callback requirement. If the primary listener covers the requested -address, the gateway reuses it and relies on sandbox JWT authorization to limit -the supervisor's RPCs. Otherwise, it creates an additional listener that -exposes only the gateway's sandbox-callable gRPC methods. Operator, health, -reflection, and HTTP requests must use the primary listener. - -### Layer 3 Inner Sandbox Network Namespace - -Inside the container, the supervisor creates another network namespace for the -user workload: - -```text -Container on the Podman bridge - | - Supervisor process, running in container's default netns - | - +-- Proxy listener at the inner namespace gateway address - | - +-- veth pair - | - +-- Inner network namespace - | - sandbox-side veth address - | - default route -> supervisor-side veth address - | - user code runs here - | - nftables rules: - ACCEPT -> proxy TCP - ACCEPT -> loopback - ACCEPT -> established/related - LOG -> TCP SYN bypass attempts - REJECT -> TCP - LOG -> UDP bypass attempts - REJECT -> UDP -``` - -The supervisor uses `nsenter --net=` rather than `ip netns exec` to avoid sysfs -remount issues that arise under rootless Podman where real host -`CAP_SYS_ADMIN` is unavailable. - -For a policy with explicit `protocol: tcp` endpoints, this same inner namespace -also hosts policy DNS and transparent TCP capture. The supervisor answers only -policy-eligible names with epoch-scoped synthetic addresses, redirects TCP to -those synthetic ranges into its transparent listener, and leaves direct real-IP -dials subject to the terminal bypass fence. The Podman driver advertises this -substrate through its driver-owned runtime capability; sandbox image and policy -environment values cannot opt into it independently. - -The container spec preserves Podman's resolver search domains and options. -Policy DNS captures both UDP and TCP in the inner namespace, so it does not -depend on libc honoring `use-vc` and does not change ordinary short-name -resolution for sandboxes that do not use native TCP. - -A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can -create named network namespaces. In rootless Podman this directory does not -exist on the host, so a private tmpfs gives the supervisor its own writable -`/run/netns` without needing host filesystem access. - -## Complete Data Paths - -### SSH Session - -```text -Client, openshell CLI - | - 1. gRPC: CreateSshSession -> gateway, returns token and connect_path - 2. HTTP CONNECT /connect/ssh to gateway - headers: x-sandbox-id, x-sandbox-token - | -Gateway - | - 3. Looks up SupervisorSession for sandbox_id - 4. Sends RelayOpen{channel_id} over ConnectSupervisor bidi stream - | - gRPC traverses host -> pasta translation -> container bridge - | -Supervisor inside container - | - 5. Receives RelayOpen, opens new RelayStream RPC back to gateway - 6. Sends RelayInit{channel_id} on the stream - 7. Connects to Unix socket /run/openshell/ssh.sock - 8. Bidirectional bridge: RelayStream <-> Unix socket - | -SSH daemon inside container, Unix socket only - | - 9. Authenticates. Access is gated by the relay chain. - 10. Spawns shell process - 11. Shell enters inner netns via setns(fd, CLONE_NEWNET) - | -User shell in sandbox netns -``` - -The SSH daemon listens on a Unix socket with restrictive permissions. The -published TCP port mapping exists in the container spec for compatibility and -health/debug paths. Normal SSH communication uses the gRPC reverse-connect relay -pattern. - -### Outbound HTTP Request - -```text -User code in inner netns - | - 1. curl https://api.example.com - HTTP_PROXY points at the local sandbox proxy - | - 2. TCP connect to proxy - allowed by nftables as the only ordinary egress destination - | - 3. HTTP CONNECT api.example.com:443 - | -Supervisor proxy in container netns - | - 4. Policy evaluation with process identity - 5. SSRF check - 6. Optional L7 TLS intercept and HTTP method/path inspection - | - 7. If allowed, TCP connect to api.example.com:443 - from the container netns - | - 8. Through Podman bridge -> pasta -> host -> internet -``` - -### Supervisor gRPC Callback - -The Podman driver auto-detects the callback endpoint scheme based on whether -TLS client certificates are configured. When the RPM's auto-generated PKI is in -place, the endpoint is `https://host.containers.internal:17670` and the -supervisor connects with mTLS. Without TLS configuration, it falls back to -`http://host.containers.internal:`. - -```text -Supervisor in container netns - | - 1. Connects to host.containers.internal: - with mTLS when OPENSHELL_TLS_* paths are set - | - 2. Routed through container default gateway - | - 3. Pasta translates L2 frame -> host L4 socket when rootless backend uses pasta - | - 4. Host TCP socket connects to gateway - | -Gateway - | - 5. TLS handshake when enabled - 6. ConnectSupervisor bidirectional stream established - 7. Heartbeats at the interval accepted by the gateway - 8. Reconnects with exponential backoff on failure - 9. Same gRPC channel reused for RelayStream calls -``` +The configured `network_name`, host-gateway aliases, upstream corporate proxy, +and published SSH port apply only to the supervisor companion. The gateway's +SSH tunnel still uses the supervisor relay, not the published port. -The gateway binds to `127.0.0.1:17670` by default in the RPM packaging. Client -certificates are auto-generated by `openshell-gateway generate-certs` on first -start and bind-mounted into sandbox containers by the Podman driver. +Rootful Podman uses the configured bridge and its gateway address. Rootless +local callbacks require the existing pasta path; slirp4netns or unknown helpers +require an explicitly remote `grpc_endpoint`. On macOS, Podman Machine provides +the runtime and host-loopback forwarding. -## Differences from the Kubernetes Driver +These runtime-managed network helpers are outside the workload trust boundary. +Sharing the workload's user namespace preserves volume UID/GID mapping; it +does not share the workload's PID, mount, or network namespaces. -| Aspect | Kubernetes | Podman, rootless pasta | -|---|---|---| -| Container or pod IP | Routable cluster-wide | Non-routable from the host in common rootless setups. | -| Network reachability | Pod IPs reachable from gateway | Bridge not reliably routable from host; requires host aliases or published ports. | -| Sandbox to gateway | Direct TCP to Kubernetes service or endpoint | `host.containers.internal` through bridge and rootless backend. | -| SSH transport | Reverse gRPC relay | Reverse gRPC relay. | -| Port publishing | Not needed for relay | Ephemeral host port remains in the container spec for compatibility and debug paths. | -| TLS | mTLS via Kubernetes secrets | mTLS via mounted client files, RPM defaults, or explicit configuration. | -| DNS | Kubernetes CoreDNS | Podman bridge DNS through aardvark-dns when DNS is enabled. | -| Network policy | Kubernetes network policy for pod ingress plus supervisor policy | nftables inside inner sandbox netns plus supervisor policy. | -| Supervisor delivery | Kubernetes driver managed pod image or template | OCI image volume mount. | -| Secrets | Kubernetes Secret volume and env vars | Per-sandbox JWT via Podman secret; TLS client materials from configured host files. | +## Troubleshooting -Both drivers use the same reverse gRPC relay for SSH transport. The most -important Podman-specific difference is network reachability: in rootless -Podman, the bridge network is not reliably routable from the host, so -host-to-container and container-to-host communication must use host aliases, -published ports, or the supervisor relay. +Inspect both containers with the same sandbox-ID label, distinguishing +`openshell.io/isolation-role=sandbox` from +`openshell.io/isolation-role=supervisor`. -## Port Assignments +- Sandbox fails its qualification probe: use its log to identify the denied + kernel/runtime primitive. Do not add capabilities or disable runtime seccomp. +- Sandbox cannot authenticate to supervisor: check the private channel volume, + matching user namespace mappings, and shared SELinux label. +- Supervisor cannot call back: inspect its configured gateway endpoint, + credentials, Podman network, and gateway callback listener. +- DNS or egress denied: inspect supervisor policy decisions. Do not add a + workload network, resolver bypass, or direct gateway route. +- Pair is not Ready: check the supervisor health socket and gateway session. + A running workload container alone does not establish readiness. -| Port | Component | Purpose | -|---|---|---| -| `17670` | Gateway | Default local gRPC and HTTP multiplexed server port. | -| `2222` | Sandbox | Container port mapping default for the SSH compatibility port. | -| `3128` | Sandbox proxy | HTTP CONNECT proxy inside the sandbox network model. | -| `0` | Host | Ephemeral host port requested for the container SSH compatibility port. | +See the [driver overview](README.md) and +[Podman runtime documentation](https://docs.podman.io/en/latest/markdown/podman-run.1.html) +for runtime options. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index e53ddc9f3f..7c33b00c23 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -1,474 +1,132 @@ # openshell-driver-podman -The Podman compute driver manages sandbox containers via the Podman REST API -over a Unix socket. It targets single-machine and developer environments where -rootless container isolation is preferred over a full Kubernetes cluster. The -driver runs in-process within the gateway server and delegates all sandbox -isolation enforcement to the `openshell-sandbox` supervisor binary, which is -sideloaded into each container via an OCI image volume mount. +Image admission checks the reserved `/.openshell` path using archive metadata +only. Downloads of `/etc/passwd` and `/etc/group` are limited to 1 MiB each, +including tar overhead, and extracted file sizes are checked independently. -When the gateway configures `[openshell.gateway.otlp]`, Podman compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-podman`. The driver preserves the gateway trace context and -uses the same compute-driver RPC span names in its in-process and standalone -forms. +The Podman compute driver runs inside the gateway and uses the native libpod +REST API over a Unix socket. Each sandbox has two independent containers: -`mise run gateway:podman` enables this export only when a local collector is -listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration -so the development gateway does not repeatedly report export failures. +- `openshell-sandbox` owns the agent process in the workload container. +- `openshell-supervisor` evaluates policy, holds gateway credentials, and + proxies approved egress in a separate companion container. -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID with pulling disabled, preventing a mutable tag from changing -between inspection and launch. The supervisor runs as root, resolves omitted -policy identity fields from the image declaration, and drops only agent -children to the completed identity. Named OCI components remain names after -validation; a missing group is filled with the user's numeric primary GID. Explicit -`process.run_as_user` and `process.run_as_group` values take precedence -independently. +The driver provisions placement, identity, credentials, transport, and lifecycle. +The shared isolation interface supplies exec, attach, signal, terminate, binary +identity, DNS, TCP, and loopback-forwarding semantics. -For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). +## Runtime posture -## Stop and Start - -Stop stops the managed container without deleting it. The per-sandbox named -workspace volume, token and proxy-auth secrets, labels, and container metadata -remain intact. Start starts the same container and reuses the same named -volume. Stopped managed containers remain visible through list and watch -reconciliation. Delete remains responsible for removing the container, -driver-owned secrets, and workspace volume. - -The stop call waits until Podman reports the container as stopped or exited. -This keeps an immediate start from racing a rootless Podman stop that is still -finishing after its API request returns. - -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -## Architecture - -The Podman driver communicates with the Podman daemon over a Unix socket and -delegates sandbox isolation to the supervisor binary running inside each -container. - -```mermaid -graph TB - CLI["openshell CLI"] -->|gRPC| GW["Gateway Server
(openshell-server)"] - GW -->|in-process| PD["PodmanComputeDriver"] - PD -->|HTTP/1.1
Unix socket| PA["Podman API"] - PA -->|OCI runtime
crun/runc| C["Sandbox Container"] - C -->|image volume
read-only| SV["Supervisor Binary
/opt/openshell/bin/openshell-sandbox"] - SV -->|creates| NS["Nested Network Namespace
veth pair + proxy"] - SV -->|enforces| LL["Landlock + seccomp"] - SV -->|gRPC callback| GW -``` - -## Isolation Model - -The Podman driver provides the same protection layers as the other compute -drivers. The driver itself does not implement isolation primitives directly. It -configures the container so that the `openshell-sandbox` supervisor can enforce -them at runtime. - -### Container Security Configuration - -The container spec in `container.rs` sets these security-critical fields: - -| Setting | Value | Rationale | +| Property | Workload | Supervisor | |---|---|---| -| `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. | -| `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. | -| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP`, `KILL` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, child bounding-set cleanup, and forwarding shutdown signals to a workload that runs as the sandbox user. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. | -| `no_new_privileges` | `true` | Prevents privilege escalation after exec. | -| `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. | -| `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. | -| CDI GPU devices | Opaque `driver_config.cdi_devices` values when set, otherwise the requested count of NVIDIA CDI GPUs selected in round-robin order. Local `/dev/dxg` permits `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. | Exposes requested GPUs to GPU-enabled sandbox containers. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | - -The restricted agent child does not retain these supervisor privileges. - -## Driver Config Mounts - -The gateway forwards the `podman` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Podman -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.podman]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Podman named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. Podman - local-driver volumes created with bind options are treated as host bind - mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. -- `image`: mounts an OCI image through Podman's image-volume API. The driver - pulls the image during provisioning using the sandbox image pull policy. - -Host bind mounts are disabled by default because they expose gateway host paths -to sandbox requests. The driver still uses internal bind mounts for configured -TLS material; per-sandbox gateway JWTs are delivered through Podman secrets. - -Podman `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. User-supplied bind and volume mounts are -read-only by default; set `read_only: false` to make them writable. Podman -image and volume mounts do not support `subpath` in OpenShell driver config. -Mount `source` and `target` values must not contain surrounding whitespace. -Mount targets must be absolute container paths and must not replace -the workspace root (`/sandbox`) or overlap OpenShell supervisor files, -`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. - -Example named-volume usage: - -```shell -podman volume create openshell-work - -openshell sandbox create \ - --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work"}]}}' \ - -- claude -``` - -### Capability Breakdown - -| Capability | Purpose | -|---|---| -| `SYS_ADMIN` | seccomp filter installation, namespace creation, and Landlock setup. | -| `NET_ADMIN` | Network namespace veth setup, IP address assignment, routes, and nftables. | -| `SYS_PTRACE` | Reading `/proc//exe` and walking process ancestry for binary identity. | -| `SYSLOG` | Reading `/dev/kmsg` for bypass-detection diagnostics. | -| `DAC_READ_SEARCH` | Reading `/proc//fd/` across UIDs so the proxy can resolve the binary responsible for a connection. | -| `SETPCAP` | Clearing the restricted child process capability bounding set before exec. | - -The driver intentionally keeps Podman's default `SETUID`, `SETGID`, `CHOWN`, -and `FOWNER` capabilities because the supervisor needs them to drop privileges -and prepare writable sandbox directories. It also keeps `SETPCAP` until child -setup so `drop_privileges()` can clear the child capability bounding set before -exec. It drops unneeded defaults such as -`DAC_OVERRIDE`, `FSETID`, `KILL`, `NET_RAW`, `SETFCAP`, -and `SYS_CHROOT`. - -## Supervisor Sideloading - -The supervisor binary is delivered to sandbox containers via Podman's OCI image -volume mechanism, distinct from both the Kubernetes pod-volume approach and the -VM's embedded guest bundle. - -```mermaid -sequenceDiagram - participant D as PodmanComputeDriver - participant P as Podman API - participant C as Sandbox Container - - D->>P: pull_image(supervisor, "missing") - D->>P: create_container(spec with image_volumes) - Note over P: Podman resolves image_volumes at
libpod layer before OCI spec generation - P->>C: Mount supervisor image at /opt/openshell/bin (read-only) - D->>P: start_container - C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox -``` - -The supervisor image from `deploy/docker/Dockerfile.supervisor` provides the -static `openshell-sandbox` binary at `/openshell-sandbox`. -Mounting that image at `/opt/openshell/bin` makes the binary available as -`/opt/openshell/bin/openshell-sandbox`. - -The container spec sets that binary as the entrypoint. This avoids relying on -the sandbox image entrypoint or command, which might otherwise append the -supervisor path as an argument to an image-provided shell. - -## TLS - -When all three Podman TLS paths are set, the driver treats sandbox callbacks as -mTLS callbacks: - -- `OPENSHELL_PODMAN_TLS_CA` -- `OPENSHELL_PODMAN_TLS_CERT` -- `OPENSHELL_PODMAN_TLS_KEY` - -The driver validates that the TLS paths are provided as a complete set. Partial -configuration fails early instead of silently falling back to plaintext. - -When enabled, the driver: - -1. Switches the auto-detected endpoint scheme from `http://` to `https://`. -2. Bind-mounts the client cert files read-only into the container at - `/etc/openshell/tls/client/`. -3. Sets `OPENSHELL_TLS_CA`, `OPENSHELL_TLS_CERT`, and `OPENSHELL_TLS_KEY` to - the container-side paths. - -The supervisor reads these env vars and uses them to establish an mTLS -connection back to the gateway. On SELinux systems, the bind mounts include -Podman's shared relabel option so the container process can read the files. - -The RPM packaging auto-generates a self-signed PKI on first start via -`openshell-gateway generate-certs`. Client certs are placed in the CLI -auto-discovery directory (`~/.config/openshell/gateways/openshell/mtls/`) so -the CLI connects with mTLS without manual configuration. See -`deploy/rpm/CONFIGURATION.md` for the full RPM configuration reference. - -## Network Model - -Sandbox network isolation uses a two-layer approach: a Podman bridge network -for container-to-host communication, and a nested network namespace created by -the supervisor for sandbox process isolation. - -```mermaid -graph TB - subgraph Host - GW["Gateway Server
127.0.0.1:17670"] - PS["Podman Socket"] - end - - subgraph Bridge["Podman Bridge Network (10.89.x.x)"] - subgraph Container["Sandbox Container"] - SV["Supervisor
(root in user ns)"] - subgraph NestedNS["Nested Network Namespace"] - SP["Sandbox Process
(resolved non-root identity)"] - VE2["veth1: 10.200.0.2"] - end - VE1["veth0: 10.200.0.1
(CONNECT proxy)"] - SV --- VE1 - VE1 ---|veth pair| VE2 - end - end - - GW -.->|SSH via supervisor relay
gRPC session| SV - SV -->|gRPC callback via
host.containers.internal| GW - SP -->|all egress via proxy| VE1 -``` - -Key points: - -- Bridge network: created by `client.ensure_network()` with DNS enabled. - Containers on the bridge can see each other at L3, but sandbox processes - cannot because they are isolated inside the nested netns. -- Nested netns: the supervisor creates a private `NetworkNamespace` with a veth - pair. Sandbox processes enter this netns via `setns(fd, CLONE_NEWNET)` in the - `pre_exec` hook, forcing ordinary traffic through the CONNECT proxy. -- Policy DNS and transparent TCP: the driver advertises the complete - `policy-dns-transparent-tcp` substrate. For explicit `protocol: tcp` - endpoints, the supervisor installs namespace-local DNS listeners, synthetic - routes, and TCP redirect rules before starting the workload. The container - disables Podman's implicit DNS search suffix so policy DNS evaluates the - exact endpoint name requested by the workload, and asks libc to use the - policy DNS TCP listener to avoid rootless Podman's nested UDP NAT return - path. -- Port publishing: the container spec still requests `host_port: 0` for the - configured SSH port. The gateway SSH tunnel uses the supervisor relay rather - than connecting directly to the published port. -- Host gateway: `host.containers.internal` and `host.openshell.internal` are - injected into `/etc/hosts` so containers can reach services on the gateway - host. Linux defaults to Podman's `host-gateway` resolver. macOS Podman - machine defaults to gvproxy's host-loopback IP, `192.168.127.254`, because - stale Podman machines may fail to resolve `host-gateway`. -- nsenter: the supervisor uses `nsenter --net=` instead of `ip netns exec` for - namespace operations, avoiding the sysfs remount path that fails in rootless - containers. - -See [NETWORKING.md](NETWORKING.md) for the rootless Podman networking deep dive. - -## Supervisor Relay - -Podman follows the same end-to-end contract as the Kubernetes and VM drivers -for the in-container SSH relay: gateway config to `PodmanComputeConfig` to -sandbox environment to supervisor session registration on that path. - -1. `openshell-core` `Config::sandbox_ssh_socket_path` is copied into - `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the - in-process driver. -2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that - value, alongside required vars such as `OPENSHELL_ENDPOINT` and - `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template - environment variables to prevent spoofing. -3. The supervisor reads `OPENSHELL_SSH_SOCKET_PATH` and uses it for the Unix - socket the gateway's SSH stack bridges to. - -The standalone `openshell-driver-podman` binary sets the same struct field from -`OPENSHELL_SANDBOX_SSH_SOCKET_PATH`. - -## Credential Injection - -Sandboxes authenticate to the gateway via mTLS using client materials bind- -mounted into the container from a Podman secret. No shared per-request secret -is injected as an environment variable. - -| Credential | Mechanism | Visible in `inspect`? | Visible in `/proc//environ`? | -|---|---|---|---| -| mTLS client cert/key | Bind-mounted file paths (`OPENSHELL_TLS_*` env vars point at them) | Yes (paths only) | Yes (paths only) | -| Sandbox identity | Plaintext env var | Yes | Yes | -| gRPC endpoint | Plaintext env var, override-protected | Yes | Yes | -| Supervisor relay socket path | Plaintext env var, override-protected | Yes | Yes | - -The `build_env()` function inserts user-supplied variables first, then -unconditionally overwrites all security-critical variables to prevent spoofing -via sandbox templates: - -- `OPENSHELL_SANDBOX` -- `OPENSHELL_SANDBOX_ID` -- `OPENSHELL_ENDPOINT` -- `OPENSHELL_SSH_SOCKET_PATH` -- `OPENSHELL_CONTAINER_IMAGE` -- `OPENSHELL_MAIN_PROCESS_SPEC` - -## Sandbox Lifecycle - -### Creation Flow - -```mermaid -sequenceDiagram - participant GW as Gateway - participant D as PodmanComputeDriver - participant P as Podman API - - GW->>D: create_sandbox(DriverSandbox) - D->>D: validate name + id - D->>D: validated_container_name() - - D->>P: pull_image(supervisor, "missing") - D->>P: pull_image(sandbox_image, policy) - - D->>P: create_volume(workspace) - Note over D: On failure below, rollback volume - - D->>P: create_container(spec) - alt Conflict (409) - D->>P: remove_volume - D-->>GW: AlreadyExists - end - Note over D: On failure below, rollback container + volume - - D->>P: start_container - D-->>GW: Ok +| UID/GID | Pinned non-root workload identity | Same mapped identity | +| Capabilities | Drop all; add none | Drop all; add none | +| Seccomp | Runtime default plus sandbox-installed filters | Runtime default | +| Network | `none`; loopback only | Configured Podman network | +| Gateway JWT and upstream credentials | Never mounted | Podman secrets | +| User volumes and CDI devices | Workload only | Never mounted | +| Channel | Private named volume, writable | Same volume, read-only | + +Podman creates the namespaces and volume ownership before the workload runs. +Rootless operation uses the operator's Podman service and subordinate-ID +configuration; it does not require adding capabilities to either container. +The supervisor joins the workload's **user namespace only** to preserve UID/GID +mapping for shared-volume access. PID, mount, and network namespaces remain +separate. The channel volume uses shared SELinux relabeling (`:z`). + +Before starting either container, the driver uploads volume-relative archives +directly to the channel and workspace volume destinations. A rootfs upload on a +stopped Podman container does not populate nested named volumes. Each restart +rotates the generation, session, bootstrap token, and mutual-TLS credentials, +then stages the matching channel bootstrap and supervisor topology while both +containers are stopped. Failed staging leaves both stopped; retry rotates again. +Restart preserves the workspace. The workload starts before the supervisor so its user namespace exists +when the supervisor joins it; a stopped supervisor resolves that namespace again +on its next start. + +Both containers have a private, writable 64 MiB `/run` tmpfs. This lets the +non-root sandbox install `/run/openshell-proxy-ca` without capabilities, an +init process, or an image-provided writable runtime directory. Podman's mount +API does not accept tmpfs `uid`/`gid` options, so the isolated mount uses the +standard sticky world-writable mode instead. + +The runtime must pass the sandbox's unprivileged enforcement probe, including +nested seccomp notification and Landlock. Unsupported runtime defaults fail +closed; do not switch to an unconfined profile or add capabilities. + +## Protected channel and network enforcement + +```text +agent -> openshell-sandbox === authenticated gRPC / private UDS === supervisor -> network + network=none TCP, DNS, control streams policy + gateway JWT ``` -Each step rolls back previously-created resources on failure. The Conflict path -cleans up the volume because it is keyed by the new sandbox's ID, not the -conflicting container's ID. - -### Readiness and Health - -The container `healthconfig` marks the sandbox healthy when any of these -signals succeeds: - -- Legacy marker file `/var/run/openshell-ssh-ready`. -- `test -S` on the configured supervisor Unix socket path. -- The prior TCP check for a listener on the in-container SSH port. - -The Unix socket check allows relay-only readiness when the supervisor exposes -the socket without the old marker or published-port signal. - -### Deletion Flow - -1. Validate `sandbox_name` and stable `sandbox_id` from `DeleteSandboxRequest`. -2. Best-effort inspect cross-checks the container label when present, but - cleanup remains keyed by the request `sandbox_id`. -3. Best-effort stop, ignoring the stop result. -4. Force-remove the container. -5. Remove workspace volume derived from the request `sandbox_id`, warning on - failure and continuing. - -If the container is already gone during inspect or remove, the driver still -performs idempotent volume cleanup using the request `sandbox_id` and -returns `Ok(false)` for the container-delete result. This prevents leaked -Podman resources after out-of-band container removal or label drift. - -## Configuration - -| Environment Variable | CLI Flag | Default | Description | -|---|---|---|---| -| `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | -| `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, or `newer`. | -| `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | -| `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | -| `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | -| `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | -| `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | -| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | -| `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | -| `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_KEY` | `--podman-tls-key` | unset | Host path to the client private key mounted for sandbox mTLS. | -| `OPENSHELL_SANDBOX_HTTPS_PROXY` | `--sandbox-https-proxy` | unset | Corporate forward proxy URL for the supervisor's upstream TLS dials, chained with HTTP CONNECT. Credential-free `http://host:port` and `https://host:port` URLs are supported (scheme and port required). For an `https://` proxy the supervisor TLS-wraps the proxy connection, verifying the proxy certificate against the built-in and system roots plus `--sandbox-proxy-ca-bundle`. Plain-HTTP requests always dial directly. | -| `OPENSHELL_SANDBOX_NO_PROXY` | `--sandbox-no-proxy` | unset | Comma-separated `NO_PROXY` list (hostnames, domain suffixes, IPs, CIDRs, each with an optional `:port` qualifier) dialed directly instead of through the corporate proxy. IP/CIDR entries also match hostnames through their validated DNS resolution. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_FILE` | `--sandbox-proxy-auth-file` | unset | Path to a file containing the proxy credentials as `user:pass`. Staged as a root-only Podman secret so credentials never appear in config or container metadata. Requires the insecure-auth acknowledgement below. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE` | `--sandbox-proxy-auth-allow-insecure` | unset | Explicit acknowledgement (`true`) that the credential is sent as cleartext Basic auth over the plain-TCP connection to the `http://` proxy. Required when the auth file is set with an `http://` proxy; not required for `https://` proxies (the credential travels inside the verified TLS session) but tolerated if set. Rejected when no auth file is configured. | -| `OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME` | `--sandbox-proxy-connect-by-hostname` | unset | Send the destination hostname in CONNECT requests instead of a validated IP. Last resort for proxies whose ACLs filter on hostnames: the proxy then resolves the name itself, so sandbox SSRF/`allowed_ips` validation no longer binds the connection. | -| `OPENSHELL_PODMAN_USERNS` | `--userns` | unset | User namespace mode for sandbox containers (e.g. `auto`). When unset, containers use the default user namespace. | -| `OPENSHELL_SANDBOX_PROXY_CA_BUNDLE` | `--sandbox-proxy-ca-bundle` | unset | Path (on the gateway host) to a PEM CA bundle trusted for the corporate proxy. Bind-mounted read-only into the sandbox (a CA certificate is not secret). Trusted for the `https://` proxy TLS handshake and, because TLS-intercepting proxies re-sign tunneled certificates, folded into the sandbox trust bundle and upstream verification. Requires a proxy URL; the file must exist and hold at least one certificate. | - -Through the gateway, the same settings are the `https_proxy`, `no_proxy`, -`proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, -and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see -`docs/reference/gateway-config.mdx`. - -This is an operator-owned egress boundary: the driver passes the settings on -the supervisor's command line, so sandbox and template environment — and any -`ENV` baked into the sandbox image — cannot override them, and the -conventional `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` variables a sandbox -controls do not steer it. Credentials must be supplied through -`proxy_auth_file`; an inline `user:pass@` in the URL is rejected at startup. - -Basic auth over an `http://` proxy is cleartext on the wire: anyone on the -network path between the sandbox host and the proxy can recover the -credential. Setting `proxy_auth_file` therefore requires -`proxy_auth_allow_insecure = true`; both the driver and the in-container -supervisor reject credentials without that explicit acknowledgement. - -CONNECT requests target a validated resolved IP by default, so the proxy -performs no DNS resolution and the tunnel stays bound to the address that -passed the sandbox's SSRF and `allowed_ips` checks; the hostname still -travels inside the tunnel (TLS SNI, application `Host`). In split-horizon -networks, point the gateway host at the corporate resolver. Set -`proxy_connect_by_hostname = true` only when the proxy's ACLs filter on -hostnames and reject IP CONNECT targets — it re-opens proxy-side DNS -resolution, making the proxy's ACLs the effective egress control. - -## Rootless-Specific Adaptations - -The Podman driver is designed for rootless operation. The following adaptations -matter compared to cluster or rootful runtimes: - -1. subuid/subgid preflight check: on non-macOS hosts, `check_subuid_range()` in - `driver.rs` warns operators if `/etc/subuid` or `/etc/subgid` entries are - missing for the current user. This is not a hard error because some systems - use LDAP or other mechanisms. macOS skips the check because `podman machine` - runs the Podman service inside a Linux VM. -2. cgroups v2 requirement: the driver refuses to start if cgroups v1 is - detected. Rootless Podman requires the unified cgroup hierarchy. -3. `nsenter` for namespace operations: `openshell-sandbox` uses - `nsenter --net=` instead of `ip netns exec` to avoid the sysfs remount path - that requires real `CAP_SYS_ADMIN` in the host user namespace. -4. `DAC_READ_SEARCH` capability: required for the proxy to read - `/proc//fd/` across UIDs within the user namespace. -5. `SETUID` and `SETGID` capabilities: kept from Podman's default capability - set so `drop_privileges()` can call `setuid()` and `setgid()`. -6. `host.containers.internal`: used instead of Docker's `host.docker.internal` - for container-to-host communication. The driver also injects the - OpenShell-owned `host.openshell.internal` alias. -7. Ephemeral port publishing: the SSH compatibility port uses `host_port: 0` - because the bridge network IP is not reliably routable from the host in - rootless mode. -8. tmpfs at `/run/netns`: a private tmpfs lets the supervisor create named - network namespaces via `ip netns add`. - -## Implementation References - -- Gateway integration: `crates/openshell-gateway/src/lib.rs` registers the - driver factory and constructs `PodmanComputeConfig` from the generic server - build context. -- Server configuration: `crates/openshell-server/src/lib.rs` exposes the - backend-agnostic registry and factory context. -- Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in - `crates/openshell-core/src/config.rs`. -- SSRF mitigation: `crates/openshell-core/src/net.rs`, - `crates/openshell-sandbox/src/proxy.rs`, and - `crates/openshell-server/src/grpc/policy.rs`. -- Sandbox supervisor: `crates/openshell-sandbox/src/` for Landlock, seccomp, - netns, proxy, and relay behavior shared by all drivers. -- Container engine abstraction: `tasks/scripts/container-engine.sh` for - build/deploy support across Docker and Podman. -- Supervisor image build: `deploy/docker/Dockerfile.supervisor`. +The workload has no external interface or published port. Seccomp socket +mediation carries TCP and DNS through one authenticated gRPC connection. +DNS remains supervisor-mediated; general UDP is unsupported. The driver sets +`net.ipv4.ip_unprivileged_port_start=0` in the isolated workload network +namespace so the sandbox's loopback DNS relay can bind port 53 without a +capability. No nftables or nested network namespace setup runs in the sandbox. + +The channel contains the sandbox bootstrap and sandbox-side TLS identity only. +Supervisor private keys and topology stay in the companion's private filesystem. +Landlock denies agent access to the top-level `/.openshell` control hierarchy. +The driver verifies Podman's reported `network=none` fence before launch and +restart. `host.containers.internal` and callback networking apply to the +supervisor, not the agent. + +Gateway callbacks use the existing sandbox JWT and optional configured mTLS +bundle. The sandbox/supervisor channel always uses its separate, per-sandbox +mutual TLS material. These are distinct authentication relationships. + +## Identity and trusted binaries + +Both workload and supervisor images are pinned by immutable image ID. The +driver reads account files from a stopped workload-image container; it never +executes the image to resolve an account. Policy identity fields override OCI +`USER` independently. Named users/groups resolve against that image, including +supplementary groups. Root and unresolved identities fail before provisioning. +Images must not prepopulate the reserved `/.openshell` hierarchy; this prevents +image-controlled symlinks from aliasing private control state into user mounts. + +The trusted runtime image supplies `/openshell-sandbox` and +`/openshell-supervisor`. Podman's read-only image volume delivers the sandbox +binary; user-namespace modes that cannot use image volumes retain the existing +trusted binary extraction path. Image and request environment belong to agent +children, never the supervisor process. + +## Lifecycle and readiness + +Create builds both stopped containers and stages the private archives before +starting either container. The sandbox does not execute the agent until the +supervisor authenticates and confirms the common boundary contract. Failed +creation removes only containers created by that attempt, then cleans up +driver-owned volumes and secrets. + +Stop retains both containers, workspace, channel, and secrets. Start restores +the consumed sandbox bootstrap from a copy in the supervisor's private +filesystem, verifies the fence, and starts the same pair. A failed supervisor +start stops the workload. Delete removes the companion first, then the workload, +channel, workspace, and driver-owned secrets. User-owned volumes are retained. + +Only workload containers appear in sandbox list/watch results. Readiness uses +the supervisor's private health socket; there is no shell, legacy marker, or +TCP-listener shortcut. Watch reconciliation and supervisor exit/removal events +stop a running workload whose companion is unavailable. The gateway also +requires the authenticated supervisor session before publishing Ready. + +## Mounts, GPUs, and configuration + +User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain +native Podman features and apply only to the workload. Bind mounts require the +operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace +root cannot be replaced. User-owned volumes are never created or deleted. + +See [gateway configuration](../../docs/reference/gateway-config.mdx) for +operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. +The configured network and upstream proxy belong to the supervisor. +`health_check_interval_secs=0` uses a one-second check rather than disabling +the readiness check required by this topology. + +Gateway OTLP configuration continues to export compute-driver spans under the +`openshell-driver-podman` service, preserving gateway trace context. diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 508b604ce7..8b4da5354b 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -26,6 +26,9 @@ const API_TIMEOUT: Duration = Duration::from_secs(30); /// Maximum allowed size for the event stream line buffer (1 MB). const MAX_EVENT_BUFFER: usize = 1_048_576; +/// Image account databases are metadata, not unrestricted archive downloads. +const MAX_IMAGE_ACCOUNT_BYTES: usize = 1_048_576; + #[derive(Debug, thiserror::Error)] pub enum PodmanApiError { #[error("podman API not found (404): {0}")] @@ -177,6 +180,8 @@ pub struct ImageInspect { pub struct ImageConfig { #[serde(default)] pub user: String, + #[serde(default)] + pub env: Vec, } /// A container summary returned by the list API. @@ -279,6 +284,38 @@ pub struct SecurityInfo { // ── Client ─────────────────────────────────────────────────────────────── +fn extract_image_account_file(archive: &[u8]) -> Result, PodmanApiError> { + use std::io::Read as _; + + let invalid = |error: std::io::Error| { + PodmanApiError::InvalidInput(format!("invalid image account archive: {error}")) + }; + let mut archive = tar::Archive::new(archive); + let mut entries = archive.entries().map_err(invalid)?; + let entry = entries + .next() + .ok_or_else(|| PodmanApiError::InvalidInput("empty image account archive".into()))? + .map_err(invalid)?; + if !entry.header().entry_type().is_file() || entry.size() > MAX_IMAGE_ACCOUNT_BYTES as u64 { + return Err(PodmanApiError::InvalidInput(format!( + "image account must be a regular file of at most {MAX_IMAGE_ACCOUNT_BYTES} bytes" + ))); + } + let mut bytes = Vec::new(); + // Bound extraction independently: sparse/PAX tar metadata must not expand + // a small wire archive into an unbounded account database. + entry + .take(MAX_IMAGE_ACCOUNT_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(invalid)?; + if bytes.len() > MAX_IMAGE_ACCOUNT_BYTES { + return Err(PodmanApiError::InvalidInput( + "image account payload exceeds size limit".into(), + )); + } + Ok(bytes) +} + /// Async Podman REST API client communicating over a Unix socket. #[derive(Debug, Clone)] pub struct PodmanClient { @@ -446,6 +483,93 @@ impl PodmanClient { .await } + pub(crate) async fn create_typed_container( + &self, + spec: &(impl serde::Serialize + Sync), + ) -> Result { + #[derive(serde::Deserialize)] + struct Created { + #[serde(rename = "Id", alias = "ID")] + id: String, + } + let body = + serde_json::to_vec(spec).map_err(|error| PodmanApiError::Json(error.to_string()))?; + let (status, bytes) = self + .request_raw( + hyper::Method::POST, + "/libpod/containers/create", + "application/json", + body.into(), + ) + .await?; + if !status.is_success() { + return Err(error_from_response(status.as_u16(), &bytes)); + } + let created: Created = serde_json::from_slice(&bytes) + .map_err(|error| PodmanApiError::Json(error.to_string()))?; + validate_name(&created.id)?; + Ok(created.id) + } + + pub(crate) async fn copy_to_container( + &self, + name: &str, + destination: &str, + archive: Vec, + ) -> Result<(), PodmanApiError> { + validate_name(name)?; + let (status, bytes) = self + .request_raw( + hyper::Method::PUT, + &format!( + "/libpod/containers/{name}/archive?path={}", + url_encode(destination) + ), + "application/x-tar", + archive.into(), + ) + .await?; + if status.is_success() { + Ok(()) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + + pub(crate) async fn verify_isolation_fence(&self, id: &str) -> Result<(), PodmanApiError> { + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: String, + privileged: bool, + } + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct FenceInspect { + host_config: HostConfig, + network_settings: NetworkSettings, + } + validate_name(id)?; + let inspected: FenceInspect = self + .request_json( + hyper::Method::GET, + &format!("/libpod/containers/{id}/json"), + None, + ) + .await?; + if inspected.host_config.network_mode != "none" + || inspected.host_config.privileged + || inspected + .network_settings + .networks + .keys() + .any(|name| name != "none") + { + return Err(PodmanApiError::InvalidInput("sandbox requires an unprivileged container with network mode none and no attached networks".into())); + } + Ok(()) + } + /// Start a container by name or ID. pub async fn start_container(&self, name: &str) -> Result<(), PodmanApiError> { validate_name(name)?; @@ -539,6 +663,94 @@ impl PodmanClient { } } + /// Fetch archive metadata without reading an image-controlled response body. + pub(crate) async fn container_path_exists( + &self, + name: &str, + path: &str, + ) -> Result { + let response = self + .image_metadata_response(name, path, hyper::Method::HEAD) + .await?; + // Libpod returns 404 for a dangling symlink, but still supplies its + // lstat metadata. Such a path exists and must never pass admission. + if response + .headers() + .contains_key("x-docker-container-path-stat") + { + return Ok(true); + } + match response.status().as_u16() { + 200 => Ok(true), + 404 => Ok(false), + status => Err(PodmanApiError::Api { + status, + message: "container image path metadata probe failed".into(), + }), + } + } + + /// Bound both the wire archive and its extracted account-file payload. + pub(crate) async fn copy_image_account_file( + &self, + name: &str, + path: &str, + ) -> Result, PodmanApiError> { + let response = self + .image_metadata_response(name, path, hyper::Method::GET) + .await?; + let status = response.status(); + if status == hyper::StatusCode::NOT_FOUND { + return Ok(Vec::new()); + } + if !status.is_success() { + return Err(PodmanApiError::Api { + status: status.as_u16(), + message: "container image account download failed".into(), + }); + } + let archive = tokio::time::timeout( + API_TIMEOUT, + http_body_util::Limited::new(response.into_body(), MAX_IMAGE_ACCOUNT_BYTES).collect(), + ) + .await + .map_err(|_| PodmanApiError::Timeout(API_TIMEOUT))? + .map_err(|error| { + PodmanApiError::InvalidInput(format!( + "image account archive must fit within {MAX_IMAGE_ACCOUNT_BYTES} bytes: {error}" + )) + })? + .to_bytes(); + extract_image_account_file(&archive) + } + + async fn image_metadata_response( + &self, + name: &str, + path: &str, + method: hyper::Method, + ) -> Result, PodmanApiError> { + validate_name(name)?; + let request = Self::build_request( + method, + &format!( + "/{API_VERSION}/libpod/containers/{name}/archive?path={}", + url_encode(path) + ), + Full::new(Bytes::new()), + None, + ); + tokio::time::timeout(API_TIMEOUT, async { + let mut sender = self.connect().await?; + sender + .send_request(request) + .await + .map_err(|error| PodmanApiError::Connection(error.to_string())) + }) + .await + .map_err(|_| PodmanApiError::Timeout(API_TIMEOUT))? + } + /// Inspect a container by name or ID. pub async fn inspect_container(&self, name: &str) -> Result { validate_name(name)?; @@ -912,6 +1124,144 @@ mod tests { use crate::test_utils::{StubResponse, spawn_podman_stub}; use hyper::StatusCode; + #[tokio::test] + async fn image_path_probe_uses_head_and_fails_closed_on_server_errors() { + let (path, requests, server) = spawn_podman_stub( + "image-path-head", + vec![ + StubResponse::new(StatusCode::OK, vec![0; MAX_IMAGE_ACCOUNT_BYTES + 1]), + StubResponse::new(StatusCode::NOT_FOUND, ""), + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, ""), + ], + ); + let client = PodmanClient::new(path); + assert!( + client + .container_path_exists("image", "/.openshell") + .await + .unwrap() + ); + assert!( + !client + .container_path_exists("image", "/.openshell") + .await + .unwrap() + ); + assert!( + client + .container_path_exists("image", "/.openshell") + .await + .is_err() + ); + server.await.unwrap(); + assert!( + requests.lock().unwrap().iter().all(|request| request + == "HEAD /v5.0.0/libpod/containers/image/archive?path=%2F.openshell") + ); + } + + #[tokio::test] + async fn image_path_probe_rejects_dangling_symlink_metadata_on_404() { + let path = crate::test_utils::unique_socket_path("dangling-root-head"); + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection( + TokioIo::new(stream), + hyper::service::service_fn(|request| async move { + assert_eq!(request.method(), hyper::Method::HEAD); + Ok::<_, std::convert::Infallible>( + hyper::Response::builder() + .status(StatusCode::NOT_FOUND) + // Its contents need not be decoded: any stat metadata + // proves this is not an absent directory entry. + .header("x-docker-container-path-stat", "present") + .body(Full::new(Bytes::new())) + .unwrap(), + ) + }), + ) + .await; + }); + let client = PodmanClient::new(path.clone()); + assert!( + client + .container_path_exists("image", "/.openshell") + .await + .unwrap() + ); + server.await.unwrap(); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn image_account_download_rejects_oversized_archive() { + let (path, _, server) = spawn_podman_stub( + "account-wire-limit", + vec![StubResponse::new( + StatusCode::OK, + vec![0; MAX_IMAGE_ACCOUNT_BYTES + 1], + )], + ); + let client = PodmanClient::new(path); + let error = client + .copy_image_account_file("image", "/etc/passwd") + .await + .unwrap_err(); + assert!(error.to_string().contains("archive must fit within")); + server.await.unwrap(); + } + + #[test] + fn image_account_extraction_rejects_oversized_header_before_allocating_payload() { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_path("passwd").unwrap(); + header.set_mode(0o644); + header.set_size(u64::MAX / 2); + header.set_cksum(); + let error = extract_image_account_file(header.as_bytes()).unwrap_err(); + assert!(error.to_string().contains("regular file of at most")); + } + + #[tokio::test] + async fn image_account_download_accepts_regular_file_and_missing_database() { + let mut archive = tar::Builder::new(Vec::new()); + let payload = b"agent:x:1000:1000::/sandbox:/bin/sh\n"; + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(0o644); + header.set_size(payload.len() as u64); + header.set_cksum(); + archive + .append_data(&mut header, "passwd", payload.as_slice()) + .unwrap(); + let (path, _, server) = spawn_podman_stub( + "account-regular", + vec![ + StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), + StubResponse::new(StatusCode::NOT_FOUND, ""), + ], + ); + let client = PodmanClient::new(path); + assert_eq!( + client + .copy_image_account_file("image", "/etc/passwd") + .await + .unwrap(), + payload + ); + assert!( + client + .copy_image_account_file("image", "/etc/group") + .await + .unwrap() + .is_empty() + ); + server.await.unwrap(); + } + #[test] fn url_encode_encodes_special_characters() { assert_eq!(url_encode("hello world"), "hello%20world"); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a81ee13e1d..0f4f858b0e 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -197,7 +197,7 @@ pub fn short_id(id: &str) -> String { // --------------------------------------------------------------------------- #[derive(Serialize)] -struct ContainerSpec { +pub struct ContainerSpec { name: String, image: String, labels: BTreeMap, @@ -212,10 +212,17 @@ struct ContainerSpec { entrypoint: Vec, command: Vec, user: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + groups: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + unsetenv: Vec, cap_drop: Vec, cap_add: Vec, no_new_privileges: bool, + #[serde(skip_serializing_if = "String::is_empty")] seccomp_profile_path: String, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + sysctl: BTreeMap, image_pull_policy: String, healthconfig: HealthConfig, resource_limits: ResourceLimits, @@ -473,7 +480,7 @@ fn build_env( config: &PodmanComputeConfig, image: &str, oci_user: &str, -) -> BTreeMap { +) -> Result, ComputeDriverError> { let spec = sandbox.spec.as_ref(); let template = spec.and_then(|s| s.template.as_ref()); @@ -498,10 +505,11 @@ fn build_env( user_env.insert(k.clone(), v.clone()); } } - env.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { + // User environment belongs exclusively to mediated workload children. In + // particular, never activate loader or policy overrides in the supervisor. + if !user_env.is_empty() { + let json = serde_json::to_string(&user_env) + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert(openshell_core::sandbox_env::USER_ENVIRONMENT.into(), json); } @@ -530,7 +538,7 @@ fn build_env( ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) - .expect("main process config serialization cannot fail"); + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert( openshell_core::sandbox_env::MAIN_PROCESS_SPEC.into(), main_process, @@ -602,7 +610,7 @@ fn build_env( ); } - env + Ok(env) } /// Merge labels from the sandbox template with required managed labels. @@ -869,6 +877,7 @@ fn validate_podman_driver_mounts( } }; driver_mounts::validate_container_mount_target(target)?; + driver_mounts::validate_mount_control_path(target, "/.openshell")?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -1006,6 +1015,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( } #[allow(clippy::too_many_arguments)] +#[cfg(test)] pub fn build_container_spec_for_image( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1017,10 +1027,36 @@ pub fn build_container_spec_for_image( supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 3]>, ) -> Result { + serde_json::to_value(build_base_spec( + sandbox, + config, + token_secret_name, + gpu_device_ids, + requested_image, + image_id, + oci_user, + supervisor_bin_path, + tls_secret_names, + )?) + .map_err(|error| ComputeDriverError::Message(format!("encode Podman spec: {error}"))) +} + +#[allow(clippy::too_many_arguments)] +fn build_base_spec( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + token_secret_name: Option<&str>, + gpu_device_ids: Option<&[String]>, + requested_image: &str, + image_id: &str, + oci_user: &str, + supervisor_bin_path: Option<&Path>, + tls_secret_names: Option<&[String; 3]>, +) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); - let env = build_env(sandbox, config, requested_image, oci_user); + let env = build_env(sandbox, config, requested_image, oci_user)?; let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) @@ -1098,83 +1134,16 @@ pub fn build_container_spec_for_image( // corporate proxy flags follow it; the workload command comes from // the reserved environment variable. command, - // Force the supervisor to run as root (UID 0). Sandbox images may - // set a non-root USER directive (e.g. `USER sandbox`), but the - // supervisor needs root to create network namespaces, set up the - // proxy, and configure Landlock/seccomp. This matches the K8s - // driver's runAsUser: 0. - user: "0:0".into(), - // Podman's default container capability set is already restricted: - // CHOWN DAC_OVERRIDE FOWNER FSETID KILL SETGID SETUID SETPCAP - // NET_BIND_SERVICE SYS_CHROOT SETFCAP - // We add what the supervisor needs and drop what it doesn't. - cap_drop: vec![ - // Not needed: standard file permission bits are sufficient; dropping - // prevents the supervisor from bypassing DAC checks it shouldn't need. - "DAC_OVERRIDE".into(), - // Not needed: the supervisor does not create setuid/setgid executables. - "FSETID".into(), - // Not needed: the supervisor does not bind privileged ports (<1024). - "NET_BIND_SERVICE".into(), - // Not in Podman's default set but explicitly denied in case the image - // or runtime adds it; raw sockets are not required. - "NET_RAW".into(), - // Not needed: the supervisor does not manipulate file capabilities. - "SETFCAP".into(), - // Not needed: the supervisor does not call chroot(). - "SYS_CHROOT".into(), - ], - cap_add: vec![ - // seccomp filter installation, namespace creation, Landlock setup. - "SYS_ADMIN".into(), - // Network namespace veth setup, IP/route configuration. - "NET_ADMIN".into(), - // Reading /proc//exe and ancestor walk for process identity in policy. - "SYS_PTRACE".into(), - // Reading /dev/kmsg for bypass-detection diagnostics. - "SYSLOG".into(), - // Reading /proc//fd/ across UIDs for process identity resolution. - // In rootless Podman the supervisor runs as UID 0 inside a user namespace - // while sandbox processes run as the sandbox user. The kernel's - // proc_fd_permission() calls generic_permission() which denies cross-UID - // access to the dr-x------ fd directory unless this cap is present. - // Without it the proxy cannot determine which binary made each outbound - // connection and all traffic is denied. - "DAC_READ_SEARCH".into(), - // Child setup clears the capability bounding set before exec, which - // requires CAP_SETPCAP in the supervisor until drop_privileges(). - "SETPCAP".into(), - // Forwarding shutdown signals to the canonical workload process - // group after it drops to the sandbox UID requires CAP_KILL. - "KILL".into(), - ], - // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from - // Podman's default set and not dropped: - // SETUID/SETGID – drop_privileges(): setuid()/setgid()/initgroups() to the - // sandbox user. In rootless Podman cap_drop:ALL removes them - // from the bounding set even though uid=0 owns the user - // namespace — so we keep them by not dropping them explicitly. - // SETPCAP – drop_privileges(): clears the child capability - // bounding set before the sandbox user execs. - // CHOWN – prepare_filesystem(): chown(path, uid, gid) on newly - // created read_write directories so the sandbox user can - // write to them. - // FOWNER – chown on files where the supervisor is not the owner - // (e.g. pre-existing directories owned by another user). - // - // Disable the container-level seccomp profile. The sandbox supervisor The sandbox supervisor - // installs its own policy-aware BPF seccomp filter at runtime via - // seccompiler (two-phase: clone3 blocker + main filter). The runtime - // filter is more restrictive than Podman's default — it blocks 20+ - // dangerous syscalls and conditionally restricts socket domains based - // on network policy. The filter self-seals by blocking further - // seccomp(SET_MODE_FILTER) calls after installation. - // - // A container-level profile would interfere by blocking the landlock - // and seccomp syscalls the supervisor needs during setup, before it - // locks itself down. + // The paired builder supplies the immutable non-root identity. + user: String::new(), + groups: Vec::new(), + unsetenv: Vec::new(), + cap_drop: vec!["ALL".into()], + cap_add: Vec::new(), no_new_privileges: true, - seccomp_profile_path: "unconfined".into(), + // Omission selects the runtime default, never an unconfined profile. + seccomp_profile_path: String::new(), + sysctl: BTreeMap::new(), image_pull_policy: "never".to_string(), healthconfig: HealthConfig { test: vec![ @@ -1389,7 +1358,193 @@ pub fn build_container_spec_for_image( }, }; - Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) + Ok(container_spec) +} + +/// Driver-owned inputs for the two independent runtime containers. +pub struct IsolationSpecInput<'a> { + pub sandbox: &'a DriverSandbox, + pub config: &'a PodmanComputeConfig, + pub token_secret: Option<&'a str>, + pub gpu_devices: Option<&'a [String]>, + pub requested_image: &'a str, + pub image_id: &'a str, + pub image_user: &'a str, + pub image_env: &'a [String], + pub supervisor_bin: Option<&'a Path>, + pub tls_secrets: Option<&'a [String; 3]>, + pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, +} + +pub struct IsolationSpecs { + pub workload: ContainerSpec, + pub supervisor: ContainerSpec, +} + +impl ContainerSpec { + pub(crate) fn join_user_namespace(&mut self, container_id: &str) { + self.userns = Some(UserNS { + nsmode: "container".to_string(), + value: Some(container_id.to_string()), + }); + self.idmappings = None; + } +} + +pub fn build_isolation_specs( + input: IsolationSpecInput<'_>, +) -> Result { + let base = || { + build_base_spec( + input.sandbox, + input.config, + input.token_secret, + input.gpu_devices, + input.requested_image, + input.image_id, + input.image_user, + input.supervisor_bin, + input.tls_secrets, + ) + }; + let mut workload = base()?; + let mut supervisor = base()?; + let user = format!("{}:{}", input.identity.uid, input.identity.gid); + let channel = crate::isolation::channel_volume_name(&input.sandbox.id); + + workload + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); + workload.env = BTreeMap::new(); + workload.unsetenv = input + .image_env + .iter() + .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) + .collect(); + workload.command = vec![ + "--bootstrap".into(), + crate::isolation::BOOTSTRAP_PATH.into(), + ]; + workload.user.clone_from(&user); + workload.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + workload.cap_drop = vec!["ALL".into()]; + workload.cap_add.clear(); + workload.seccomp_profile_path.clear(); + workload + .sysctl + .insert("net.ipv4.ip_unprivileged_port_start".into(), "0".into()); + workload.netns.nsmode = "none".into(); + workload.networks.clear(); + workload.portmappings.clear(); + workload.hostadd.clear(); + workload.secret_env.clear(); + workload.secrets.clear(); + workload.healthconfig.test = vec!["NONE".into()]; + workload + .mounts + .retain(|mount| !trusted_mount(&mount.destination)); + // The capability-free sandbox installs interception CA material below + // /run before launching the agent. Image-owned /run is commonly root-only. + workload.mounts.push(private_writable_tmpfs("/run")); + workload.volumes.push(NamedVolume { + name: channel.clone(), + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["rw".into(), "nocopy".into(), "z".into()], + }); + + supervisor.name = crate::isolation::supervisor_name(&input.sandbox.id); + supervisor + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "supervisor".into()); + supervisor.image.clone_from(&input.config.supervisor_image); + supervisor.entrypoint = vec!["/openshell-supervisor".into()]; + supervisor.command.extend([ + "--topology-backend-name=podman".into(), + format!( + "--topology-payload-file={}", + crate::isolation::TOPOLOGY_PATH + ), + "--health-socket-path=/run/openshell/supervisor-health.sock".into(), + ]); + supervisor.env.insert( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND.into(), + "podman".into(), + ); + supervisor.user = user; + supervisor.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + supervisor.cap_drop = vec!["ALL".into()]; + supervisor.cap_add.clear(); + supervisor.seccomp_profile_path.clear(); + supervisor.devices = None; + supervisor.image_volumes.clear(); + supervisor.volumes = vec![NamedVolume { + name: channel, + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["ro".into(), "nocopy".into(), "z".into()], + }]; + supervisor.mounts.retain(|mount| { + trusted_mount(&mount.destination) + && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT + }); + for destination in ["/run", "/var/log", "/tmp"] { + supervisor.mounts.push(private_writable_tmpfs(destination)); + } + for secret in &mut supervisor.secrets { + secret.uid = input.identity.uid; + secret.gid = input.identity.gid; + } + supervisor.healthconfig.test = vec![ + "CMD".into(), + "/openshell-supervisor".into(), + "health".into(), + "--socket".into(), + "/run/openshell/supervisor-health.sock".into(), + ]; + supervisor.healthconfig.interval = + input.config.health_check_interval_secs.max(1) * 1_000_000_000; + Ok(IsolationSpecs { + workload, + supervisor, + }) +} + +fn private_writable_tmpfs(destination: &str) -> Mount { + Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: destination.into(), + options: vec![ + "rw".into(), + "nosuid".into(), + "nodev".into(), + // Podman's SpecGenerator mount parser rejects uid/gid tmpfs + // options. Each mount is private to one container, so a writable + // mode grants no access across the workload/supervisor boundary. + "mode=1777".into(), + "size=64m".into(), + ], + } +} + +fn trusted_mount(destination: &str) -> bool { + matches!( + destination, + TLS_CA_MOUNT_PATH + | TLS_CERT_MOUNT_PATH + | TLS_KEY_MOUNT_PATH + | PROXY_CA_MOUNT_PATH + | PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + ) || destination == openshell_core::container_paths::NETNS_MOUNT_ROOT } fn provider_spiffe_workload_api_socket_env_value(config: &PodmanComputeConfig) -> Option { @@ -1496,6 +1651,83 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + #[test] + fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { + let sandbox = DriverSandbox { + id: "pair".into(), + name: "agent".into(), + ..Default::default() + }; + let config = PodmanComputeConfig::default(); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![2000], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let env = vec![ + "LD_PRELOAD=/hostile.so".into(), + "HTTP_PROXY=http://bypass".into(), + ]; + let specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: Some("jwt"), + gpu_devices: None, + requested_image: "image:latest", + image_id: "sha256:image", + image_user: "1000:1001", + image_env: &env, + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + }) + .unwrap(); + for spec in [&specs.workload, &specs.supervisor] { + assert_eq!(spec.user, "1000:1001"); + assert_eq!(spec.groups, vec!["2000"]); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); + let runtime_mounts: Vec<_> = spec + .mounts + .iter() + .filter(|mount| mount.destination == "/run") + .collect(); + assert_eq!(runtime_mounts.len(), 1); + let runtime = runtime_mounts[0]; + assert_eq!(runtime.kind, "tmpfs"); + assert_eq!(runtime.source, "tmpfs"); + assert_eq!( + runtime.options, + vec!["rw", "nosuid", "nodev", "mode=1777", "size=64m"] + ); + } + assert_eq!(specs.workload.netns.nsmode, "none"); + assert!(specs.workload.networks.is_empty()); + assert!(specs.workload.portmappings.is_empty()); + assert!(specs.workload.env.is_empty()); + assert_eq!(specs.workload.unsetenv, vec!["LD_PRELOAD", "HTTP_PROXY"]); + assert!(specs.workload.secrets.is_empty()); + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| !trusted_mount(&mount.destination)) + ); + assert_eq!(specs.supervisor.secrets.len(), 1); + assert_eq!(specs.supervisor.secrets[0].source, "jwt"); + assert_eq!(specs.supervisor.secrets[0].uid, 1000); + assert_eq!(specs.supervisor.volumes.len(), 1); + assert!(specs.supervisor.volumes[0].options.contains(&"ro".into())); + assert!(specs.supervisor.volumes[0].options.contains(&"z".into())); + assert_eq!(specs.supervisor.entrypoint, vec!["/openshell-supervisor"]); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -1626,7 +1858,8 @@ mod tests { container["env"]["OPENSHELL_CONTAINER_IMAGE"].as_str(), Some("registry.example/app:latest") ); - assert_eq!(container["user"].as_str(), Some("0:0")); + // Only the paired builder materializes the immutable non-root user. + assert_eq!(container["user"].as_str(), Some("")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!([])); assert_eq!(container["dns_option"], serde_json::json!([])); @@ -1866,66 +2099,25 @@ mod tests { } #[test] - fn container_spec_includes_required_capabilities() { + fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); - let spec = build_container_spec(&sandbox, &config); - - let added: Vec<&str> = spec["cap_add"] - .as_array() - .expect("cap_add should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); - assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); - assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); - assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); - assert!( - added.contains(&"DAC_READ_SEARCH"), - "missing DAC_READ_SEARCH" - ); - assert!(added.contains(&"SETPCAP"), "missing SETPCAP"); - assert!(added.contains(&"KILL"), "missing KILL"); - - // SETUID and SETGID are NOT in cap_add — they remain available from the - // default bounding set because we no longer use cap_drop:ALL. Verify they - // are also not explicitly dropped. Similarly SETPCAP, CHOWN and FOWNER - // must not be dropped because child setup clears the bounding set and - // prepare_filesystem() calls chown() on newly created read_write - // directories before the supervisor drops privileges. - let dropped: Vec<&str> = spec["cap_drop"] - .as_array() - .expect("cap_drop should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(!dropped.contains(&"SETUID"), "SETUID must not be dropped"); - assert!(!dropped.contains(&"SETGID"), "SETGID must not be dropped"); - assert!( - dropped.contains(&"NET_BIND_SERVICE"), - "NET_BIND_SERVICE must stay dropped; policy DNS binds an unprivileged port" - ); - assert!( - !dropped.contains(&"CHOWN"), - "CHOWN must not be dropped (needed for prepare_filesystem chown)" - ); - assert!( - !dropped.contains(&"FOWNER"), - "FOWNER must not be dropped (needed for chown on non-owned files)" - ); - assert!( - !dropped.contains(&"SETPCAP"), - "SETPCAP must not be dropped (needed for child bounding-set clear)" - ); - assert!( - !dropped.contains(&"KILL"), - "KILL must not be dropped (needed to signal the sandbox workload on shutdown)" - ); - assert!( - !dropped.contains(&"ALL"), - "must not use cap_drop:ALL in rootless Podman" - ); + let spec = build_base_spec( + &sandbox, + &config, + None, + None, + "image", + "sha256:image", + "", + None, + None, + ) + .unwrap(); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); } #[test] @@ -2867,6 +3059,27 @@ mod tests { assert!(err.to_string().contains("reserved OpenShell path")); } + #[test] + fn user_mounts_cannot_replace_private_channel_hierarchy() { + for target in [ + "/.openshell", + "/.openshell/channel", + "/.openshell/channel/sandbox", + "/.openshell/supervisor", + ] { + let mount = PodmanDriverMountConfig::Volume { + source: "user-owned".into(), + target: target.into(), + read_only: false, + subpath: None, + }; + assert!( + validate_podman_driver_mounts(&[mount], false).is_err(), + "{target}" + ); + } + } + #[test] fn container_spec_uses_configured_host_gateway_ip() { let sandbox = test_sandbox("test-id", "test-name"); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 50eb014691..aceb3feab1 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -780,7 +780,7 @@ impl PodmanComputeDriver { "Creating sandbox container" ); - let (image, immutable_image_id, image_user) = async { + let (image, immutable_image_id, image_user, image_env) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { // The supervisor binary is shipped in a standalone OCI image and @@ -840,7 +840,8 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; } - Ok((image.to_string(), inspected_image.id, image_user)) + let image_env = inspected_image.config.as_ref().map_or_else(Vec::new, |config| config.env.clone()); + Ok((image.to_string(), inspected_image.id, image_user, image_env)) } .await; phase_status.finish(result) @@ -859,6 +860,22 @@ impl PodmanComputeDriver { // content at startup. validate_sandbox_proxy_ca_bundle(&self.config).await?; + let identity = self + .resolve_workload_identity(sandbox, &immutable_image_id, &image_user) + .await?; + let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); + let mut runtime_config = self.config.clone(); + runtime_config.supervisor_image = self + .client + .inspect_image(&self.config.supervisor_image) + .await? + .id; + if runtime_config.supervisor_image.is_empty() { + return Err(ComputeDriverError::Precondition( + "supervisor image inspection returned no immutable image ID".into(), + )); + } + // Create workspace volume and per-sandbox token secret. let (token_secret_name, proxy_auth_secret_name) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); @@ -904,6 +921,7 @@ impl PodmanComputeDriver { // Clean up the volume and both per-sandbox secrets on any failure past // this point. let cleanup_created = || async { + let _ = self.client.remove_volume(&channel_volume).await; let _ = self.client.remove_volume(&vol_name).await; if let Some(secret) = token_secret_name.as_deref() { cleanup_sandbox_token_secret(&self.client, secret).await; @@ -914,7 +932,7 @@ impl PodmanComputeDriver { }; // Prepare and create the container. - let tls_secret_names = async { + async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { let gpu_devices = match self.resolve_gpu_cdi_devices( @@ -930,7 +948,7 @@ impl PodmanComputeDriver { }; let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) { - match extract_supervisor_bin(&self.client, &self.config).await { + match extract_supervisor_bin(&self.client, &runtime_config).await { Ok(path) => Some(path), Err(e) => { cleanup_created().await; @@ -941,9 +959,7 @@ impl PodmanComputeDriver { None }; - let tls_secret_names = if userns_remaps_uids(self.config.userns.as_deref()) - && self.config.tls_enabled() - { + let tls_secret_names = if self.config.tls_enabled() { let names = container::tls_secret_names(&sandbox.id); if let Err(e) = create_tls_secrets(&self.client, &self.config, &names).await { cleanup_created().await; @@ -961,32 +977,91 @@ impl PodmanComputeDriver { } }; - let spec = match container::build_container_spec_for_image( + let specs = container::build_isolation_specs(container::IsolationSpecInput { sandbox, - &self.config, - token_secret_name.as_deref(), - gpu_devices.as_deref(), - &image, - &immutable_image_id, - &image_user, - supervisor_bin_path.as_deref(), - tls_secret_names.as_ref(), - ) { + config: &runtime_config, + token_secret: token_secret_name.as_deref(), + gpu_devices: gpu_devices.as_deref(), + requested_image: &image, + image_id: &immutable_image_id, + image_user: &image_user, + image_env: &image_env, + supervisor_bin: supervisor_bin_path.as_deref(), + tls_secrets: tls_secret_names.as_ref(), + identity: &identity, + }); + let mut specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; return Err(e); } }; - match self.client.create_container(&spec).await { - Ok(_) => Ok(tls_secret_names), - Err(PodmanApiError::Conflict(_)) => { - cleanup_all().await; - Err(ComputeDriverError::AlreadyExists) + let mut created_workload = None; + let mut created_supervisor = None; + let create_result = async { + self.client.create_volume(&channel_volume).await?; + let workload_id = self.client.create_typed_container(&specs.workload).await?; + created_workload = Some(workload_id.clone()); + self.client.verify_isolation_fence(&workload_id).await?; + let child_env = image_env + .iter() + .filter_map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.into(), value.into())) + }) + .collect(); + let archives = crate::isolation::bootstrap_archives( + &sandbox.id, + &workload_id, + &identity, + child_env, + )?; + self.client + .copy_to_container( + &workload_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&workload_id, "/sandbox", archives.workspace) + .await?; + // Rootless Podman places default/host-mode containers in + // the same user namespace already. Asking crun to re-enter + // that namespace fails with EINVAL. Only an explicitly + // isolated userns mode needs the supervisor to join the + // workload's namespace. + if userns_needs_extraction(self.config.userns.as_deref()) { + specs.supervisor.join_user_namespace(&workload_id); } + let supervisor_id = self + .client + .create_typed_container(&specs.supervisor) + .await?; + created_supervisor = Some(supervisor_id.clone()); + self.client + .copy_to_container(&supervisor_id, "/", archives.supervisor) + .await?; + // Both resources and private files exist before either + // container can run. Only the trusted sandbox starts here; + // authenticated confirmation gates subsequent agent exec. + self.client.start_container(&workload_id).await?; + self.client.start_container(&supervisor_id).await?; + Ok::<(), ComputeDriverError>(()) + } + .await; + if create_result.is_err() { + for id in [created_supervisor, created_workload].into_iter().flatten() { + let _ = self.client.remove_container(&id, 0).await; + } + } + match create_result { + Ok(()) => Ok(()), Err(e) => { cleanup_all().await; - Err(ComputeDriverError::from(e)) + Err(e) } } } @@ -1001,44 +1076,6 @@ impl PodmanComputeDriver { )) .await?; - let cleanup_all = || async { - cleanup_created().await; - if let Some(names) = &tls_secret_names { - cleanup_tls_secrets(&self.client, names).await; - } - }; - - // Start container. - let start_result = async { - let phase_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .client - .start_container(&name) - .await - .map_err(ComputeDriverError::from); - phase_status.finish(result) - } - .instrument(tracing::info_span!( - "podman.start_container", - otel.name = "podman.start_container", - otel.status_code = tracing::field::Empty, - container.name = %name, - )) - .await; - if let Err(e) = start_result { - warn!( - sandbox_name = %sandbox.name, - error = %e, - "Failed to start container; cleaning up" - ); - let _ = self - .client - .remove_container(&name, self.config.stop_timeout_secs) - .await; - cleanup_all().await; - return Err(e); - } - info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -1048,7 +1085,61 @@ impl PodmanComputeDriver { span_status.finish(Ok(())) } - /// Find the Podman container ID for a sandbox by its sandbox ID using label lookup. + /// Resolve image accounts without executing any image-supplied program. + async fn resolve_workload_identity( + &self, + sandbox: &DriverSandbox, + image: &str, + image_user: &str, + ) -> Result + { + #[derive(serde::Serialize)] + struct InspectionSpec<'a> { + name: String, + image: &'a str, + } + // Inspect a stopped, unexecuted container pinned to the final image ID. + let id = self + .client + .create_typed_container(&InspectionSpec { + name: format!("openshell-identity-{}", uuid::Uuid::new_v4()), + image, + }) + .await?; + let result = async { + // Do not let image-controlled symlinks alias the protected channel + // into an agent-readable subtree before Podman mounts it. + if self + .client + .container_path_exists(&id, "/.openshell") + .await? + { + return Err(ComputeDriverError::Precondition( + "workload images must not prepopulate the reserved /.openshell hierarchy" + .into(), + )); + } + let mut accounts = Vec::new(); + for path in ["/etc/passwd", "/etc/group"] { + let content = self.client.copy_image_account_file(&id, path).await?; + accounts.push(content); + } + let [passwd, group] = accounts.as_slice() else { + return Err(ComputeDriverError::Precondition( + "image account inspection was incomplete".into(), + )); + }; + crate::isolation::resolve_identity(sandbox, image, image_user, passwd, group) + } + .await; + let cleanup = self.client.remove_container(&id, 0).await; + if let Err(error) = cleanup { + warn!(container = %id, %error, "Failed to remove stopped identity inspection container"); + } + result + } + + /// Find only the workload, never its supervisor companion. async fn find_container_id( &self, sandbox_id: &str, @@ -1063,7 +1154,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(entries.into_iter().next()) @@ -1116,6 +1211,15 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; let container_id = container.id; + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } if container.state == "stopping" { let result = async { let finished_at = self @@ -1179,7 +1283,19 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; if container.state == "running" { - return span_status.finish(Ok(())); + let supervisor = self + .client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if supervisor + .as_ref() + .is_ok_and(|inspect| inspect.state.running) + { + return span_status.finish(Ok(())); + } + self.client.stop_container(&container.id, 0).await?; + self.wait_for_container_stopped(sandbox_id, &container.id) + .await?; } let container_id = container.id; info!(sandbox_id = %sandbox_id, container = %container_id, "Starting sandbox container"); @@ -1196,11 +1312,42 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous.state.finished_at.as_deref()); - let result = self - .client - .start_container(&container_id) - .await - .map_err(ComputeDriverError::from); + let result = async { + let supervisor = crate::isolation::supervisor_name(sandbox_id); + self.client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await?; + self.wait_for_container_stopped(sandbox_id, &supervisor) + .await?; + let archive = self + .client + .copy_from_container(&supervisor, crate::isolation::RESTART_BUNDLE_PATH) + .await?; + let bundle = + extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + let archives = + crate::isolation::rotate_bootstrap_archives(sandbox_id, &container_id, &bundle)?; + // Update both endpoints while stopped. A failed upload leaves no + // running peer with a mismatched generation; retry rotates again. + self.client + .copy_to_container( + &container_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&supervisor, "/", archives.supervisor) + .await?; + self.client.verify_isolation_fence(&container_id).await?; + self.client.start_container(&container_id).await?; + if let Err(error) = self.client.start_container(&supervisor).await { + let _ = self.client.stop_container(&container_id, 0).await; + return Err(error.into()); + } + Ok(()) + } + .await; span_status.finish(result) } @@ -1222,6 +1369,25 @@ impl PodmanComputeDriver { )); } + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .remove_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } + match self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + // The workload still owns the volume until its removal below. + Err(error) => debug!(%error, "Channel volume is still attached to workload"), + } + let Some(container_id) = self.find_container_id(sandbox_id).await? else { debug!(sandbox_id = %sandbox_id, "Sandbox container not found (already deleted)"); let vol = container::volume_name(sandbox_id); @@ -1255,6 +1421,13 @@ impl PodmanComputeDriver { }; // Remove workspace volume. + if let Err(error) = self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + warn!(%sandbox_id, %error, "Failed to remove private channel volume"); + } let vol = container::volume_name(sandbox_id); if let Err(e) = self.client.remove_volume(&vol).await { warn!( @@ -1281,7 +1454,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(!entries.is_empty()) @@ -1295,16 +1472,18 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; let Some(entry) = entries.first() else { return Ok(None); }; if entry.state == "running" { - Ok(self - .client - .inspect_container(&entry.id) + Ok(watcher::inspect_workload(&self.client, &entry.id) .await .ok() .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) @@ -1321,7 +1500,7 @@ impl PodmanComputeDriver { pub async fn list_sandboxes(&self) -> Result, ComputeDriverError> { let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER]) + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) .await .map_err(ComputeDriverError::from)?; @@ -1329,7 +1508,7 @@ impl PodmanComputeDriver { for entry in &entries { if entry.state == "running" { // Running containers need inspect for health check status. - match self.client.inspect_container(&entry.id).await { + match watcher::inspect_workload(&self.client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { sandboxes.push(sandbox); @@ -1592,6 +1771,7 @@ fn userns_needs_extraction(userns: Option<&str>) -> bool { /// Returns `true` when userns remaps all UIDs, making host-owned bind mounts /// unreadable from inside the container. `auto` and `no-map` remap every UID; /// `keep-id` preserves the host user's UID; `host` uses the host namespace. +#[cfg(test)] fn userns_remaps_uids(userns: Option<&str>) -> bool { userns.is_some_and(|mode| { let base = mode.split(':').next().unwrap_or(mode); @@ -1688,6 +1868,7 @@ mod tests { "lifecycle-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1703,7 +1884,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[1], + .expect("request log lock should not be poisoned")[2], format!( "POST {}", api_path("/libpod/containers/ctr-1/stop?timeout=10") @@ -1712,7 +1893,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[2], + .expect("request log lock should not be poisoned")[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1724,14 +1905,37 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); test_driver(start_socket.clone()) .start_sandbox("sandbox-1") .await .expect("start should succeed"); start_handle.await.expect("start stub should finish"); + let restart_requests = start_requests.lock().unwrap().clone(); + assert_eq!( + restart_requests + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect::>(), + vec![ + format!( + "PUT {}", + api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") + ), + format!( + "PUT {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/archive?path=%2F") + ), + ] + ); + assert!( + !restart_requests + .iter() + .any(|request| request.contains("/volumes/create") + || request.contains("/containers/create")) + ); assert_eq!( start_requests .lock() @@ -1742,19 +1946,84 @@ mod tests { start_requests .lock() .expect("request log lock should not be poisoned")[2], - format!("POST {}", api_path("/libpod/containers/ctr-1/start")) + format!( + "POST {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/stop?timeout=10") + ) ); let _ = fs::remove_file(stop_socket); let _ = fs::remove_file(start_socket); } + #[tokio::test] + async fn restart_upload_failures_leave_both_peers_stopped() { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Summary { + id: &'static str, + state: &'static str, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct State { + status: &'static str, + running: bool, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Inspect { + id: &'static str, + name: &'static str, + state: State, + } + for upload_index in [3, 4] { + let mut responses = vec![ + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&[Summary { + id: "ctr-1", + state: "stopped", + }]) + .unwrap(), + ), + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Inspect { + id: "ctr-1", + name: "sandbox", + state: State { + status: "exited", + running: false, + }, + }) + .unwrap(), + ), + ]; + let mut restart = restart_responses(); + restart[upload_index] = + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "upload failed"); + responses.extend(restart.into_iter().take(upload_index + 1)); + let (path, requests, server) = spawn_podman_stub("restart-upload-failure", responses); + assert!(test_driver(path).start_sandbox("sandbox-1").await.is_err()); + server.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|request| !request.ends_with("/start")) + ); + } + } + #[tokio::test] async fn stop_waits_for_the_container_to_leave_stopping_state() { let (socket, requests, handle) = spawn_podman_stub( "lifecycle-stop-wait", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1776,12 +2045,12 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 4); + assert_eq!(requests.len(), 5); assert_eq!( - requests[2], + requests[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); - assert_eq!(requests[3], requests[2]); + assert_eq!(requests[4], requests[3]); let _ = fs::remove_file(socket); } @@ -1792,6 +2061,7 @@ mod tests { "lifecycle-stop-retry", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"stopping"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new( StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, @@ -1808,9 +2078,9 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 2); + assert_eq!(requests.len(), 3); assert_eq!( - requests[1], + requests[2], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1828,6 +2098,7 @@ mod tests { "trace-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1874,19 +2145,12 @@ mod tests { use tracing_subscriber::layer::SubscriberExt as _; let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; - let (socket_path, _requests, handle) = spawn_podman_stub( + let (socket_path, requests, handle) = spawn_podman_stub( "trace-create", - vec![ - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + create_setup_responses(false) + .into_iter() + .chain(create_launch_responses()) + .collect(), ); let exporter = InMemorySpanExporterBuilder::new().build(); let provider = SdkTracerProvider::builder() @@ -1901,6 +2165,22 @@ mod tests { .await .expect("create should succeed"); handle.await.expect("stub should finish"); + let uploads: Vec<_> = requests + .lock() + .unwrap() + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect(); + assert_eq!( + uploads, + [ + "/libpod/containers/workload/archive?path=%2F.openshell%2Fchannel", + "/libpod/containers/workload/archive?path=%2Fsandbox", + "/libpod/containers/supervisor/archive?path=%2F", + ] + .map(|path| format!("PUT {}", api_path(path))) + ); provider.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); @@ -1912,7 +2192,6 @@ mod tests { "podman.prepare_images", "podman.prepare_storage", "podman.prepare_container", - "podman.start_container", ] { let child = spans .iter() @@ -1992,8 +2271,7 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); test_driver(start_socket.clone()) .start_sandbox("sandbox-1") @@ -2005,6 +2283,8 @@ mod tests { let (delete_socket, _requests, delete_handle) = spawn_podman_stub( "trace-delete", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached StubResponse::new(StatusCode::OK, "[]"), StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -2855,6 +3135,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-not-found", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -2874,9 +3156,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) @@ -2924,6 +3206,190 @@ mod tests { ) } + fn restart_responses() -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let bundle = + crate::isolation::bootstrap_archives("sandbox-1", "ctr-1", &identity, HashMap::new()) + .unwrap() + .channel; + let mut header = tar::Header::new_gnu(); + header.set_size(bundle.len() as u64); + header.set_mode(0o600); + header.set_cksum(); + archive + .append_data(&mut header, "sandbox-bundle.tar", bundle.as_slice()) + .unwrap(); + vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor stop + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&[ + ".openshell", + ".openshell/supervisor", + ".openshell/supervisor/topology.payload", + ".openshell/supervisor/sandbox-bundle.tar", + ]), + fence_response(), + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + fn fence_response() -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: &'static str, + privileged: bool, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Networks { + networks: std::collections::BTreeMap, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Fence { + host_config: HostConfig, + network_settings: Networks, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Fence { + host_config: HostConfig { + network_mode: "none", + privileged: false, + }, + network_settings: Networks { + networks: std::collections::BTreeMap::default(), + }, + }) + .unwrap(), + ) + } + + fn created_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + struct Created { + #[serde(rename = "Id")] + id: &'static str, + } + StubResponse::new( + StatusCode::CREATED, + serde_json::to_vec(&Created { id }).unwrap(), + ) + } + + fn image_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Config { + user: &'static str, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Image { + id: &'static str, + config: Config, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Image { + id, + config: Config { user: "1234:1235" }, + }) + .unwrap(), + ) + } + + fn create_setup_responses(proxy_secret: bool) -> Vec { + let mut responses = vec![ + StubResponse::new(StatusCode::OK, "{}"), // supervisor pull + StubResponse::new(StatusCode::OK, "{}"), // workload pull + image_response("sha256:sandbox"), + created_response("identity-reader"), + StubResponse::new(StatusCode::NOT_FOUND, ""), // reserved hierarchy absent + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional passwd + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional group + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader + image_response("sha256:supervisor"), + StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume + ]; + if proxy_secret { + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); + } + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume + responses + } + + fn create_launch_responses() -> Vec { + vec![ + created_response("workload"), + fence_response(), + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), + created_response("supervisor"), + StubResponse::new(StatusCode::OK, ""), // supervisor archive + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + fn channel_archive_members() -> &'static [&'static str] { + &[ + ".", + "sandbox", + "sandbox/bootstrap.json", + "sandbox/server.crt", + "sandbox/server.key", + "sandbox/client-ca.crt", + ] + } + + #[tokio::test] + async fn reserved_image_control_root_fails_before_workload_or_secrets() { + let (path, requests, handle) = spawn_podman_stub( + "reserved-control-root", + vec![ + StubResponse::new(StatusCode::OK, "{}"), + StubResponse::new(StatusCode::OK, "{}"), + image_response("sha256:image"), + created_response("identity-reader"), + StubResponse::new(StatusCode::OK, "existing reserved path"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let error = test_driver(path.clone()) + .create_sandbox(&plain_sandbox("id", "name")) + .await + .unwrap_err(); + assert!(error.to_string().contains("reserved /.openshell")); + handle.await.unwrap(); + assert!(requests.lock().unwrap().iter().any(|request| request + == "HEAD /v5.0.0/libpod/containers/identity-reader/archive?path=%2F.openshell")); + assert!( + !requests + .lock() + .unwrap() + .iter() + .any(|request| request.contains("/libpod/volumes") + || request.contains("/libpod/secrets")) + ); + let _ = fs::remove_file(path); + } + #[tokio::test] async fn create_sandbox_removes_proxy_auth_secret_on_container_create_failure() { // A credential secret is staged before the container is created, so a @@ -2932,19 +3398,15 @@ mod tests { let auth_file = write_proxy_auth_file("create-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-container-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // create container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); @@ -2974,21 +3436,18 @@ mod tests { let auth_file = write_proxy_auth_file("start-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-start-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::CREATED, "{}"), // create container - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // start container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain(create_launch_responses().into_iter().take(7)) + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); @@ -3018,7 +3477,9 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-proxy-auth", vec![ - StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached + StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) StubResponse::new(StatusCode::NO_CONTENT, ""), // remove volume StubResponse::new(StatusCode::NO_CONTENT, ""), // remove token secret StubResponse::new(StatusCode::NO_CONTENT, ""), // remove proxy-auth secret @@ -3061,10 +3522,14 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-label-lookup", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers by label StubResponse::new(StatusCode::OK, list_body), // single timed remove_container operation StubResponse::new(StatusCode::NO_CONTENT, ""), + // channel volume, now detached + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove_volume StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -3082,9 +3547,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!( @@ -3093,7 +3558,7 @@ mod tests { ) ); assert_eq!( - requests[2], + requests[5], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index fedeea3068..d18ad5d88d 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -162,8 +162,7 @@ impl ComputeDriver for ComputeDriverService { .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) + Box::pin(self.driver.create_sandbox(&sandbox)) .await .map_err(Status::from)?; Ok(Response::new(CreateSandboxResponse {})) @@ -680,6 +679,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "forward-id", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -708,9 +709,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs new file mode 100644 index 0000000000..38226e9809 --- /dev/null +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -0,0 +1,493 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman-owned provisioning for the common authenticated isolation channel. + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; + +use openshell_core::ComputeDriverError; +use openshell_core::proto::compute::v1::DriverSandbox; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, + BoundaryTransport, generate_boundary_mutual_tls_material, +}; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; + +pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; +pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; +pub const CHANNEL_ROOT: &str = "/.openshell/channel"; +pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; +pub const TOPOLOGY_PATH: &str = "/.openshell/supervisor/topology.payload"; +pub const RESTART_BUNDLE_PATH: &str = "/.openshell/supervisor/sandbox-bundle.tar"; +const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; + +pub fn supervisor_name(id: &str) -> String { + format!("openshell-supervisor-{id}") +} +pub fn channel_volume_name(id: &str) -> String { + format!("openshell-channel-{id}") +} + +fn invalid(error: impl std::fmt::Display) -> ComputeDriverError { + ComputeDriverError::Precondition(error.to_string()) +} + +/// Resolve policy names against the pinned workload image, never the gateway. +pub fn resolve_identity( + sandbox: &DriverSandbox, + image_id: &str, + image_user: &str, + passwd: &[u8], + group: &[u8], +) -> Result { + let passwd = std::str::from_utf8(passwd).map_err(invalid)?; + let group = std::str::from_utf8(group).map_err(invalid)?; + let accounts: Vec<_> = passwd + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some(( + name, + fields.next()?.parse::().ok()?, + fields.next()?.parse::().ok()?, + )) + }) + .collect(); + let groups: Vec<_> = group + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some((name, fields.next()?.parse::().ok()?, fields.next()?)) + }) + .collect(); + let request = sandbox + .spec + .as_ref() + .and_then(|spec| spec.workload_identity.as_ref()); + let requested_user = request.map_or("", |identity| identity.user.trim()); + let requested_group = request.map_or("", |identity| identity.group.trim()); + let (image_user, image_group) = image_user.split_once(':').unwrap_or((image_user, "")); + let user = if requested_user.is_empty() { + image_user + } else { + requested_user + }; + let group = if requested_group.is_empty() { + image_group + } else { + requested_group + }; + let account = accounts + .iter() + .find(|(name, uid, _)| *name == user || user.parse::().ok() == Some(*uid)); + let uid = user + .parse() + .ok() + .or_else(|| account.map(|(_, uid, _)| *uid)) + .ok_or_else(|| invalid("configure a non-root workload user present in the pinned image"))?; + let gid = if group.is_empty() { + account.map(|(_, _, gid)| *gid) + } else { + group.parse().ok().or_else(|| { + groups + .iter() + .find(|(name, _, _)| *name == group) + .map(|(_, gid, _)| *gid) + }) + } + .ok_or_else(|| { + invalid("configure an explicit workload group for a UID without an image passwd entry") + })?; + let supplemental = account.map_or_else(Vec::new, |(username, _, _)| { + groups + .iter() + .filter(|(_, id, members)| { + *id != gid && members.split(',').any(|member| member == *username) + }) + .map(|(_, gid, _)| *gid) + .collect() + }); + let source = if requested_user.is_empty() && requested_group.is_empty() { + "image" + } else { + "policy" + }; + ResolvedWorkloadIdentity::new(uid, gid, supplemental, source.into(), image_id.into()) + .map_err(invalid) +} + +pub struct BootstrapArchives { + pub channel: Vec, + pub workspace: Vec, + pub supervisor: Vec, +} + +/// The shared volume contains only sandbox credentials. Supervisor credentials, +/// gateway authorization, and the restart copy never enter that volume. +pub fn bootstrap_archives( + sandbox_id: &str, + container_id: &str, + identity: &ResolvedWorkloadIdentity, + child_env: HashMap, +) -> Result { + let tls = generate_boundary_mutual_tls_material().map_err(invalid)?; + let resource_claims = BTreeMap::from([ + ("podman.container_id".into(), container_id.into()), + ( + "podman.image_identity".into(), + identity.resource_digest.clone(), + ), + ]); + let driver_fence = DriverFenceEvidence::Podman { + container_id: container_id.into(), + network_mode: "none".into(), + unexpected_networks: Vec::new(), + }; + let generation = uuid::Uuid::new_v4().to_string(); + let session_epoch = uuid::Uuid::new_v4().to_string(); + let bootstrap_token = format!( + "{}{}", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ); + let config = BoundaryConfig { + boundary_id: sandbox_id.into(), + generation: generation.clone(), + session_epoch: session_epoch.clone(), + bootstrap_token: bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/.openshell/channel/sandbox/server.crt"), + private_key_path: PathBuf::from("/.openshell/channel/sandbox/server.key"), + client_ca_certificate_path: PathBuf::from( + "/.openshell/channel/sandbox/client-ca.crt", + ), + }, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: identity.clone(), + driver_fence: driver_fence.clone(), + child_env, + }; + let topology = BoundaryTopology { + boundary_id: sandbox_id.into(), + generation, + session_epoch, + bootstrap_token, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: BoundaryClientTls { + server_name: tls.server_name, + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem, + private_key_pem: tls.supervisor_private_key_pem, + }, + }, + host_gateway_ip: None, + resource_claims, + workload_identity: identity.clone(), + driver_fence, + }; + // Libpod resolves the requested upload destination once for a stopped + // container. Archive entries must be relative to the selected named volume, + // not rootfs paths that the volume would shadow on container start. + let mut channel = Archive::new(identity); + channel.directory(".", 0o755, false)?; + channel.directory("sandbox", 0o711, true)?; + channel.file( + "sandbox/bootstrap.json", + &serde_json::to_vec(&config).map_err(invalid)?, + )?; + channel.file("sandbox/server.crt", tls.sandbox_certificate_pem.as_bytes())?; + channel.file("sandbox/server.key", tls.sandbox_private_key_pem.as_bytes())?; + channel.file("sandbox/client-ca.crt", tls.ca_certificate_pem.as_bytes())?; + let channel = channel.finish()?; + let mut workspace = Archive::new(identity); + workspace.directory(".", 0o700, true)?; + let mut supervisor = Archive::new(identity); + supervisor.directory(".openshell", 0o755, false)?; + supervisor.directory(".openshell/supervisor", 0o700, true)?; + supervisor.file( + TOPOLOGY_PATH, + &serde_json::to_vec(&topology).map_err(invalid)?, + )?; + supervisor.file(RESTART_BUNDLE_PATH, &channel)?; + Ok(BootstrapArchives { + channel, + workspace: workspace.finish()?, + supervisor: supervisor.finish()?, + }) +} + +/// Retain only stable launch inputs from the supervisor-private restart copy. +/// Every container restart establishes a new authenticated listener generation. +pub fn rotate_bootstrap_archives( + sandbox_id: &str, + container_id: &str, + previous_channel: &[u8], +) -> Result { + let mut archive = tar::Archive::new(previous_channel); + for entry in archive.entries().map_err(invalid)? { + let entry = entry.map_err(invalid)?; + if entry.path().map_err(invalid)?.as_ref() != std::path::Path::new("sandbox/bootstrap.json") + { + continue; + } + if !entry.header().entry_type().is_file() { + return Err(invalid("restart bootstrap must be a regular file")); + } + let previous: BoundaryConfig = serde_json::from_reader(entry).map_err(invalid)?; + if previous.boundary_id != sandbox_id + || previous + .resource_claims + .get("podman.container_id") + .map(String::as_str) + != Some(container_id) + || !matches!(&previous.driver_fence, DriverFenceEvidence::Podman { container_id: bound, .. } if bound == container_id) + { + return Err(invalid( + "restart bootstrap does not belong to this Podman workload", + )); + } + previous + .driver_fence + .validate_for_backend("podman") + .map_err(invalid)?; + return bootstrap_archives( + sandbox_id, + container_id, + &previous.workload_identity, + previous.child_env, + ); + } + Err(invalid( + "supervisor restart archive is missing sandbox/bootstrap.json", + )) +} + +struct Archive<'a> { + builder: tar::Builder>, + identity: &'a ResolvedWorkloadIdentity, +} +impl<'a> Archive<'a> { + fn new(identity: &'a ResolvedWorkloadIdentity) -> Self { + Self { + builder: tar::Builder::new(Vec::new()), + identity, + } + } + fn directory(&mut self, path: &str, mode: u32, owned: bool) -> Result<(), ComputeDriverError> { + self.append(path, mode, owned, tar::EntryType::Directory, &[]) + } + fn file(&mut self, path: &str, content: &[u8]) -> Result<(), ComputeDriverError> { + self.append( + path.trim_start_matches('/'), + 0o600, + true, + tar::EntryType::Regular, + content, + ) + } + fn append( + &mut self, + path: &str, + mode: u32, + owned: bool, + kind: tar::EntryType, + content: &[u8], + ) -> Result<(), ComputeDriverError> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_mode(mode); + header.set_uid(if owned { + u64::from(self.identity.uid) + } else { + 0 + }); + header.set_gid(if owned { + u64::from(self.identity.gid) + } else { + 0 + }); + header.set_size(content.len() as u64); + header.set_mtime(0); + header.set_cksum(); + self.builder + .append_data(&mut header, path, content) + .map_err(invalid) + } + fn finish(self) -> Result, ComputeDriverError> { + self.builder.into_inner().map_err(invalid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read as _; + + #[test] + fn identity_uses_pinned_image_accounts_and_rejects_root() { + let sandbox = DriverSandbox::default(); + let passwd = b"root:x:0:0:root:/root:/bin/sh\nagent:x:1000:1001::/home/agent:/bin/sh\n"; + let groups = b"agent:x:1001:\ndata:x:2000:agent\n"; + let identity = + resolve_identity(&sandbox, "sha256:pinned", "agent", passwd, groups).unwrap(); + assert_eq!((identity.uid, identity.gid), (1000, 1001)); + assert_eq!(identity.supplementary_gids, vec![2000]); + assert_eq!(identity.resource_digest, "sha256:pinned"); + assert!(resolve_identity(&sandbox, "sha256:pinned", "root", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "2000", passwd, groups).is_err()); + } + + fn files(bytes: &[u8]) -> BTreeMap> { + tar::Archive::new(bytes) + .entries() + .unwrap() + .filter_map(|entry| { + let mut entry = entry.unwrap(); + if !entry.header().entry_type().is_file() { + return None; + } + let path = entry.path().unwrap().into_owned(); + assert_eq!(entry.header().mode().unwrap(), 0o600); + assert_eq!(entry.header().uid().unwrap(), 1000); + let mut content = Vec::new(); + entry.read_to_end(&mut content).unwrap(); + Some((path, content)) + }) + .collect() + } + + #[test] + fn archives_separate_supervisor_credentials_and_bind_one_channel() { + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let archives = + bootstrap_archives("sandbox", "container", &identity, HashMap::new()).unwrap(); + let workload = files(&archives.channel); + let supervisor = files(&archives.supervisor); + let mut workspace = tar::Archive::new(archives.workspace.as_slice()); + let mut entries = workspace.entries().unwrap(); + let root = entries.next().unwrap().unwrap(); + assert_eq!(root.path().unwrap().as_ref(), std::path::Path::new(".")); + assert!(root.header().entry_type().is_dir()); + assert_eq!(root.header().uid().unwrap(), u64::from(identity.uid)); + assert_eq!(root.header().gid().unwrap(), u64::from(identity.gid)); + assert_eq!(root.header().mode().unwrap(), 0o700); + assert!(entries.next().is_none()); + assert_eq!(workload.len(), 4); + assert_eq!(supervisor.len(), 2); + assert!(workload.keys().all(|path| path.starts_with("sandbox"))); + assert!( + supervisor + .keys() + .all(|path| path.starts_with(".openshell/supervisor")) + ); + let config: BoundaryConfig = serde_json::from_slice( + workload + .get(&PathBuf::from("sandbox/bootstrap.json")) + .unwrap(), + ) + .unwrap(); + let topology: BoundaryTopology = serde_json::from_slice( + supervisor + .get(&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))) + .unwrap(), + ) + .unwrap(); + assert_eq!(config.boundary_id, topology.boundary_id); + assert_eq!(config.bootstrap_token, topology.bootstrap_token); + assert_eq!(config.driver_fence, topology.driver_fence); + assert_eq!(config.workload_identity, identity); + topology + .driver_fence + .validate_for_backend("podman") + .unwrap(); + assert_eq!( + supervisor + .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) + .unwrap(), + &archives.channel + ); + } + + #[test] + fn restart_rotates_every_authentication_value_while_preserving_launch_inputs() { + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let original = bootstrap_archives( + "sandbox", + "container", + &identity, + HashMap::from([("PATH".into(), "/usr/bin".into())]), + ) + .unwrap(); + let renewed = rotate_bootstrap_archives("sandbox", "container", &original.channel).unwrap(); + let original_files = files(&original.channel); + let renewed_files = files(&renewed.channel); + let supervisor_files = files(&renewed.supervisor); + let old: BoundaryConfig = + serde_json::from_slice(&original_files[&PathBuf::from("sandbox/bootstrap.json")]) + .unwrap(); + let new: BoundaryConfig = + serde_json::from_slice(&renewed_files[&PathBuf::from("sandbox/bootstrap.json")]) + .unwrap(); + let supervisor: BoundaryTopology = serde_json::from_slice( + &supervisor_files[&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))], + ) + .unwrap(); + assert_ne!(new.generation, old.generation); + assert_ne!(new.session_epoch, old.session_epoch); + assert_ne!(new.bootstrap_token, old.bootstrap_token); + assert_eq!(new.generation, supervisor.generation); + assert_eq!(new.session_epoch, supervisor.session_epoch); + assert_eq!(new.bootstrap_token, supervisor.bootstrap_token); + assert_eq!(new.workload_identity, old.workload_identity); + assert_eq!(new.resource_claims, old.resource_claims); + assert_eq!(new.child_env, old.child_env); + for path in [ + "sandbox/server.crt", + "sandbox/server.key", + "sandbox/client-ca.crt", + ] { + assert_ne!( + renewed_files[&PathBuf::from(path)], + original_files[&PathBuf::from(path)] + ); + } + let BoundaryTransport::Unix { tls, .. } = supervisor.transport else { + panic!("expected Unix channel") + }; + assert_eq!( + tls.ca_certificate_pem.as_bytes(), + &renewed_files[&PathBuf::from("sandbox/client-ca.crt")] + ); + assert_eq!( + supervisor_files[&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))], + renewed.channel + ); + assert!(rotate_bootstrap_archives("other", "container", &original.channel).is_err()); + assert!(rotate_bootstrap_archives("sandbox", "other", &original.channel).is_err()); + } +} diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index 115e64eb2f..fa06cf4864 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub(crate) mod container; pub mod driver; pub mod grpc; +mod isolation; pub mod otel_tracing; mod socket_discovery; #[cfg(test)] diff --git a/crates/openshell-driver-podman/src/test_utils.rs b/crates/openshell-driver-podman/src/test_utils.rs index ec5c8f7f11..25cbcb4ac2 100644 --- a/crates/openshell-driver-podman/src/test_utils.rs +++ b/crates/openshell-driver-podman/src/test_utils.rs @@ -3,7 +3,7 @@ //! Shared test helpers for openshell-driver-podman unit tests. -use http_body_util::Full; +use http_body_util::{BodyExt as _, Full}; use hyper::StatusCode; use hyper::body::Bytes; use hyper::server::conn::http1; @@ -20,16 +20,18 @@ use tokio::net::UnixListener; #[derive(Clone)] pub struct StubResponse { pub status: StatusCode, - pub body: String, + pub body: Bytes, pub delay: Duration, + pub archive_members: Option>, } impl StubResponse { - pub fn new(status: StatusCode, body: impl Into) -> Self { + pub fn new(status: StatusCode, body: impl Into) -> Self { Self { status, body: body.into(), delay: Duration::ZERO, + archive_members: None, } } @@ -37,6 +39,11 @@ impl StubResponse { self.delay = delay; self } + + pub fn with_archive_members(mut self, members: &[&str]) -> Self { + self.archive_members = Some(members.iter().map(PathBuf::from).collect()); + self + } } /// Generate a unique Unix socket path for a test. @@ -88,7 +95,7 @@ pub fn spawn_podman_stub( let result = http1::Builder::new() .serve_connection( TokioIo::new(stream), - service_fn(move |req| { + service_fn(move |req: hyper::Request| { let log = log.clone(); let queue = queue.clone(); async move { @@ -104,11 +111,22 @@ pub fn spawn_podman_stub( .expect("response queue lock should not be poisoned") .pop_front() .expect("stub response should exist"); + if let Some(expected_members) = &response.archive_members { + assert_eq!(req.method(), hyper::Method::PUT); + let body = req.into_body().collect().await.unwrap().to_bytes(); + let mut archive = tar::Archive::new(body.as_ref()); + let members: Vec<_> = archive + .entries() + .unwrap() + .map(|entry| entry.unwrap().path().unwrap().into_owned()) + .collect(); + assert_eq!(&members, expected_members); + } tokio::time::sleep(response.delay).await; Ok::<_, Infallible>( hyper::Response::builder() .status(response.status) - .body(Full::new(Bytes::from(response.body))) + .body(Full::new(response.body)) .expect("stub response should build"), ) } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index cc98e29641..299f61cf18 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -138,14 +138,16 @@ pub async fn start_watch( let mut event_rx = client.events_stream(LABEL_MANAGED_FILTER).await?; // 2. List existing containers for initial state sync. - let existing = client.list_containers(&[LABEL_MANAGED_FILTER]).await?; + let existing = client + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) + .await?; for entry in &existing { // For running containers, use inspect to get full state including // health check status — matching the same condition derivation used // for live events. if entry.state == "running" { - match client.inspect_container(&entry.id).await { + match inspect_workload(&client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { if tx.send(Ok(sandbox_event(sandbox))).await.is_err() { @@ -245,11 +247,34 @@ async fn map_podman_event( return None; } + if event + .actor + .attributes + .get(crate::isolation::LABEL_ROLE) + .is_some_and(|role| role == "supervisor") + { + let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); + let workloads = client + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) + .await + .ok()?; + let workload = workloads.first()?; + return inspect_workload(client, &workload.id) + .await + .ok() + .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) + .map(sandbox_event); + } + match event.action.as_str() { "remove" => Some(deleted_event(sandbox_id.clone())), "create" | "start" | "stop" | "die" | "health_status" => { // Inspect the container to get current state. - match client.inspect_container(container_id).await { + match inspect_workload(client, container_id).await { Ok(inspect) => { if lifecycle_event_fences.matches_previous_exit( event, @@ -326,6 +351,54 @@ async fn map_podman_event( } } +/// A workload is ready only when its independent supervisor is healthy. This +/// check runs both on watch reconciliation and on events, and contains a lost +/// supervisor even when the gateway missed the original exit event. +pub async fn inspect_workload( + client: &PodmanClient, + id: &str, +) -> Result { + let mut workload = client.inspect_container(id).await?; + if workload + .config + .labels + .get(crate::isolation::LABEL_ROLE) + .is_none_or(|role| role != "sandbox") + { + return Ok(workload); + } + let Some(sandbox_id) = workload.config.labels.get(LABEL_SANDBOX_ID) else { + return Ok(workload); + }; + let supervisor = client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if workload.state.running { + match supervisor { + Ok(supervisor) if supervisor.state.running => { + workload.state.health = supervisor.state.health; + } + Ok(supervisor) + if supervisor.state.status == "configured" + || supervisor.state.status == "created" => + { + workload.state.health = Some(HealthState { + status: "starting".into(), + }); + } + // Both containers exist before initial start. A missing or exited + // companion therefore requires containment, including after a + // gateway restart that missed the original Podman exit event. + Ok(_) | Err(PodmanApiError::NotFound(_)) => { + client.stop_container(&workload.id, 0).await?; + workload = client.inspect_container(&workload.id).await?; + } + Err(error) => return Err(error), + } + } + Ok(workload) +} + /// Construct a `DriverSandbox` from common fields. /// /// Centralises the boilerplate that every event/inspect/list path shares: @@ -499,6 +572,65 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { mod tests { use super::*; + #[tokio::test] + async fn missing_supervisor_stops_workload_during_reconciliation() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "lost-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new(StatusCode::NOT_FOUND, "missing companion"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(!inspected.state.running); + handle.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|request| request.ends_with("/libpod/containers/workload/stop?timeout=0")) + ); + let _ = std::fs::remove_file(path); + } + + #[tokio::test] + async fn created_supervisor_keeps_bootstrapping_workload_starting() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "starting-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"configured","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(inspected.state.running); + assert_eq!(inspected.state.health.unwrap().status, "starting"); + handle.await.unwrap(); + assert_eq!(requests.lock().unwrap().len(), 2); + let _ = std::fs::remove_file(path); + } + fn podman_event(action: &str, sandbox_id: &str, time_nano: i64) -> PodmanEvent { PodmanEvent { event_type: "container".to_string(), diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index fa2d9ba835..813ea93dfe 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -618,7 +618,7 @@ sandbox_pids_limit = 2048 ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Each Podman sandbox has a workload container running `openshell-sandbox` with `network=none`, and a separate `openshell-supervisor` companion on the configured network. Both use non-root identities, drop all capabilities, and keep the runtime's default seccomp profile. A private named volume carries their authenticated gRPC Unix socket. Gateway JWTs, optional gateway mTLS material, and upstream proxy credentials are delivered only to the supervisor; user mounts and GPU devices stay with the workload. ```toml [openshell] @@ -659,7 +659,8 @@ enable_bind_mounts = false sandbox_pids_limit = 2048 # Health check interval in seconds. Lower values detect readiness faster # but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to disable health checks entirely. Default: 10. +# Set to 0 to use a one-second check. Readiness checks cannot be disabled. +# Default: 10. health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -670,7 +671,7 @@ health_check_interval_secs = 10 # rootful Podman uses absolute host IDs (0:1000:1, 1:100000:65536). # uidmap = ["0:0:1", "1:1:65535"] # gidmap = ["0:0:1", "1:1:65535"] -# Corporate forward proxy for sandbox egress. When set, the in-container +# Corporate forward proxy for sandbox egress. When set, the external # supervisor chains policy-approved TLS tunnels through this proxy with HTTP # CONNECT instead of dialing destinations directly. Plain-HTTP requests are # not proxied and always dial the destination directly. http:// and https:// diff --git a/e2e/rust/tests/podman_gateway_start.rs b/e2e/rust/tests/podman_gateway_start.rs index 28a9ceb15c..7e159ffa78 100644 --- a/e2e/rust/tests/podman_gateway_start.rs +++ b/e2e/rust/tests/podman_gateway_start.rs @@ -38,10 +38,10 @@ const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; /// harness), fall back to plain `podman`, leaving Linux behavior unchanged. fn podman_command() -> Command { let mut command = Command::new("podman"); - if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - if !socket.is_empty() { - command.arg("--url").arg(format!("unix://{socket}")); - } + if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !socket.is_empty() + { + command.arg("--url").arg(format!("unix://{socket}")); } command } @@ -49,7 +49,15 @@ fn podman_command() -> Command { fn sandbox_container_running(sandbox_name: &str) -> Result { let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); let output = podman_command() - .args(["ps", "-aq", "--filter", MANAGED_BY_LABEL_FILTER, "--filter"]) + .args([ + "ps", + "-aq", + "--filter", + MANAGED_BY_LABEL_FILTER, + "--filter", + "label=openshell.io/isolation-role=sandbox", + "--filter", + ]) .arg(sandbox_name_filter) .stdout(Stdio::piped()) .stderr(Stdio::piped()) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index e30516bf09..15ea13a77f 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -22,7 +22,7 @@ const BASE_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:late const READY_MARKER: &str = "podman-oci-identity-ready"; const OCI_UID: &str = "2345"; const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r#"version: 1 +const OCI_FALLBACK_POLICY: &str = r"version: 1 filesystem_policy: include_workdir: true @@ -32,7 +32,7 @@ landlock: compatibility: best_effort network_policies: {} -"#; +"; struct ImageGuard { engine: ContainerEngine, @@ -128,7 +128,16 @@ fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result } fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result { + container_id_for_role(engine, sandbox_name, "sandbox") +} + +fn container_id_for_role( + engine: &ContainerEngine, + sandbox_name: &str, + role: &str, +) -> Result { let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); + let role_filter = format!("label=openshell.io/isolation-role={role}"); let stdout = run_engine( engine, &[ @@ -138,6 +147,8 @@ fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result< "label=openshell.managed=true", "--filter", &name_filter, + "--filter", + &role_filter, ], )?; let ids = stdout @@ -233,5 +244,54 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { "Podman sandbox must launch the immutable image ID inspected before creation" ); + assert_isolated_pair(&image, &sandbox, &container_id).await; sandbox.cleanup().await; } + +async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, container_id: &str) { + let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") + .expect("find separate supervisor companion"); + assert_ne!(supervisor_id, container_id); + for id in [container_id, &supervisor_id] { + let user = run_engine( + &image.engine, + &["inspect", "--format", "{{.Config.User}}", id], + ) + .unwrap(); + assert_eq!(user, format!("{OCI_UID}:{OCI_GID}")); + let caps = run_engine( + &image.engine, + &["inspect", "--format", "{{.EffectiveCaps}}", id], + ) + .unwrap(); + assert_eq!( + caps, "[]", + "neither container may have effective capabilities" + ); + } + let network = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{.HostConfig.NetworkMode}}", + container_id, + ], + ) + .unwrap(); + assert_eq!(network, "none"); + let mounts = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{range .Mounts}}{{println .Destination}}{{end}}", + container_id, + ], + ) + .unwrap(); + assert!(!mounts.contains("/etc/openshell/tls")); + assert!(!mounts.contains("/.openshell/supervisor")); + let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/topology.payload"]).await.expect("workload cannot read either control credential set"); + assert!(posture.contains("0000000000000000")); +} diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fc3419e182..0868cd9883 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -160,9 +160,24 @@ cleanup() { for id in ${sandbox_ids}; do local sandbox_id sandbox_id="$(podman_cmd inspect --format '{{ index .Config.Labels "openshell.ai/sandbox-id" }}' "${id}" 2>/dev/null || true)" - podman_cmd rm -f "${id}" >/dev/null 2>&1 || true if [ -n "${sandbox_id}" ] && [ "${sandbox_id}" != "" ]; then + # Only the companion is attached to the test network. Remove it first + # (it depends on the workload user namespace), then locate the isolated + # network=none workload by this test sandbox's immutable label. + podman_cmd rm -f "openshell-supervisor-${sandbox_id}" >/dev/null 2>&1 || true + local workload_ids workload_id + workload_ids="$(podman_cmd ps -aq --filter "label=openshell.managed=true" \ + --filter "label=openshell.ai/sandbox-id=${sandbox_id}" \ + --filter "label=openshell.io/isolation-role=sandbox" 2>/dev/null || true)" + for workload_id in ${workload_ids}; do + podman_cmd rm -f "${workload_id}" >/dev/null 2>&1 || true + done + podman_cmd volume rm "openshell-channel-${sandbox_id}" >/dev/null 2>&1 || true podman_cmd volume rm -f "openshell-sandbox-${sandbox_id}-workspace" >/dev/null 2>&1 || true + local secret_prefix + for secret_prefix in openshell-token openshell-proxy-auth openshell-tls-ca openshell-tls-cert openshell-tls-key; do + podman_cmd secret rm "${secret_prefix}-${sandbox_id}" >/dev/null 2>&1 || true + done fi done fi @@ -411,6 +426,9 @@ export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) +while [ "${HEALTH_PORT}" = "${HOST_PORT}" ]; do + HEALTH_PORT=$(e2e_pick_port) +done if [ "$(uname -s)" = "Darwin" ]; then # Podman Machine reserves IPv4 loopback for its callback-only listener. PRIMARY_BIND_IP="::1" diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ce676148aa..89f6533c83 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -225,10 +225,14 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. -- A sandbox with explicit `protocol: tcp` endpoints fails before readiness: - inspect supervisor logs for policy DNS port-53 binding, synthetic-route, or - nftables redirect failures. Rootless Podman must provide these primitives - inside the supervisor-owned nested network namespace; setup fails closed. +- Inspect both Podman containers for the sandbox: the `sandbox` isolation role + must have network mode `none`; the `supervisor` role owns gateway callbacks + and egress. Both run non-root with all capabilities dropped. Check the private + channel volume and shared user-namespace mapping if authentication fails. +- If a sandbox fails before readiness, inspect its unprivileged enforcement + probe and the companion supervisor's private health check. Do not add + capabilities, attach a workload network, or disable the runtime seccomp + profile. There is no sandbox nftables or nested-network setup to repair. - Gateway exits before becoming healthy with a callback-listener discovery error: inspect `podman info --debug`, the configured Podman network, and the host's IPv4 default route. Rootless pasta uses the private source address