From d42a7ccd1d7d9b94396ad0226fd3f48f865d00c8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:09:05 -0700 Subject: [PATCH 1/5] feat(podman): adopt isolated sandbox and supervisor containers Signed-off-by: Drew Newberry --- Cargo.lock | 3 + architecture/compute-runtimes.md | 9 + crates/openshell-driver-podman/Cargo.toml | 3 + crates/openshell-driver-podman/NETWORKING.md | 494 ++------------- crates/openshell-driver-podman/README.md | 568 ++++------------- crates/openshell-driver-podman/src/client.rs | 85 +++ .../openshell-driver-podman/src/container.rs | 482 ++++++++++----- crates/openshell-driver-podman/src/driver.rs | 578 ++++++++++++++---- crates/openshell-driver-podman/src/grpc.rs | 9 +- .../openshell-driver-podman/src/isolation.rs | 381 ++++++++++++ crates/openshell-driver-podman/src/lib.rs | 1 + .../openshell-driver-podman/src/test_utils.rs | 6 +- crates/openshell-driver-podman/src/watcher.rs | 138 ++++- docs/reference/gateway-config.mdx | 7 +- e2e/rust/tests/podman_gateway_start.rs | 18 +- e2e/rust/tests/podman_oci_identity.rs | 64 +- e2e/with-podman-gateway.sh | 17 +- skills/debug-openshell-cluster/SKILL.md | 12 +- 18 files changed, 1658 insertions(+), 1217 deletions(-) create mode 100644 crates/openshell-driver-podman/src/isolation.rs diff --git a/Cargo.lock b/Cargo.lock index 87021d5eea..9bb6846e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4040,6 +4040,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -4048,6 +4049,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_json", + "tar", "temp-env", "thiserror 2.0.18", "tokio", @@ -4058,6 +4060,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "uuid", ] [[package]] diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0396f51bc0..a502589dde 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -5,6 +5,15 @@ gateway. A supported runtime provisions `openshell-sandbox` inside the workload, `openshell-supervisor` outside it, a protected channel between them, and an independent outer network fence. Drivers do not implement policy evaluation. +Podman provisions a paired workload and supervisor container using its native +libpod API. The workload uses `network=none`; the external supervisor alone joins +the configured network. A per-sandbox named volume carries their mutually +authenticated gRPC Unix socket, with supervisor credentials kept in its separate +filesystem. Both containers run as the resolved non-root identity with all +capabilities dropped. They share only a user namespace for volume ownership, +not PID, mount, or network namespaces. Podman owns paired lifecycle and health; +the common protocol owns process, identity, TCP, DNS, and forwarding semantics. + ## Driver Contract Each runtime receives a sandbox spec and canonical policy from the gateway and diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 8b3e014e8c..cd463b86fe 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -17,6 +17,9 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } openshell-otel = { path = "../openshell-otel" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +tar = "0.4" +uuid = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 567abcbfcd..9a0f96475f 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -1,463 +1,61 @@ -# Rootless Podman Networking +# Podman Networking -Deep-dive into how networking works in the Podman compute driver when running -rootless with pasta as the network backend. Covers the external tooling -(Podman, Netavark, pasta, aardvark-dns), the three nested namespace layers, and -the complete data paths for SSH, outbound traffic, and supervisor-to-gateway -communication. - -For the general Podman driver architecture, lifecycle, API surface, and driver -comparison, see [README.md](README.md). - -## Component Stack - -Podman's networking is composed of four independent projects: - -| Component | Language | Role | -|---|---|---| -| Podman | Go | Container runtime; orchestrates network lifecycle. | -| Netavark | Rust | Network backend; creates interfaces, bridges, firewall rules. | -| aardvark-dns | Rust | Authoritative DNS server for container name resolution. | -| pasta, part of passt | C | User-mode networking; L2-to-L4 socket translation for rootless containers. | - -The key split: rootful containers default to Netavark bridge networking with -real kernel interfaces, while rootless containers commonly use pasta user-mode -networking without needing host privileges. - -## How Netavark Works - -Netavark is invoked by Podman as an external binary. It reads a JSON network -configuration from STDIN and executes one of three commands: - -- `netavark setup ` creates interfaces, assigns IPs, and sets up - firewall rules for NAT and port-forwarding. -- `netavark teardown ` reverses setup and removes interfaces and - firewall rules. -- `netavark create` takes a partial network config and completes it by - assigning subnets and gateways. - -For rootful bridge networking: - -1. Podman creates a network namespace for the container. -2. Podman invokes `netavark setup` with the network config JSON. -3. Netavark creates a bridge, such as `podman0`, if it does not exist. The - default subnet is `10.88.0.0/16`. -4. Netavark creates a veth pair. One end goes into the container's netns and - the other attaches to the bridge. -5. Netavark assigns an IP from the subnet to the container's veth interface. -6. Netavark configures iptables or nftables rules for masquerade and port - mappings. -7. Netavark starts aardvark-dns when DNS is enabled, listening on the bridge - gateway address. - -```text -Host Kernel - | - +-- Bridge interface, such as "podman0" - | | - | +-- veth pair endpoint, host side, container 1 - | +-- veth pair endpoint, host side, container 2 - | - +-- Host physical interface, such as eth0 - | - +-- NAT, iptables or nftables rules managed by Netavark -``` - -Netavark also supports macvlan networks, where the container gets a -sub-interface of a physical host NIC with its own MAC address, and external -plugins via a documented JSON API. - -## How Pasta Works - -Unprivileged users cannot create network interfaces on the host. They cannot -create veth pairs, bridges, or iptables rules. Netavark's bridge approach -cannot work directly for rootless containers without an additional rootless -networking layer. - -Pasta, part of the `passt` project, operates in userspace and translates -between the container's L2 TAP interface and the host's L4 sockets. It requires -no capabilities or privileges. - -```text -Container Network Namespace - | - +-- TAP device, such as "eth0" - | ^ - | | L2 frames, Ethernet - | v - +-- pasta process, userspace - | - | Translation: L2 frames <-> L4 sockets - | - v - Host Network Stack, native TCP/UDP/ICMP sockets -``` - -For an outbound TCP connection from a container: - -1. The application calls `connect()` to an external address. -2. The kernel routes the packet through the default gateway to the TAP device. -3. Pasta reads the raw Ethernet frame from the TAP file descriptor. -4. Pasta parses L2/L3/L4 headers and identifies the TCP SYN. -5. Pasta opens a native TCP socket on the host and calls `connect()` to the - same destination. -6. When the host socket connects, pasta reflects the SYN-ACK back through the - TAP as an L2 frame. -7. For ongoing data transfer, pasta translates between TAP frames and the host - socket, coordinating TCP windows and acknowledgments between the two sides. - -Pasta does not maintain per-connection packet buffers. It reflects observed -sending windows and ACKs directly between peers. This is a thinner translation -layer than a full TCP/IP stack. - -### Built-in Services - -Pasta includes minimal network services so the container stack can -auto-configure: - -| Service | Purpose | -|---|---| -| ARP proxy | Resolves the gateway address to the host's MAC address. | -| DHCP server | Hands out a single IPv4 address, usually matching the host's upstream interface. | -| NDP proxy | Handles IPv6 neighbor discovery and SLAAC prefix advertisement. | -| DHCPv6 server | Hands out a single IPv6 address, usually matching the host's upstream interface. | - -By default there is no NAT. Pasta copies the host's IP addresses into the -container namespace. - -### Local Connection Bypass - -For connections between the container and the host, pasta implements a local -bypass path: - -- Packets with a local destination skip L2 translation. -- TCP uses `splice(2)`. -- UDP uses `recvmmsg(2)` and `sendmmsg(2)`. - -### Port Forwarding - -By default, pasta uses auto-detection. It scans `/proc/net/tcp` and -`/proc/net/tcp6` periodically and automatically forwards ports that are bound -and listening. Port forwarding is configurable through pasta options. - -### Security Properties - -Pasta is designed for rootless use: - -- No dynamic memory allocation after startup. -- All capabilities dropped, except `CAP_NET_BIND_SERVICE` when granted. -- Restrictive seccomp profile. -- Detaches into its own user, mount, IPC, UTS, and PID namespaces. -- No external dependencies beyond libc. - -### Inter-Container Limitation - -Unlike bridge networking, pasta containers are isolated from each other by -default. No virtual bridge connects them. Communication requires port mappings -through the host, pods with a shared network namespace, or opting into rootless -Netavark bridge networking with `podman network create`. - -## Three Nested Namespaces - -The Podman compute driver creates three layers of network isolation: +Only the external supervisor has external network connectivity. The workload +container uses `network=none`; its loopback DNS relay and TCP socket mediation +reach the supervisor through a protected Unix socket, not a veth or proxy +environment variable. ```text -Namespace 1: Host - | - pasta manages port forwarding, such as 127.0.0.1: - gateway listens on its configured bind address and port - | -Namespace 2: Rootless Podman network namespace, managed by pasta - | - Bridge "openshell", often 10.89.x.0/24 - aardvark-dns for container name resolution - | - Container netns - supervisor, proxy, and relay client run here - | -Namespace 3: Inner sandbox netns, created by supervisor - | - veth pair, such as 10.200.0.1 <-> 10.200.0.2 - nftables forces ordinary traffic through proxy - user workload runs here +workload container supervisor container +agent -> sandbox -- private UDS / gRPC -> policy proxy -> Podman network -> destination + | + +-- authenticated gateway callback ``` -Pasta bridges namespace 1 and 2. The veth pair bridges namespace 2 and 3. The -proxy at the boundary of namespace 2 and 3 enforces network policy. +## Outer network fence -### Layer 1 Pasta +The driver creates the workload without networks, host aliases, published +ports, or added capabilities. It checks the Podman inspect response before +launch and restart. The sandbox installs seccomp mediation and Landlock before +executing the agent. It does not create network namespaces, configure nftables, +or require `CAP_NET_ADMIN`. -At driver startup, the driver ensures a Podman bridge network exists: +TCP opens, TCP byte streams, DNS requests/replies, and lifecycle operations +share the authenticated gRPC channel. DNS is resolved and authorized by the +supervisor. General UDP is unsupported. -```rust -client.ensure_network(&config.network_name).await?; -``` +## Supervisor callback network -This creates a bridge network named `openshell` by default, with DNS enabled. -In rootless mode, this bridge can exist inside a user namespace managed by -pasta. The bridge IP range is not reliably routable from the host. - -```text -Host - | - 127.0.0.1:, pasta binds this on the host - | - pasta process, translates L4 sockets <-> L2 TAP frames - | - rootless network namespace - | - Bridge "openshell", such as 10.89.1.0/24 - | - +-- 10.89.1.1, bridge gateway and aardvark-dns - | - +-- veth to container netns - | - 10.89.1.2, container IP -``` - -### Layer 2 Container Networking - -The container spec configures: - -- `nsmode: "bridge"` to use the Podman bridge network. -- `networks` to attach to the configured bridge, `openshell` by default. -- `portmappings` with `host_port: 0`, `container_port: 2222`, and `protocol: - "tcp"` to publish the SSH compatibility port on an ephemeral host port. -- `hostadd` entries for `host.containers.internal` and - `host.openshell.internal`, using Podman's `host-gateway` resolver or the - configured `host_gateway_ip`. - -Pasta is not explicitly configured by the driver. The driver requests bridge -mode and logs the network backend that Podman reports at startup. - -The `host.containers.internal` hostname is injected into `/etc/hosts` so the -supervisor can reach the gateway on the host. Linux defaults to -`host-gateway`; macOS Podman machine defaults to `192.168.127.254`, gvproxy's -host-loopback IP, because older Podman machine images can fail to resolve -`host-gateway`. Override this with `host_gateway_ip` or -`OPENSHELL_PODMAN_HOST_GATEWAY_IP` when a Podman machine uses a non-standard -host-loopback address. - -If `OPENSHELL_GRPC_ENDPOINT` is empty, the driver auto-detects: - -```rust -if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = - format!("{scheme}://host.containers.internal:{}", config.gateway_port); -} -``` - -The bridge gateway IP is not a stable substitute in rootless mode because it -can live inside the user namespace rather than on the host. - -Before the gateway binds its serving sockets, the driver reports the callback -listener required by the selected topology: - -- Rootful Linux Podman reports the configured bridge's gateway address exactly. -- Rootless Linux Podman explicitly reporting pasta requests the private IPv4 - source address selected by the host's default route. This avoids guessing - among private interfaces on a multihomed host. -- Rootless Linux Podman reporting slirp4netns, another named helper, or no - helper cannot use a direct local callback listener. The driver fails startup - unless `grpc_endpoint` names an explicitly remote endpoint. Supporting - slirp4netns requires a relay inside Podman's rootless network namespace. -- Podman Machine requests IPv4 loopback because gvproxy terminates the host - forwarding path there. -- An explicitly remote callback endpoint requests no additional local listener. - -On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman -and rootless pasta because the driver maps both local callback aliases to that -literal. Other rootless helpers still fail closed. Podman Machine requests -gateway loopback because its configured address is guest-visible and gvproxy -terminates that route on host loopback. The gateway validates and binds every -accepted callback requirement. If the primary listener covers the requested -address, the gateway reuses it and relies on sandbox JWT authorization to limit -the supervisor's RPCs. Otherwise, it creates an additional listener that -exposes only the gateway's sandbox-callable gRPC methods. Operator, health, -reflection, and HTTP requests must use the primary listener. - -### Layer 3 Inner Sandbox Network Namespace - -Inside the container, the supervisor creates another network namespace for the -user workload: - -```text -Container on the Podman bridge - | - Supervisor process, running in container's default netns - | - +-- Proxy listener at the inner namespace gateway address - | - +-- veth pair - | - +-- Inner network namespace - | - sandbox-side veth address - | - default route -> supervisor-side veth address - | - user code runs here - | - nftables rules: - ACCEPT -> proxy TCP - ACCEPT -> loopback - ACCEPT -> established/related - LOG -> TCP SYN bypass attempts - REJECT -> TCP - LOG -> UDP bypass attempts - REJECT -> UDP -``` - -The supervisor uses `nsenter --net=` rather than `ip netns exec` to avoid sysfs -remount issues that arise under rootless Podman where real host -`CAP_SYS_ADMIN` is unavailable. - -For a policy with explicit `protocol: tcp` endpoints, this same inner namespace -also hosts policy DNS and transparent TCP capture. The supervisor answers only -policy-eligible names with epoch-scoped synthetic addresses, redirects TCP to -those synthetic ranges into its transparent listener, and leaves direct real-IP -dials subject to the terminal bypass fence. The Podman driver advertises this -substrate through its driver-owned runtime capability; sandbox image and policy -environment values cannot opt into it independently. - -The container spec preserves Podman's resolver search domains and options. -Policy DNS captures both UDP and TCP in the inner namespace, so it does not -depend on libc honoring `use-vc` and does not change ordinary short-name -resolution for sandboxes that do not use native TCP. - -A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can -create named network namespaces. In rootless Podman this directory does not -exist on the host, so a private tmpfs gives the supervisor its own writable -`/run/netns` without needing host filesystem access. - -## Complete Data Paths - -### SSH Session - -```text -Client, openshell CLI - | - 1. gRPC: CreateSshSession -> gateway, returns token and connect_path - 2. HTTP CONNECT /connect/ssh to gateway - headers: x-sandbox-id, x-sandbox-token - | -Gateway - | - 3. Looks up SupervisorSession for sandbox_id - 4. Sends RelayOpen{channel_id} over ConnectSupervisor bidi stream - | - gRPC traverses host -> pasta translation -> container bridge - | -Supervisor inside container - | - 5. Receives RelayOpen, opens new RelayStream RPC back to gateway - 6. Sends RelayInit{channel_id} on the stream - 7. Connects to Unix socket /run/openshell/ssh.sock - 8. Bidirectional bridge: RelayStream <-> Unix socket - | -SSH daemon inside container, Unix socket only - | - 9. Authenticates. Access is gated by the relay chain. - 10. Spawns shell process - 11. Shell enters inner netns via setns(fd, CLONE_NEWNET) - | -User shell in sandbox netns -``` - -The SSH daemon listens on a Unix socket with restrictive permissions. The -published TCP port mapping exists in the container spec for compatibility and -health/debug paths. Normal SSH communication uses the gRPC reverse-connect relay -pattern. - -### Outbound HTTP Request - -```text -User code in inner netns - | - 1. curl https://api.example.com - HTTP_PROXY points at the local sandbox proxy - | - 2. TCP connect to proxy - allowed by nftables as the only ordinary egress destination - | - 3. HTTP CONNECT api.example.com:443 - | -Supervisor proxy in container netns - | - 4. Policy evaluation with process identity - 5. SSRF check - 6. Optional L7 TLS intercept and HTTP method/path inspection - | - 7. If allowed, TCP connect to api.example.com:443 - from the container netns - | - 8. Through Podman bridge -> pasta -> host -> internet -``` - -### Supervisor gRPC Callback - -The Podman driver auto-detects the callback endpoint scheme based on whether -TLS client certificates are configured. When the RPM's auto-generated PKI is in -place, the endpoint is `https://host.containers.internal:17670` and the -supervisor connects with mTLS. Without TLS configuration, it falls back to -`http://host.containers.internal:`. - -```text -Supervisor in container netns - | - 1. Connects to host.containers.internal: - with mTLS when OPENSHELL_TLS_* paths are set - | - 2. Routed through container default gateway - | - 3. Pasta translates L2 frame -> host L4 socket when rootless backend uses pasta - | - 4. Host TCP socket connects to gateway - | -Gateway - | - 5. TLS handshake when enabled - 6. ConnectSupervisor bidirectional stream established - 7. Heartbeats at the interval accepted by the gateway - 8. Reconnects with exponential backoff on failure - 9. Same gRPC channel reused for RelayStream calls -``` +The configured `network_name`, host-gateway aliases, upstream corporate proxy, +and published SSH port apply only to the supervisor companion. The gateway's +SSH tunnel still uses the supervisor relay, not the published port. -The gateway binds to `127.0.0.1:17670` by default in the RPM packaging. Client -certificates are auto-generated by `openshell-gateway generate-certs` on first -start and bind-mounted into sandbox containers by the Podman driver. +Rootful Podman uses the configured bridge and its gateway address. Rootless +local callbacks require the existing pasta path; slirp4netns or unknown helpers +require an explicitly remote `grpc_endpoint`. On macOS, Podman Machine provides +the runtime and host-loopback forwarding. -## Differences from the Kubernetes Driver +These runtime-managed network helpers are outside the workload trust boundary. +Sharing the workload's user namespace preserves volume UID/GID mapping; it +does not share the workload's PID, mount, or network namespaces. -| Aspect | Kubernetes | Podman, rootless pasta | -|---|---|---| -| Container or pod IP | Routable cluster-wide | Non-routable from the host in common rootless setups. | -| Network reachability | Pod IPs reachable from gateway | Bridge not reliably routable from host; requires host aliases or published ports. | -| Sandbox to gateway | Direct TCP to Kubernetes service or endpoint | `host.containers.internal` through bridge and rootless backend. | -| SSH transport | Reverse gRPC relay | Reverse gRPC relay. | -| Port publishing | Not needed for relay | Ephemeral host port remains in the container spec for compatibility and debug paths. | -| TLS | mTLS via Kubernetes secrets | mTLS via mounted client files, RPM defaults, or explicit configuration. | -| DNS | Kubernetes CoreDNS | Podman bridge DNS through aardvark-dns when DNS is enabled. | -| Network policy | Kubernetes network policy for pod ingress plus supervisor policy | nftables inside inner sandbox netns plus supervisor policy. | -| Supervisor delivery | Kubernetes driver managed pod image or template | OCI image volume mount. | -| Secrets | Kubernetes Secret volume and env vars | Per-sandbox JWT via Podman secret; TLS client materials from configured host files. | +## Troubleshooting -Both drivers use the same reverse gRPC relay for SSH transport. The most -important Podman-specific difference is network reachability: in rootless -Podman, the bridge network is not reliably routable from the host, so -host-to-container and container-to-host communication must use host aliases, -published ports, or the supervisor relay. +Inspect both containers with the same sandbox-ID label, distinguishing +`openshell.io/isolation-role=sandbox` from +`openshell.io/isolation-role=supervisor`. -## Port Assignments +- Sandbox fails its qualification probe: use its log to identify the denied + kernel/runtime primitive. Do not add capabilities or disable runtime seccomp. +- Sandbox cannot authenticate to supervisor: check the private channel volume, + matching user namespace mappings, and shared SELinux label. +- Supervisor cannot call back: inspect its configured gateway endpoint, + credentials, Podman network, and gateway callback listener. +- DNS or egress denied: inspect supervisor policy decisions. Do not add a + workload network, resolver bypass, or direct gateway route. +- Pair is not Ready: check the supervisor health socket and gateway session. + A running workload container alone does not establish readiness. -| Port | Component | Purpose | -|---|---|---| -| `17670` | Gateway | Default local gRPC and HTTP multiplexed server port. | -| `2222` | Sandbox | Container port mapping default for the SSH compatibility port. | -| `3128` | Sandbox proxy | HTTP CONNECT proxy inside the sandbox network model. | -| `0` | Host | Ephemeral host port requested for the container SSH compatibility port. | +See the [driver overview](README.md) and +[Podman runtime documentation](https://docs.podman.io/en/latest/markdown/podman-run.1.html) +for runtime options. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index e53ddc9f3f..17bacce1ae 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -1,474 +1,112 @@ # openshell-driver-podman -The Podman compute driver manages sandbox containers via the Podman REST API -over a Unix socket. It targets single-machine and developer environments where -rootless container isolation is preferred over a full Kubernetes cluster. The -driver runs in-process within the gateway server and delegates all sandbox -isolation enforcement to the `openshell-sandbox` supervisor binary, which is -sideloaded into each container via an OCI image volume mount. +The Podman compute driver runs inside the gateway and uses the native libpod +REST API over a Unix socket. Each sandbox has two independent containers: -When the gateway configures `[openshell.gateway.otlp]`, Podman compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-podman`. The driver preserves the gateway trace context and -uses the same compute-driver RPC span names in its in-process and standalone -forms. +- `openshell-sandbox` owns the agent process in the workload container. +- `openshell-supervisor` evaluates policy, holds gateway credentials, and + proxies approved egress in a separate companion container. -`mise run gateway:podman` enables this export only when a local collector is -listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration -so the development gateway does not repeatedly report export failures. +The driver provisions placement, identity, credentials, transport, and lifecycle. +The shared isolation interface supplies exec, attach, signal, terminate, binary +identity, DNS, TCP, and loopback-forwarding semantics. -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID with pulling disabled, preventing a mutable tag from changing -between inspection and launch. The supervisor runs as root, resolves omitted -policy identity fields from the image declaration, and drops only agent -children to the completed identity. Named OCI components remain names after -validation; a missing group is filled with the user's numeric primary GID. Explicit -`process.run_as_user` and `process.run_as_group` values take precedence -independently. +## Runtime posture -For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). - -## Stop and Start - -Stop stops the managed container without deleting it. The per-sandbox named -workspace volume, token and proxy-auth secrets, labels, and container metadata -remain intact. Start starts the same container and reuses the same named -volume. Stopped managed containers remain visible through list and watch -reconciliation. Delete remains responsible for removing the container, -driver-owned secrets, and workspace volume. - -The stop call waits until Podman reports the container as stopped or exited. -This keeps an immediate start from racing a rootless Podman stop that is still -finishing after its API request returns. - -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -## Architecture - -The Podman driver communicates with the Podman daemon over a Unix socket and -delegates sandbox isolation to the supervisor binary running inside each -container. - -```mermaid -graph TB - CLI["openshell CLI"] -->|gRPC| GW["Gateway Server
(openshell-server)"] - GW -->|in-process| PD["PodmanComputeDriver"] - PD -->|HTTP/1.1
Unix socket| PA["Podman API"] - PA -->|OCI runtime
crun/runc| C["Sandbox Container"] - C -->|image volume
read-only| SV["Supervisor Binary
/opt/openshell/bin/openshell-sandbox"] - SV -->|creates| NS["Nested Network Namespace
veth pair + proxy"] - SV -->|enforces| LL["Landlock + seccomp"] - SV -->|gRPC callback| GW -``` - -## Isolation Model - -The Podman driver provides the same protection layers as the other compute -drivers. The driver itself does not implement isolation primitives directly. It -configures the container so that the `openshell-sandbox` supervisor can enforce -them at runtime. - -### Container Security Configuration - -The container spec in `container.rs` sets these security-critical fields: - -| Setting | Value | Rationale | +| Property | Workload | Supervisor | |---|---|---| -| `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. | -| `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. | -| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP`, `KILL` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, child bounding-set cleanup, and forwarding shutdown signals to a workload that runs as the sandbox user. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. | -| `no_new_privileges` | `true` | Prevents privilege escalation after exec. | -| `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. | -| `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. | -| CDI GPU devices | Opaque `driver_config.cdi_devices` values when set, otherwise the requested count of NVIDIA CDI GPUs selected in round-robin order. Local `/dev/dxg` permits `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. | Exposes requested GPUs to GPU-enabled sandbox containers. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | - -The restricted agent child does not retain these supervisor privileges. - -## Driver Config Mounts - -The gateway forwards the `podman` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Podman -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.podman]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Podman named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. Podman - local-driver volumes created with bind options are treated as host bind - mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. -- `image`: mounts an OCI image through Podman's image-volume API. The driver - pulls the image during provisioning using the sandbox image pull policy. - -Host bind mounts are disabled by default because they expose gateway host paths -to sandbox requests. The driver still uses internal bind mounts for configured -TLS material; per-sandbox gateway JWTs are delivered through Podman secrets. - -Podman `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. User-supplied bind and volume mounts are -read-only by default; set `read_only: false` to make them writable. Podman -image and volume mounts do not support `subpath` in OpenShell driver config. -Mount `source` and `target` values must not contain surrounding whitespace. -Mount targets must be absolute container paths and must not replace -the workspace root (`/sandbox`) or overlap OpenShell supervisor files, -`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. - -Example named-volume usage: - -```shell -podman volume create openshell-work - -openshell sandbox create \ - --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work"}]}}' \ - -- claude +| UID/GID | Pinned non-root workload identity | Same mapped identity | +| Capabilities | Drop all; add none | Drop all; add none | +| Seccomp | Runtime default plus sandbox-installed filters | Runtime default | +| Network | `none`; loopback only | Configured Podman network | +| Gateway JWT and upstream credentials | Never mounted | Podman secrets | +| User volumes and CDI devices | Workload only | Never mounted | +| Channel | Private named volume, writable | Same volume, read-only | + +Podman creates the namespaces and volume ownership before the workload runs. +Rootless operation uses the operator's Podman service and subordinate-ID +configuration; it does not require adding capabilities to either container. +The supervisor joins the workload's **user namespace only** to preserve UID/GID +mapping for shared-volume access. PID, mount, and network namespaces remain +separate. The channel volume uses shared SELinux relabeling (`:z`). + +The runtime must pass the sandbox's unprivileged enforcement probe, including +nested seccomp notification and Landlock. Unsupported runtime defaults fail +closed; do not switch to an unconfined profile or add capabilities. + +## Protected channel and network enforcement + +```text +agent -> openshell-sandbox === authenticated gRPC / private UDS === supervisor -> network + network=none TCP, DNS, control streams policy + gateway JWT ``` -### Capability Breakdown - -| Capability | Purpose | -|---|---| -| `SYS_ADMIN` | seccomp filter installation, namespace creation, and Landlock setup. | -| `NET_ADMIN` | Network namespace veth setup, IP address assignment, routes, and nftables. | -| `SYS_PTRACE` | Reading `/proc//exe` and walking process ancestry for binary identity. | -| `SYSLOG` | Reading `/dev/kmsg` for bypass-detection diagnostics. | -| `DAC_READ_SEARCH` | Reading `/proc//fd/` across UIDs so the proxy can resolve the binary responsible for a connection. | -| `SETPCAP` | Clearing the restricted child process capability bounding set before exec. | - -The driver intentionally keeps Podman's default `SETUID`, `SETGID`, `CHOWN`, -and `FOWNER` capabilities because the supervisor needs them to drop privileges -and prepare writable sandbox directories. It also keeps `SETPCAP` until child -setup so `drop_privileges()` can clear the child capability bounding set before -exec. It drops unneeded defaults such as -`DAC_OVERRIDE`, `FSETID`, `KILL`, `NET_RAW`, `SETFCAP`, -and `SYS_CHROOT`. - -## Supervisor Sideloading - -The supervisor binary is delivered to sandbox containers via Podman's OCI image -volume mechanism, distinct from both the Kubernetes pod-volume approach and the -VM's embedded guest bundle. - -```mermaid -sequenceDiagram - participant D as PodmanComputeDriver - participant P as Podman API - participant C as Sandbox Container - - D->>P: pull_image(supervisor, "missing") - D->>P: create_container(spec with image_volumes) - Note over P: Podman resolves image_volumes at
libpod layer before OCI spec generation - P->>C: Mount supervisor image at /opt/openshell/bin (read-only) - D->>P: start_container - C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox -``` - -The supervisor image from `deploy/docker/Dockerfile.supervisor` provides the -static `openshell-sandbox` binary at `/openshell-sandbox`. -Mounting that image at `/opt/openshell/bin` makes the binary available as -`/opt/openshell/bin/openshell-sandbox`. - -The container spec sets that binary as the entrypoint. This avoids relying on -the sandbox image entrypoint or command, which might otherwise append the -supervisor path as an argument to an image-provided shell. - -## TLS - -When all three Podman TLS paths are set, the driver treats sandbox callbacks as -mTLS callbacks: - -- `OPENSHELL_PODMAN_TLS_CA` -- `OPENSHELL_PODMAN_TLS_CERT` -- `OPENSHELL_PODMAN_TLS_KEY` - -The driver validates that the TLS paths are provided as a complete set. Partial -configuration fails early instead of silently falling back to plaintext. - -When enabled, the driver: - -1. Switches the auto-detected endpoint scheme from `http://` to `https://`. -2. Bind-mounts the client cert files read-only into the container at - `/etc/openshell/tls/client/`. -3. Sets `OPENSHELL_TLS_CA`, `OPENSHELL_TLS_CERT`, and `OPENSHELL_TLS_KEY` to - the container-side paths. - -The supervisor reads these env vars and uses them to establish an mTLS -connection back to the gateway. On SELinux systems, the bind mounts include -Podman's shared relabel option so the container process can read the files. - -The RPM packaging auto-generates a self-signed PKI on first start via -`openshell-gateway generate-certs`. Client certs are placed in the CLI -auto-discovery directory (`~/.config/openshell/gateways/openshell/mtls/`) so -the CLI connects with mTLS without manual configuration. See -`deploy/rpm/CONFIGURATION.md` for the full RPM configuration reference. - -## Network Model - -Sandbox network isolation uses a two-layer approach: a Podman bridge network -for container-to-host communication, and a nested network namespace created by -the supervisor for sandbox process isolation. - -```mermaid -graph TB - subgraph Host - GW["Gateway Server
127.0.0.1:17670"] - PS["Podman Socket"] - end - - subgraph Bridge["Podman Bridge Network (10.89.x.x)"] - subgraph Container["Sandbox Container"] - SV["Supervisor
(root in user ns)"] - subgraph NestedNS["Nested Network Namespace"] - SP["Sandbox Process
(resolved non-root identity)"] - VE2["veth1: 10.200.0.2"] - end - VE1["veth0: 10.200.0.1
(CONNECT proxy)"] - SV --- VE1 - VE1 ---|veth pair| VE2 - end - end - - GW -.->|SSH via supervisor relay
gRPC session| SV - SV -->|gRPC callback via
host.containers.internal| GW - SP -->|all egress via proxy| VE1 -``` - -Key points: - -- Bridge network: created by `client.ensure_network()` with DNS enabled. - Containers on the bridge can see each other at L3, but sandbox processes - cannot because they are isolated inside the nested netns. -- Nested netns: the supervisor creates a private `NetworkNamespace` with a veth - pair. Sandbox processes enter this netns via `setns(fd, CLONE_NEWNET)` in the - `pre_exec` hook, forcing ordinary traffic through the CONNECT proxy. -- Policy DNS and transparent TCP: the driver advertises the complete - `policy-dns-transparent-tcp` substrate. For explicit `protocol: tcp` - endpoints, the supervisor installs namespace-local DNS listeners, synthetic - routes, and TCP redirect rules before starting the workload. The container - disables Podman's implicit DNS search suffix so policy DNS evaluates the - exact endpoint name requested by the workload, and asks libc to use the - policy DNS TCP listener to avoid rootless Podman's nested UDP NAT return - path. -- Port publishing: the container spec still requests `host_port: 0` for the - configured SSH port. The gateway SSH tunnel uses the supervisor relay rather - than connecting directly to the published port. -- Host gateway: `host.containers.internal` and `host.openshell.internal` are - injected into `/etc/hosts` so containers can reach services on the gateway - host. Linux defaults to Podman's `host-gateway` resolver. macOS Podman - machine defaults to gvproxy's host-loopback IP, `192.168.127.254`, because - stale Podman machines may fail to resolve `host-gateway`. -- nsenter: the supervisor uses `nsenter --net=` instead of `ip netns exec` for - namespace operations, avoiding the sysfs remount path that fails in rootless - containers. - -See [NETWORKING.md](NETWORKING.md) for the rootless Podman networking deep dive. - -## Supervisor Relay - -Podman follows the same end-to-end contract as the Kubernetes and VM drivers -for the in-container SSH relay: gateway config to `PodmanComputeConfig` to -sandbox environment to supervisor session registration on that path. - -1. `openshell-core` `Config::sandbox_ssh_socket_path` is copied into - `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the - in-process driver. -2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that - value, alongside required vars such as `OPENSHELL_ENDPOINT` and - `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template - environment variables to prevent spoofing. -3. The supervisor reads `OPENSHELL_SSH_SOCKET_PATH` and uses it for the Unix - socket the gateway's SSH stack bridges to. - -The standalone `openshell-driver-podman` binary sets the same struct field from -`OPENSHELL_SANDBOX_SSH_SOCKET_PATH`. - -## Credential Injection - -Sandboxes authenticate to the gateway via mTLS using client materials bind- -mounted into the container from a Podman secret. No shared per-request secret -is injected as an environment variable. - -| Credential | Mechanism | Visible in `inspect`? | Visible in `/proc//environ`? | -|---|---|---|---| -| mTLS client cert/key | Bind-mounted file paths (`OPENSHELL_TLS_*` env vars point at them) | Yes (paths only) | Yes (paths only) | -| Sandbox identity | Plaintext env var | Yes | Yes | -| gRPC endpoint | Plaintext env var, override-protected | Yes | Yes | -| Supervisor relay socket path | Plaintext env var, override-protected | Yes | Yes | - -The `build_env()` function inserts user-supplied variables first, then -unconditionally overwrites all security-critical variables to prevent spoofing -via sandbox templates: - -- `OPENSHELL_SANDBOX` -- `OPENSHELL_SANDBOX_ID` -- `OPENSHELL_ENDPOINT` -- `OPENSHELL_SSH_SOCKET_PATH` -- `OPENSHELL_CONTAINER_IMAGE` -- `OPENSHELL_MAIN_PROCESS_SPEC` - -## Sandbox Lifecycle - -### Creation Flow - -```mermaid -sequenceDiagram - participant GW as Gateway - participant D as PodmanComputeDriver - participant P as Podman API - - GW->>D: create_sandbox(DriverSandbox) - D->>D: validate name + id - D->>D: validated_container_name() - - D->>P: pull_image(supervisor, "missing") - D->>P: pull_image(sandbox_image, policy) - - D->>P: create_volume(workspace) - Note over D: On failure below, rollback volume - - D->>P: create_container(spec) - alt Conflict (409) - D->>P: remove_volume - D-->>GW: AlreadyExists - end - Note over D: On failure below, rollback container + volume - - D->>P: start_container - D-->>GW: Ok -``` - -Each step rolls back previously-created resources on failure. The Conflict path -cleans up the volume because it is keyed by the new sandbox's ID, not the -conflicting container's ID. - -### Readiness and Health - -The container `healthconfig` marks the sandbox healthy when any of these -signals succeeds: - -- Legacy marker file `/var/run/openshell-ssh-ready`. -- `test -S` on the configured supervisor Unix socket path. -- The prior TCP check for a listener on the in-container SSH port. - -The Unix socket check allows relay-only readiness when the supervisor exposes -the socket without the old marker or published-port signal. - -### Deletion Flow - -1. Validate `sandbox_name` and stable `sandbox_id` from `DeleteSandboxRequest`. -2. Best-effort inspect cross-checks the container label when present, but - cleanup remains keyed by the request `sandbox_id`. -3. Best-effort stop, ignoring the stop result. -4. Force-remove the container. -5. Remove workspace volume derived from the request `sandbox_id`, warning on - failure and continuing. - -If the container is already gone during inspect or remove, the driver still -performs idempotent volume cleanup using the request `sandbox_id` and -returns `Ok(false)` for the container-delete result. This prevents leaked -Podman resources after out-of-band container removal or label drift. - -## Configuration - -| Environment Variable | CLI Flag | Default | Description | -|---|---|---|---| -| `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | -| `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, or `newer`. | -| `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | -| `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | -| `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | -| `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | -| `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | -| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | -| `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | -| `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_KEY` | `--podman-tls-key` | unset | Host path to the client private key mounted for sandbox mTLS. | -| `OPENSHELL_SANDBOX_HTTPS_PROXY` | `--sandbox-https-proxy` | unset | Corporate forward proxy URL for the supervisor's upstream TLS dials, chained with HTTP CONNECT. Credential-free `http://host:port` and `https://host:port` URLs are supported (scheme and port required). For an `https://` proxy the supervisor TLS-wraps the proxy connection, verifying the proxy certificate against the built-in and system roots plus `--sandbox-proxy-ca-bundle`. Plain-HTTP requests always dial directly. | -| `OPENSHELL_SANDBOX_NO_PROXY` | `--sandbox-no-proxy` | unset | Comma-separated `NO_PROXY` list (hostnames, domain suffixes, IPs, CIDRs, each with an optional `:port` qualifier) dialed directly instead of through the corporate proxy. IP/CIDR entries also match hostnames through their validated DNS resolution. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_FILE` | `--sandbox-proxy-auth-file` | unset | Path to a file containing the proxy credentials as `user:pass`. Staged as a root-only Podman secret so credentials never appear in config or container metadata. Requires the insecure-auth acknowledgement below. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE` | `--sandbox-proxy-auth-allow-insecure` | unset | Explicit acknowledgement (`true`) that the credential is sent as cleartext Basic auth over the plain-TCP connection to the `http://` proxy. Required when the auth file is set with an `http://` proxy; not required for `https://` proxies (the credential travels inside the verified TLS session) but tolerated if set. Rejected when no auth file is configured. | -| `OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME` | `--sandbox-proxy-connect-by-hostname` | unset | Send the destination hostname in CONNECT requests instead of a validated IP. Last resort for proxies whose ACLs filter on hostnames: the proxy then resolves the name itself, so sandbox SSRF/`allowed_ips` validation no longer binds the connection. | -| `OPENSHELL_PODMAN_USERNS` | `--userns` | unset | User namespace mode for sandbox containers (e.g. `auto`). When unset, containers use the default user namespace. | -| `OPENSHELL_SANDBOX_PROXY_CA_BUNDLE` | `--sandbox-proxy-ca-bundle` | unset | Path (on the gateway host) to a PEM CA bundle trusted for the corporate proxy. Bind-mounted read-only into the sandbox (a CA certificate is not secret). Trusted for the `https://` proxy TLS handshake and, because TLS-intercepting proxies re-sign tunneled certificates, folded into the sandbox trust bundle and upstream verification. Requires a proxy URL; the file must exist and hold at least one certificate. | - -Through the gateway, the same settings are the `https_proxy`, `no_proxy`, -`proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, -and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see -`docs/reference/gateway-config.mdx`. - -This is an operator-owned egress boundary: the driver passes the settings on -the supervisor's command line, so sandbox and template environment — and any -`ENV` baked into the sandbox image — cannot override them, and the -conventional `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` variables a sandbox -controls do not steer it. Credentials must be supplied through -`proxy_auth_file`; an inline `user:pass@` in the URL is rejected at startup. - -Basic auth over an `http://` proxy is cleartext on the wire: anyone on the -network path between the sandbox host and the proxy can recover the -credential. Setting `proxy_auth_file` therefore requires -`proxy_auth_allow_insecure = true`; both the driver and the in-container -supervisor reject credentials without that explicit acknowledgement. - -CONNECT requests target a validated resolved IP by default, so the proxy -performs no DNS resolution and the tunnel stays bound to the address that -passed the sandbox's SSRF and `allowed_ips` checks; the hostname still -travels inside the tunnel (TLS SNI, application `Host`). In split-horizon -networks, point the gateway host at the corporate resolver. Set -`proxy_connect_by_hostname = true` only when the proxy's ACLs filter on -hostnames and reject IP CONNECT targets — it re-opens proxy-side DNS -resolution, making the proxy's ACLs the effective egress control. - -## Rootless-Specific Adaptations - -The Podman driver is designed for rootless operation. The following adaptations -matter compared to cluster or rootful runtimes: - -1. subuid/subgid preflight check: on non-macOS hosts, `check_subuid_range()` in - `driver.rs` warns operators if `/etc/subuid` or `/etc/subgid` entries are - missing for the current user. This is not a hard error because some systems - use LDAP or other mechanisms. macOS skips the check because `podman machine` - runs the Podman service inside a Linux VM. -2. cgroups v2 requirement: the driver refuses to start if cgroups v1 is - detected. Rootless Podman requires the unified cgroup hierarchy. -3. `nsenter` for namespace operations: `openshell-sandbox` uses - `nsenter --net=` instead of `ip netns exec` to avoid the sysfs remount path - that requires real `CAP_SYS_ADMIN` in the host user namespace. -4. `DAC_READ_SEARCH` capability: required for the proxy to read - `/proc//fd/` across UIDs within the user namespace. -5. `SETUID` and `SETGID` capabilities: kept from Podman's default capability - set so `drop_privileges()` can call `setuid()` and `setgid()`. -6. `host.containers.internal`: used instead of Docker's `host.docker.internal` - for container-to-host communication. The driver also injects the - OpenShell-owned `host.openshell.internal` alias. -7. Ephemeral port publishing: the SSH compatibility port uses `host_port: 0` - because the bridge network IP is not reliably routable from the host in - rootless mode. -8. tmpfs at `/run/netns`: a private tmpfs lets the supervisor create named - network namespaces via `ip netns add`. - -## Implementation References - -- Gateway integration: `crates/openshell-gateway/src/lib.rs` registers the - driver factory and constructs `PodmanComputeConfig` from the generic server - build context. -- Server configuration: `crates/openshell-server/src/lib.rs` exposes the - backend-agnostic registry and factory context. -- Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in - `crates/openshell-core/src/config.rs`. -- SSRF mitigation: `crates/openshell-core/src/net.rs`, - `crates/openshell-sandbox/src/proxy.rs`, and - `crates/openshell-server/src/grpc/policy.rs`. -- Sandbox supervisor: `crates/openshell-sandbox/src/` for Landlock, seccomp, - netns, proxy, and relay behavior shared by all drivers. -- Container engine abstraction: `tasks/scripts/container-engine.sh` for - build/deploy support across Docker and Podman. -- Supervisor image build: `deploy/docker/Dockerfile.supervisor`. +The workload has no external interface or published port. Seccomp socket +mediation carries TCP and DNS through one authenticated gRPC connection. +DNS remains supervisor-mediated; general UDP is unsupported. The driver sets +`net.ipv4.ip_unprivileged_port_start=0` in the isolated workload network +namespace so the sandbox's loopback DNS relay can bind port 53 without a +capability. No nftables or nested network namespace setup runs in the sandbox. + +The channel contains the sandbox bootstrap and sandbox-side TLS identity only. +Supervisor private keys and topology stay in the companion's private filesystem. +Landlock denies agent access to the top-level `/.openshell` control hierarchy. +The driver verifies Podman's reported `network=none` fence before launch and +restart. `host.containers.internal` and callback networking apply to the +supervisor, not the agent. + +Gateway callbacks use the existing sandbox JWT and optional configured mTLS +bundle. The sandbox/supervisor channel always uses its separate, per-sandbox +mutual TLS material. These are distinct authentication relationships. + +## Identity and trusted binaries + +Both workload and supervisor images are pinned by immutable image ID. The +driver reads account files from a stopped workload-image container; it never +executes the image to resolve an account. Policy identity fields override OCI +`USER` independently. Named users/groups resolve against that image, including +supplementary groups. Root and unresolved identities fail before provisioning. +Images must not prepopulate the reserved `/.openshell` hierarchy; this prevents +image-controlled symlinks from aliasing private control state into user mounts. + +The trusted runtime image supplies `/openshell-sandbox` and +`/openshell-supervisor`. Podman's read-only image volume delivers the sandbox +binary; user-namespace modes that cannot use image volumes retain the existing +trusted binary extraction path. Image and request environment belong to agent +children, never the supervisor process. + +## Lifecycle and readiness + +Create builds both stopped containers and stages both private archives before +starting either container. The sandbox does not execute the agent until the +supervisor authenticates and confirms the common boundary contract. Failed +creation removes only containers created by that attempt, then cleans up +driver-owned volumes and secrets. + +Stop retains both containers, workspace, channel, and secrets. Start restores +the consumed sandbox bootstrap from a copy in the supervisor's private +filesystem, verifies the fence, and starts the same pair. A failed supervisor +start stops the workload. Delete removes the companion first, then the workload, +channel, workspace, and driver-owned secrets. User-owned volumes are retained. + +Only workload containers appear in sandbox list/watch results. Readiness uses +the supervisor's private health socket; there is no shell, legacy marker, or +TCP-listener shortcut. Watch reconciliation and supervisor exit/removal events +stop a running workload whose companion is unavailable. The gateway also +requires the authenticated supervisor session before publishing Ready. + +## Mounts, GPUs, and configuration + +User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain +native Podman features and apply only to the workload. Bind mounts require the +operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace +root cannot be replaced. User-owned volumes are never created or deleted. + +See [gateway configuration](../../docs/reference/gateway-config.mdx) for +operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. +The configured network and upstream proxy belong to the supervisor. +`health_check_interval_secs=0` uses a one-second check rather than disabling +the readiness check required by this topology. + +Gateway OTLP configuration continues to export compute-driver spans under the +`openshell-driver-podman` service, preserving gateway trace context. diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 508b604ce7..1cb1e12bfb 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -177,6 +177,8 @@ pub struct ImageInspect { pub struct ImageConfig { #[serde(default)] pub user: String, + #[serde(default)] + pub env: Vec, } /// A container summary returned by the list API. @@ -446,6 +448,89 @@ impl PodmanClient { .await } + pub(crate) async fn create_typed_container( + &self, + spec: &(impl serde::Serialize + Sync), + ) -> Result { + #[derive(serde::Deserialize)] + struct Created { + #[serde(rename = "Id", alias = "ID")] + id: String, + } + let body = + serde_json::to_vec(spec).map_err(|error| PodmanApiError::Json(error.to_string()))?; + let (status, bytes) = self + .request_raw( + hyper::Method::POST, + "/libpod/containers/create", + "application/json", + body.into(), + ) + .await?; + if !status.is_success() { + return Err(error_from_response(status.as_u16(), &bytes)); + } + let created: Created = serde_json::from_slice(&bytes) + .map_err(|error| PodmanApiError::Json(error.to_string()))?; + validate_name(&created.id)?; + Ok(created.id) + } + + pub(crate) async fn copy_to_container( + &self, + name: &str, + archive: Vec, + ) -> Result<(), PodmanApiError> { + validate_name(name)?; + let (status, bytes) = self + .request_raw( + hyper::Method::PUT, + &format!("/libpod/containers/{name}/archive?path=/"), + "application/x-tar", + archive.into(), + ) + .await?; + if status.is_success() { + Ok(()) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + + pub(crate) async fn verify_isolation_fence(&self, id: &str) -> Result<(), PodmanApiError> { + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: String, + privileged: bool, + } + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct FenceInspect { + host_config: HostConfig, + network_settings: NetworkSettings, + } + validate_name(id)?; + let inspected: FenceInspect = self + .request_json( + hyper::Method::GET, + &format!("/libpod/containers/{id}/json"), + None, + ) + .await?; + if inspected.host_config.network_mode != "none" + || inspected.host_config.privileged + || inspected + .network_settings + .networks + .keys() + .any(|name| name != "none") + { + return Err(PodmanApiError::InvalidInput("sandbox requires an unprivileged container with network mode none and no attached networks".into())); + } + Ok(()) + } + /// Start a container by name or ID. pub async fn start_container(&self, name: &str) -> Result<(), PodmanApiError> { validate_name(name)?; diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a81ee13e1d..7091fbb7a2 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -197,7 +197,7 @@ pub fn short_id(id: &str) -> String { // --------------------------------------------------------------------------- #[derive(Serialize)] -struct ContainerSpec { +pub struct ContainerSpec { name: String, image: String, labels: BTreeMap, @@ -212,10 +212,17 @@ struct ContainerSpec { entrypoint: Vec, command: Vec, user: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + groups: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + unsetenv: Vec, cap_drop: Vec, cap_add: Vec, no_new_privileges: bool, + #[serde(skip_serializing_if = "String::is_empty")] seccomp_profile_path: String, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + sysctl: BTreeMap, image_pull_policy: String, healthconfig: HealthConfig, resource_limits: ResourceLimits, @@ -473,7 +480,7 @@ fn build_env( config: &PodmanComputeConfig, image: &str, oci_user: &str, -) -> BTreeMap { +) -> Result, ComputeDriverError> { let spec = sandbox.spec.as_ref(); let template = spec.and_then(|s| s.template.as_ref()); @@ -498,10 +505,11 @@ fn build_env( user_env.insert(k.clone(), v.clone()); } } - env.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { + // User environment belongs exclusively to mediated workload children. In + // particular, never activate loader or policy overrides in the supervisor. + if !user_env.is_empty() { + let json = serde_json::to_string(&user_env) + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert(openshell_core::sandbox_env::USER_ENVIRONMENT.into(), json); } @@ -530,7 +538,7 @@ fn build_env( ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) - .expect("main process config serialization cannot fail"); + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert( openshell_core::sandbox_env::MAIN_PROCESS_SPEC.into(), main_process, @@ -602,7 +610,7 @@ fn build_env( ); } - env + Ok(env) } /// Merge labels from the sandbox template with required managed labels. @@ -869,6 +877,7 @@ fn validate_podman_driver_mounts( } }; driver_mounts::validate_container_mount_target(target)?; + driver_mounts::validate_mount_control_path(target, "/.openshell")?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -1006,6 +1015,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( } #[allow(clippy::too_many_arguments)] +#[cfg(test)] pub fn build_container_spec_for_image( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1017,10 +1027,36 @@ pub fn build_container_spec_for_image( supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 3]>, ) -> Result { + serde_json::to_value(build_base_spec( + sandbox, + config, + token_secret_name, + gpu_device_ids, + requested_image, + image_id, + oci_user, + supervisor_bin_path, + tls_secret_names, + )?) + .map_err(|error| ComputeDriverError::Message(format!("encode Podman spec: {error}"))) +} + +#[allow(clippy::too_many_arguments)] +fn build_base_spec( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + token_secret_name: Option<&str>, + gpu_device_ids: Option<&[String]>, + requested_image: &str, + image_id: &str, + oci_user: &str, + supervisor_bin_path: Option<&Path>, + tls_secret_names: Option<&[String; 3]>, +) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); - let env = build_env(sandbox, config, requested_image, oci_user); + let env = build_env(sandbox, config, requested_image, oci_user)?; let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) @@ -1098,83 +1134,16 @@ pub fn build_container_spec_for_image( // corporate proxy flags follow it; the workload command comes from // the reserved environment variable. command, - // Force the supervisor to run as root (UID 0). Sandbox images may - // set a non-root USER directive (e.g. `USER sandbox`), but the - // supervisor needs root to create network namespaces, set up the - // proxy, and configure Landlock/seccomp. This matches the K8s - // driver's runAsUser: 0. - user: "0:0".into(), - // Podman's default container capability set is already restricted: - // CHOWN DAC_OVERRIDE FOWNER FSETID KILL SETGID SETUID SETPCAP - // NET_BIND_SERVICE SYS_CHROOT SETFCAP - // We add what the supervisor needs and drop what it doesn't. - cap_drop: vec![ - // Not needed: standard file permission bits are sufficient; dropping - // prevents the supervisor from bypassing DAC checks it shouldn't need. - "DAC_OVERRIDE".into(), - // Not needed: the supervisor does not create setuid/setgid executables. - "FSETID".into(), - // Not needed: the supervisor does not bind privileged ports (<1024). - "NET_BIND_SERVICE".into(), - // Not in Podman's default set but explicitly denied in case the image - // or runtime adds it; raw sockets are not required. - "NET_RAW".into(), - // Not needed: the supervisor does not manipulate file capabilities. - "SETFCAP".into(), - // Not needed: the supervisor does not call chroot(). - "SYS_CHROOT".into(), - ], - cap_add: vec![ - // seccomp filter installation, namespace creation, Landlock setup. - "SYS_ADMIN".into(), - // Network namespace veth setup, IP/route configuration. - "NET_ADMIN".into(), - // Reading /proc//exe and ancestor walk for process identity in policy. - "SYS_PTRACE".into(), - // Reading /dev/kmsg for bypass-detection diagnostics. - "SYSLOG".into(), - // Reading /proc//fd/ across UIDs for process identity resolution. - // In rootless Podman the supervisor runs as UID 0 inside a user namespace - // while sandbox processes run as the sandbox user. The kernel's - // proc_fd_permission() calls generic_permission() which denies cross-UID - // access to the dr-x------ fd directory unless this cap is present. - // Without it the proxy cannot determine which binary made each outbound - // connection and all traffic is denied. - "DAC_READ_SEARCH".into(), - // Child setup clears the capability bounding set before exec, which - // requires CAP_SETPCAP in the supervisor until drop_privileges(). - "SETPCAP".into(), - // Forwarding shutdown signals to the canonical workload process - // group after it drops to the sandbox UID requires CAP_KILL. - "KILL".into(), - ], - // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from - // Podman's default set and not dropped: - // SETUID/SETGID – drop_privileges(): setuid()/setgid()/initgroups() to the - // sandbox user. In rootless Podman cap_drop:ALL removes them - // from the bounding set even though uid=0 owns the user - // namespace — so we keep them by not dropping them explicitly. - // SETPCAP – drop_privileges(): clears the child capability - // bounding set before the sandbox user execs. - // CHOWN – prepare_filesystem(): chown(path, uid, gid) on newly - // created read_write directories so the sandbox user can - // write to them. - // FOWNER – chown on files where the supervisor is not the owner - // (e.g. pre-existing directories owned by another user). - // - // Disable the container-level seccomp profile. The sandbox supervisor The sandbox supervisor - // installs its own policy-aware BPF seccomp filter at runtime via - // seccompiler (two-phase: clone3 blocker + main filter). The runtime - // filter is more restrictive than Podman's default — it blocks 20+ - // dangerous syscalls and conditionally restricts socket domains based - // on network policy. The filter self-seals by blocking further - // seccomp(SET_MODE_FILTER) calls after installation. - // - // A container-level profile would interfere by blocking the landlock - // and seccomp syscalls the supervisor needs during setup, before it - // locks itself down. + // The paired builder supplies the immutable non-root identity. + user: String::new(), + groups: Vec::new(), + unsetenv: Vec::new(), + cap_drop: vec!["ALL".into()], + cap_add: Vec::new(), no_new_privileges: true, - seccomp_profile_path: "unconfined".into(), + // Omission selects the runtime default, never an unconfined profile. + seccomp_profile_path: String::new(), + sysctl: BTreeMap::new(), image_pull_policy: "never".to_string(), healthconfig: HealthConfig { test: vec![ @@ -1389,7 +1358,185 @@ pub fn build_container_spec_for_image( }, }; - Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) + Ok(container_spec) +} + +/// Driver-owned inputs for the two independent runtime containers. +pub struct IsolationSpecInput<'a> { + pub sandbox: &'a DriverSandbox, + pub config: &'a PodmanComputeConfig, + pub token_secret: Option<&'a str>, + pub gpu_devices: Option<&'a [String]>, + pub requested_image: &'a str, + pub image_id: &'a str, + pub image_user: &'a str, + pub image_env: &'a [String], + pub supervisor_bin: Option<&'a Path>, + pub tls_secrets: Option<&'a [String; 3]>, + pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, +} + +pub struct IsolationSpecs { + pub workload: ContainerSpec, + pub supervisor: ContainerSpec, +} + +impl ContainerSpec { + pub(crate) fn join_user_namespace(&mut self, container_id: &str) { + self.userns = Some(UserNS { + nsmode: "container".to_string(), + value: Some(container_id.to_string()), + }); + self.idmappings = None; + } +} + +pub fn build_isolation_specs( + input: IsolationSpecInput<'_>, +) -> Result { + let base = || { + build_base_spec( + input.sandbox, + input.config, + input.token_secret, + input.gpu_devices, + input.requested_image, + input.image_id, + input.image_user, + input.supervisor_bin, + input.tls_secrets, + ) + }; + let mut workload = base()?; + let mut supervisor = base()?; + let user = format!("{}:{}", input.identity.uid, input.identity.gid); + let channel = crate::isolation::channel_volume_name(&input.sandbox.id); + + workload + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); + workload.env = BTreeMap::new(); + workload.unsetenv = input + .image_env + .iter() + .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) + .collect(); + workload.command = vec![ + "--bootstrap".into(), + crate::isolation::BOOTSTRAP_PATH.into(), + ]; + workload.user.clone_from(&user); + workload.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + workload.cap_drop = vec!["ALL".into()]; + workload.cap_add.clear(); + workload.seccomp_profile_path.clear(); + workload + .sysctl + .insert("net.ipv4.ip_unprivileged_port_start".into(), "0".into()); + workload.netns.nsmode = "none".into(); + workload.networks.clear(); + workload.portmappings.clear(); + workload.hostadd.clear(); + workload.secret_env.clear(); + workload.secrets.clear(); + workload.healthconfig.test = vec!["NONE".into()]; + workload + .mounts + .retain(|mount| !trusted_mount(&mount.destination)); + workload.volumes.push(NamedVolume { + name: channel.clone(), + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["rw".into(), "nocopy".into(), "z".into()], + }); + + supervisor.name = crate::isolation::supervisor_name(&input.sandbox.id); + supervisor + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "supervisor".into()); + supervisor.image.clone_from(&input.config.supervisor_image); + supervisor.entrypoint = vec!["/openshell-supervisor".into()]; + supervisor.command.extend([ + "--topology-backend-name=podman".into(), + format!( + "--topology-payload-file={}", + crate::isolation::TOPOLOGY_PATH + ), + "--health-socket-path=/run/openshell/supervisor-health.sock".into(), + ]); + supervisor.env.insert( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND.into(), + "podman".into(), + ); + supervisor.user = user; + supervisor.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + supervisor.cap_drop = vec!["ALL".into()]; + supervisor.cap_add.clear(); + supervisor.seccomp_profile_path.clear(); + supervisor.devices = None; + supervisor.image_volumes.clear(); + supervisor.volumes = vec![NamedVolume { + name: channel, + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["ro".into(), "nocopy".into(), "z".into()], + }]; + supervisor.mounts.retain(|mount| { + trusted_mount(&mount.destination) + && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT + }); + for destination in ["/run", "/var/log", "/tmp"] { + supervisor.mounts.push(Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: destination.into(), + options: vec![ + "rw".into(), + "nosuid".into(), + "nodev".into(), + format!("uid={}", input.identity.uid), + format!("gid={}", input.identity.gid), + "mode=0700".into(), + "size=64m".into(), + ], + }); + } + for secret in &mut supervisor.secrets { + secret.uid = input.identity.uid; + secret.gid = input.identity.gid; + } + supervisor.healthconfig.test = vec![ + "CMD".into(), + "/openshell-supervisor".into(), + "health".into(), + "--socket".into(), + "/run/openshell/supervisor-health.sock".into(), + ]; + supervisor.healthconfig.interval = + input.config.health_check_interval_secs.max(1) * 1_000_000_000; + Ok(IsolationSpecs { + workload, + supervisor, + }) +} + +fn trusted_mount(destination: &str) -> bool { + matches!( + destination, + TLS_CA_MOUNT_PATH + | TLS_CERT_MOUNT_PATH + | TLS_KEY_MOUNT_PATH + | PROXY_CA_MOUNT_PATH + | PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + ) || destination == openshell_core::container_paths::NETNS_MOUNT_ROOT } fn provider_spiffe_workload_api_socket_env_value(config: &PodmanComputeConfig) -> Option { @@ -1496,6 +1643,70 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + #[test] + fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { + let sandbox = DriverSandbox { + id: "pair".into(), + name: "agent".into(), + ..Default::default() + }; + let config = PodmanComputeConfig::default(); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![2000], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let env = vec![ + "LD_PRELOAD=/hostile.so".into(), + "HTTP_PROXY=http://bypass".into(), + ]; + let specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: Some("jwt"), + gpu_devices: None, + requested_image: "image:latest", + image_id: "sha256:image", + image_user: "1000:1001", + image_env: &env, + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + }) + .unwrap(); + for spec in [&specs.workload, &specs.supervisor] { + assert_eq!(spec.user, "1000:1001"); + assert_eq!(spec.groups, vec!["2000"]); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); + } + assert_eq!(specs.workload.netns.nsmode, "none"); + assert!(specs.workload.networks.is_empty()); + assert!(specs.workload.portmappings.is_empty()); + assert!(specs.workload.env.is_empty()); + assert_eq!(specs.workload.unsetenv, vec!["LD_PRELOAD", "HTTP_PROXY"]); + assert!(specs.workload.secrets.is_empty()); + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| !trusted_mount(&mount.destination)) + ); + assert_eq!(specs.supervisor.secrets.len(), 1); + assert_eq!(specs.supervisor.secrets[0].source, "jwt"); + assert_eq!(specs.supervisor.secrets[0].uid, 1000); + assert_eq!(specs.supervisor.volumes.len(), 1); + assert!(specs.supervisor.volumes[0].options.contains(&"ro".into())); + assert!(specs.supervisor.volumes[0].options.contains(&"z".into())); + assert_eq!(specs.supervisor.entrypoint, vec!["/openshell-supervisor"]); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -1626,7 +1837,8 @@ mod tests { container["env"]["OPENSHELL_CONTAINER_IMAGE"].as_str(), Some("registry.example/app:latest") ); - assert_eq!(container["user"].as_str(), Some("0:0")); + // Only the paired builder materializes the immutable non-root user. + assert_eq!(container["user"].as_str(), Some("")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!([])); assert_eq!(container["dns_option"], serde_json::json!([])); @@ -1866,66 +2078,25 @@ mod tests { } #[test] - fn container_spec_includes_required_capabilities() { + fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); - let spec = build_container_spec(&sandbox, &config); - - let added: Vec<&str> = spec["cap_add"] - .as_array() - .expect("cap_add should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); - assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); - assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); - assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); - assert!( - added.contains(&"DAC_READ_SEARCH"), - "missing DAC_READ_SEARCH" - ); - assert!(added.contains(&"SETPCAP"), "missing SETPCAP"); - assert!(added.contains(&"KILL"), "missing KILL"); - - // SETUID and SETGID are NOT in cap_add — they remain available from the - // default bounding set because we no longer use cap_drop:ALL. Verify they - // are also not explicitly dropped. Similarly SETPCAP, CHOWN and FOWNER - // must not be dropped because child setup clears the bounding set and - // prepare_filesystem() calls chown() on newly created read_write - // directories before the supervisor drops privileges. - let dropped: Vec<&str> = spec["cap_drop"] - .as_array() - .expect("cap_drop should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(!dropped.contains(&"SETUID"), "SETUID must not be dropped"); - assert!(!dropped.contains(&"SETGID"), "SETGID must not be dropped"); - assert!( - dropped.contains(&"NET_BIND_SERVICE"), - "NET_BIND_SERVICE must stay dropped; policy DNS binds an unprivileged port" - ); - assert!( - !dropped.contains(&"CHOWN"), - "CHOWN must not be dropped (needed for prepare_filesystem chown)" - ); - assert!( - !dropped.contains(&"FOWNER"), - "FOWNER must not be dropped (needed for chown on non-owned files)" - ); - assert!( - !dropped.contains(&"SETPCAP"), - "SETPCAP must not be dropped (needed for child bounding-set clear)" - ); - assert!( - !dropped.contains(&"KILL"), - "KILL must not be dropped (needed to signal the sandbox workload on shutdown)" - ); - assert!( - !dropped.contains(&"ALL"), - "must not use cap_drop:ALL in rootless Podman" - ); + let spec = build_base_spec( + &sandbox, + &config, + None, + None, + "image", + "sha256:image", + "", + None, + None, + ) + .unwrap(); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); } #[test] @@ -2867,6 +3038,27 @@ mod tests { assert!(err.to_string().contains("reserved OpenShell path")); } + #[test] + fn user_mounts_cannot_replace_private_channel_hierarchy() { + for target in [ + "/.openshell", + "/.openshell/channel", + "/.openshell/channel/sandbox", + "/.openshell/supervisor", + ] { + let mount = PodmanDriverMountConfig::Volume { + source: "user-owned".into(), + target: target.into(), + read_only: false, + subpath: None, + }; + assert!( + validate_podman_driver_mounts(&[mount], false).is_err(), + "{target}" + ); + } + } + #[test] fn container_spec_uses_configured_host_gateway_ip() { let sandbox = test_sandbox("test-id", "test-name"); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 50eb014691..310b47adb0 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -780,7 +780,7 @@ impl PodmanComputeDriver { "Creating sandbox container" ); - let (image, immutable_image_id, image_user) = async { + let (image, immutable_image_id, image_user, image_env) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { // The supervisor binary is shipped in a standalone OCI image and @@ -840,7 +840,8 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; } - Ok((image.to_string(), inspected_image.id, image_user)) + let image_env = inspected_image.config.as_ref().map_or_else(Vec::new, |config| config.env.clone()); + Ok((image.to_string(), inspected_image.id, image_user, image_env)) } .await; phase_status.finish(result) @@ -859,6 +860,22 @@ impl PodmanComputeDriver { // content at startup. validate_sandbox_proxy_ca_bundle(&self.config).await?; + let identity = self + .resolve_workload_identity(sandbox, &immutable_image_id, &image_user) + .await?; + let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); + let mut runtime_config = self.config.clone(); + runtime_config.supervisor_image = self + .client + .inspect_image(&self.config.supervisor_image) + .await? + .id; + if runtime_config.supervisor_image.is_empty() { + return Err(ComputeDriverError::Precondition( + "supervisor image inspection returned no immutable image ID".into(), + )); + } + // Create workspace volume and per-sandbox token secret. let (token_secret_name, proxy_auth_secret_name) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); @@ -904,6 +921,7 @@ impl PodmanComputeDriver { // Clean up the volume and both per-sandbox secrets on any failure past // this point. let cleanup_created = || async { + let _ = self.client.remove_volume(&channel_volume).await; let _ = self.client.remove_volume(&vol_name).await; if let Some(secret) = token_secret_name.as_deref() { cleanup_sandbox_token_secret(&self.client, secret).await; @@ -914,7 +932,7 @@ impl PodmanComputeDriver { }; // Prepare and create the container. - let tls_secret_names = async { + async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { let gpu_devices = match self.resolve_gpu_cdi_devices( @@ -930,7 +948,7 @@ impl PodmanComputeDriver { }; let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) { - match extract_supervisor_bin(&self.client, &self.config).await { + match extract_supervisor_bin(&self.client, &runtime_config).await { Ok(path) => Some(path), Err(e) => { cleanup_created().await; @@ -941,9 +959,7 @@ impl PodmanComputeDriver { None }; - let tls_secret_names = if userns_remaps_uids(self.config.userns.as_deref()) - && self.config.tls_enabled() - { + let tls_secret_names = if self.config.tls_enabled() { let names = container::tls_secret_names(&sandbox.id); if let Err(e) = create_tls_secrets(&self.client, &self.config, &names).await { cleanup_created().await; @@ -961,32 +977,77 @@ impl PodmanComputeDriver { } }; - let spec = match container::build_container_spec_for_image( + let specs = container::build_isolation_specs(container::IsolationSpecInput { sandbox, - &self.config, - token_secret_name.as_deref(), - gpu_devices.as_deref(), - &image, - &immutable_image_id, - &image_user, - supervisor_bin_path.as_deref(), - tls_secret_names.as_ref(), - ) { + config: &runtime_config, + token_secret: token_secret_name.as_deref(), + gpu_devices: gpu_devices.as_deref(), + requested_image: &image, + image_id: &immutable_image_id, + image_user: &image_user, + image_env: &image_env, + supervisor_bin: supervisor_bin_path.as_deref(), + tls_secrets: tls_secret_names.as_ref(), + identity: &identity, + }); + let mut specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; return Err(e); } }; - match self.client.create_container(&spec).await { - Ok(_) => Ok(tls_secret_names), - Err(PodmanApiError::Conflict(_)) => { - cleanup_all().await; - Err(ComputeDriverError::AlreadyExists) + let mut created_workload = None; + let mut created_supervisor = None; + let create_result = async { + self.client.create_volume(&channel_volume).await?; + let workload_id = self.client.create_typed_container(&specs.workload).await?; + created_workload = Some(workload_id.clone()); + self.client.verify_isolation_fence(&workload_id).await?; + let child_env = image_env + .iter() + .filter_map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.into(), value.into())) + }) + .collect(); + let archives = crate::isolation::bootstrap_archives( + &sandbox.id, + &workload_id, + &identity, + child_env, + )?; + self.client + .copy_to_container(&workload_id, archives.workload) + .await?; + specs.supervisor.join_user_namespace(&workload_id); + let supervisor_id = self + .client + .create_typed_container(&specs.supervisor) + .await?; + created_supervisor = Some(supervisor_id.clone()); + self.client + .copy_to_container(&supervisor_id, archives.supervisor) + .await?; + // Both resources and private files exist before either + // container can run. Only the trusted sandbox starts here; + // authenticated confirmation gates subsequent agent exec. + self.client.start_container(&workload_id).await?; + self.client.start_container(&supervisor_id).await?; + Ok::<(), ComputeDriverError>(()) + } + .await; + if create_result.is_err() { + for id in [created_supervisor, created_workload].into_iter().flatten() { + let _ = self.client.remove_container(&id, 0).await; } + } + match create_result { + Ok(()) => Ok(()), Err(e) => { cleanup_all().await; - Err(ComputeDriverError::from(e)) + Err(e) } } } @@ -1001,44 +1062,6 @@ impl PodmanComputeDriver { )) .await?; - let cleanup_all = || async { - cleanup_created().await; - if let Some(names) = &tls_secret_names { - cleanup_tls_secrets(&self.client, names).await; - } - }; - - // Start container. - let start_result = async { - let phase_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .client - .start_container(&name) - .await - .map_err(ComputeDriverError::from); - phase_status.finish(result) - } - .instrument(tracing::info_span!( - "podman.start_container", - otel.name = "podman.start_container", - otel.status_code = tracing::field::Empty, - container.name = %name, - )) - .await; - if let Err(e) = start_result { - warn!( - sandbox_name = %sandbox.name, - error = %e, - "Failed to start container; cleaning up" - ); - let _ = self - .client - .remove_container(&name, self.config.stop_timeout_secs) - .await; - cleanup_all().await; - return Err(e); - } - info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -1048,7 +1071,65 @@ impl PodmanComputeDriver { span_status.finish(Ok(())) } - /// Find the Podman container ID for a sandbox by its sandbox ID using label lookup. + /// Resolve image accounts without executing any image-supplied program. + async fn resolve_workload_identity( + &self, + sandbox: &DriverSandbox, + image: &str, + image_user: &str, + ) -> Result + { + #[derive(serde::Serialize)] + struct InspectionSpec<'a> { + name: String, + image: &'a str, + } + // Inspect a stopped, unexecuted container pinned to the final image ID. + let id = self + .client + .create_typed_container(&InspectionSpec { + name: format!("openshell-identity-{}", uuid::Uuid::new_v4()), + image, + }) + .await?; + let result = + async { + // Do not let image-controlled symlinks alias the protected channel + // into an agent-readable subtree before Podman mounts it. + match self.client.copy_from_container(&id, "/.openshell").await { + Err(PodmanApiError::NotFound(_)) => {} + Ok(_) => return Err(ComputeDriverError::Precondition( + "workload images must not prepopulate the reserved /.openshell hierarchy" + .into(), + )), + Err(error) => return Err(error.into()), + } + let mut accounts = Vec::new(); + for path in ["/etc/passwd", "/etc/group"] { + let content = match self.client.copy_from_container(&id, path).await { + Ok(archive) => extract_first_tar_entry(&archive) + .map_err(ComputeDriverError::Precondition)?, + Err(PodmanApiError::NotFound(_)) => Vec::new(), + Err(error) => return Err(error.into()), + }; + accounts.push(content); + } + let [passwd, group] = accounts.as_slice() else { + return Err(ComputeDriverError::Precondition( + "image account inspection was incomplete".into(), + )); + }; + crate::isolation::resolve_identity(sandbox, image, image_user, passwd, group) + } + .await; + let cleanup = self.client.remove_container(&id, 0).await; + if let Err(error) = cleanup { + warn!(container = %id, %error, "Failed to remove stopped identity inspection container"); + } + result + } + + /// Find only the workload, never its supervisor companion. async fn find_container_id( &self, sandbox_id: &str, @@ -1063,7 +1144,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(entries.into_iter().next()) @@ -1116,6 +1201,15 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; let container_id = container.id; + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } if container.state == "stopping" { let result = async { let finished_at = self @@ -1179,7 +1273,19 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; if container.state == "running" { - return span_status.finish(Ok(())); + let supervisor = self + .client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if supervisor + .as_ref() + .is_ok_and(|inspect| inspect.state.running) + { + return span_status.finish(Ok(())); + } + self.client.stop_container(&container.id, 0).await?; + self.wait_for_container_stopped(sandbox_id, &container.id) + .await?; } let container_id = container.id; info!(sandbox_id = %sandbox_id, container = %container_id, "Starting sandbox container"); @@ -1196,11 +1302,29 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous.state.finished_at.as_deref()); - let result = self - .client - .start_container(&container_id) - .await - .map_err(ComputeDriverError::from); + let result = async { + let supervisor = crate::isolation::supervisor_name(sandbox_id); + self.client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await?; + self.wait_for_container_stopped(sandbox_id, &supervisor) + .await?; + let archive = self + .client + .copy_from_container(&supervisor, crate::isolation::RESTART_BUNDLE_PATH) + .await?; + let bundle = + extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + self.client.copy_to_container(&container_id, bundle).await?; + self.client.verify_isolation_fence(&container_id).await?; + self.client.start_container(&container_id).await?; + if let Err(error) = self.client.start_container(&supervisor).await { + let _ = self.client.stop_container(&container_id, 0).await; + return Err(error.into()); + } + Ok(()) + } + .await; span_status.finish(result) } @@ -1222,6 +1346,25 @@ impl PodmanComputeDriver { )); } + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .remove_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } + match self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + // The workload still owns the volume until its removal below. + Err(error) => debug!(%error, "Channel volume is still attached to workload"), + } + let Some(container_id) = self.find_container_id(sandbox_id).await? else { debug!(sandbox_id = %sandbox_id, "Sandbox container not found (already deleted)"); let vol = container::volume_name(sandbox_id); @@ -1255,6 +1398,13 @@ impl PodmanComputeDriver { }; // Remove workspace volume. + if let Err(error) = self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + warn!(%sandbox_id, %error, "Failed to remove private channel volume"); + } let vol = container::volume_name(sandbox_id); if let Err(e) = self.client.remove_volume(&vol).await { warn!( @@ -1281,7 +1431,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(!entries.is_empty()) @@ -1295,16 +1449,18 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; let Some(entry) = entries.first() else { return Ok(None); }; if entry.state == "running" { - Ok(self - .client - .inspect_container(&entry.id) + Ok(watcher::inspect_workload(&self.client, &entry.id) .await .ok() .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) @@ -1321,7 +1477,7 @@ impl PodmanComputeDriver { pub async fn list_sandboxes(&self) -> Result, ComputeDriverError> { let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER]) + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) .await .map_err(ComputeDriverError::from)?; @@ -1329,7 +1485,7 @@ impl PodmanComputeDriver { for entry in &entries { if entry.state == "running" { // Running containers need inspect for health check status. - match self.client.inspect_container(&entry.id).await { + match watcher::inspect_workload(&self.client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { sandboxes.push(sandbox); @@ -1592,6 +1748,7 @@ fn userns_needs_extraction(userns: Option<&str>) -> bool { /// Returns `true` when userns remaps all UIDs, making host-owned bind mounts /// unreadable from inside the container. `auto` and `no-map` remap every UID; /// `keep-id` preserves the host user's UID; `host` uses the host namespace. +#[cfg(test)] fn userns_remaps_uids(userns: Option<&str>) -> bool { userns.is_some_and(|mode| { let base = mode.split(':').next().unwrap_or(mode); @@ -1688,6 +1845,7 @@ mod tests { "lifecycle-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1703,7 +1861,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[1], + .expect("request log lock should not be poisoned")[2], format!( "POST {}", api_path("/libpod/containers/ctr-1/stop?timeout=10") @@ -1712,7 +1870,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[2], + .expect("request log lock should not be poisoned")[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1724,8 +1882,7 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); test_driver(start_socket.clone()) .start_sandbox("sandbox-1") @@ -1742,7 +1899,10 @@ mod tests { start_requests .lock() .expect("request log lock should not be poisoned")[2], - format!("POST {}", api_path("/libpod/containers/ctr-1/start")) + format!( + "POST {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/stop?timeout=10") + ) ); let _ = fs::remove_file(stop_socket); @@ -1755,6 +1915,7 @@ mod tests { "lifecycle-stop-wait", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1776,12 +1937,12 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 4); + assert_eq!(requests.len(), 5); assert_eq!( - requests[2], + requests[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); - assert_eq!(requests[3], requests[2]); + assert_eq!(requests[4], requests[3]); let _ = fs::remove_file(socket); } @@ -1792,6 +1953,7 @@ mod tests { "lifecycle-stop-retry", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"stopping"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new( StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, @@ -1808,9 +1970,9 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 2); + assert_eq!(requests.len(), 3); assert_eq!( - requests[1], + requests[2], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1828,6 +1990,7 @@ mod tests { "trace-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1876,17 +2039,10 @@ mod tests { let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; let (socket_path, _requests, handle) = spawn_podman_stub( "trace-create", - vec![ - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + create_setup_responses(false) + .into_iter() + .chain(create_launch_responses()) + .collect(), ); let exporter = InMemorySpanExporterBuilder::new().build(); let provider = SdkTracerProvider::builder() @@ -1912,7 +2068,6 @@ mod tests { "podman.prepare_images", "podman.prepare_storage", "podman.prepare_container", - "podman.start_container", ] { let child = spans .iter() @@ -1992,8 +2147,7 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); test_driver(start_socket.clone()) .start_sandbox("sandbox-1") @@ -2005,6 +2159,8 @@ mod tests { let (delete_socket, _requests, delete_handle) = spawn_podman_stub( "trace-delete", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached StubResponse::new(StatusCode::OK, "[]"), StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -2855,6 +3011,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-not-found", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -2874,9 +3032,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) @@ -2924,6 +3082,159 @@ mod tests { ) } + fn restart_responses() -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + let bundle = tar::Builder::new(Vec::new()).into_inner().unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(bundle.len() as u64); + header.set_mode(0o600); + header.set_cksum(); + archive + .append_data(&mut header, "sandbox-bundle.tar", bundle.as_slice()) + .unwrap(); + vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor stop + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), + StubResponse::new(StatusCode::OK, ""), // restore bootstrap + fence_response(), + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + fn fence_response() -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: &'static str, + privileged: bool, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Networks { + networks: std::collections::BTreeMap, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Fence { + host_config: HostConfig, + network_settings: Networks, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Fence { + host_config: HostConfig { + network_mode: "none", + privileged: false, + }, + network_settings: Networks { + networks: std::collections::BTreeMap::default(), + }, + }) + .unwrap(), + ) + } + + fn created_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + struct Created { + #[serde(rename = "Id")] + id: &'static str, + } + StubResponse::new( + StatusCode::CREATED, + serde_json::to_vec(&Created { id }).unwrap(), + ) + } + + fn image_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Config { + user: &'static str, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Image { + id: &'static str, + config: Config, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Image { + id, + config: Config { user: "1234:1235" }, + }) + .unwrap(), + ) + } + + fn create_setup_responses(proxy_secret: bool) -> Vec { + let mut responses = vec![ + StubResponse::new(StatusCode::OK, "{}"), // supervisor pull + StubResponse::new(StatusCode::OK, "{}"), // workload pull + image_response("sha256:sandbox"), + created_response("identity-reader"), + StubResponse::new(StatusCode::NOT_FOUND, ""), // reserved hierarchy absent + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional passwd + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional group + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader + image_response("sha256:supervisor"), + StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume + ]; + if proxy_secret { + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); + } + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume + responses + } + + fn create_launch_responses() -> Vec { + vec![ + created_response("workload"), + fence_response(), + StubResponse::new(StatusCode::OK, ""), // workload archive + created_response("supervisor"), + StubResponse::new(StatusCode::OK, ""), // supervisor archive + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + #[tokio::test] + async fn reserved_image_control_root_fails_before_workload_or_secrets() { + let (path, requests, handle) = spawn_podman_stub( + "reserved-control-root", + vec![ + StubResponse::new(StatusCode::OK, "{}"), + StubResponse::new(StatusCode::OK, "{}"), + image_response("sha256:image"), + created_response("identity-reader"), + StubResponse::new(StatusCode::OK, "existing reserved path"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let error = test_driver(path.clone()) + .create_sandbox(&plain_sandbox("id", "name")) + .await + .unwrap_err(); + assert!(error.to_string().contains("reserved /.openshell")); + handle.await.unwrap(); + assert!( + !requests + .lock() + .unwrap() + .iter() + .any(|request| request.contains("/libpod/volumes") + || request.contains("/libpod/secrets")) + ); + let _ = fs::remove_file(path); + } + #[tokio::test] async fn create_sandbox_removes_proxy_auth_secret_on_container_create_failure() { // A credential secret is staged before the container is created, so a @@ -2932,19 +3243,15 @@ mod tests { let auth_file = write_proxy_auth_file("create-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-container-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // create container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); @@ -2974,21 +3281,18 @@ mod tests { let auth_file = write_proxy_auth_file("start-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-start-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::CREATED, "{}"), // create container - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // start container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain(create_launch_responses().into_iter().take(6)) + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); @@ -3018,7 +3322,9 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-proxy-auth", vec![ - StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached + StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) StubResponse::new(StatusCode::NO_CONTENT, ""), // remove volume StubResponse::new(StatusCode::NO_CONTENT, ""), // remove token secret StubResponse::new(StatusCode::NO_CONTENT, ""), // remove proxy-auth secret @@ -3061,10 +3367,14 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-label-lookup", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers by label StubResponse::new(StatusCode::OK, list_body), // single timed remove_container operation StubResponse::new(StatusCode::NO_CONTENT, ""), + // channel volume, now detached + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove_volume StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -3082,9 +3392,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!( @@ -3093,7 +3403,7 @@ mod tests { ) ); assert_eq!( - requests[2], + requests[5], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index fedeea3068..d18ad5d88d 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -162,8 +162,7 @@ impl ComputeDriver for ComputeDriverService { .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) + Box::pin(self.driver.create_sandbox(&sandbox)) .await .map_err(Status::from)?; Ok(Response::new(CreateSandboxResponse {})) @@ -680,6 +679,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "forward-id", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -708,9 +709,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs new file mode 100644 index 0000000000..9a33e6559f --- /dev/null +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -0,0 +1,381 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman-owned provisioning for the common authenticated isolation channel. + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; + +use openshell_core::ComputeDriverError; +use openshell_core::proto::compute::v1::DriverSandbox; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, + BoundaryTransport, generate_boundary_mutual_tls_material, +}; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; + +pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; +pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; +pub const CHANNEL_ROOT: &str = "/.openshell/channel"; +pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; +pub const TOPOLOGY_PATH: &str = "/.openshell/supervisor/topology.payload"; +pub const RESTART_BUNDLE_PATH: &str = "/.openshell/supervisor/sandbox-bundle.tar"; +const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; + +pub fn supervisor_name(id: &str) -> String { + format!("openshell-supervisor-{id}") +} +pub fn channel_volume_name(id: &str) -> String { + format!("openshell-channel-{id}") +} + +fn invalid(error: impl std::fmt::Display) -> ComputeDriverError { + ComputeDriverError::Precondition(error.to_string()) +} + +/// Resolve policy names against the pinned workload image, never the gateway. +pub fn resolve_identity( + sandbox: &DriverSandbox, + image_id: &str, + image_user: &str, + passwd: &[u8], + group: &[u8], +) -> Result { + let passwd = std::str::from_utf8(passwd).map_err(invalid)?; + let group = std::str::from_utf8(group).map_err(invalid)?; + let accounts: Vec<_> = passwd + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some(( + name, + fields.next()?.parse::().ok()?, + fields.next()?.parse::().ok()?, + )) + }) + .collect(); + let groups: Vec<_> = group + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some((name, fields.next()?.parse::().ok()?, fields.next()?)) + }) + .collect(); + let request = sandbox + .spec + .as_ref() + .and_then(|spec| spec.workload_identity.as_ref()); + let requested_user = request.map_or("", |identity| identity.user.trim()); + let requested_group = request.map_or("", |identity| identity.group.trim()); + let (image_user, image_group) = image_user.split_once(':').unwrap_or((image_user, "")); + let user = if requested_user.is_empty() { + image_user + } else { + requested_user + }; + let group = if requested_group.is_empty() { + image_group + } else { + requested_group + }; + let account = accounts + .iter() + .find(|(name, uid, _)| *name == user || user.parse::().ok() == Some(*uid)); + let uid = user + .parse() + .ok() + .or_else(|| account.map(|(_, uid, _)| *uid)) + .ok_or_else(|| invalid("configure a non-root workload user present in the pinned image"))?; + let gid = if group.is_empty() { + account.map(|(_, _, gid)| *gid) + } else { + group.parse().ok().or_else(|| { + groups + .iter() + .find(|(name, _, _)| *name == group) + .map(|(_, gid, _)| *gid) + }) + } + .ok_or_else(|| { + invalid("configure an explicit workload group for a UID without an image passwd entry") + })?; + let supplemental = account.map_or_else(Vec::new, |(username, _, _)| { + groups + .iter() + .filter(|(_, id, members)| { + *id != gid && members.split(',').any(|member| member == *username) + }) + .map(|(_, gid, _)| *gid) + .collect() + }); + let source = if requested_user.is_empty() && requested_group.is_empty() { + "image" + } else { + "policy" + }; + ResolvedWorkloadIdentity::new(uid, gid, supplemental, source.into(), image_id.into()) + .map_err(invalid) +} + +pub struct BootstrapArchives { + pub workload: Vec, + pub supervisor: Vec, +} + +/// The shared volume contains only sandbox credentials. Supervisor credentials, +/// gateway authorization, and the restart copy never enter that volume. +pub fn bootstrap_archives( + sandbox_id: &str, + container_id: &str, + identity: &ResolvedWorkloadIdentity, + child_env: HashMap, +) -> Result { + let tls = generate_boundary_mutual_tls_material().map_err(invalid)?; + let resource_claims = BTreeMap::from([ + ("podman.container_id".into(), container_id.into()), + ( + "podman.image_identity".into(), + identity.resource_digest.clone(), + ), + ]); + let driver_fence = DriverFenceEvidence::Podman { + container_id: container_id.into(), + network_mode: "none".into(), + unexpected_networks: Vec::new(), + }; + let generation = uuid::Uuid::new_v4().to_string(); + let session_epoch = uuid::Uuid::new_v4().to_string(); + let bootstrap_token = format!( + "{}{}", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ); + let config = BoundaryConfig { + boundary_id: sandbox_id.into(), + generation: generation.clone(), + session_epoch: session_epoch.clone(), + bootstrap_token: bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/.openshell/channel/sandbox/server.crt"), + private_key_path: PathBuf::from("/.openshell/channel/sandbox/server.key"), + client_ca_certificate_path: PathBuf::from( + "/.openshell/channel/sandbox/client-ca.crt", + ), + }, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: identity.clone(), + driver_fence: driver_fence.clone(), + child_env, + }; + let topology = BoundaryTopology { + boundary_id: sandbox_id.into(), + generation, + session_epoch, + bootstrap_token, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: BoundaryClientTls { + server_name: tls.server_name, + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem, + private_key_pem: tls.supervisor_private_key_pem, + }, + }, + host_gateway_ip: None, + resource_claims, + workload_identity: identity.clone(), + driver_fence, + }; + let mut workload = Archive::new(identity); + workload.directory(".openshell", 0o755, false)?; + workload.directory(".openshell/channel", 0o755, false)?; + workload.directory(".openshell/channel/sandbox", 0o711, true)?; + workload.directory("sandbox", 0o700, true)?; + workload.file( + BOOTSTRAP_PATH, + &serde_json::to_vec(&config).map_err(invalid)?, + )?; + workload.file( + "/.openshell/channel/sandbox/server.crt", + tls.sandbox_certificate_pem.as_bytes(), + )?; + workload.file( + "/.openshell/channel/sandbox/server.key", + tls.sandbox_private_key_pem.as_bytes(), + )?; + workload.file( + "/.openshell/channel/sandbox/client-ca.crt", + tls.ca_certificate_pem.as_bytes(), + )?; + let workload = workload.finish()?; + let mut supervisor = Archive::new(identity); + supervisor.directory(".openshell", 0o755, false)?; + supervisor.directory(".openshell/supervisor", 0o700, true)?; + supervisor.file( + TOPOLOGY_PATH, + &serde_json::to_vec(&topology).map_err(invalid)?, + )?; + supervisor.file(RESTART_BUNDLE_PATH, &workload)?; + Ok(BootstrapArchives { + workload, + supervisor: supervisor.finish()?, + }) +} + +struct Archive<'a> { + builder: tar::Builder>, + identity: &'a ResolvedWorkloadIdentity, +} +impl<'a> Archive<'a> { + fn new(identity: &'a ResolvedWorkloadIdentity) -> Self { + Self { + builder: tar::Builder::new(Vec::new()), + identity, + } + } + fn directory(&mut self, path: &str, mode: u32, owned: bool) -> Result<(), ComputeDriverError> { + self.append(path, mode, owned, tar::EntryType::Directory, &[]) + } + fn file(&mut self, path: &str, content: &[u8]) -> Result<(), ComputeDriverError> { + self.append( + path.trim_start_matches('/'), + 0o600, + true, + tar::EntryType::Regular, + content, + ) + } + fn append( + &mut self, + path: &str, + mode: u32, + owned: bool, + kind: tar::EntryType, + content: &[u8], + ) -> Result<(), ComputeDriverError> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_mode(mode); + header.set_uid(if owned { + u64::from(self.identity.uid) + } else { + 0 + }); + header.set_gid(if owned { + u64::from(self.identity.gid) + } else { + 0 + }); + header.set_size(content.len() as u64); + header.set_mtime(0); + header.set_cksum(); + self.builder + .append_data(&mut header, path, content) + .map_err(invalid) + } + fn finish(self) -> Result, ComputeDriverError> { + self.builder.into_inner().map_err(invalid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read as _; + + #[test] + fn identity_uses_pinned_image_accounts_and_rejects_root() { + let sandbox = DriverSandbox::default(); + let passwd = b"root:x:0:0:root:/root:/bin/sh\nagent:x:1000:1001::/home/agent:/bin/sh\n"; + let groups = b"agent:x:1001:\ndata:x:2000:agent\n"; + let identity = + resolve_identity(&sandbox, "sha256:pinned", "agent", passwd, groups).unwrap(); + assert_eq!((identity.uid, identity.gid), (1000, 1001)); + assert_eq!(identity.supplementary_gids, vec![2000]); + assert_eq!(identity.resource_digest, "sha256:pinned"); + assert!(resolve_identity(&sandbox, "sha256:pinned", "root", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "2000", passwd, groups).is_err()); + } + + fn files(bytes: &[u8]) -> BTreeMap> { + tar::Archive::new(bytes) + .entries() + .unwrap() + .filter_map(|entry| { + let mut entry = entry.unwrap(); + if !entry.header().entry_type().is_file() { + return None; + } + let path = entry.path().unwrap().into_owned(); + assert_eq!(entry.header().mode().unwrap(), 0o600); + assert_eq!(entry.header().uid().unwrap(), 1000); + let mut content = Vec::new(); + entry.read_to_end(&mut content).unwrap(); + Some((path, content)) + }) + .collect() + } + + #[test] + fn archives_separate_supervisor_credentials_and_bind_one_channel() { + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let archives = + bootstrap_archives("sandbox", "container", &identity, HashMap::new()).unwrap(); + let workload = files(&archives.workload); + let supervisor = files(&archives.supervisor); + assert_eq!(workload.len(), 4); + assert_eq!(supervisor.len(), 2); + assert!( + workload + .keys() + .all(|path| path.starts_with(".openshell/channel/sandbox")) + ); + assert!( + supervisor + .keys() + .all(|path| path.starts_with(".openshell/supervisor")) + ); + let config: BoundaryConfig = serde_json::from_slice( + workload + .get(&PathBuf::from(BOOTSTRAP_PATH.trim_start_matches('/'))) + .unwrap(), + ) + .unwrap(); + let topology: BoundaryTopology = serde_json::from_slice( + supervisor + .get(&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))) + .unwrap(), + ) + .unwrap(); + assert_eq!(config.boundary_id, topology.boundary_id); + assert_eq!(config.bootstrap_token, topology.bootstrap_token); + assert_eq!(config.driver_fence, topology.driver_fence); + assert_eq!(config.workload_identity, identity); + topology + .driver_fence + .validate_for_backend("podman") + .unwrap(); + assert_eq!( + supervisor + .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) + .unwrap(), + &archives.workload + ); + } +} diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index 115e64eb2f..fa06cf4864 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub(crate) mod container; pub mod driver; pub mod grpc; +mod isolation; pub mod otel_tracing; mod socket_discovery; #[cfg(test)] diff --git a/crates/openshell-driver-podman/src/test_utils.rs b/crates/openshell-driver-podman/src/test_utils.rs index ec5c8f7f11..24e9d4d8ed 100644 --- a/crates/openshell-driver-podman/src/test_utils.rs +++ b/crates/openshell-driver-podman/src/test_utils.rs @@ -20,12 +20,12 @@ use tokio::net::UnixListener; #[derive(Clone)] pub struct StubResponse { pub status: StatusCode, - pub body: String, + pub body: Bytes, pub delay: Duration, } impl StubResponse { - pub fn new(status: StatusCode, body: impl Into) -> Self { + pub fn new(status: StatusCode, body: impl Into) -> Self { Self { status, body: body.into(), @@ -108,7 +108,7 @@ pub fn spawn_podman_stub( Ok::<_, Infallible>( hyper::Response::builder() .status(response.status) - .body(Full::new(Bytes::from(response.body))) + .body(Full::new(response.body)) .expect("stub response should build"), ) } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index cc98e29641..299f61cf18 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -138,14 +138,16 @@ pub async fn start_watch( let mut event_rx = client.events_stream(LABEL_MANAGED_FILTER).await?; // 2. List existing containers for initial state sync. - let existing = client.list_containers(&[LABEL_MANAGED_FILTER]).await?; + let existing = client + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) + .await?; for entry in &existing { // For running containers, use inspect to get full state including // health check status — matching the same condition derivation used // for live events. if entry.state == "running" { - match client.inspect_container(&entry.id).await { + match inspect_workload(&client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { if tx.send(Ok(sandbox_event(sandbox))).await.is_err() { @@ -245,11 +247,34 @@ async fn map_podman_event( return None; } + if event + .actor + .attributes + .get(crate::isolation::LABEL_ROLE) + .is_some_and(|role| role == "supervisor") + { + let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); + let workloads = client + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) + .await + .ok()?; + let workload = workloads.first()?; + return inspect_workload(client, &workload.id) + .await + .ok() + .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) + .map(sandbox_event); + } + match event.action.as_str() { "remove" => Some(deleted_event(sandbox_id.clone())), "create" | "start" | "stop" | "die" | "health_status" => { // Inspect the container to get current state. - match client.inspect_container(container_id).await { + match inspect_workload(client, container_id).await { Ok(inspect) => { if lifecycle_event_fences.matches_previous_exit( event, @@ -326,6 +351,54 @@ async fn map_podman_event( } } +/// A workload is ready only when its independent supervisor is healthy. This +/// check runs both on watch reconciliation and on events, and contains a lost +/// supervisor even when the gateway missed the original exit event. +pub async fn inspect_workload( + client: &PodmanClient, + id: &str, +) -> Result { + let mut workload = client.inspect_container(id).await?; + if workload + .config + .labels + .get(crate::isolation::LABEL_ROLE) + .is_none_or(|role| role != "sandbox") + { + return Ok(workload); + } + let Some(sandbox_id) = workload.config.labels.get(LABEL_SANDBOX_ID) else { + return Ok(workload); + }; + let supervisor = client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if workload.state.running { + match supervisor { + Ok(supervisor) if supervisor.state.running => { + workload.state.health = supervisor.state.health; + } + Ok(supervisor) + if supervisor.state.status == "configured" + || supervisor.state.status == "created" => + { + workload.state.health = Some(HealthState { + status: "starting".into(), + }); + } + // Both containers exist before initial start. A missing or exited + // companion therefore requires containment, including after a + // gateway restart that missed the original Podman exit event. + Ok(_) | Err(PodmanApiError::NotFound(_)) => { + client.stop_container(&workload.id, 0).await?; + workload = client.inspect_container(&workload.id).await?; + } + Err(error) => return Err(error), + } + } + Ok(workload) +} + /// Construct a `DriverSandbox` from common fields. /// /// Centralises the boilerplate that every event/inspect/list path shares: @@ -499,6 +572,65 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { mod tests { use super::*; + #[tokio::test] + async fn missing_supervisor_stops_workload_during_reconciliation() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "lost-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new(StatusCode::NOT_FOUND, "missing companion"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(!inspected.state.running); + handle.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|request| request.ends_with("/libpod/containers/workload/stop?timeout=0")) + ); + let _ = std::fs::remove_file(path); + } + + #[tokio::test] + async fn created_supervisor_keeps_bootstrapping_workload_starting() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "starting-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"configured","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(inspected.state.running); + assert_eq!(inspected.state.health.unwrap().status, "starting"); + handle.await.unwrap(); + assert_eq!(requests.lock().unwrap().len(), 2); + let _ = std::fs::remove_file(path); + } + fn podman_event(action: &str, sandbox_id: &str, time_nano: i64) -> PodmanEvent { PodmanEvent { event_type: "container".to_string(), diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index fa2d9ba835..813ea93dfe 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -618,7 +618,7 @@ sandbox_pids_limit = 2048 ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Each Podman sandbox has a workload container running `openshell-sandbox` with `network=none`, and a separate `openshell-supervisor` companion on the configured network. Both use non-root identities, drop all capabilities, and keep the runtime's default seccomp profile. A private named volume carries their authenticated gRPC Unix socket. Gateway JWTs, optional gateway mTLS material, and upstream proxy credentials are delivered only to the supervisor; user mounts and GPU devices stay with the workload. ```toml [openshell] @@ -659,7 +659,8 @@ enable_bind_mounts = false sandbox_pids_limit = 2048 # Health check interval in seconds. Lower values detect readiness faster # but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to disable health checks entirely. Default: 10. +# Set to 0 to use a one-second check. Readiness checks cannot be disabled. +# Default: 10. health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -670,7 +671,7 @@ health_check_interval_secs = 10 # rootful Podman uses absolute host IDs (0:1000:1, 1:100000:65536). # uidmap = ["0:0:1", "1:1:65535"] # gidmap = ["0:0:1", "1:1:65535"] -# Corporate forward proxy for sandbox egress. When set, the in-container +# Corporate forward proxy for sandbox egress. When set, the external # supervisor chains policy-approved TLS tunnels through this proxy with HTTP # CONNECT instead of dialing destinations directly. Plain-HTTP requests are # not proxied and always dial the destination directly. http:// and https:// diff --git a/e2e/rust/tests/podman_gateway_start.rs b/e2e/rust/tests/podman_gateway_start.rs index 28a9ceb15c..7e159ffa78 100644 --- a/e2e/rust/tests/podman_gateway_start.rs +++ b/e2e/rust/tests/podman_gateway_start.rs @@ -38,10 +38,10 @@ const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; /// harness), fall back to plain `podman`, leaving Linux behavior unchanged. fn podman_command() -> Command { let mut command = Command::new("podman"); - if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - if !socket.is_empty() { - command.arg("--url").arg(format!("unix://{socket}")); - } + if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !socket.is_empty() + { + command.arg("--url").arg(format!("unix://{socket}")); } command } @@ -49,7 +49,15 @@ fn podman_command() -> Command { fn sandbox_container_running(sandbox_name: &str) -> Result { let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); let output = podman_command() - .args(["ps", "-aq", "--filter", MANAGED_BY_LABEL_FILTER, "--filter"]) + .args([ + "ps", + "-aq", + "--filter", + MANAGED_BY_LABEL_FILTER, + "--filter", + "label=openshell.io/isolation-role=sandbox", + "--filter", + ]) .arg(sandbox_name_filter) .stdout(Stdio::piped()) .stderr(Stdio::piped()) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index e30516bf09..15ea13a77f 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -22,7 +22,7 @@ const BASE_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:late const READY_MARKER: &str = "podman-oci-identity-ready"; const OCI_UID: &str = "2345"; const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r#"version: 1 +const OCI_FALLBACK_POLICY: &str = r"version: 1 filesystem_policy: include_workdir: true @@ -32,7 +32,7 @@ landlock: compatibility: best_effort network_policies: {} -"#; +"; struct ImageGuard { engine: ContainerEngine, @@ -128,7 +128,16 @@ fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result } fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result { + container_id_for_role(engine, sandbox_name, "sandbox") +} + +fn container_id_for_role( + engine: &ContainerEngine, + sandbox_name: &str, + role: &str, +) -> Result { let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); + let role_filter = format!("label=openshell.io/isolation-role={role}"); let stdout = run_engine( engine, &[ @@ -138,6 +147,8 @@ fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result< "label=openshell.managed=true", "--filter", &name_filter, + "--filter", + &role_filter, ], )?; let ids = stdout @@ -233,5 +244,54 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { "Podman sandbox must launch the immutable image ID inspected before creation" ); + assert_isolated_pair(&image, &sandbox, &container_id).await; sandbox.cleanup().await; } + +async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, container_id: &str) { + let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") + .expect("find separate supervisor companion"); + assert_ne!(supervisor_id, container_id); + for id in [container_id, &supervisor_id] { + let user = run_engine( + &image.engine, + &["inspect", "--format", "{{.Config.User}}", id], + ) + .unwrap(); + assert_eq!(user, format!("{OCI_UID}:{OCI_GID}")); + let caps = run_engine( + &image.engine, + &["inspect", "--format", "{{.EffectiveCaps}}", id], + ) + .unwrap(); + assert_eq!( + caps, "[]", + "neither container may have effective capabilities" + ); + } + let network = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{.HostConfig.NetworkMode}}", + container_id, + ], + ) + .unwrap(); + assert_eq!(network, "none"); + let mounts = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{range .Mounts}}{{println .Destination}}{{end}}", + container_id, + ], + ) + .unwrap(); + assert!(!mounts.contains("/etc/openshell/tls")); + assert!(!mounts.contains("/.openshell/supervisor")); + let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/topology.payload"]).await.expect("workload cannot read either control credential set"); + assert!(posture.contains("0000000000000000")); +} diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fc3419e182..f87230a1a9 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -160,9 +160,24 @@ cleanup() { for id in ${sandbox_ids}; do local sandbox_id sandbox_id="$(podman_cmd inspect --format '{{ index .Config.Labels "openshell.ai/sandbox-id" }}' "${id}" 2>/dev/null || true)" - podman_cmd rm -f "${id}" >/dev/null 2>&1 || true if [ -n "${sandbox_id}" ] && [ "${sandbox_id}" != "" ]; then + # Only the companion is attached to the test network. Remove it first + # (it depends on the workload user namespace), then locate the isolated + # network=none workload by this test sandbox's immutable label. + podman_cmd rm -f "openshell-supervisor-${sandbox_id}" >/dev/null 2>&1 || true + local workload_ids workload_id + workload_ids="$(podman_cmd ps -aq --filter "label=openshell.managed=true" \ + --filter "label=openshell.ai/sandbox-id=${sandbox_id}" \ + --filter "label=openshell.io/isolation-role=sandbox" 2>/dev/null || true)" + for workload_id in ${workload_ids}; do + podman_cmd rm -f "${workload_id}" >/dev/null 2>&1 || true + done + podman_cmd volume rm "openshell-channel-${sandbox_id}" >/dev/null 2>&1 || true podman_cmd volume rm -f "openshell-sandbox-${sandbox_id}-workspace" >/dev/null 2>&1 || true + local secret_prefix + for secret_prefix in openshell-token openshell-proxy-auth openshell-tls-ca openshell-tls-cert openshell-tls-key; do + podman_cmd secret rm "${secret_prefix}-${sandbox_id}" >/dev/null 2>&1 || true + done fi done fi diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ce676148aa..89f6533c83 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -225,10 +225,14 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. -- A sandbox with explicit `protocol: tcp` endpoints fails before readiness: - inspect supervisor logs for policy DNS port-53 binding, synthetic-route, or - nftables redirect failures. Rootless Podman must provide these primitives - inside the supervisor-owned nested network namespace; setup fails closed. +- Inspect both Podman containers for the sandbox: the `sandbox` isolation role + must have network mode `none`; the `supervisor` role owns gateway callbacks + and egress. Both run non-root with all capabilities dropped. Check the private + channel volume and shared user-namespace mapping if authentication fails. +- If a sandbox fails before readiness, inspect its unprivileged enforcement + probe and the companion supervisor's private health check. Do not add + capabilities, attach a workload network, or disable the runtime seccomp + profile. There is no sandbox nftables or nested-network setup to repair. - Gateway exits before becoming healthy with a callback-listener discovery error: inspect `podman info --debug`, the configured Podman network, and the host's IPv4 default route. Rootless pasta uses the private source address From 5314ed3a75e56194e4c29ab556bac2a14935533f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:23:28 -0700 Subject: [PATCH 2/5] fix(podman): stage bootstrap archives at named volume destinations Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 10 ++- crates/openshell-driver-podman/src/client.rs | 6 +- crates/openshell-driver-podman/src/driver.rs | 82 +++++++++++++++++-- .../openshell-driver-podman/src/isolation.rs | 63 +++++++------- .../openshell-driver-podman/src/test_utils.rs | 22 ++++- 5 files changed, 140 insertions(+), 43 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 17bacce1ae..0768b9c712 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -30,6 +30,14 @@ The supervisor joins the workload's **user namespace only** to preserve UID/GID mapping for shared-volume access. PID, mount, and network namespaces remain separate. The channel volume uses shared SELinux relabeling (`:z`). +Before starting either container, the driver uploads volume-relative archives +directly to the channel and workspace volume destinations. A rootfs upload on a +stopped Podman container does not populate nested named volumes. Restart restores +only the channel bootstrap into the existing channel volume, preserving the +workspace. The workload starts before the supervisor so its user namespace exists +when the supervisor joins it; a stopped supervisor resolves that namespace again +on its next start. + The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail closed; do not switch to an unconfined profile or add capabilities. @@ -77,7 +85,7 @@ children, never the supervisor process. ## Lifecycle and readiness -Create builds both stopped containers and stages both private archives before +Create builds both stopped containers and stages the private archives before starting either container. The sandbox does not execute the agent until the supervisor authenticates and confirms the common boundary contract. Failed creation removes only containers created by that attempt, then cleans up diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 1cb1e12bfb..9f37864373 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -479,13 +479,17 @@ impl PodmanClient { pub(crate) async fn copy_to_container( &self, name: &str, + destination: &str, archive: Vec, ) -> Result<(), PodmanApiError> { validate_name(name)?; let (status, bytes) = self .request_raw( hyper::Method::PUT, - &format!("/libpod/containers/{name}/archive?path=/"), + &format!( + "/libpod/containers/{name}/archive?path={}", + url_encode(destination) + ), "application/x-tar", archive.into(), ) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 310b47adb0..2452fe922e 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1019,7 +1019,14 @@ impl PodmanComputeDriver { child_env, )?; self.client - .copy_to_container(&workload_id, archives.workload) + .copy_to_container( + &workload_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&workload_id, "/sandbox", archives.workspace) .await?; specs.supervisor.join_user_namespace(&workload_id); let supervisor_id = self @@ -1028,7 +1035,7 @@ impl PodmanComputeDriver { .await?; created_supervisor = Some(supervisor_id.clone()); self.client - .copy_to_container(&supervisor_id, archives.supervisor) + .copy_to_container(&supervisor_id, "/", archives.supervisor) .await?; // Both resources and private files exist before either // container can run. Only the trusted sandbox starts here; @@ -1315,7 +1322,9 @@ impl PodmanComputeDriver { .await?; let bundle = extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; - self.client.copy_to_container(&container_id, bundle).await?; + self.client + .copy_to_container(&container_id, crate::isolation::CHANNEL_ROOT, bundle) + .await?; self.client.verify_isolation_fence(&container_id).await?; self.client.start_container(&container_id).await?; if let Err(error) = self.client.start_container(&supervisor).await { @@ -1889,6 +1898,24 @@ mod tests { .await .expect("start should succeed"); start_handle.await.expect("start stub should finish"); + let restart_requests = start_requests.lock().unwrap().clone(); + assert_eq!( + restart_requests + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect::>(), + vec![format!( + "PUT {}", + api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") + )] + ); + assert!( + !restart_requests + .iter() + .any(|request| request.contains("/volumes/create") + || request.contains("/containers/create")) + ); assert_eq!( start_requests .lock() @@ -2037,7 +2064,7 @@ mod tests { use tracing_subscriber::layer::SubscriberExt as _; let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; - let (socket_path, _requests, handle) = spawn_podman_stub( + let (socket_path, requests, handle) = spawn_podman_stub( "trace-create", create_setup_responses(false) .into_iter() @@ -2057,6 +2084,22 @@ mod tests { .await .expect("create should succeed"); handle.await.expect("stub should finish"); + let uploads: Vec<_> = requests + .lock() + .unwrap() + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect(); + assert_eq!( + uploads, + [ + "/libpod/containers/workload/archive?path=%2F.openshell%2Fchannel", + "/libpod/containers/workload/archive?path=%2Fsandbox", + "/libpod/containers/supervisor/archive?path=%2F", + ] + .map(|path| format!("PUT {}", api_path(path))) + ); provider.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); @@ -3084,7 +3127,18 @@ mod tests { fn restart_responses() -> Vec { let mut archive = tar::Builder::new(Vec::new()); - let bundle = tar::Builder::new(Vec::new()).into_inner().unwrap(); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let bundle = + crate::isolation::bootstrap_archives("sandbox-1", "ctr-1", &identity, HashMap::new()) + .unwrap() + .channel; let mut header = tar::Header::new_gnu(); header.set_size(bundle.len() as u64); header.set_mode(0o600); @@ -3099,7 +3153,7 @@ mod tests { r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"exited","Running":false},"Config":{}}"#, ), StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), - StubResponse::new(StatusCode::OK, ""), // restore bootstrap + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), fence_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start @@ -3197,7 +3251,8 @@ mod tests { vec![ created_response("workload"), fence_response(), - StubResponse::new(StatusCode::OK, ""), // workload archive + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), created_response("supervisor"), StubResponse::new(StatusCode::OK, ""), // supervisor archive StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start @@ -3205,6 +3260,17 @@ mod tests { ] } + fn channel_archive_members() -> &'static [&'static str] { + &[ + ".", + "sandbox", + "sandbox/bootstrap.json", + "sandbox/server.crt", + "sandbox/server.key", + "sandbox/client-ca.crt", + ] + } + #[tokio::test] async fn reserved_image_control_root_fails_before_workload_or_secrets() { let (path, requests, handle) = spawn_podman_stub( @@ -3283,7 +3349,7 @@ mod tests { "create-start-fail", create_setup_responses(true) .into_iter() - .chain(create_launch_responses().into_iter().take(6)) + .chain(create_launch_responses().into_iter().take(7)) .chain([ StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 9a33e6559f..1ba6aae055 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -122,7 +122,8 @@ pub fn resolve_identity( } pub struct BootstrapArchives { - pub workload: Vec, + pub channel: Vec, + pub workspace: Vec, pub supervisor: Vec, } @@ -194,28 +195,22 @@ pub fn bootstrap_archives( workload_identity: identity.clone(), driver_fence, }; - let mut workload = Archive::new(identity); - workload.directory(".openshell", 0o755, false)?; - workload.directory(".openshell/channel", 0o755, false)?; - workload.directory(".openshell/channel/sandbox", 0o711, true)?; - workload.directory("sandbox", 0o700, true)?; - workload.file( - BOOTSTRAP_PATH, + // Libpod resolves the requested upload destination once for a stopped + // container. Archive entries must be relative to the selected named volume, + // not rootfs paths that the volume would shadow on container start. + let mut channel = Archive::new(identity); + channel.directory(".", 0o755, false)?; + channel.directory("sandbox", 0o711, true)?; + channel.file( + "sandbox/bootstrap.json", &serde_json::to_vec(&config).map_err(invalid)?, )?; - workload.file( - "/.openshell/channel/sandbox/server.crt", - tls.sandbox_certificate_pem.as_bytes(), - )?; - workload.file( - "/.openshell/channel/sandbox/server.key", - tls.sandbox_private_key_pem.as_bytes(), - )?; - workload.file( - "/.openshell/channel/sandbox/client-ca.crt", - tls.ca_certificate_pem.as_bytes(), - )?; - let workload = workload.finish()?; + channel.file("sandbox/server.crt", tls.sandbox_certificate_pem.as_bytes())?; + channel.file("sandbox/server.key", tls.sandbox_private_key_pem.as_bytes())?; + channel.file("sandbox/client-ca.crt", tls.ca_certificate_pem.as_bytes())?; + let channel = channel.finish()?; + let mut workspace = Archive::new(identity); + workspace.directory(".", 0o700, true)?; let mut supervisor = Archive::new(identity); supervisor.directory(".openshell", 0o755, false)?; supervisor.directory(".openshell/supervisor", 0o700, true)?; @@ -223,9 +218,10 @@ pub fn bootstrap_archives( TOPOLOGY_PATH, &serde_json::to_vec(&topology).map_err(invalid)?, )?; - supervisor.file(RESTART_BUNDLE_PATH, &workload)?; + supervisor.file(RESTART_BUNDLE_PATH, &channel)?; Ok(BootstrapArchives { - workload, + channel, + workspace: workspace.finish()?, supervisor: supervisor.finish()?, }) } @@ -337,15 +333,20 @@ mod tests { .unwrap(); let archives = bootstrap_archives("sandbox", "container", &identity, HashMap::new()).unwrap(); - let workload = files(&archives.workload); + let workload = files(&archives.channel); let supervisor = files(&archives.supervisor); + let mut workspace = tar::Archive::new(archives.workspace.as_slice()); + let mut entries = workspace.entries().unwrap(); + let root = entries.next().unwrap().unwrap(); + assert_eq!(root.path().unwrap().as_ref(), std::path::Path::new(".")); + assert!(root.header().entry_type().is_dir()); + assert_eq!(root.header().uid().unwrap(), u64::from(identity.uid)); + assert_eq!(root.header().gid().unwrap(), u64::from(identity.gid)); + assert_eq!(root.header().mode().unwrap(), 0o700); + assert!(entries.next().is_none()); assert_eq!(workload.len(), 4); assert_eq!(supervisor.len(), 2); - assert!( - workload - .keys() - .all(|path| path.starts_with(".openshell/channel/sandbox")) - ); + assert!(workload.keys().all(|path| path.starts_with("sandbox"))); assert!( supervisor .keys() @@ -353,7 +354,7 @@ mod tests { ); let config: BoundaryConfig = serde_json::from_slice( workload - .get(&PathBuf::from(BOOTSTRAP_PATH.trim_start_matches('/'))) + .get(&PathBuf::from("sandbox/bootstrap.json")) .unwrap(), ) .unwrap(); @@ -375,7 +376,7 @@ mod tests { supervisor .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) .unwrap(), - &archives.workload + &archives.channel ); } } diff --git a/crates/openshell-driver-podman/src/test_utils.rs b/crates/openshell-driver-podman/src/test_utils.rs index 24e9d4d8ed..25cbcb4ac2 100644 --- a/crates/openshell-driver-podman/src/test_utils.rs +++ b/crates/openshell-driver-podman/src/test_utils.rs @@ -3,7 +3,7 @@ //! Shared test helpers for openshell-driver-podman unit tests. -use http_body_util::Full; +use http_body_util::{BodyExt as _, Full}; use hyper::StatusCode; use hyper::body::Bytes; use hyper::server::conn::http1; @@ -22,6 +22,7 @@ pub struct StubResponse { pub status: StatusCode, pub body: Bytes, pub delay: Duration, + pub archive_members: Option>, } impl StubResponse { @@ -30,6 +31,7 @@ impl StubResponse { status, body: body.into(), delay: Duration::ZERO, + archive_members: None, } } @@ -37,6 +39,11 @@ impl StubResponse { self.delay = delay; self } + + pub fn with_archive_members(mut self, members: &[&str]) -> Self { + self.archive_members = Some(members.iter().map(PathBuf::from).collect()); + self + } } /// Generate a unique Unix socket path for a test. @@ -88,7 +95,7 @@ pub fn spawn_podman_stub( let result = http1::Builder::new() .serve_connection( TokioIo::new(stream), - service_fn(move |req| { + service_fn(move |req: hyper::Request| { let log = log.clone(); let queue = queue.clone(); async move { @@ -104,6 +111,17 @@ pub fn spawn_podman_stub( .expect("response queue lock should not be poisoned") .pop_front() .expect("stub response should exist"); + if let Some(expected_members) = &response.archive_members { + assert_eq!(req.method(), hyper::Method::PUT); + let body = req.into_body().collect().await.unwrap().to_bytes(); + let mut archive = tar::Archive::new(body.as_ref()); + let members: Vec<_> = archive + .entries() + .unwrap() + .map(|entry| entry.unwrap().path().unwrap().into_owned()) + .collect(); + assert_eq!(&members, expected_members); + } tokio::time::sleep(response.delay).await; Ok::<_, Infallible>( hyper::Response::builder() From 515b70987d2de1d715a2e6ccf9ea0c7368a9c398 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:53:16 -0700 Subject: [PATCH 3/5] fix(podman): bound image metadata inspection and reject dangling roots Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 4 + crates/openshell-driver-podman/src/client.rs | 261 +++++++++++++++++++ crates/openshell-driver-podman/src/driver.rs | 56 ++-- 3 files changed, 292 insertions(+), 29 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 0768b9c712..9754271015 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -1,5 +1,9 @@ # openshell-driver-podman +Image admission checks the reserved `/.openshell` path using archive metadata +only. Downloads of `/etc/passwd` and `/etc/group` are limited to 1 MiB each, +including tar overhead, and extracted file sizes are checked independently. + The Podman compute driver runs inside the gateway and uses the native libpod REST API over a Unix socket. Each sandbox has two independent containers: diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 9f37864373..8b4da5354b 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -26,6 +26,9 @@ const API_TIMEOUT: Duration = Duration::from_secs(30); /// Maximum allowed size for the event stream line buffer (1 MB). const MAX_EVENT_BUFFER: usize = 1_048_576; +/// Image account databases are metadata, not unrestricted archive downloads. +const MAX_IMAGE_ACCOUNT_BYTES: usize = 1_048_576; + #[derive(Debug, thiserror::Error)] pub enum PodmanApiError { #[error("podman API not found (404): {0}")] @@ -281,6 +284,38 @@ pub struct SecurityInfo { // ── Client ─────────────────────────────────────────────────────────────── +fn extract_image_account_file(archive: &[u8]) -> Result, PodmanApiError> { + use std::io::Read as _; + + let invalid = |error: std::io::Error| { + PodmanApiError::InvalidInput(format!("invalid image account archive: {error}")) + }; + let mut archive = tar::Archive::new(archive); + let mut entries = archive.entries().map_err(invalid)?; + let entry = entries + .next() + .ok_or_else(|| PodmanApiError::InvalidInput("empty image account archive".into()))? + .map_err(invalid)?; + if !entry.header().entry_type().is_file() || entry.size() > MAX_IMAGE_ACCOUNT_BYTES as u64 { + return Err(PodmanApiError::InvalidInput(format!( + "image account must be a regular file of at most {MAX_IMAGE_ACCOUNT_BYTES} bytes" + ))); + } + let mut bytes = Vec::new(); + // Bound extraction independently: sparse/PAX tar metadata must not expand + // a small wire archive into an unbounded account database. + entry + .take(MAX_IMAGE_ACCOUNT_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(invalid)?; + if bytes.len() > MAX_IMAGE_ACCOUNT_BYTES { + return Err(PodmanApiError::InvalidInput( + "image account payload exceeds size limit".into(), + )); + } + Ok(bytes) +} + /// Async Podman REST API client communicating over a Unix socket. #[derive(Debug, Clone)] pub struct PodmanClient { @@ -628,6 +663,94 @@ impl PodmanClient { } } + /// Fetch archive metadata without reading an image-controlled response body. + pub(crate) async fn container_path_exists( + &self, + name: &str, + path: &str, + ) -> Result { + let response = self + .image_metadata_response(name, path, hyper::Method::HEAD) + .await?; + // Libpod returns 404 for a dangling symlink, but still supplies its + // lstat metadata. Such a path exists and must never pass admission. + if response + .headers() + .contains_key("x-docker-container-path-stat") + { + return Ok(true); + } + match response.status().as_u16() { + 200 => Ok(true), + 404 => Ok(false), + status => Err(PodmanApiError::Api { + status, + message: "container image path metadata probe failed".into(), + }), + } + } + + /// Bound both the wire archive and its extracted account-file payload. + pub(crate) async fn copy_image_account_file( + &self, + name: &str, + path: &str, + ) -> Result, PodmanApiError> { + let response = self + .image_metadata_response(name, path, hyper::Method::GET) + .await?; + let status = response.status(); + if status == hyper::StatusCode::NOT_FOUND { + return Ok(Vec::new()); + } + if !status.is_success() { + return Err(PodmanApiError::Api { + status: status.as_u16(), + message: "container image account download failed".into(), + }); + } + let archive = tokio::time::timeout( + API_TIMEOUT, + http_body_util::Limited::new(response.into_body(), MAX_IMAGE_ACCOUNT_BYTES).collect(), + ) + .await + .map_err(|_| PodmanApiError::Timeout(API_TIMEOUT))? + .map_err(|error| { + PodmanApiError::InvalidInput(format!( + "image account archive must fit within {MAX_IMAGE_ACCOUNT_BYTES} bytes: {error}" + )) + })? + .to_bytes(); + extract_image_account_file(&archive) + } + + async fn image_metadata_response( + &self, + name: &str, + path: &str, + method: hyper::Method, + ) -> Result, PodmanApiError> { + validate_name(name)?; + let request = Self::build_request( + method, + &format!( + "/{API_VERSION}/libpod/containers/{name}/archive?path={}", + url_encode(path) + ), + Full::new(Bytes::new()), + None, + ); + tokio::time::timeout(API_TIMEOUT, async { + let mut sender = self.connect().await?; + sender + .send_request(request) + .await + .map_err(|error| PodmanApiError::Connection(error.to_string())) + }) + .await + .map_err(|_| PodmanApiError::Timeout(API_TIMEOUT))? + } + /// Inspect a container by name or ID. pub async fn inspect_container(&self, name: &str) -> Result { validate_name(name)?; @@ -1001,6 +1124,144 @@ mod tests { use crate::test_utils::{StubResponse, spawn_podman_stub}; use hyper::StatusCode; + #[tokio::test] + async fn image_path_probe_uses_head_and_fails_closed_on_server_errors() { + let (path, requests, server) = spawn_podman_stub( + "image-path-head", + vec![ + StubResponse::new(StatusCode::OK, vec![0; MAX_IMAGE_ACCOUNT_BYTES + 1]), + StubResponse::new(StatusCode::NOT_FOUND, ""), + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, ""), + ], + ); + let client = PodmanClient::new(path); + assert!( + client + .container_path_exists("image", "/.openshell") + .await + .unwrap() + ); + assert!( + !client + .container_path_exists("image", "/.openshell") + .await + .unwrap() + ); + assert!( + client + .container_path_exists("image", "/.openshell") + .await + .is_err() + ); + server.await.unwrap(); + assert!( + requests.lock().unwrap().iter().all(|request| request + == "HEAD /v5.0.0/libpod/containers/image/archive?path=%2F.openshell") + ); + } + + #[tokio::test] + async fn image_path_probe_rejects_dangling_symlink_metadata_on_404() { + let path = crate::test_utils::unique_socket_path("dangling-root-head"); + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection( + TokioIo::new(stream), + hyper::service::service_fn(|request| async move { + assert_eq!(request.method(), hyper::Method::HEAD); + Ok::<_, std::convert::Infallible>( + hyper::Response::builder() + .status(StatusCode::NOT_FOUND) + // Its contents need not be decoded: any stat metadata + // proves this is not an absent directory entry. + .header("x-docker-container-path-stat", "present") + .body(Full::new(Bytes::new())) + .unwrap(), + ) + }), + ) + .await; + }); + let client = PodmanClient::new(path.clone()); + assert!( + client + .container_path_exists("image", "/.openshell") + .await + .unwrap() + ); + server.await.unwrap(); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn image_account_download_rejects_oversized_archive() { + let (path, _, server) = spawn_podman_stub( + "account-wire-limit", + vec![StubResponse::new( + StatusCode::OK, + vec![0; MAX_IMAGE_ACCOUNT_BYTES + 1], + )], + ); + let client = PodmanClient::new(path); + let error = client + .copy_image_account_file("image", "/etc/passwd") + .await + .unwrap_err(); + assert!(error.to_string().contains("archive must fit within")); + server.await.unwrap(); + } + + #[test] + fn image_account_extraction_rejects_oversized_header_before_allocating_payload() { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_path("passwd").unwrap(); + header.set_mode(0o644); + header.set_size(u64::MAX / 2); + header.set_cksum(); + let error = extract_image_account_file(header.as_bytes()).unwrap_err(); + assert!(error.to_string().contains("regular file of at most")); + } + + #[tokio::test] + async fn image_account_download_accepts_regular_file_and_missing_database() { + let mut archive = tar::Builder::new(Vec::new()); + let payload = b"agent:x:1000:1000::/sandbox:/bin/sh\n"; + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(0o644); + header.set_size(payload.len() as u64); + header.set_cksum(); + archive + .append_data(&mut header, "passwd", payload.as_slice()) + .unwrap(); + let (path, _, server) = spawn_podman_stub( + "account-regular", + vec![ + StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), + StubResponse::new(StatusCode::NOT_FOUND, ""), + ], + ); + let client = PodmanClient::new(path); + assert_eq!( + client + .copy_image_account_file("image", "/etc/passwd") + .await + .unwrap(), + payload + ); + assert!( + client + .copy_image_account_file("image", "/etc/group") + .await + .unwrap() + .is_empty() + ); + server.await.unwrap(); + } + #[test] fn url_encode_encodes_special_characters() { assert_eq!(url_encode("hello world"), "hello%20world"); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 2452fe922e..883d06b0b2 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1099,36 +1099,32 @@ impl PodmanComputeDriver { image, }) .await?; - let result = - async { - // Do not let image-controlled symlinks alias the protected channel - // into an agent-readable subtree before Podman mounts it. - match self.client.copy_from_container(&id, "/.openshell").await { - Err(PodmanApiError::NotFound(_)) => {} - Ok(_) => return Err(ComputeDriverError::Precondition( - "workload images must not prepopulate the reserved /.openshell hierarchy" - .into(), - )), - Err(error) => return Err(error.into()), - } - let mut accounts = Vec::new(); - for path in ["/etc/passwd", "/etc/group"] { - let content = match self.client.copy_from_container(&id, path).await { - Ok(archive) => extract_first_tar_entry(&archive) - .map_err(ComputeDriverError::Precondition)?, - Err(PodmanApiError::NotFound(_)) => Vec::new(), - Err(error) => return Err(error.into()), - }; - accounts.push(content); - } - let [passwd, group] = accounts.as_slice() else { - return Err(ComputeDriverError::Precondition( - "image account inspection was incomplete".into(), - )); - }; - crate::isolation::resolve_identity(sandbox, image, image_user, passwd, group) + let result = async { + // Do not let image-controlled symlinks alias the protected channel + // into an agent-readable subtree before Podman mounts it. + if self + .client + .container_path_exists(&id, "/.openshell") + .await? + { + return Err(ComputeDriverError::Precondition( + "workload images must not prepopulate the reserved /.openshell hierarchy" + .into(), + )); } - .await; + let mut accounts = Vec::new(); + for path in ["/etc/passwd", "/etc/group"] { + let content = self.client.copy_image_account_file(&id, path).await?; + accounts.push(content); + } + let [passwd, group] = accounts.as_slice() else { + return Err(ComputeDriverError::Precondition( + "image account inspection was incomplete".into(), + )); + }; + crate::isolation::resolve_identity(sandbox, image, image_user, passwd, group) + } + .await; let cleanup = self.client.remove_container(&id, 0).await; if let Err(error) = cleanup { warn!(container = %id, %error, "Failed to remove stopped identity inspection container"); @@ -3290,6 +3286,8 @@ mod tests { .unwrap_err(); assert!(error.to_string().contains("reserved /.openshell")); handle.await.unwrap(); + assert!(requests.lock().unwrap().iter().any(|request| request + == "HEAD /v5.0.0/libpod/containers/identity-reader/archive?path=%2F.openshell")); assert!( !requests .lock() From a820116ffe7a6278d484f16e55eecb1e903d3edf Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 21:10:52 -0700 Subject: [PATCH 4/5] fix(podman): provision runtime state and rotate restart channel identity Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 12 +- .../openshell-driver-podman/src/container.rs | 63 +++++++--- crates/openshell-driver-podman/src/driver.rs | 94 ++++++++++++++- .../openshell-driver-podman/src/isolation.rs | 111 ++++++++++++++++++ 4 files changed, 258 insertions(+), 22 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 9754271015..506266a3f7 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -36,12 +36,18 @@ separate. The channel volume uses shared SELinux relabeling (`:z`). Before starting either container, the driver uploads volume-relative archives directly to the channel and workspace volume destinations. A rootfs upload on a -stopped Podman container does not populate nested named volumes. Restart restores -only the channel bootstrap into the existing channel volume, preserving the -workspace. The workload starts before the supervisor so its user namespace exists +stopped Podman container does not populate nested named volumes. Each restart +rotates the generation, session, bootstrap token, and mutual-TLS credentials, +then stages the matching channel bootstrap and supervisor topology while both +containers are stopped. Failed staging leaves both stopped; retry rotates again. +Restart preserves the workspace. The workload starts before the supervisor so its user namespace exists when the supervisor joins it; a stopped supervisor resolves that namespace again on its next start. +Both containers have an identity-owned, 64 MiB `/run` tmpfs. This lets the +non-root sandbox install `/run/openshell-proxy-ca` without capabilities or an +image-provided writable runtime directory. + The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail closed; do not switch to an unconfined profile or add capabilities. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 7091fbb7a2..429545886a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1448,6 +1448,11 @@ pub fn build_isolation_specs( workload .mounts .retain(|mount| !trusted_mount(&mount.destination)); + // The capability-free sandbox installs interception CA material below + // /run before launching the agent. Image-owned /run is commonly root-only. + workload + .mounts + .push(identity_owned_tmpfs("/run", input.identity)); workload.volumes.push(NamedVolume { name: channel.clone(), dest: crate::isolation::CHANNEL_ROOT.into(), @@ -1494,20 +1499,9 @@ pub fn build_isolation_specs( && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT }); for destination in ["/run", "/var/log", "/tmp"] { - supervisor.mounts.push(Mount { - kind: "tmpfs".into(), - source: "tmpfs".into(), - destination: destination.into(), - options: vec![ - "rw".into(), - "nosuid".into(), - "nodev".into(), - format!("uid={}", input.identity.uid), - format!("gid={}", input.identity.gid), - "mode=0700".into(), - "size=64m".into(), - ], - }); + supervisor + .mounts + .push(identity_owned_tmpfs(destination, input.identity)); } for secret in &mut supervisor.secrets { secret.uid = input.identity.uid; @@ -1528,6 +1522,26 @@ pub fn build_isolation_specs( }) } +fn identity_owned_tmpfs( + destination: &str, + identity: &openshell_isolation_interface::contract::ResolvedWorkloadIdentity, +) -> Mount { + Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: destination.into(), + options: vec![ + "rw".into(), + "nosuid".into(), + "nodev".into(), + format!("uid={}", identity.uid), + format!("gid={}", identity.gid), + "mode=0700".into(), + "size=64m".into(), + ], + } +} + fn trusted_mount(destination: &str) -> bool { matches!( destination, @@ -1684,6 +1698,27 @@ mod tests { assert!(spec.cap_add.is_empty()); assert!(spec.seccomp_profile_path.is_empty()); assert!(spec.no_new_privileges); + let runtime_mounts: Vec<_> = spec + .mounts + .iter() + .filter(|mount| mount.destination == "/run") + .collect(); + assert_eq!(runtime_mounts.len(), 1); + let runtime = runtime_mounts[0]; + assert_eq!(runtime.kind, "tmpfs"); + assert_eq!(runtime.source, "tmpfs"); + assert_eq!( + runtime.options, + vec![ + "rw", + "nosuid", + "nodev", + "uid=1000", + "gid=1001", + "mode=0700", + "size=64m" + ] + ); } assert_eq!(specs.workload.netns.nsmode, "none"); assert!(specs.workload.networks.is_empty()); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 883d06b0b2..e64918a58f 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1318,8 +1318,19 @@ impl PodmanComputeDriver { .await?; let bundle = extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + let archives = + crate::isolation::rotate_bootstrap_archives(sandbox_id, &container_id, &bundle)?; + // Update both endpoints while stopped. A failed upload leaves no + // running peer with a mismatched generation; retry rotates again. self.client - .copy_to_container(&container_id, crate::isolation::CHANNEL_ROOT, bundle) + .copy_to_container( + &container_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&supervisor, "/", archives.supervisor) .await?; self.client.verify_isolation_fence(&container_id).await?; self.client.start_container(&container_id).await?; @@ -1901,10 +1912,16 @@ mod tests { .filter(|request| request.starts_with("PUT ")) .cloned() .collect::>(), - vec![format!( - "PUT {}", - api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") - )] + vec![ + format!( + "PUT {}", + api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") + ), + format!( + "PUT {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/archive?path=%2F") + ), + ] ); assert!( !restart_requests @@ -1932,6 +1949,67 @@ mod tests { let _ = fs::remove_file(start_socket); } + #[tokio::test] + async fn restart_upload_failures_leave_both_peers_stopped() { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Summary { + id: &'static str, + state: &'static str, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct State { + status: &'static str, + running: bool, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Inspect { + id: &'static str, + name: &'static str, + state: State, + } + for upload_index in [3, 4] { + let mut responses = vec![ + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&[Summary { + id: "ctr-1", + state: "stopped", + }]) + .unwrap(), + ), + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Inspect { + id: "ctr-1", + name: "sandbox", + state: State { + status: "exited", + running: false, + }, + }) + .unwrap(), + ), + ]; + let mut restart = restart_responses(); + restart[upload_index] = + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "upload failed"); + responses.extend(restart.into_iter().take(upload_index + 1)); + let (path, requests, server) = spawn_podman_stub("restart-upload-failure", responses); + assert!(test_driver(path).start_sandbox("sandbox-1").await.is_err()); + server.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|request| !request.ends_with("/start")) + ); + } + } + #[tokio::test] async fn stop_waits_for_the_container_to_leave_stopping_state() { let (socket, requests, handle) = spawn_podman_stub( @@ -3150,6 +3228,12 @@ mod tests { ), StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&[ + ".openshell", + ".openshell/supervisor", + ".openshell/supervisor/topology.payload", + ".openshell/supervisor/sandbox-bundle.tar", + ]), fence_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 1ba6aae055..38226e9809 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -226,6 +226,52 @@ pub fn bootstrap_archives( }) } +/// Retain only stable launch inputs from the supervisor-private restart copy. +/// Every container restart establishes a new authenticated listener generation. +pub fn rotate_bootstrap_archives( + sandbox_id: &str, + container_id: &str, + previous_channel: &[u8], +) -> Result { + let mut archive = tar::Archive::new(previous_channel); + for entry in archive.entries().map_err(invalid)? { + let entry = entry.map_err(invalid)?; + if entry.path().map_err(invalid)?.as_ref() != std::path::Path::new("sandbox/bootstrap.json") + { + continue; + } + if !entry.header().entry_type().is_file() { + return Err(invalid("restart bootstrap must be a regular file")); + } + let previous: BoundaryConfig = serde_json::from_reader(entry).map_err(invalid)?; + if previous.boundary_id != sandbox_id + || previous + .resource_claims + .get("podman.container_id") + .map(String::as_str) + != Some(container_id) + || !matches!(&previous.driver_fence, DriverFenceEvidence::Podman { container_id: bound, .. } if bound == container_id) + { + return Err(invalid( + "restart bootstrap does not belong to this Podman workload", + )); + } + previous + .driver_fence + .validate_for_backend("podman") + .map_err(invalid)?; + return bootstrap_archives( + sandbox_id, + container_id, + &previous.workload_identity, + previous.child_env, + ); + } + Err(invalid( + "supervisor restart archive is missing sandbox/bootstrap.json", + )) +} + struct Archive<'a> { builder: tar::Builder>, identity: &'a ResolvedWorkloadIdentity, @@ -379,4 +425,69 @@ mod tests { &archives.channel ); } + + #[test] + fn restart_rotates_every_authentication_value_while_preserving_launch_inputs() { + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let original = bootstrap_archives( + "sandbox", + "container", + &identity, + HashMap::from([("PATH".into(), "/usr/bin".into())]), + ) + .unwrap(); + let renewed = rotate_bootstrap_archives("sandbox", "container", &original.channel).unwrap(); + let original_files = files(&original.channel); + let renewed_files = files(&renewed.channel); + let supervisor_files = files(&renewed.supervisor); + let old: BoundaryConfig = + serde_json::from_slice(&original_files[&PathBuf::from("sandbox/bootstrap.json")]) + .unwrap(); + let new: BoundaryConfig = + serde_json::from_slice(&renewed_files[&PathBuf::from("sandbox/bootstrap.json")]) + .unwrap(); + let supervisor: BoundaryTopology = serde_json::from_slice( + &supervisor_files[&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))], + ) + .unwrap(); + assert_ne!(new.generation, old.generation); + assert_ne!(new.session_epoch, old.session_epoch); + assert_ne!(new.bootstrap_token, old.bootstrap_token); + assert_eq!(new.generation, supervisor.generation); + assert_eq!(new.session_epoch, supervisor.session_epoch); + assert_eq!(new.bootstrap_token, supervisor.bootstrap_token); + assert_eq!(new.workload_identity, old.workload_identity); + assert_eq!(new.resource_claims, old.resource_claims); + assert_eq!(new.child_env, old.child_env); + for path in [ + "sandbox/server.crt", + "sandbox/server.key", + "sandbox/client-ca.crt", + ] { + assert_ne!( + renewed_files[&PathBuf::from(path)], + original_files[&PathBuf::from(path)] + ); + } + let BoundaryTransport::Unix { tls, .. } = supervisor.transport else { + panic!("expected Unix channel") + }; + assert_eq!( + tls.ca_certificate_pem.as_bytes(), + &renewed_files[&PathBuf::from("sandbox/client-ca.crt")] + ); + assert_eq!( + supervisor_files[&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))], + renewed.channel + ); + assert!(rotate_bootstrap_archives("other", "container", &original.channel).is_err()); + assert!(rotate_bootstrap_archives("sandbox", "other", &original.channel).is_err()); + } } From 5e464ee7e22aa2ec9507c22b8a7c5eecaf6d1dd6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 23:08:43 -0700 Subject: [PATCH 5/5] fix(podman): use portable private tmpfs mounts Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 8 +++-- .../openshell-driver-podman/src/container.rs | 30 +++++-------------- crates/openshell-driver-podman/src/driver.rs | 9 +++++- e2e/with-podman-gateway.sh | 3 ++ 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 506266a3f7..7c33b00c23 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -44,9 +44,11 @@ Restart preserves the workspace. The workload starts before the supervisor so it when the supervisor joins it; a stopped supervisor resolves that namespace again on its next start. -Both containers have an identity-owned, 64 MiB `/run` tmpfs. This lets the -non-root sandbox install `/run/openshell-proxy-ca` without capabilities or an -image-provided writable runtime directory. +Both containers have a private, writable 64 MiB `/run` tmpfs. This lets the +non-root sandbox install `/run/openshell-proxy-ca` without capabilities, an +init process, or an image-provided writable runtime directory. Podman's mount +API does not accept tmpfs `uid`/`gid` options, so the isolated mount uses the +standard sticky world-writable mode instead. The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 429545886a..0f4f858b0e 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1450,9 +1450,7 @@ pub fn build_isolation_specs( .retain(|mount| !trusted_mount(&mount.destination)); // The capability-free sandbox installs interception CA material below // /run before launching the agent. Image-owned /run is commonly root-only. - workload - .mounts - .push(identity_owned_tmpfs("/run", input.identity)); + workload.mounts.push(private_writable_tmpfs("/run")); workload.volumes.push(NamedVolume { name: channel.clone(), dest: crate::isolation::CHANNEL_ROOT.into(), @@ -1499,9 +1497,7 @@ pub fn build_isolation_specs( && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT }); for destination in ["/run", "/var/log", "/tmp"] { - supervisor - .mounts - .push(identity_owned_tmpfs(destination, input.identity)); + supervisor.mounts.push(private_writable_tmpfs(destination)); } for secret in &mut supervisor.secrets { secret.uid = input.identity.uid; @@ -1522,10 +1518,7 @@ pub fn build_isolation_specs( }) } -fn identity_owned_tmpfs( - destination: &str, - identity: &openshell_isolation_interface::contract::ResolvedWorkloadIdentity, -) -> Mount { +fn private_writable_tmpfs(destination: &str) -> Mount { Mount { kind: "tmpfs".into(), source: "tmpfs".into(), @@ -1534,9 +1527,10 @@ fn identity_owned_tmpfs( "rw".into(), "nosuid".into(), "nodev".into(), - format!("uid={}", identity.uid), - format!("gid={}", identity.gid), - "mode=0700".into(), + // Podman's SpecGenerator mount parser rejects uid/gid tmpfs + // options. Each mount is private to one container, so a writable + // mode grants no access across the workload/supervisor boundary. + "mode=1777".into(), "size=64m".into(), ], } @@ -1709,15 +1703,7 @@ mod tests { assert_eq!(runtime.source, "tmpfs"); assert_eq!( runtime.options, - vec![ - "rw", - "nosuid", - "nodev", - "uid=1000", - "gid=1001", - "mode=0700", - "size=64m" - ] + vec!["rw", "nosuid", "nodev", "mode=1777", "size=64m"] ); } assert_eq!(specs.workload.netns.nsmode, "none"); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index e64918a58f..aceb3feab1 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1028,7 +1028,14 @@ impl PodmanComputeDriver { self.client .copy_to_container(&workload_id, "/sandbox", archives.workspace) .await?; - specs.supervisor.join_user_namespace(&workload_id); + // Rootless Podman places default/host-mode containers in + // the same user namespace already. Asking crun to re-enter + // that namespace fails with EINVAL. Only an explicitly + // isolated userns mode needs the supervisor to join the + // workload's namespace. + if userns_needs_extraction(self.config.userns.as_deref()) { + specs.supervisor.join_user_namespace(&workload_id); + } let supervisor_id = self .client .create_typed_container(&specs.supervisor) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index f87230a1a9..0868cd9883 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -426,6 +426,9 @@ export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) +while [ "${HEALTH_PORT}" = "${HOST_PORT}" ]; do + HEALTH_PORT=$(e2e_pick_port) +done if [ "$(uname -s)" = "Darwin" ]; then # Podman Machine reserves IPv4 loopback for its callback-only listener. PRIMARY_BIND_IP="::1"