From 2a4ba0ef8a504c60ae79d43a2a10b6799fde61f9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 15:27:12 -0700 Subject: [PATCH 01/10] feat(isolation): split supervisor and sandbox runtimes Signed-off-by: Drew Newberry --- .github/actions/build-docker-image/action.yml | 19 + .github/workflows/branch-checks.yml | 10 +- .github/workflows/branch-e2e.yml | 1 + .github/workflows/build-sandbox-binaries.yml | 18 +- .github/workflows/docker-build.yml | 5 + .github/workflows/release-dev.yml | 1 + .github/workflows/release-tag.yml | 1 + Cargo.lock | 78 +- architecture/build.md | 18 +- architecture/compute-runtimes.md | 189 +- architecture/sandbox.md | 231 +- architecture/security-policy.md | 16 +- crates/openshell-core/src/sandbox_env.rs | 29 +- crates/openshell-core/src/shell.rs | 7 +- crates/openshell-router/src/lib.rs | 52 +- crates/openshell-sandbox/Cargo.toml | 66 +- crates/openshell-sandbox/src/boundary_exec.rs | 723 ++ crates/openshell-sandbox/src/boundary_io.rs | 311 + .../openshell-sandbox/src/boundary_server.rs | 3874 ++++++++++ crates/openshell-sandbox/src/child_env.rs | 53 + crates/openshell-sandbox/src/delegated.rs | 243 + .../src/google_cloud_metadata.rs | 536 -- crates/openshell-sandbox/src/identity.rs | 833 +++ crates/openshell-sandbox/src/lib.rs | 6337 +---------------- crates/openshell-sandbox/src/main.rs | 2460 +++++-- crates/openshell-sandbox/src/main_session.rs | 1067 +++ .../openshell-sandbox/src/managed_children.rs | 122 + .../openshell-sandbox/src/metadata_server.rs | 231 - .../openshell-sandbox/src/network_broker.rs | 2038 ++++++ crates/openshell-sandbox/src/process.rs | 3844 ++++++++++ crates/openshell-sandbox/src/pty.rs | 144 + .../src/sandbox/linux/landlock.rs | 764 ++ .../src/sandbox/linux/mod.rs | 221 + .../src/sandbox/linux/seccomp.rs | 892 +++ crates/openshell-sandbox/src/sandbox/mod.rs | 57 + .../openshell-sandbox/src/sidecar_control.rs | 1210 ---- .../openshell-sandbox/tests/stdout_logging.rs | 21 +- crates/openshell-server/src/compute/mod.rs | 418 +- crates/openshell-server/src/grpc/sandbox.rs | 19 +- crates/openshell-server/src/lib.rs | 22 +- .../src/supervisor_session.rs | 28 +- .../openshell-supervisor-network/Cargo.toml | 6 +- .../data/sandbox-policy.rego | 13 +- .../src/identity_source.rs | 137 + .../src/inference_routes.rs | 27 +- .../src/l7/rest.rs | 59 +- .../src/l7/tls.rs | 101 +- .../openshell-supervisor-network/src/lib.rs | 2 + .../openshell-supervisor-network/src/opa.rs | 47 +- .../src/policy_dns/mod.rs | 158 +- .../src/policy_dns/runtime.rs | 94 + .../src/policy_dns/store.rs | 11 + .../openshell-supervisor-network/src/proxy.rs | 1118 ++- .../src/proxy/destination.rs | 70 +- .../src/proxy/egress.rs | 2 + .../src/proxy/tests/compatibility.rs | 207 +- .../openshell-supervisor-network/src/run.rs | 58 +- .../src/spiffe_endpoint.rs | 18 + .../src/upstream_proxy.rs | 47 +- .../openshell-supervisor-process/Cargo.toml | 14 +- .../src/bypass_monitor/mod.rs | 651 -- .../src/bypass_monitor/procfs.rs | 318 - .../src/delegated.rs | 269 + .../openshell-supervisor-process/src/lib.rs | 22 +- .../src/main_session.rs | 267 +- .../src/managed_children.rs | 53 - .../src/netns/mod.rs | 1239 ---- .../src/netns/nft_ruleset.rs | 825 --- .../openshell-supervisor-process/src/run.rs | 830 --- .../openshell-supervisor-process/src/ssh.rs | 2418 ++----- .../src/supervisor_session.rs | 160 +- crates/openshell-supervisor/Cargo.toml | 58 + .../src/activity_aggregator.rs | 224 + .../src/denial_aggregator.rs | 210 + crates/openshell-supervisor/src/lib.rs | 5602 +++++++++++++++ crates/openshell-supervisor/src/main.rs | 327 + .../src/mechanistic_mapper.rs | 790 ++ deploy/docker/Dockerfile.supervisor | 20 +- e2e/rust/tests/bypass_detection.rs | 33 +- e2e/rust/tests/credential_gating.rs | 26 +- e2e/rust/tests/forward_proxy_graphql_l7.rs | 92 +- e2e/rust/tests/forward_proxy_jsonrpc_l7.rs | 66 +- e2e/rust/tests/live_policy_update.rs | 178 - e2e/rust/tests/no_proxy.rs | 25 +- e2e/rust/tests/websocket_conformance.rs | 77 +- rfc/0012-isolation-backend/README.md | 164 +- rfc/0012-isolation-backend/topology-matrix.md | 25 +- tasks/rust.toml | 12 +- tasks/scripts/docker-build-image.sh | 2 +- tasks/scripts/stage-prebuilt-binaries.sh | 13 +- .../verify-defaults-without-telemetry.sh | 2 +- 91 files changed, 28403 insertions(+), 15993 deletions(-) create mode 100644 crates/openshell-sandbox/src/boundary_exec.rs create mode 100644 crates/openshell-sandbox/src/boundary_io.rs create mode 100644 crates/openshell-sandbox/src/boundary_server.rs create mode 100644 crates/openshell-sandbox/src/child_env.rs create mode 100644 crates/openshell-sandbox/src/delegated.rs delete mode 100644 crates/openshell-sandbox/src/google_cloud_metadata.rs create mode 100644 crates/openshell-sandbox/src/identity.rs create mode 100644 crates/openshell-sandbox/src/main_session.rs create mode 100644 crates/openshell-sandbox/src/managed_children.rs delete mode 100644 crates/openshell-sandbox/src/metadata_server.rs create mode 100644 crates/openshell-sandbox/src/network_broker.rs create mode 100644 crates/openshell-sandbox/src/process.rs create mode 100644 crates/openshell-sandbox/src/pty.rs create mode 100644 crates/openshell-sandbox/src/sandbox/linux/landlock.rs create mode 100644 crates/openshell-sandbox/src/sandbox/linux/mod.rs create mode 100644 crates/openshell-sandbox/src/sandbox/linux/seccomp.rs create mode 100644 crates/openshell-sandbox/src/sandbox/mod.rs delete mode 100644 crates/openshell-sandbox/src/sidecar_control.rs create mode 100644 crates/openshell-supervisor-network/src/identity_source.rs create mode 100644 crates/openshell-supervisor-network/src/spiffe_endpoint.rs delete mode 100644 crates/openshell-supervisor-process/src/bypass_monitor/mod.rs delete mode 100644 crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs create mode 100644 crates/openshell-supervisor-process/src/delegated.rs delete mode 100644 crates/openshell-supervisor-process/src/managed_children.rs delete mode 100644 crates/openshell-supervisor-process/src/netns/mod.rs delete mode 100644 crates/openshell-supervisor-process/src/netns/nft_ruleset.rs delete mode 100644 crates/openshell-supervisor-process/src/run.rs create mode 100644 crates/openshell-supervisor/Cargo.toml create mode 100644 crates/openshell-supervisor/src/activity_aggregator.rs create mode 100644 crates/openshell-supervisor/src/denial_aggregator.rs create mode 100644 crates/openshell-supervisor/src/lib.rs create mode 100644 crates/openshell-supervisor/src/main.rs create mode 100644 crates/openshell-supervisor/src/mechanistic_mapper.rs diff --git a/.github/actions/build-docker-image/action.yml b/.github/actions/build-docker-image/action.yml index 08557fbfd4..578ee6091d 100644 --- a/.github/actions/build-docker-image/action.yml +++ b/.github/actions/build-docker-image/action.yml @@ -11,6 +11,10 @@ inputs: binary: description: Binary staged in the Docker build context required: true + additional-binary: + description: Optional second binary staged in the Docker build context + required: false + default: "" triple: description: Binary artifact target triple required: true @@ -53,6 +57,21 @@ runs: INPUTS_BINARY: ${{ inputs.binary }} INPUTS_ARCH: ${{ inputs.arch }} + - name: Download ${{ inputs.additional-binary }} + if: inputs.additional-binary != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.additional-binary }}-${{ inputs.triple }} + path: additional-artifact + + - name: Stage ${{ inputs.additional-binary }} + if: inputs.additional-binary != '' + shell: bash + run: install -Dm0755 additional-artifact/${INPUTS_ADDITIONAL_BINARY} deploy/docker/.build/prebuilt-binaries/${INPUTS_ARCH}/${INPUTS_ADDITIONAL_BINARY} + env: + INPUTS_ADDITIONAL_BINARY: ${{ inputs.additional-binary }} + INPUTS_ARCH: ${{ inputs.arch }} + - name: Build ${{ inputs.component }} image shell: bash env: diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 5d4a98c61a..089f1b9b4c 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -179,20 +179,20 @@ jobs: tasks/scripts/verify-telemetry-compiled-out.sh present target/debug/openshell-gateway cargo build -p openshell-gateway --bin openshell-gateway --no-default-features --features defaults-without-telemetry tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway - cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features defaults-without-telemetry - tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox + cargo build -p openshell-supervisor --bin openshell-supervisor --no-default-features --features defaults-without-telemetry + tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-supervisor - name: Verify the defaults-without-telemetry feature alias tracks the default feature set run: tasks/scripts/verify-defaults-without-telemetry.sh - name: Verify system CA roots build mode compiles and excludes bundled Mozilla roots run: | - cargo check -p openshell-sandbox --all-targets --no-default-features --features system-ca-roots - if cargo tree -p openshell-sandbox -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then + cargo check -p openshell-supervisor --all-targets --no-default-features --features system-ca-roots + if cargo tree -p openshell-supervisor -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then echo "ERROR: webpki-roots found in system CA roots build" >&2 exit 1 fi - if cargo tree -p openshell-sandbox -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then + if cargo tree -p openshell-supervisor -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then echo "ERROR: webpki-root-certs found in system CA roots build" >&2 exit 1 fi diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 2e316f88bd..4c894e3ea3 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -262,6 +262,7 @@ jobs: with: component: supervisor binary: openshell-sandbox + additional-binary: openshell-supervisor target-suffix: unknown-linux-musl secrets: inherit diff --git a/.github/workflows/build-sandbox-binaries.yml b/.github/workflows/build-sandbox-binaries.yml index 53f81bec40..bdfccc345b 100644 --- a/.github/workflows/build-sandbox-binaries.yml +++ b/.github/workflows/build-sandbox-binaries.yml @@ -32,13 +32,27 @@ jobs: - triple: x86_64-unknown-linux-musl runner: linux-amd64-cpu8 dev_shell: .#devShells.x86_64-linux.musl + package: openshell-sandbox + binary: openshell-sandbox + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.musl + package: openshell-supervisor + binary: openshell-supervisor + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.musl + package: openshell-sandbox + binary: openshell-sandbox - triple: aarch64-unknown-linux-musl runner: linux-arm64-cpu8 dev_shell: .#devShells.aarch64-linux.musl + package: openshell-supervisor + binary: openshell-supervisor uses: ./.github/workflows/build-binaries.yml with: - package: openshell-sandbox - binary: openshell-sandbox + package: ${{ matrix.package }} + binary: ${{ matrix.binary }} triple: ${{ matrix.triple }} runner: ${{ matrix.runner }} dev-shell: ${{ matrix.dev_shell }} diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d27ba1e984..c89bf6fe0e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -12,6 +12,10 @@ on: binary: required: true type: string + additional-binary: + required: false + type: string + default: "" target-suffix: required: true type: string @@ -54,6 +58,7 @@ jobs: with: component: ${{ inputs.component }} binary: ${{ inputs.binary }} + additional-binary: ${{ inputs['additional-binary'] }} triple: ${{ matrix.rust_arch }}-${{ inputs['target-suffix'] }} arch: ${{ matrix.arch }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 18b5f33a2e..14eee4e0b3 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -145,6 +145,7 @@ jobs: with: component: supervisor binary: openshell-sandbox + additional-binary: openshell-supervisor target-suffix: unknown-linux-musl secrets: inherit diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index b3c45c97fb..80c053eb54 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -184,6 +184,7 @@ jobs: with: component: supervisor binary: openshell-sandbox + additional-binary: openshell-supervisor target-suffix: unknown-linux-musl image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} diff --git a/Cargo.lock b/Cargo.lock index 32e64ab9cd..73d4b1ab3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4308,32 +4308,40 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "bytes", + "capctl", "clap", - "futures", + "hex", + "ipnet", + "landlock", + "libc", "miette", "nix 0.29.0", + "openshell-binary-identity", "openshell-core", - "openshell-extension-core", + "openshell-isolation-interface", "openshell-ocsf", "openshell-policy", - "openshell-supervisor-middleware", - "openshell-supervisor-middleware-builtins", - "openshell-supervisor-network", - "openshell-supervisor-process", - "prost", - "prost-types", + "rand 0.10.2", + "rcgen", + "rustix 1.1.4", "rustls", + "rustls-pemfile", + "seccompiler", "serde", "serde_json", - "temp-env", + "sha2 0.10.9", + "socket2", "tempfile", "tokio", - "tokio-tungstenite 0.26.2", + "tokio-rustls", + "tokio-stream", "tonic", "tracing", - "tracing-appender", "tracing-subscriber", - "uuid", ] [[package]] @@ -4460,6 +4468,40 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "openshell-supervisor" +version = "0.0.0" +dependencies = [ + "clap", + "futures", + "miette", + "nix 0.29.0", + "openshell-core", + "openshell-extension-core", + "openshell-isolation-interface", + "openshell-ocsf", + "openshell-policy", + "openshell-supervisor-middleware", + "openshell-supervisor-middleware-builtins", + "openshell-supervisor-network", + "openshell-supervisor-process", + "prost", + "prost-types", + "rustix 1.1.4", + "rustls", + "serde", + "serde_json", + "temp-env", + "tempfile", + "tokio", + "tokio-tungstenite 0.26.2", + "tonic", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", +] + [[package]] name = "openshell-supervisor-middleware" version = "0.0.0" @@ -4497,6 +4539,7 @@ name = "openshell-supervisor-network" version = "0.0.0" dependencies = [ "apollo-parser", + "async-trait", "aws-credential-types", "aws-sigv4", "aws-smithy-runtime-api", @@ -4512,7 +4555,9 @@ dependencies = [ "libc", "miette", "noyalib", + "openshell-binary-identity", "openshell-core", + "openshell-isolation-interface", "openshell-ocsf", "openshell-policy", "openshell-router", @@ -4550,25 +4595,20 @@ name = "openshell-supervisor-process" version = "0.0.0" dependencies = [ "anyhow", + "async-trait", "base64 0.22.1", "bytes", - "capctl", "hex", - "ipnet", - "landlock", "libc", "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation-interface", "openshell-ocsf", - "openshell-policy", "rand 0.10.2", "russh", - "rustix 1.1.4", - "seccompiler", "serde_json", "sha2 0.10.9", - "socket2", "tempfile", "tokio", "tokio-stream", diff --git a/architecture/build.md b/architecture/build.md index 972f847bb9..69ddba420a 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -47,8 +47,7 @@ Cargo cannot subtract a single default feature, so each of the three binary crates also defines a `defaults-without-telemetry` alias listing every default except `telemetry`. Telemetry-free builds use `--no-default-features --features defaults-without-telemetry` and stay correct -as the default set grows, instead of dropping unrelated defaults the way a bare -`--no-default-features` does on `openshell-sandbox`. The alias is a keep-list, +as the default set grows. The alias is a keep-list, not a switch: enabling it on top of the defaults would otherwise yield a telemetry-on binary that reads as telemetry-free, so each crate root carries a `compile_error!` for the `telemetry` + `defaults-without-telemetry` combination. @@ -63,7 +62,7 @@ roots through `webpki-roots` plus locally-installed CAs from the system bundle. Building without `bundled-ca-roots` switches to the platform trust store via `rustls-native-certs` and excludes bundled Mozilla root crates such as `webpki-roots` and `webpki-root-certs` from the dependency graph. The -`system-ca-roots` feature alias on `openshell-sandbox` includes all other +`system-ca-roots` feature alias on `openshell-supervisor` includes all other defaults (currently `telemetry`) except `bundled-ca-roots`, so Linux distribution builds (e.g. RPM) can use `--no-default-features --features system-ca-roots` without manually re-adding @@ -196,13 +195,12 @@ Runtime layout: cache action runs. An explicitly configured VM runtime bundle is required to contain every non-empty embedding input; the driver build fails before packaging when an input is absent or empty. -- **Supervisor**: Alpine base with `nftables`, static binary at - `/openshell-sandbox` (musl by default; see `SUPERVISOR_LIBC` above). Static - linkage keeps the binary usable when the image is mounted/extracted into - sandbox environments (Docker extraction, Podman image volumes, Kubernetes - init-container copy-self), whose libc and glibc version are not known at build - time, while `nftables` supports Kubernetes supervisor sidecar egress - enforcement. The VM driver bundles its own supervisor build +- **Sandbox and supervisor**: Alpine base with separate static + `/openshell-sandbox` and `/openshell-supervisor` binaries (musl by default; + see `SUPERVISOR_LIBC` above). Static linkage keeps the sandbox executable + usable when a driver stages it into an arbitrary workload image. The image + entrypoint is the external supervisor; drivers copy only the sandbox binary + into the workload trust domain. The VM driver bundles both builds (`tasks/scripts/vm/build-supervisor-bundle.sh`) and does not read `SUPERVISOR_LIBC`. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..0396f51bc0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -1,10 +1,9 @@ # Compute Runtimes Compute runtimes create, stop, start, delete, and watch sandbox workloads for the -gateway. Supervisor-controlled runtimes start a workload that runs the -`openshell-sandbox` supervisor, which enforces the sandbox contract locally. -Driver-controlled runtimes apply the canonical sandbox policy while -provisioning and report workload readiness directly. +gateway. A supported runtime provisions `openshell-sandbox` inside the workload, +`openshell-supervisor` outside it, a protected channel between them, and an +independent outer network fence. Drivers do not implement policy evaluation. ## Driver Contract @@ -12,11 +11,15 @@ Each runtime receives a sandbox spec and canonical policy from the gateway and is responsible for: - Selecting the sandbox image. -- For supervisor-controlled runtimes, injecting sandbox identity and gateway - callback configuration, supplying callback credentials, and providing the - supervisor binary or image. -- For runtimes without the standard supervisor, validating and applying the - canonical policy before launching the workload. +- Resolving an immutable non-root sandbox identity before workload creation. +- Supplying separate sandbox and supervisor bootstrap material. +- Delivering `openshell-sandbox` to the workload and `openshell-supervisor` only + to the external supervisor placement. +- Provisioning protected control and boundary configs plus a private Unix socket, + TLS-authenticated TCP, or vsock transport when the supervisor is separated. + Runtime-specific code supplies immutable resource claims and transport + coordinates; the shared boundary protocol supplies lifecycle, exec, signaling, + forwarding, and binary identity semantics. - Forwarding the exact canonical main-process argv and TTY mode without shell reconstruction. The sandbox-level environment and policy workspace apply to the main process. @@ -247,10 +250,10 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| -| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | -| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| Docker | Local development with Docker available. | Capability-free workload container. | Uses `network_mode=none`; a separate capability-free supervisor container mediates egress and access over a private daemon-local Unix socket volume. | +| Podman | Existing rootless driver. | Container. | Not converted by this isolation stack. | +| Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Uses empty-egress NetworkPolicy, paired-only supervisor ingress, and a separate capability-free supervisor Deployment over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace. | +| VM | Experimental microVM isolation. | Per-sandbox libkrun or QEMU VM. | The NIC-less guest runs `openshell-sandbox` as PID 1; host `openshell-supervisor` owns gateway networking and reaches the guest over vsock. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through @@ -278,25 +281,17 @@ operator override because they place gateway-host filesystem state inside the sandbox and can negate OpenShell workspace isolation and filesystem-policy controls. Driver-owned supervisor, token, and TLS bind mounts stay reserved. -Network features follow the existing driver/substrate split. Compute drivers -advertise only the runtime mechanics they can guarantee: namespace and -capability ownership, DNS/TCP capture installation, and coupled -restart ordering. The shared supervisor remains the sole owner of DNS -eligibility, synthetic mappings, process authorization, destination filtering, -pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise -`policy-dns-transparent-tcp`; other runtimes reject explicit TCP policy until -they implement and validate the same complete contract. The capability marker -is driver-owned supervisor input and is removed from workload environments. - -Kubernetes deployments may set an AppArmor profile on sandbox agent containers -through the driver configuration. The Helm chart defaults sandbox agents to -`Unconfined` so runtime/default AppArmor profiles do not block supervisor -network namespace setup on AppArmor-enabled nodes. +Network features follow the driver/substrate split. Drivers own only the outer +fence and protected channel. The sandbox owns seccomp notification, local DNS, +socket virtualization, process observation, and binary identity. The supervisor +owns DNS eligibility, policy authorization, destination filtering, upstream +dials, relay behavior, credential rewriting, and OCSF decisions. No supported +path requires nftables, a workload network namespace, proxy environment +variables, added capabilities, or an unconfined AppArmor profile. The Kubernetes deployment packaging has two ownership boundaries. The gateway chart owns the gateway workload, configuration, Services, PKI, and -cluster-scoped gateway resources. It can retain the legacy combined behavior, -or omit workspace resources. The workspace chart is installed into a +cluster-scoped gateway resources. The workspace chart is installed into a pre-provisioned sandbox namespace and owns only the sandbox ServiceAccount, namespaced RBAC, and sandbox ingress NetworkPolicy. Its RoleBinding names the gateway ServiceAccount and namespace explicitly, so the two releases have @@ -321,113 +316,57 @@ Runtime-specific implementation notes belong in the driver crate README: - `crates/openshell-driver-kubernetes/README.md` - `crates/openshell-driver-vm/README.md` -The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun -executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves -that identity when it execs the supervisor after mounting and network setup. +The VM guest bootstrap runs once as root to prepare mounts, loopback, and the +safe port-53 sysctl. It then drops to the resolved identity with empty +capability sets and executes `openshell-sandbox` as guest PID 1. ## Supervisor Delivery -The supervisor must be available inside each sandbox workload: +Drivers deliver the two binaries to separate trust domains: | Runtime | Delivery model | |---|---| -| Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | -| Podman | Read-only OCI image volume by default; host-cached bind mount when `userns` is configured. | -| Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | -| VM | Embedded in the guest rootfs bundle. | +| Docker | A digest-pinned daemon-local volume supplies `openshell-sandbox`; the companion image runs `openshell-supervisor`. | +| Podman | Existing driver behavior; not converted by this stack. | +| Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; the separate Deployment image runs `openshell-supervisor`. | +| VM | `openshell-sandbox` is embedded in the guest rootfs; a separately digest-checked native `openshell-supervisor` runs on the host. | | Extension | Defined by the out-of-tree driver. | -Driver-controlled environment variables must override sandbox image or template -values for sandbox ID, sandbox name, gateway endpoint, relay socket path, TLS -paths, and command metadata. +Driver-controlled sandbox bootstrap must override image or template values for +sandbox identity, command metadata, resolver configuration, and public trust +paths. Gateway endpoints, callback credentials, policy, and private TLS material +belong only to the supervisor placement. ## Process Identity -The gateway preserves whether each policy process field was omitted. The active -driver then supplies one authoritative identity input to the supervisor: - -- Docker and Podman inspect the final sandbox image, pin container creation to - its immutable image ID, and pass its raw OCI `Config.User`. Docker also - resolves the workspace from OCI `Config.WorkingDir` during that inspection. -- Kubernetes passes its platform-resolved numeric UID/GID, including OpenShift - SCC-derived values. -- VM keeps its existing guest identity behavior. - -Explicit numeric workload identities may use any Linux UID/GID from `1` -through `u32::MAX - 1`. UID/GID `0` remains prohibited as root, and -`u32::MAX` remains prohibited because Linux APIs and POSIX ACLs use it as an -invalid identity sentinel. Infrastructure identities use separate validation: -the Kubernetes network proxy UID remains at least `1000` and must not match the -workload UID because its traffic bypasses the pod egress fence. - -For Docker and Podman, policy values take precedence independently. An omitted -`run_as_user` or `run_as_group` falls back to the corresponding identity from -the image. The supervisor resolves names from the image's `/etc/passwd` and -`/etc/group` before readiness, preserves declared name or numeric components, -and uses the same privilege-drop path for direct and SSH children. When a -declaration omits the group, the supervisor fills it with the user's numeric -primary GID. It does not rewrite the account files. - -Docker uses an absolute OCI working directory as the workspace. An -empty, root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which -OpenShell creates and owns as a compatibility workspace. Any other workdir must already -exist in the immutable image without symlink components. The completed -identity, including supplementary groups, must already be able to traverse -every parent and write and enter the workdir; OpenShell does not change that -directory's ownership or mode. A one-shot validator drops to that identity and -uses kernel effective-access checks so POSIX ACL and LSM decisions are honored. -Path checks reserve the standard OCI runtime namespaces under `/proc`, `/sys`, -and `/dev`, while separate collision checks are derived from actual OpenShell -control paths. -Docker performs the check in the final container before workload launch and -rejects image `VOLUME` declarations that would mask the workdir ancestry. The -resolved workspace is the child cwd and `HOME`; when -`filesystem.include_workdir` is enabled, it becomes the automatic writable -policy path. Podman, Kubernetes/OpenShift, and VM retain their existing -`/sandbox` workspace behavior. - -Sandbox creation fails before the workload becomes ready when a required image -identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. -The supervisor itself remains root so it can establish isolation before -starting unprivileged children. - -Kubernetes can run the supervisor in the default combined topology or in a -sidecar topology. Combined mode keeps network and process supervision in the -agent container. Sidecar mode runs network enforcement, the proxy, and gateway -session in a dedicated sidecar, while the agent container runs only the -process-supervision leaf and launches the user workload after the sidecar -serves bootstrap state over a local control socket. The network sidecar owns -gateway credentials and sends policy plus workload-facing provider environment -state to the process leaf over that socket. It also streams provider -environment updates after settings polls so future process sessions see -updated provider env without giving the process leaf gateway access. The -pre-workload process supervisor is the only accepted control client: the -network sidecar verifies its UID, GID, and PID with peer credentials, removes -the listener after accepting it, and ignores workload-supplied relay targets. -SSH relays use a Linux abstract socket and verify its peer PID against that -authenticated process-supervisor connection, so workload filesystem access -cannot replace the relay endpoint. Either supervisor exits when this control -connection closes. This couples their restart lifecycle and prevents a workload -that survives an isolated network-sidecar restart from becoming the next -authoritative control client. In sidecar mode, an init container performs the -privileged pod-network nftables setup with -`NET_ADMIN`. The default binary-aware network sidecar runs as UID 0 without -`NET_ADMIN` and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` so it can resolve -cross-UID workload process/binary identity through shared `/proc`. Operators -can set the sidecar `process_binary_aware_network_policy` flag false to run the -sidecar as the configured non-root proxy UID, omit both inspection capabilities, -and downgrade network policy to endpoint/L7 matching without `policy.binaries`. -The init path applies nftables as individual commands so optional conntrack and -log expressions can fail without rolling back the required table, chain, and -reject rules. -The agent container runs as the resolved sandbox UID/GID with no added Linux -capabilities. Sidecar mode preserves gateway session and SSH behavior, but -treats the process leaf as network-only: Landlock filesystem policy and child -seccomp still apply where supported, while process privilege dropping and -supervisor identity mount isolation do not run because the agent container is -already unprivileged. Sidecar pods use a shared process namespace so the -network sidecar can resolve workload process and binary identity through -`/proc/`. +The gateway preserves whether each policy process field was omitted and passes +the admitted selectors to the driver. The driver resolves one exact UID, GID, +and supplementary-group set before creating the immutable workload: + +- Docker pins the image ID, resolves policy selectors against the image's + `/etc/passwd` and `/etc/group`, and validates its OCI working directory. +- Kubernetes uses platform-resolved numeric values, including OpenShift + namespace ranges. +- VM uses the configured numeric guest identity. + +UID/GID zero and `u32::MAX` are invalid. The sandbox and every child start with +the resolved identity and zero capability masks; neither process performs an +in-workload UID transition. Identity-changing policy updates require sandbox +recreation, while other policy updates remain live. + +Docker uses an absolute OCI working directory as the workspace. Empty, root, +and explicit `/sandbox` values select `/sandbox`; other paths must already +exist without symlink or reserved-mount collisions and must be usable by the +resolved identity. Kubernetes and VM use `/sandbox`. + +Kubernetes uses only the proxy-pod topology. The driver creates the empty-egress +workload fence before a suspended Sandbox CR, then provisions split immutable +bootstrap Secrets, the boundary Service, and the supervisor Deployment. A +non-root init container stages `openshell-sandbox` and one-use bootstrap files +into memory volumes. The workload Pod never mounts supervisor or gateway +credentials. The driver removes its scheduling gate only after the companions +exist; measured confirmation and supervisor-session registration gate public +readiness. ## Images diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 75263865e3..7f4a5d17a5 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -1,39 +1,63 @@ # Sandbox -A sandbox is the runtime boundary where agent code executes. It is created by a -compute runtime and managed inside the workload by `openshell-sandbox`, the -sandbox supervisor. +A sandbox is the runtime boundary where agent code executes. A compute driver +creates it and connects two dedicated components: `openshell-sandbox` inside +the workload boundary and `openshell-supervisor` outside it. ## Runtime Model -Each sandbox workload has two trust levels: +Each sandbox has three trust levels: -| Process | Role | +| Component | Role | |---|---| -| Supervisor | Starts as root inside the workload, prepares isolation, runs the proxy, fetches config, injects credentials, serves the relay socket, and launches child processes. | -| Agent child | Runs as an unprivileged user with filesystem, process, and network restrictions applied. | - -The supervisor keeps enough privilege to manage the sandbox, but the agent child -loses that privilege before user code runs. On Linux, child setup clears the -capability bounding set during privilege drop so later execs cannot regain -container-granted capabilities. This is fail-closed: the supervisor retains -`CAP_SETPCAP` solely to perform the clear, and spawning the workload or SSH shell -aborts unless the bounding set ends up empty. A `setpcap` `EPERM` is tolerated -only when the set is already empty; any other outcome fails the spawn. +| Supervisor | Owns gateway credentials, admitted policy, L7 proxying, SSH, and gateway relays. It never executes inside the agent workload. | +| Sandbox | Runs as the same non-root identity as the agent, installs the workload seccomp listener, applies the Landlock baseline, owns child processes, and mediates the protected supervisor channel. | +| Agent child | Inherits the sandbox network listener and runs with zero capabilities, `no_new_privs`, Landlock, and the final syscall filter. | + +The runtime grants neither trusted component nor agent child any Linux +capability inside the workload. Drivers resolve one exact non-root UID, GID, +and supplementary-group set before launch. The sandbox and all of its children +use that immutable identity, so no in-workload privilege transition is needed. +The supervisor uses its own driver-defined identity and has no workload-creation +or backend-admin authority. + +The compute driver provisions separate protected configurations and one +mutually authenticated gRPC connection over a private Unix socket, Kubernetes +TCP Service, or VM vsock channel. Independent bidirectional `Exchange` RPCs +carry lifecycle, exec, TCP, and forwarding traffic, while one persistent +bidirectional `Mediate` RPC carries multiplexed DNS and UDP traffic. +NetworkPolicy is an outer reachability fence, not a confidentiality boundary. +Each sandbox generation receives a fresh CA and distinct server/client leaves; +both endpoints bind the same workload identity and immutable driver resource +claims. Driver crates do not appear in generic process, network, SSH, or +session code. + +The supervisor exposes readiness only after the sandbox is confirmed and the +gateway access plane is registered. Driver-owned channel directories limit +reachability, while mutual authentication and channel epochs prevent endpoint +replacement from granting authority. ## Startup Flow -1. The compute runtime starts the workload with sandbox identity, callback - endpoint, TLS or secret material, image metadata, and initial command. -2. The supervisor loads policy and runtime settings from local files or the - gateway, depending on mode. -3. It prepares filesystem access, process restrictions, network namespace - routing, trust stores, provider credential resolution, and inference routes. -4. It launches the persisted canonical main-process argv and retains its PTY - or pipes in the main-session multiplexer. -5. It starts the policy proxy and local SSH server. -6. It opens a supervisor session back to the gateway for connect, exec, file - sync, config polling, and log push. +1. The driver resolves the immutable workload identity, installs the outer + network fence, and starts `openshell-sandbox` with one-use bootstrap state. +2. The sandbox consumes and unlinks bootstrap material, proves the admitted + runtime posture, and listens on the protected driver channel. It does not + run untrusted code yet. +3. `openshell-supervisor` loads policy and runtime settings from the gateway, + attaches to the sandbox, and verifies the driver's generation and evidence. +4. The sandbox installs its seccomp notification broker and Landlock baseline, + then reports measured confirmation. The supervisor must accept that evidence + before it sends the launch permit. +5. The sandbox starts the canonical process through its single workload + launcher. The supervisor starts SSH and registers its gateway session. +6. Exec, signaling, PTY, DNS, TCP, and loopback-forwarding operations cross the + authenticated channel for the lifetime of the sandbox generation. + +When the admitted main process exits, its status and retained terminal output +remain available. The confirmed sandbox and supervisor-owned access plane continue +to serve policy-authorized exec and loopback forwarding until explicit stop or +delete tears down the boundary and terminates any remaining workload processes. ## Isolation Layers @@ -42,9 +66,9 @@ OpenShell uses overlapping controls rather than a single sandbox primitive: | Layer | Purpose | |---|---| | Filesystem policy | Landlock restricts the paths the agent can read or write. | -| Process policy | The child process runs as a non-root user with reduced privileges. | -| Seccomp | Blocks dangerous syscalls, including raw socket paths that bypass the proxy. | -| Network namespace | Forces ordinary agent egress through the local CONNECT proxy. | +| Process policy | Sandbox and children run as one immutable non-root identity with zero capabilities. | +| Seccomp notification | Virtualizes supported INET sockets and sends DNS/TCP decisions to the supervisor without nftables or proxy environment variables. | +| Driver outer fence | Docker `network_mode=none`, a NIC-less VM, or Kubernetes NetworkPolicy prevents any missed or unsupported kernel path from escaping. | | Policy proxy | Evaluates destination, binary identity, TLS/L7 rules, SSRF checks, and inference interception. | The supervisor may enrich baseline filesystem allowances for runtime-required @@ -55,12 +79,27 @@ paths, such as proxy support files or GPU device paths when a GPU is present. See [Sandbox Limits](sandbox-limits.md) for the current numeric safety ceilings, their ownership, terminal behavior, and known gaps. -All ordinary agent egress is routed through the sandbox proxy. The proxy -identifies the calling binary, checks trust-on-first-use binary identity, rejects -unsafe internal destinations, and evaluates the active policy. On Linux, it -maps an accepted proxy connection back to the workload socket by matching the -complete local-to-remote TCP tuple before resolving every process that owns the -socket inode. +The sandbox installs one seccomp user-notification listener on a dedicated +launcher thread. Every canonical and exec process inherits that listener. It +virtualizes supported INET sockets before they enter the agent FD table, copies +bounded syscall inputs from the notifying task, resolves the calling binary, +and blocks external `connect` until the supervisor returns a policy decision +and relay stream. Connected data stays on ordinary kernel sockets, so the +notification path is limited to socket setup and pointer-bearing operations. +This topology requires Linux 5.19 or newer: the sandbox treats +`SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` as mandatory so cancelled +notifications cannot race task-memory writes. + +DNS uses an exact sandbox-local resolver at `127.0.0.53:53`. The driver sets the +nameserver and permits an unprivileged bind to port 53. UDP and TCP DNS requests +are attributed to the calling binary and forwarded through the supervisor; the +kernel delivers replies from the configured nameserver address, including for +strict musl and c-ares resolvers. No proxy environment variable, nftables rule, +or workload network namespace setup is part of enforcement. + +The outer fence remains mandatory. If notification handling misses a syscall, +loses the supervisor, exceeds a bound, or encounters an unsupported socket +type, the request fails and the driver-owned fence still blocks direct egress. CONNECT and absolute-form forward HTTP are explicit-proxy adapters over the same egress pipeline. Each adapter normalizes its request into an egress intent, and @@ -86,29 +125,6 @@ captured before the bypass fence, mapped back to its workload process, authorize through the same egress pipeline, and dialed only through the pinned addresses. Omitted protocol endpoints retain explicit-proxy behavior. -The DNS store is in-memory and sandbox-local. A combined-supervisor restart also -restarts its workload; before execution, the supervisor advances a persisted -boot epoch and installs only that epoch's synthetic capture ranges. An address -cached from the preceding epoch therefore falls through to the bypass fence -instead of inheriting a new mapping. Policy reload, expiry, wrong ports, direct real-IP access, missing -mappings, or pool exhaustion fail closed. Resolver injection, DNS listeners, -capture rules, and the transparent listener are all ready before workload -execution. A runtime that cannot provide the complete contract rejects a policy -containing explicit TCP endpoints rather than partially activating it. Because -that substrate is startup infrastructure, a sandbox created without explicit -TCP endpoints rejects a hot reload that introduces one and keeps its complete -previous policy active; recreating the sandbox installs the substrate before -the workload starts. A sandbox that started with the substrate may continue to -remove and re-add TCP endpoints through ordinary atomic policy reloads. -Workload DNS targets port 53, while nftables redirects eligible IPv4 DNS traffic -to an unprivileged supervisor listener. The filter admits DNS and transparent -TCP only when the kernel records the traffic as DNATed to the corresponding -supervisor listener, so direct dials to either unprivileged listener port remain -fenced. `SO_ORIGINAL_DST`, synthetic mapping lookup, endpoint correlation, and -generation-pinned authorization form the transparent TCP security boundary. -Docker and Podman do not currently advertise usable IPv6 egress for this -substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. - Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static credential resolves only when the request host, port, and path match an endpoint @@ -121,12 +137,12 @@ partially active or last-known-good static set. Invalid metadata preserves the supplied dynamic snapshot, while a fetch failure preserves the currently active dynamic snapshot. -In the Kubernetes sidecar topology, the provider environment revision remains +Across the protected sandbox/supervisor channel, the provider environment revision remains an opaque content fingerprint and has no numeric ordering semantics. The network supervisor assigns a separate, connection-local monotonic generation to each distinct environment it publishes. The process supervisor applies only newer generations, which accepts descending fingerprint values while rejecting -duplicate or delayed sidecar messages. +duplicate or delayed supervisor messages. Gateway-managed refresh credentials use an opaque workload handle derived from the sandbox, provider identity, credential key, refresh authorization epoch, @@ -263,8 +279,9 @@ last resort for proxies whose ACLs filter on hostnames and reject IP CONNECT targets — with it, the proxy resolves the name itself and its ACLs become the effective egress control for proxied TLS. (Resolving through the proxy's own DNS view, e.g. DoH tunneled via CONNECT, is a possible future -enhancement and out of scope.) The workload child's proxy variables are -unaffected — they are always rewritten to point at the local policy proxy. +enhancement and out of scope.) Workload proxy variables are removed from the +protected launch environment; transparent socket mediation does not depend on +them. Template environment is treated like user-provided sandbox environment. It can shape the workload child, but it cannot override driver-controlled identity, @@ -283,10 +300,9 @@ sandbox-create time through validators shared with the supervisor (`openshell_core::driver_utils::parse_upstream_proxy_url` and `parse_upstream_proxy_credential`). -An optional operator CA bundle (`--upstream-proxy-ca-bundle`, a PEM path the -driver bind-mounts read-only into the sandbox) extends the trust boundary for -corporate proxies. A CA certificate is not secret, so unlike the auth file it -travels as a plain read-only bind mount rather than a driver secret. It is +An optional operator CA bundle (`--upstream-proxy-ca-bundle`, a supervisor-only +PEM path) extends the trust boundary for corporate proxies. A CA certificate is +not secret, but the supervisor is still its only configuration authority. It is trusted in two places: the TLS handshake with an `https://` proxy, and — because a TLS-intercepting proxy (mitmproxy, squid `ssl-bump`) re-signs tunneled server certificates with the same CA — the sandbox combined trust @@ -301,48 +317,27 @@ plain HTTP) and is fail-closed: an unreadable or certificate-free file is fatal. Proxy credentials are never embedded in the URL: an inline `user:pass@` is rejected because it would be stored in `gateway.toml` and exposed in container metadata. Operators supply credentials via `proxy_auth_file`; the driver -stages them as a root-only secret mounted at a fixed path and passes only +stages them as a supervisor-only secret mounted at a fixed path and passes only that path on the supervisor's command line. The supervisor reads the file and builds the `Proxy-Authorization: Basic` header; a credential that is empty, contains control characters, or is not in `user:pass` form is fatal on both sides. -The VM driver has no argv seam of its own: its guest init script runs as PID 1 -and execs a fixed supervisor command line, and the libkrun and QEMU launch -backends both reach the supervisor through that script. Driver-owned -supervisor arguments therefore travel in a per-sandbox file the driver writes -into the overlay upperdir at a fixed guest path, one argument per line, which -the guest reads verbatim (no word splitting or globbing) and appends to every -supervisor exec. The file is written on **every** launch, including an empty -file when there is nothing to pass: the upperdir copy always shadows the -read-only image layer, so a sandbox image can neither supply its own -supervisor arguments by baking a file at that path nor disable the operator's -by omitting one. This mirrors the driver-authored `init.d` manifest, which -solves the same trust problem for guest init drop-ins. - -A microVM has no bind mounts or container secrets, so the VM driver stages the -credential and the CA bundle into the per-sandbox overlay disk instead — the -credential root-only, the CA world-readable, both at fixed `/opt/openshell` -paths and both removed with the sandbox state directory. The consequence, -which differs from the Podman secret model, is that the credential is at rest -inside that overlay image on the gateway host; the per-sandbox gateway JWT -already travels the same path. Proxy reachability differs by VM backend. libkrun-backed -sandboxes egress through gvproxy, so a proxy on the gateway host's loopback is -reachable through the host alias `host.openshell.internal`, which gvproxy NATs -to the host's `127.0.0.1`. QEMU/TAP sandboxes (GPU) have no equivalent: that -alias resolves to the TAP host address, and the driver's nftables `input` -chain accepts only the gateway port from the guest, so no gateway-host proxy -is reachable. The driver rejects a gateway-host proxy URL on the QEMU path at -launch rather than producing CONNECT timeouts. The guest's gateway callback is -unaffected in both backends and never traverses the proxy. - -For Kubernetes sandboxes, the operator configures a Secret name and key rather -than a gateway-host file path. Kubernetes projects that Secret only into the -container that runs network supervision. Proxy credential Secrets require the -sidecar topology, which gives them a separate container boundary from the -workload. Combined topology is rejected because Kubernetes `fsGroup` volume -permission handling can make a shared credential mount readable by the sandbox -group. +The VM driver runs `openshell-supervisor` on the host. Corporate-proxy +credentials, private CA keys, policy, and gateway credentials never enter the +guest. The NIC-less guest reaches the host supervisor only through the +authenticated vsock channel; the host supervisor performs DNS and upstream +connections. + +The Docker driver runs `openshell-supervisor` in a separate companion container. +Its private named volume contains supervisor bootstrap and channel material. +The workload container receives only `openshell-sandbox`, public interception +CA material, and the other sandbox half of the authenticated channel. + +For Kubernetes, the operator configures a Secret name and key rather than a +gateway-host file path. Kubernetes projects that Secret only into the separate +supervisor Deployment. The sandbox Pod never mounts corporate-proxy credentials +or the interception CA private key. The Basic header travels over the plain-TCP connection to the `http://` proxy, so it is readable on the network path between sandbox host and proxy. @@ -361,16 +356,14 @@ agent process and SSH child processes. Driver-controlled environment variables override template values so sandbox images cannot spoof identity, callback, or relay settings. -Supervisor bootstrap identity is not inherited by agent child processes. When -provider token grants mount a SPIFFE Workload API socket, the socket path must -live under a dedicated directory. Children also enter a private mount namespace -where that socket directory is hidden before privilege drop. +Supervisor bootstrap identity and provider workload-identity sockets never +enter the sandbox workload. The authenticated channel carries only the +policy-authorized provider environment intended for child launch and public +trust material intended for TLS clients. -Credential placeholders in proxied HTTP requests can be resolved by the proxy -when policy allows the target endpoint. For GCP providers, a loopback metadata -server inside the network namespace serves placeholders to SDKs that bypass the -proxy (e.g. Go's `cloud.google.com/go/compute/metadata`). Secrets must not be -logged in OCSF or plain tracing output. The supervisor uses revision-scoped +Credential placeholders in mediated HTTP requests can be resolved by the proxy +when policy allows the target endpoint. Secrets must not be logged in OCSF or +plain tracing output. The supervisor uses revision-scoped placeholders for unmanaged rotating credentials and identity-stable opaque handles for gateway-managed refresh credentials. Provider environment keys beginning with `v_` or `s<64 lowercase hex characters>_` are reserved @@ -514,22 +507,26 @@ refreshes and cannot permanently lose the initial acknowledgement. Only sandbox-scoped revisions (`PolicySource::Sandbox`, version greater than zero) are acknowledged. Global policies and local-file development policies do not use the sandbox revision API and produce no acknowledgement. When explicit -local Rego and data files are configured, the supervisor continues polling the -gateway for settings and provider refreshes but never replaces the local OPA -engine with a gateway policy revision. +local Rego and data files are provisioned into the supervisor, it continues +polling the gateway for settings and provider refreshes but never replaces the +local OPA engine with a gateway policy revision. Workload image files and +environment variables do not configure the separately isolated supervisor. ## Failure Behavior - If gateway config polling fails, the sandbox keeps its last-known-good policy. - If a live policy or middleware-registry update is invalid, the supervisor - rejects the combined update and keeps the current runtime pair. + rejects the update and keeps the current runtime pair. - If an operator-run middleware call fails, the selected config's `on_error` behavior decides whether to deny the request or continue without that stage. - Existing raw byte streams are connection scoped. Dynamic policy changes apply to new connections or the next parsed HTTP request where the proxy can safely re-evaluate. - If the supervisor relay drops, the sandbox can keep running, but connect and - exec operations fail until the supervisor registers again. + exec operations fail until the supervisor registers again. A replacement + supervisor replays the identical sandbox lifecycle and receives the existing + process handle. The sandbox rejects changed launch inputs and releases the + single main-process attachment when the old supervisor transport closes. - If the canonical main process exits, the supervisor durably reports the normalized result immediately. A foreground create declares a one-shot main attachment, so the supervisor accepts it even after a fast process exits, diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 62f5837e70..96e57df17b 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -160,13 +160,15 @@ policy without provenance applies neither the raw-tunnel refusal nor the WebSocket binary-frame refusal. The request-body backstop still applies, because it keys off the presence of a secret resolver rather than endpoint provenance. -Two paths load a policy without provenance. A supervisor booting from a -container-image policy is a bounded window: that policy is resynchronized to the -gateway, which then serves a stamped effective policy. An explicit local Rego and -data override is permanent, because gateway revisions are observed for settings -and providers but never replace the local policy. When that override is combined -with injected provider credentials, the supervisor emits a high-severity -detection finding at startup naming the inactive controls. +Two supervisor-local paths load a policy without provenance. A supervisor +booting from an explicitly provisioned policy file has a bounded window before +that policy is resynchronized to the gateway, which then serves a stamped +effective policy. An explicit supervisor Rego and data override is permanent, +because gateway revisions are observed for settings and providers but never +replace the local policy. Workload-image files and environment variables cannot +configure the separately isolated supervisor. When a supervisor override is +combined with injected provider credentials, the supervisor emits a +high-severity detection finding at startup naming the inactive controls. ## Live Updates diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 2ce8e4b058..c1c91822b6 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -39,7 +39,8 @@ const MAIN_PROCESS_SPEC_BASE64URL_PREFIX: &str = "base64url:"; pub struct MainProcessConfig { pub version: u32, /// Canonical command. Empty means "no command supplied": the supervisor - /// resolves the default login shell against the sandbox image. A non-empty + /// asks the sandbox boundary to resolve the default login shell against + /// the agent image. A non-empty /// command is the exact program+args and is run verbatim. pub command: Vec, pub tty: bool, @@ -51,8 +52,8 @@ impl MainProcessConfig { pub const VERSION: u32 = 1; /// Default config for a sandbox created without a command. The command is - /// left empty on purpose: the supervisor picks a login shell that exists in - /// the sandbox image (bash when present, otherwise `/bin/sh`). A TTY is + /// left empty on purpose: the sandbox boundary picks a login shell that + /// exists in the agent image (bash when present, otherwise `/bin/sh`). A TTY is /// requested because the default is an interactive login shell. #[must_use] pub fn scratch() -> Self { @@ -99,7 +100,7 @@ impl MainProcessConfig { )); } // An empty command is valid: it means "no command supplied", and the - // supervisor resolves the default login shell. Only a present-but-blank + // sandbox boundary resolves the default login shell. Only a present-but-blank // program is rejected. if !config.command.is_empty() && config.command[0].is_empty() { return Err(format!( @@ -134,6 +135,13 @@ pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; /// `"sidecar"`; the default combined supervisor path omits it. pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; +/// The isolation backend admitted by the deployment configuration (RFC 0012). +/// +/// Delivered on a channel separate from the topology descriptor so descriptor +/// verification against the admitted backend is not self-referential. Required +/// whenever a topology descriptor is supplied. +pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND"; + /// Network enforcement backend selected by the compute driver. pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; @@ -166,6 +174,17 @@ pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; /// by workload child processes. pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; +/// Optional path to a durable PEM-encoded interception CA certificate. +/// Must be configured together with [`PROXY_CA_KEY`]. +pub const PROXY_CA_CERT: &str = "OPENSHELL_PROXY_CA_CERT"; + +/// Optional path to the private key for [`PROXY_CA_CERT`]. +/// Must be configured together with the certificate path. +pub const PROXY_CA_KEY: &str = "OPENSHELL_PROXY_CA_KEY"; + +/// Whether the control-owned SSH Unix socket is shared across trusted UIDs. +pub const SSH_SOCKET_SHARED: &str = "OPENSHELL_SSH_SOCKET_SHARED"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; @@ -298,7 +317,7 @@ mod tests { #[test] fn omitted_command_stays_empty_for_supervisor_resolution() { - // No command supplied → empty command; the supervisor resolves the + // No command supplied → empty command; the sandbox boundary resolves the // default login shell against the sandbox image. let empty = crate::proto::compute::v1::DriverSandboxSpec::default(); assert!( diff --git a/crates/openshell-core/src/shell.rs b/crates/openshell-core/src/shell.rs index 09610afe1a..d24a7bced2 100644 --- a/crates/openshell-core/src/shell.rs +++ b/crates/openshell-core/src/shell.rs @@ -3,7 +3,7 @@ //! Login-shell resolution for sandbox images. //! -//! The default sandbox command and the interactive SSH session need a shell, +//! The default sandbox command and interactive SSH sessions need a shell, //! but not every base image ships the same one. Debian-based images provide //! `bash`; minimal images such as Alpine only provide `/bin/sh` (`BusyBox` //! `ash`). Hard-coding `/bin/bash` makes sandbox startup fail on those images @@ -51,8 +51,9 @@ pub fn is_executable(path: &str) -> bool { /// Resolve a login shell that exists in the current root filesystem. /// /// Tries [`SHELL_CANDIDATES`] in order and falls back to [`POSIX_SH`]. Because -/// this inspects the filesystem, call it from the supervisor (inside the -/// sandbox), never on the gateway. +/// this inspects the filesystem, call it from the sandbox boundary or another +/// process inside the workload filesystem, never from the external supervisor +/// or gateway. /// /// `$SHELL` is intentionally not consulted: it is image/user-controlled, the /// result is later invoked with `-lc`, and an executable that is not a diff --git a/crates/openshell-router/src/lib.rs b/crates/openshell-router/src/lib.rs index 79bbfe6ca3..c52239f63b 100644 --- a/crates/openshell-router/src/lib.rs +++ b/crates/openshell-router/src/lib.rs @@ -37,8 +37,20 @@ pub struct Router { impl Router { pub fn new() -> Result { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(30)) + Self::with_dns_overrides(std::iter::empty::<(&str, std::net::IpAddr)>()) + } + + /// Build a router with trusted, static DNS overrides for its upstream + /// HTTP client. URL hostnames remain unchanged for HTTP and TLS; only the + /// dial address is replaced. + pub fn with_dns_overrides<'a>( + overrides: impl IntoIterator, + ) -> Result { + let mut builder = reqwest::Client::builder().connect_timeout(Duration::from_secs(30)); + for (host, ip) in overrides { + builder = builder.resolve(host, std::net::SocketAddr::new(ip, 0)); + } + let client = builder .build() .map_err(|e| RouterError::Internal(format!("failed to build HTTP client: {e}")))?; Ok(Self { @@ -186,4 +198,40 @@ mod tests { let err = Router::from_config(&config).unwrap_err(); assert!(matches!(err, RouterError::Internal(_))); } + + #[tokio::test] + async fn trusted_dns_override_preserves_url_host_and_port() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let port = listener.local_addr().expect("server address").port(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept request"); + let mut request = vec![0_u8; 1024]; + let length = stream.read(&mut request).await.expect("read request"); + assert!( + String::from_utf8_lossy(&request[..length]) + .to_ascii_lowercase() + .contains(&format!("host: host.openshell.internal:{port}")) + ); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .expect("write response"); + }); + + let router = + Router::with_dns_overrides([("host.openshell.internal", "127.0.0.1".parse().unwrap())]) + .expect("build router"); + let response = router + .client + .get(format!("http://host.openshell.internal:{port}/health")) + .send() + .await + .expect("request through DNS override"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + server.await.expect("server task"); + } } diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 3463f03767..7972b34700 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -14,22 +14,28 @@ repository.workspace = true name = "openshell-sandbox" path = "src/main.rs" +[features] +perf-harness = [] + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } -openshell-extension-core = { path = "../openshell-extension-core" } +openshell-binary-identity = { path = "../openshell-binary-identity" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } -openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } -openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } -openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } -openshell-supervisor-process = { path = "../openshell-supervisor-process" } + +anyhow = { workspace = true } +async-trait = "0.1" +bytes = { workspace = true } +hex = "0.4" +ipnet = "2" +rand = "0.10" +sha2 = { workspace = true } # Async runtime tokio = { workspace = true } - -# gRPC (tonic::Status downcast in error mapping) -tonic = { workspace = true, features = ["channel", "tls-native-roots"] } -prost-types = { workspace = true } +tokio-stream = { workspace = true } +tonic = { workspace = true } # CLI clap = { workspace = true } @@ -37,49 +43,37 @@ clap = { workspace = true } # Error handling miette = { workspace = true } -# Unix ownership for Kubernetes sidecar init setup +# Unix identity and bootstrap ownership nix = { workspace = true } # TLS crypto provider install (main.rs) rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +base64 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -prost = { workspace = true } # Logging tracing = { workspace = true } -uuid = { workspace = true } tracing-subscriber = { workspace = true } -tracing-appender = { workspace = true } -[features] -default = ["telemetry", "bundled-ca-roots"] -## Convenience alias: all defaults except bundled CA roots. Use -## `--no-default-features --features system-ca-roots` to build a supervisor -## that uses the platform trust store with telemetry intact. -system-ca-roots = ["telemetry"] -## Convenience alias: every default feature except `telemetry`. Build a -## telemetry-free supervisor with -## `--no-default-features --features defaults-without-telemetry` and stay -## correct as new default features are added. Cargo cannot subtract a single -## default feature, so this alias must be paired with `--no-default-features`; -## enabling it alongside `telemetry` is a compile error rather than a silent -## telemetry-on build. Kept in sync with `default` by -## `rust:verify:defaults-without-telemetry`. Do not pair it with -## `system-ca-roots`, which re-enables `telemetry`; a build with neither -## telemetry nor bundled CA roots is plain `--no-default-features`. -defaults-without-telemetry = ["bundled-ca-roots"] - -telemetry = ["openshell-core/telemetry"] -bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] +[target.'cfg(unix)'.dependencies] +libc = "0.2" +rustix = { workspace = true } + +[target.'cfg(target_os = "linux")'.dependencies] +capctl = "0.2.4" +landlock = "0.4" +seccompiler = "0.5" +socket2 = { workspace = true } +tempfile = "3" [dev-dependencies] +rcgen = { workspace = true } tempfile = "3" -temp-env = "0.3" -tokio-tungstenite = { workspace = true } -futures = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs new file mode 100644 index 0000000000..5fb9ea5ab7 --- /dev/null +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -0,0 +1,723 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workload-side implementation of RFC 0012 sandbox exec. + +use std::collections::HashMap; +use std::os::fd::{AsRawFd, OwnedFd}; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; + +use async_trait::async_trait; +use nix::pty::{Winsize, openpty}; +use nix::sys::signal::{Signal, killpg}; +use nix::unistd::Pid; +use openshell_core::policy::SandboxPolicy; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{ + BackendError, BoundaryExec, BoundaryExitStatus, BoundaryInput, BoundaryOutput, BoundaryProcess, + BoundarySignal, BoundaryTerminal, ExecSession, ExecSpec, +}; + +/// The sandbox executor. Every spawn reuses the same admitted policy and +/// execution-environment controls while taking a fresh provider credential +/// snapshot. +#[derive(Clone)] +pub struct LocalBoundaryExec { + policy: SandboxPolicy, + base_workdir: Option, + ca_file_paths: Option>, + provider_credentials: ProviderCredentialState, + user_environment: HashMap, + runtime: Arc, +} + +impl LocalBoundaryExec { + /// Construct the executor owned by an active sandbox boundary. + #[must_use] + pub fn new( + policy: SandboxPolicy, + base_workdir: Option, + ca_file_paths: Option>, + provider_credentials: ProviderCredentialState, + user_environment: HashMap, + runtime: Arc, + ) -> Self { + Self { + policy, + base_workdir, + ca_file_paths, + provider_credentials, + user_environment, + runtime, + } + } + + fn command(&self, spec: &ExecSpec) -> Result { + if spec.program.is_empty() { + return Err(BackendError::Process("exec program is empty".to_string())); + } + let mut command = Command::new(&spec.program); + command.args(&spec.args); + let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); + let (session_user, session_home) = + crate::process::session_user_and_home(&self.policy, effective_workdir); + let path = std::env::var("PATH").unwrap_or_else(|_| "/usr/local/bin:/usr/bin:/bin".into()); + command + .env_clear() + .env(openshell_core::sandbox_env::SANDBOX, "1") + .env("HOME", session_home) + .env("USER", session_user) + .env("SHELL", "/bin/bash") + .env("PATH", path) + .env("TERM", if spec.pty { "xterm-256color" } else { "dumb" }); + for (key, value) in &self.user_environment { + if !key.starts_with("OPENSHELL_") { + command.env(key, value); + } + } + if let Some((ca_cert_path, combined_bundle_path)) = self.ca_file_paths.as_deref() { + for (key, value) in crate::child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { + command.env(key, value); + } + } + for (key, value) in self.provider_credentials.child_env_with_gcp_resolved() { + if !crate::process::is_supervisor_only_env_var(&key) { + command.env(key, value); + } + } + crate::process::strip_proxy_env_std(&mut command); + for (key, value) in &spec.env { + if !key.starts_with("OPENSHELL_") { + command.env(key, value); + } + } + if let Some(workdir) = spec.workdir.as_deref().or(self.base_workdir.as_deref()) { + command.current_dir(workdir); + } + Ok(command) + } + + #[cfg(target_os = "linux")] + fn prepare_sandbox( + &self, + workdir: Option<&str>, + ) -> Result, BackendError> { + crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox(&self.policy, workdir, &runtime_read_only) + .map_err(|error| BackendError::Process(error.to_string())) + } + + fn spawn_piped(&self, spec: &ExecSpec) -> Result { + self.runtime.ensure_active()?; + let mut command = self.command(spec)?; + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); + #[cfg(target_os = "linux")] + let prepared = self.prepare_sandbox(effective_workdir)?; + #[cfg(target_os = "linux")] + let child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| BackendError::Process(error.to_string()))?; + crate::pty::install_dedicated_process_group(&mut command); + crate::pty::install_pre_exec_no_pty( + &mut command, + self.policy.clone(), + effective_workdir.map(str::to_string), + #[cfg(target_os = "linux")] + prepared, + #[cfg(target_os = "linux")] + child_hardening, + ) + .map_err(|error| BackendError::Process(error.to_string()))?; + #[cfg(target_os = "linux")] + let mut child_registry = crate::managed_children::lock(); + #[cfg(target_os = "linux")] + let mut child = crate::process::spawn_std_command_with_workload_launcher(command) + .map_err(|error| BackendError::Process(error.to_string()))?; + #[cfg(not(target_os = "linux"))] + let mut child = command + .spawn() + .map_err(|error| BackendError::Process(error.to_string()))?; + let pid = child.id(); + let process_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let signal_lock = Arc::new(std::sync::Mutex::new(())); + if let Err(error) = + self.runtime + .register_process_group(pid, process_terminal.clone(), signal_lock.clone()) + { + let _ = killpg( + Pid::from_raw(i32::try_from(pid).unwrap_or(i32::MAX)), + Signal::SIGKILL, + ); + let _ = child.wait(); + return Err(error); + } + #[cfg(target_os = "linux")] + let managed_child = child_registry.register(pid); + #[cfg(target_os = "linux")] + drop(child_registry); + let stdin = child.stdin.take().map(|file| -> BoundaryInput { + let fd: OwnedFd = file.into(); + Box::new(tokio::fs::File::from_std(std::fs::File::from(fd))) + }); + let stdout = child + .stdout + .take() + .map(|file| -> BoundaryOutput { + let fd: OwnedFd = file.into(); + Box::new(tokio::fs::File::from_std(std::fs::File::from(fd))) + }) + .ok_or_else(|| BackendError::Process("exec stdout pipe missing".to_string()))?; + let stderr = child.stderr.take().map(|file| -> BoundaryOutput { + let fd: OwnedFd = file.into(); + Box::new(tokio::fs::File::from_std(std::fs::File::from(fd))) + }); + let process = Arc::new(LocalExecProcess::new( + child, + pid, + self.runtime.clone(), + process_terminal, + signal_lock, + #[cfg(target_os = "linux")] + managed_child, + )); + Ok(SpawnedExec { + session: Some(ExecSession { + process: process.clone(), + stdin, + stdout, + stderr, + terminal: None, + }), + process, + armed: true, + }) + } + + fn spawn_pty(&self, spec: &ExecSpec) -> Result { + self.runtime.ensure_active()?; + let winsize = Winsize { + ws_row: 24, + ws_col: 80, + ws_xpixel: 0, + ws_ypixel: 0, + }; + let pty = openpty(Some(&winsize), None) + .map_err(|error| BackendError::Process(error.to_string()))?; + let master = std::fs::File::from(pty.master); + let slave = std::fs::File::from(pty.slave); + let slave_fd = slave.as_raw_fd(); + let input = master + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let output = master + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let stdin = slave + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let stdout = slave + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let mut command = self.command(spec)?; + command.stdin(stdin).stdout(stdout).stderr(slave); + let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); + #[cfg(target_os = "linux")] + let prepared = self.prepare_sandbox(effective_workdir)?; + #[cfg(target_os = "linux")] + let child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| BackendError::Process(error.to_string()))?; + crate::pty::install_pre_exec( + &mut command, + self.policy.clone(), + effective_workdir.map(str::to_string), + slave_fd, + #[cfg(target_os = "linux")] + prepared, + #[cfg(target_os = "linux")] + child_hardening, + ) + .map_err(|error| BackendError::Process(error.to_string()))?; + #[cfg(target_os = "linux")] + let mut child_registry = crate::managed_children::lock(); + #[cfg(target_os = "linux")] + let mut child = crate::process::spawn_std_command_with_workload_launcher(command) + .map_err(|error| BackendError::Process(error.to_string()))?; + #[cfg(not(target_os = "linux"))] + let mut child = command + .spawn() + .map_err(|error| BackendError::Process(error.to_string()))?; + let pid = child.id(); + let process_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let signal_lock = Arc::new(std::sync::Mutex::new(())); + if let Err(error) = + self.runtime + .register_process_group(pid, process_terminal.clone(), signal_lock.clone()) + { + let _ = killpg( + Pid::from_raw(i32::try_from(pid).unwrap_or(i32::MAX)), + Signal::SIGKILL, + ); + let _ = child.wait(); + return Err(error); + } + #[cfg(target_os = "linux")] + let managed_child = child_registry.register(pid); + #[cfg(target_os = "linux")] + drop(child_registry); + let terminal: Arc = Arc::new(LocalTerminal { master }); + let process = Arc::new(LocalExecProcess::new( + child, + pid, + self.runtime.clone(), + process_terminal, + signal_lock, + #[cfg(target_os = "linux")] + managed_child, + )); + Ok(SpawnedExec { + session: Some(ExecSession { + process: process.clone(), + stdin: Some(Box::new(tokio::fs::File::from_std(input))), + stdout: Box::new(tokio::fs::File::from_std(output)), + stderr: None, + terminal: Some(terminal), + }), + process, + armed: true, + }) + } +} + +struct SpawnedExec { + session: Option, + process: Arc, + armed: bool, +} + +impl SpawnedExec { + fn into_session(mut self) -> ExecSession { + self.armed = false; + self.session.take().expect("spawned exec session") + } +} + +impl Drop for SpawnedExec { + fn drop(&mut self) { + if self.armed { + let _ = self.process.deliver(Signal::SIGKILL); + } + } +} + +#[async_trait] +impl BoundaryExec for LocalBoundaryExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let executor = self.clone(); + let (send, receive) = tokio::sync::oneshot::channel(); + tokio::task::spawn_blocking(move || { + let result = if spec.pty { + executor.spawn_pty(&spec) + } else { + executor.spawn_piped(&spec) + }; + // If the caller cancelled, either send fails and drops the armed + // process guard here, or the queued guard is dropped with the + // receiver. Both paths terminate an unobservable exec process. + let _ = send.send(result); + }); + receive + .await + .map_err(|_| BackendError::Process("exec spawn task failed".to_string()))? + .map(SpawnedExec::into_session) + } +} + +struct LocalTerminal { + master: std::fs::File, +} + +#[async_trait] +impl BoundaryTerminal for LocalTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + crate::pty::set_winsize( + self.master.as_raw_fd(), + Winsize { + ws_row: rows.max(1), + ws_col: cols.max(1), + ws_xpixel: 0, + ws_ypixel: 0, + }, + ) + .map_err(|error| BackendError::Process(error.to_string())) + } +} + +struct LocalExecProcess { + pid: u32, + result: Arc>>>, + exited: Arc, + runtime: Arc, + terminal: Arc, + signal_lock: Arc>, +} + +impl LocalExecProcess { + fn new( + child: Child, + pid: u32, + runtime: Arc, + terminal: Arc, + signal_lock: Arc>, + #[cfg(target_os = "linux")] managed_child: Option, + ) -> Self { + let result = Arc::new(std::sync::Mutex::new(None)); + let exited = Arc::new(tokio::sync::Notify::new()); + let result_for_wait = result.clone(); + let exited_for_wait = exited.clone(); + let runtime_for_wait = runtime.clone(); + let terminal_for_wait = terminal.clone(); + let registration_terminal = terminal.clone(); + #[cfg(target_os = "linux")] + let signal_lock_for_wait = signal_lock.clone(); + tokio::spawn(async move { + let waited = tokio::task::spawn_blocking(move || { + let mut child = child; + #[cfg(target_os = "linux")] + { + let terminal_observed = crate::managed_children::wait_until_terminal(pid); + let _signal_guard = signal_lock_for_wait + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let result = child.wait(); + terminal_for_wait.store(true, std::sync::atomic::Ordering::Release); + if let Some(managed_child) = managed_child { + crate::managed_children::unregister(managed_child); + } + match (terminal_observed, result) { + (_, Ok(status)) => Ok(status), + (Err(observe_error), Err(wait_error)) => Err(std::io::Error::other( + format!( + "observe exec terminal state: {observe_error}; reap exec: {wait_error}" + ), + )), + (Ok(()), Err(wait_error)) => Err(wait_error), + } + } + #[cfg(not(target_os = "linux"))] + { + let result = child.wait(); + terminal_for_wait.store(true, std::sync::atomic::Ordering::Release); + result + } + }) + .await + .map_err(|error| error.to_string()) + .and_then(|status| status.map_err(|error| error.to_string())) + .map(|status| { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(signal) = status.signal() { + return BoundaryExitStatus::Signaled(signal); + } + } + BoundaryExitStatus::Exited(status.code().unwrap_or(1)) + }); + runtime_for_wait.unregister_process_group(pid, ®istration_terminal); + if let Ok(mut slot) = result_for_wait.lock() { + *slot = Some(waited); + } + exited_for_wait.notify_waiters(); + }); + Self { + pid, + result, + exited, + runtime, + terminal, + signal_lock, + } + } + + fn deliver(&self, signal: Signal) -> Result<(), BackendError> { + self.runtime.ensure_active()?; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(std::sync::atomic::Ordering::Acquire) { + return Err(BackendError::Terminated("process has exited".to_string())); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + killpg(Pid::from_raw(pid), signal).map_err(|error| BackendError::Process(error.to_string())) + } +} + +#[async_trait] +impl BoundaryProcess for LocalExecProcess { + async fn wait(&self) -> Result { + loop { + let notified = self.exited.notified(); + let result = self + .result + .lock() + .map_err(|_| BackendError::Process("exec result lock poisoned".to_string()))? + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Process); + } + notified.await; + } + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.deliver(match signal { + BoundarySignal::Term => Signal::SIGTERM, + BoundarySignal::Kill => Signal::SIGKILL, + BoundarySignal::Int => Signal::SIGINT, + BoundarySignal::Hup => Signal::SIGHUP, + }) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.deliver(Signal::SIGKILL) + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use std::sync::Once; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn executor() -> LocalBoundaryExec { + static LAUNCHER: Once = Once::new(); + LAUNCHER.call_once(|| { + let (launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start() + .expect("start test workload launcher"); + std::thread::spawn(move || { + while let Ok(notification) = listener.receive() { + let _ = listener.respond_errno(notification.id, libc::EPERM); + } + }); + crate::process::configure_workload_launcher(launcher) + .expect("configure test workload launcher"); + }); + LocalBoundaryExec::new( + SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }, + None, + None, + ProviderCredentialState::from_environment( + 0, + HashMap::new(), + HashMap::new(), + HashMap::new(), + ), + HashMap::new(), + crate::boundary_io::BoundaryRuntimeState::new(), + ) + } + + #[tokio::test] + async fn non_pty_exec_preserves_stdin_stdout_and_stderr() { + let mut session = executor() + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec![ + "-c".to_string(), + "read line; printf 'out:%s' \"$line\"; printf 'err:%s' \"$line\" >&2" + .to_string(), + ], + env: vec![], + workdir: None, + pty: false, + }) + .await + .expect("spawn exec"); + let mut stdin = session.stdin.take().expect("stdin"); + stdin.write_all(b"value\n").await.expect("write stdin"); + drop(stdin); + let mut stdout = String::new(); + let mut stderr = String::new(); + session + .stdout + .read_to_string(&mut stdout) + .await + .expect("read stdout"); + session + .stderr + .take() + .expect("stderr") + .read_to_string(&mut stderr) + .await + .expect("read stderr"); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(0) + ); + assert_eq!(stdout, "out:value"); + assert_eq!(stderr, "err:value"); + } + + #[tokio::test] + async fn exec_rejects_after_boundary_end() { + let executor = executor(); + executor.runtime.deactivate(); + let result = executor + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "exit 0".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + .await; + assert!(matches!(result, Err(BackendError::Terminated(_)))); + } + + #[tokio::test] + async fn failed_exec_leaves_boundary_active_without_registered_processes() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let result = executor + .exec(ExecSpec { + program: "/definitely/missing/openshell-exec".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }) + .await; + assert!(matches!(result, Err(BackendError::Process(_)))); + runtime.ensure_active().expect("boundary remains active"); + assert_eq!(runtime.registered_process_group_count(), 0); + } + + #[tokio::test] + async fn cancelled_exec_does_not_leave_a_registered_process() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let task = tokio::spawn(async move { + executor + .exec(ExecSpec { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + .await + }); + tokio::task::yield_now().await; + task.abort(); + let _ = task.await; + + // Give the detached blocking setup time to reach its cancelled + // handoff, including the case where cancellation won before spawn. + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while runtime.registered_process_group_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled exec process must be terminated and reaped"); + runtime.ensure_active().expect("boundary remains active"); + } + + #[tokio::test] + async fn dropping_undelivered_exec_guard_terminates_process() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let spawned = tokio::task::spawn_blocking(move || { + executor.spawn_piped(&ExecSpec { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + }) + .await + .expect("spawn task") + .expect("spawn exec"); + assert_eq!(runtime.registered_process_group_count(), 1); + + // This is the post-send/pre-receive cancellation case: dropping the + // queued ownership guard must kill the process before it is observable. + drop(spawned); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while runtime.registered_process_group_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("undelivered exec process must be terminated and reaped"); + runtime.ensure_active().expect("boundary remains active"); + } + + #[tokio::test] + async fn completed_exec_removes_its_process_group_registration() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let session = executor + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "exit 0".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + .await + .expect("spawn exec"); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(0) + ); + assert_eq!(runtime.registered_process_group_count(), 0); + } + + #[tokio::test] + async fn pty_exec_exposes_resize_and_stable_wait() { + let session = executor() + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "exit 7".to_string()], + env: vec![], + workdir: None, + pty: true, + }) + .await + .expect("spawn pty exec"); + session + .terminal + .as_ref() + .expect("terminal") + .resize(120, 40) + .await + .expect("resize"); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(7) + ); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(7) + ); + } +} diff --git a/crates/openshell-sandbox/src/boundary_io.rs b/crates/openshell-sandbox/src/boundary_io.rs new file mode 100644 index 0000000000..27613f0df1 --- /dev/null +++ b/crates/openshell-sandbox/src/boundary_io.rs @@ -0,0 +1,311 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox-local [`BoundaryPortForward`] implementation. + +use async_trait::async_trait; +use openshell_isolation_interface::contract::{ + BackendError, BoundaryDuplexStream, BoundaryPortForward, LoopbackTarget, +}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Shared liveness and child-process ownership for one active boundary. +pub struct BoundaryRuntimeState { + state: AtomicU8, + process_groups: Mutex>, + exclusive_pid_namespace: bool, +} + +impl BoundaryRuntimeState { + #[must_use] + pub fn new() -> Arc { + Arc::new(Self { + state: AtomicU8::new(0), + process_groups: Mutex::new(HashMap::new()), + exclusive_pid_namespace: false, + }) + } + + /// Construct state for a boundary that exclusively owns its PID namespace. + #[must_use] + pub fn new_exclusive_pid_namespace() -> Arc { + Arc::new(Self { + state: AtomicU8::new(0), + process_groups: Mutex::new(HashMap::new()), + exclusive_pid_namespace: true, + }) + } + + #[must_use] + pub const fn requires_dedicated_process_group(&self) -> bool { + self.exclusive_pid_namespace + } + + pub fn ensure_active(&self) -> Result<(), BackendError> { + if self.state.load(Ordering::Acquire) == 0 { + Ok(()) + } else { + Err(BackendError::Terminated("boundary has ended".to_string())) + } + } + + #[must_use] + pub fn is_active(&self) -> bool { + self.state.load(Ordering::Acquire) == 0 + } + + #[must_use] + pub fn enforcement_was_lost(&self) -> bool { + self.state.load(Ordering::Acquire) == 2 + } + + pub fn register_process_group( + &self, + pid: u32, + terminal: Arc, + signal_lock: Arc>, + ) -> Result<(), BackendError> { + let mut groups = self + .process_groups + .lock() + .map_err(|_| BackendError::Process("boundary process registry poisoned".to_string()))?; + self.ensure_active()?; + groups.insert( + pid, + RegisteredProcessGroup { + pid, + terminal, + signal_lock, + }, + ); + Ok(()) + } + + pub fn unregister_process_group( + &self, + pid: u32, + terminal: &Arc, + ) { + if let Ok(mut groups) = self.process_groups.lock() + && groups + .get(&pid) + .is_some_and(|group| Arc::ptr_eq(&group.terminal, terminal)) + { + groups.remove(&pid); + } + } + + #[cfg(test)] + pub fn registered_process_group_count(&self) -> usize { + self.process_groups.lock().map_or(0, |groups| groups.len()) + } + + /// End the boundary and terminate every registered workload process group. + pub fn deactivate(&self) { + if self + .state + .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + self.terminate_registered_processes(); + } + } + + /// End the boundary because required standing enforcement was lost. + /// + /// Returns `true` only to the caller that won the active-to-terminated + /// transition. A concurrent normal teardown cannot later be reclassified + /// as enforcement loss. + pub fn deactivate_for_enforcement_loss(&self) -> bool { + if self + .state + .compare_exchange(0, 2, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return false; + } + self.terminate_registered_processes(); + true + } + + fn terminate_registered_processes(&self) { + let groups = self + .process_groups + .lock() + .map(|groups| groups.values().cloned().collect::>()) + .unwrap_or_default(); + for group in groups { + group.terminate(); + } + } +} + +#[derive(Clone)] +struct RegisteredProcessGroup { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +impl RegisteredProcessGroup { + fn terminate(&self) { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return; + } + if let Ok(pid) = i32::try_from(self.pid) { + let _ = nix::sys::signal::killpg( + nix::unistd::Pid::from_raw(pid), + nix::sys::signal::Signal::SIGKILL, + ); + } + } +} + +/// Loopback port-forward owned by the sandbox process. +pub struct LocalPortForward { + runtime: Option>, +} + +impl LocalPortForward { + #[must_use] + pub fn new(runtime: Option>) -> Self { + Self { runtime } + } +} + +#[async_trait] +impl BoundaryPortForward for LocalPortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + if let Some(runtime) = &self.runtime { + runtime.ensure_active()?; + } + let addr = std::net::SocketAddr::new(target.host(), target.port()); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[addr]) + .await + .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; + if let Some(runtime) = &self.runtime { + runtime.ensure_active()?; + } + Ok(Box::new(stream)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv4Addr; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + /// Stands in for the SSH server's port-forward path: connect through the + /// interface, write, and read the echo. + #[tokio::test] + async fn port_forward_connects_and_round_trips() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut buf = [0u8; 4]; + sock.read_exact(&mut buf).await.unwrap(); + sock.write_all(&buf).await.unwrap(); + }); + + let pf = LocalPortForward::new(None); + let target = + LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), addr.port()).expect("loopback target"); + let mut conn = pf.connect(target).await.expect("connect through interface"); + conn.write_all(b"ping").await.unwrap(); + let mut buf = [0u8; 4]; + conn.read_exact(&mut buf).await.unwrap(); + assert_eq!(&buf, b"ping"); + } + + /// Drive the port-forward interface through a generic `&dyn` consumer, proving a + /// kernel-separated backend (tunneling into a guest) would use the same call. + #[tokio::test] + async fn port_forward_is_driven_via_dyn() { + async fn forward_one(pf: &dyn BoundaryPortForward, target: LoopbackTarget) -> bool { + pf.connect(target).await.is_ok() + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let _ = listener.accept().await; + }); + let pf = LocalPortForward::new(None); + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), addr.port()).unwrap(); + assert!(forward_one(&pf, target).await); + } + + #[tokio::test] + async fn port_forward_rejects_after_boundary_end() { + let runtime = BoundaryRuntimeState::new(); + let pf = LocalPortForward::new(Some(runtime.clone())); + runtime.deactivate(); + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), 1).unwrap(); + assert!(matches!( + pf.connect(target).await, + Err(BackendError::Terminated(_)) + )); + } + + #[tokio::test] + async fn failed_port_forward_keeps_boundary_active() { + let runtime = BoundaryRuntimeState::new(); + let pf = LocalPortForward::new(Some(runtime.clone())); + // Port zero is never a connectable TCP destination. Reserving an ephemeral + // port and dropping its listener races other parallel tests that may bind it. + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), 0).unwrap(); + assert!(matches!( + pf.connect(target).await, + Err(BackendError::Process(_)) + )); + runtime.ensure_active().expect("boundary remains active"); + } + + #[test] + fn stale_unregister_preserves_reused_process_group_registration() { + let runtime = BoundaryRuntimeState::new(); + let first_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let second_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let pid = 42; + runtime + .register_process_group(pid, first_terminal.clone(), Arc::new(Mutex::new(()))) + .expect("first registration"); + runtime + .register_process_group(pid, second_terminal.clone(), Arc::new(Mutex::new(()))) + .expect("replacement registration"); + + runtime.unregister_process_group(pid, &first_terminal); + assert_eq!(runtime.registered_process_group_count(), 1); + + runtime.unregister_process_group(pid, &second_terminal); + assert_eq!(runtime.registered_process_group_count(), 0); + } + + #[test] + fn canonical_process_completion_does_not_end_boundary_runtime() { + let runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let terminal = Arc::new(std::sync::atomic::AtomicBool::new(true)); + runtime + .register_process_group(42, terminal.clone(), Arc::new(Mutex::new(()))) + .expect("register canonical process"); + + runtime.unregister_process_group(42, &terminal); + + runtime + .ensure_active() + .expect("canonical completion must preserve exec and forwarding"); + assert_eq!(runtime.registered_process_group_count(), 0); + runtime.deactivate(); + assert!(matches!( + runtime.ensure_active(), + Err(BackendError::Terminated(_)) + )); + } +} diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs new file mode 100644 index 0000000000..7fcd040662 --- /dev/null +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -0,0 +1,3874 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared implementation of the capability-free `openshell-sandbox` runtime. +//! +//! This is transport and lifecycle glue, not another supervisor model. When +//! the control role authorizes `start_agent`, it invokes the existing process +//! supervisor inside the driver-provisioned boundary. + +#![allow(unsafe_code)] + +use std::path::Path; + +#[cfg(target_os = "linux")] +mod linux { + use super::Path; + use std::fs::File; + use std::io::{self, Read, Write}; + use std::mem::size_of; + use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; + use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _, PermissionsExt as _}; + use std::pin::Pin; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::task::{Context, Poll}; + use std::time::Duration; + + use crate::boundary_io::BoundaryRuntimeState; + use crate::delegated::{AgentSignaler, spawn_workload}; + use crate::identity::{DriverIdentity, resolve_process_identity}; + use crate::main_session::{MainOutput, MainSession}; + use crate::network_broker::NetworkBroker; + use crate::process::ProcessStatus; + #[cfg(test)] + use openshell_core::proto::isolation::v1::isolation_boundary_client::IsolationBoundaryClient; + use openshell_core::proto::isolation::v1::{ + BoundaryChunk, + isolation_boundary_server::{IsolationBoundary, IsolationBoundaryServer}, + }; + use openshell_core::provider_credentials::ProviderCredentialState; + use openshell_isolation_interface::contract::{ + BoundaryExec, BoundaryPortForward, BoundaryProcess, BoundaryTerminal, CapabilityEvidence, + ExecSession, LoopbackTarget, ResolvedWorkloadIdentity, SandboxConfirmEvidence, + }; + use openshell_isolation_interface::mediation::{ + self, DnsQueryWire, MediationFrame, MediationFrameKind, + }; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio_stream::wrappers::ReceiverStream; + + use openshell_isolation_interface::boundary_protocol::{ + AgentSpecWire, BinaryIdentityWire, BoundaryConfig, + BoundaryListener as BoundaryListenerConfig, DnsQueryResultWire, ExecSpecWire, + ExitStatusWire, MediationTimingWire, OutputWindowWire, ProcessKindWire, + ProcessSnapshotWire, Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_DNS_ACK, + STREAM_DNS_RESPONSE, STREAM_EXIT, STREAM_NETWORK_DECISION, STREAM_STDERR, STREAM_STDIN, + STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SessionSnapshotWire, SignalWire, + encode_frame, read_frame, read_stream_frame, validate_resource_claims, write_frame, + write_stream_frame, + }; + + const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + const MAX_CONTROL_CONNECTIONS: usize = 128; + const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096; + const MAX_RETAINED_EXEC_PROCESSES: usize = 64; + + fn duration_micros(duration: Duration) -> u64 { + u64::try_from(duration.as_micros()).unwrap_or(u64::MAX) + } + + struct ControlConnectionSlot(Arc); + + impl Drop for ControlConnectionSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + + fn acquire_control_connection_slot(active: &Arc) -> Option { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < MAX_CONTROL_CONNECTIONS).then_some(current + 1) + }) + .ok() + .map(|_| ControlConnectionSlot(active.clone())) + } + static BOUNDARY_TERMINATION_REQUESTED: AtomicBool = AtomicBool::new(false); + + extern "C" fn request_boundary_termination(_signal: libc::c_int) { + BOUNDARY_TERMINATION_REQUESTED.store(true, Ordering::Release); + } + + pub fn run_boundary( + config_path: &Path, + qualification: crate::RuntimeQualification, + ) -> Result<(), String> { + install_boundary_signal_handlers()?; + make_boundary_nondumpable()?; + disable_core_dumps()?; + let bytes = std::fs::read(config_path) + .map_err(|error| format!("read boundary config {}: {error}", config_path.display()))?; + let config: BoundaryConfig = serde_json::from_slice(&bytes).map_err(|error| { + format!("decode boundary config {}: {error}", config_path.display()) + })?; + validate_config(&config)?; + validate_runtime_resource_claims(&config)?; + validate_running_identity(&config.workload_identity)?; + std::fs::remove_file(config_path).map_err(|error| { + format!("consume boundary config {}: {error}", config_path.display()) + })?; + let child_env = serde_json::to_string(&config.child_env) + .map_err(|error| format!("encode boundary workload environment: {error}"))?; + // This runs before the Tokio runtime or control threads exist. The process + // supervisor consumes the serialized map and applies values only to + // workload children. + unsafe { + std::env::set_var(openshell_core::sandbox_env::USER_ENVIRONMENT, child_env); + } + crate::sandbox::apply_supervisor_startup_hardening() + .map_err(|error| format!("install sandbox process prelude: {error}"))?; + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .map_err(|error| format!("start sandbox workload launcher: {error}"))?; + crate::process::configure_workload_launcher(launcher.clone()) + .map_err(|error| format!("configure sandbox workload launcher: {error}"))?; + let network_broker = NetworkBroker::start(listener) + .map_err(|error| format!("start sandbox network broker: {error}"))?; + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create boundary process runtime: {error}"))?; + let runtime = Arc::new(BoundaryRuntime::new( + config.clone(), + process_runtime.handle().clone(), + network_broker, + launcher, + qualification, + )); + serve(&config.listener, config.multiplexed, runtime) + } + + fn make_boundary_nondumpable() -> Result<(), String> { + // SAFETY: PR_SET_DUMPABLE accepts one scalar flag. The sandbox keeps + // bootstrap and protected-channel keys in memory after this point. + if unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) } == 0 { + Ok(()) + } else { + Err(format!( + "make sandbox process nondumpable: {}", + io::Error::last_os_error() + )) + } + } + + fn disable_core_dumps() -> Result<(), String> { + let limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `limit` is a valid immutable rlimit value. + if unsafe { libc::setrlimit(libc::RLIMIT_CORE, &raw const limit) } == 0 { + Ok(()) + } else { + Err(format!( + "disable sandbox core dumps: {}", + io::Error::last_os_error() + )) + } + } + + fn install_boundary_signal_handlers() -> Result<(), String> { + BOUNDARY_TERMINATION_REQUESTED.store(false, Ordering::Release); + let action = nix::sys::signal::SigAction::new( + nix::sys::signal::SigHandler::Handler(request_boundary_termination), + nix::sys::signal::SaFlags::empty(), + nix::sys::signal::SigSet::empty(), + ); + for signal in [ + nix::sys::signal::Signal::SIGTERM, + nix::sys::signal::Signal::SIGINT, + ] { + // SAFETY: the installed handler only performs a lock-free atomic + // store, which is async-signal-safe, and remains valid for the + // lifetime of the boundary process. + unsafe { nix::sys::signal::sigaction(signal, &action) } + .map_err(|error| format!("install boundary {signal:?} handler: {error}"))?; + } + Ok(()) + } + + fn validate_config(config: &BoundaryConfig) -> Result<(), String> { + if config.boundary_id.is_empty() { + return Err("boundary ID must not be empty".to_string()); + } + if config.generation.is_empty() || config.session_epoch.is_empty() { + return Err("boundary generation and session epoch must not be empty".to_string()); + } + if config.bootstrap_token.len() < 32 { + return Err("boundary bootstrap token must contain at least 32 bytes".to_string()); + } + validate_resource_claims(&config.resource_claims).map_err(|error| error.to_string())?; + config + .driver_fence + .validate_for_backend(config.driver_fence.backend_name()) + .map_err(|error| error.to_string())?; + for (claim, path) in &config.resource_claim_files { + if !config.resource_claims.contains_key(claim) { + return Err(format!( + "runtime resource-claim file refers to unknown claim {claim}" + )); + } + if !path.is_absolute() { + return Err(format!( + "runtime resource-claim file for {claim} must be absolute" + )); + } + } + match &config.listener { + BoundaryListenerConfig::Unix { socket_path, tls } + if !socket_path.is_absolute() || !tls_paths_are_absolute(tls) => + { + return Err("boundary Unix socket path must be absolute".to_string()); + } + BoundaryListenerConfig::TlsTcp { address, tls } + if address.port() == 0 || !tls_paths_are_absolute(tls) => + { + return Err( + "boundary TLS listener requires a nonzero port and absolute certificate paths" + .to_string(), + ); + } + BoundaryListenerConfig::Vsock { + control_port: 0, .. + } => { + return Err("boundary control port must be nonzero".to_string()); + } + BoundaryListenerConfig::Unix { .. } + | BoundaryListenerConfig::TlsTcp { .. } + | BoundaryListenerConfig::Vsock { .. } => {} + } + if config.workload_identity.uid == 0 || config.workload_identity.gid == 0 { + return Err("sandbox workload UID and GID must be nonzero".to_string()); + } + Ok(()) + } + + fn tls_paths_are_absolute( + tls: &openshell_isolation_interface::boundary_protocol::BoundaryServerTls, + ) -> bool { + tls.certificate_chain_path.is_absolute() + && tls.private_key_path.is_absolute() + && tls.client_ca_certificate_path.is_absolute() + } + + fn validate_runtime_resource_claims(config: &BoundaryConfig) -> Result<(), String> { + for (claim, path) in &config.resource_claim_files { + let expected = config + .resource_claims + .get(claim) + .expect("validated resource-claim file key"); + let observed = std::fs::read_to_string(path).map_err(|error| { + format!( + "read runtime resource claim {claim} from {}: {error}", + path.display() + ) + })?; + if observed.trim() != expected { + return Err(format!( + "runtime resource claim {claim} does not match the admitted resource" + )); + } + } + Ok(()) + } + + fn normalized_supplementary_groups(mut groups: Vec, primary_gid: u32) -> Vec { + groups.retain(|gid| *gid != primary_gid); + groups.sort_unstable(); + groups.dedup(); + groups + } + + #[allow(clippy::similar_names)] + fn validate_running_identity(expected: &ResolvedWorkloadIdentity) -> Result<(), String> { + let mut real_uid = 0; + let mut effective_uid = 0; + let mut saved_uid = 0; + let mut real_gid = 0; + let mut effective_gid = 0; + let mut saved_gid = 0; + // SAFETY: all pointers refer to live scalar output storage. + if unsafe { + libc::getresuid( + &raw mut real_uid, + &raw mut effective_uid, + &raw mut saved_uid, + ) + } != 0 + || unsafe { + libc::getresgid( + &raw mut real_gid, + &raw mut effective_gid, + &raw mut saved_gid, + ) + } != 0 + { + return Err(format!( + "measure sandbox identity: {}", + io::Error::last_os_error() + )); + } + if [real_uid, effective_uid, saved_uid] + .iter() + .any(|uid| *uid != expected.uid) + || [real_gid, effective_gid, saved_gid] + .iter() + .any(|gid| *gid != expected.gid) + { + return Err(format!( + "sandbox identity does not match resolved workload {}:{}", + expected.uid, expected.gid + )); + } + // SAFETY: a null buffer with size zero queries the group count. + let count = unsafe { libc::getgroups(0, std::ptr::null_mut()) }; + if count < 0 { + return Err(format!( + "measure sandbox supplementary groups: {}", + io::Error::last_os_error() + )); + } + let mut groups = vec![0_u32; usize::try_from(count).unwrap_or(0)]; + if count > 0 { + // SAFETY: groups has capacity for exactly `count` gid_t values. + if unsafe { libc::getgroups(count, groups.as_mut_ptr()) } != count { + return Err(format!( + "read sandbox supplementary groups: {}", + io::Error::last_os_error() + )); + } + } + let groups = normalized_supplementary_groups(groups, expected.gid); + if groups != expected.supplementary_gids { + return Err(format!( + "sandbox supplementary groups {groups:?} do not match resolved workload {:?}", + expected.supplementary_gids + )); + } + Ok(()) + } + + fn serve( + config: &BoundaryListenerConfig, + multiplexed: bool, + runtime: Arc, + ) -> Result<(), String> { + let listener = ControlListener::bind(config) + .map_err(|error| format!("bind boundary control listener: {error}"))?; + let active_connections = Arc::new(AtomicUsize::new(0)); + tracing::info!(?config, "Boundary control listener ready"); + loop { + if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { + runtime.shutdown(); + return Ok(()); + } + match listener.accept() { + Ok(stream) => { + let Some(slot) = acquire_control_connection_slot(&active_connections) else { + tracing::warn!( + limit = MAX_CONTROL_CONNECTIONS, + "Boundary control connection limit reached" + ); + continue; + }; + let runtime = runtime.clone(); + std::thread::spawn(move || { + let _slot = slot; + let stream = match stream.establish(&runtime.process_runtime) { + Ok(stream) => stream, + Err(error) => { + tracing::warn!(%error, "Boundary control transport handshake failed"); + return; + } + }; + let result = if multiplexed { + let stream = match stream.into_tokio() { + Ok(stream) => stream, + Err(error) => { + tracing::warn!(%error, "prepare boundary gRPC session"); + return; + } + }; + runtime + .process_runtime + .block_on(serve_grpc(stream, runtime.clone())) + } else { + serve_one(stream, &runtime) + }; + if let Err(error) = result { + tracing::warn!(%error, "Boundary control session failed: {error}"); + } + }); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(format!("accept boundary control connection: {error}")), + } + } + } + + async fn serve_grpc( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + runtime: Arc, + ) -> Result<(), String> { + let incoming = tokio_stream::iter([Ok::<_, io::Error>(GrpcServerIo(stream))]); + tonic::transport::Server::builder() + .max_concurrent_streams( + u32::try_from(MAX_CONTROL_CONNECTIONS) + .expect("control connection limit fits in HTTP/2 settings"), + ) + .initial_stream_window_size(16 * 1024 * 1024) + .initial_connection_window_size(16 * 1024 * 1024) + .add_service( + IsolationBoundaryServer::new(GrpcBoundaryService { runtime }) + .max_decoding_message_size(64 * 1024) + .max_encoding_message_size(64 * 1024), + ) + .serve_with_incoming(incoming) + .await + .map_err(|error| format!("serve boundary gRPC connection: {error}")) + } + + struct GrpcServerIo(openshell_isolation_interface::contract::BoundaryDuplexStream); + + impl tokio::io::AsyncRead for GrpcServerIo { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_read(context, buffer) + } + } + + impl tokio::io::AsyncWrite for GrpcServerIo { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + Pin::new(&mut self.0).poll_write(context, buffer) + } + + fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(context) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(context) + } + } + + impl tonic::transport::server::Connected for GrpcServerIo { + type ConnectInfo = (); + + fn connect_info(&self) -> Self::ConnectInfo {} + } + + #[derive(Clone)] + struct GrpcBoundaryService { + runtime: Arc, + } + + type GrpcResponseStream = ReceiverStream>; + + #[tonic::async_trait] + impl IsolationBoundary for GrpcBoundaryService { + type ExchangeStream = GrpcResponseStream; + type MediateStream = GrpcResponseStream; + + async fn exchange( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + let (stream, response) = bridge_grpc_server_stream(request.into_inner()); + let runtime = self.runtime.clone(); + tokio::task::spawn_blocking(move || { + let stream = ControlStream::Grpc { + stream: Some(stream), + runtime: runtime.process_runtime.clone(), + }; + if let Err(error) = serve_one(stream, &runtime) { + tracing::warn!(%error, "Boundary gRPC exchange failed"); + } + }); + Ok(tonic::Response::new(response)) + } + + async fn mediate( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + let (stream, response) = bridge_grpc_server_stream(request.into_inner()); + let runtime = self.runtime.clone(); + tokio::spawn(async move { + if let Err(error) = serve_persistent_mediation(stream, runtime).await { + tracing::warn!(%error, "Persistent boundary mediation ended"); + } + }); + Ok(tonic::Response::new(response)) + } + } + + fn bridge_grpc_server_stream( + mut inbound: tonic::Streaming, + ) -> (tokio::io::DuplexStream, GrpcResponseStream) { + let (application, bridge) = tokio::io::duplex(256 * 1024); + let (mut reader, mut writer) = tokio::io::split(bridge); + let (outbound, outbound_rx) = + tokio::sync::mpsc::channel::>(64); + tokio::spawn(async move { + loop { + match inbound.message().await { + Ok(Some(chunk)) => { + if writer.write_all(&chunk.data).await.is_err() { + return; + } + } + Ok(None) => { + let _ = writer.shutdown().await; + return; + } + Err(error) => { + tracing::debug!(%error, "Boundary gRPC request stream ended"); + return; + } + } + } + }); + tokio::spawn(async move { + let mut buffer = vec![0_u8; 16 * 1024]; + loop { + let read = match reader.read(&mut buffer).await { + Ok(read) => read, + Err(error) => { + tracing::debug!(%error, "Boundary gRPC response reader ended"); + return; + } + }; + if read == 0 { + return; + } + if outbound + .send(Ok(BoundaryChunk { + data: buffer[..read].to_vec(), + })) + .await + .is_err() + { + return; + } + } + }); + (application, ReceiverStream::new(outbound_rx)) + } + + const MEDIATION_EVENT_QUEUE: usize = 256; + const MEDIATION_ROUTE_QUEUE: usize = 64; + + struct BoundaryOutboundFrame { + kind: MediationFrameKind, + stream_id: u64, + payload: Vec, + } + + type BoundaryMediationRoutes = Arc< + tokio::sync::Mutex< + std::collections::HashMap>, + >, + >; + + struct MediationLease(Arc); + + impl Drop for MediationLease { + fn drop(&mut self) { + self.0.mediation_active.store(false, Ordering::Release); + } + } + + async fn serve_persistent_mediation( + mut stream: tokio::io::DuplexStream, + runtime: Arc, + ) -> Result<(), String> { + let request: RequestEnvelope = + openshell_isolation_interface::boundary_protocol::read_frame_async(&mut stream) + .await + .map_err(|error| format!("read mediation attach: {error}"))?; + let request_id = request.request_id.clone(); + if !matches!(request.request, Request::OpenMediation) { + return Err("persistent mediation stream omitted OpenMediation".to_string()); + } + let mut response = runtime.dispatch(request); + let lease = if matches!(response, Response::MediationReady) { + if runtime + .mediation_active + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + Some(MediationLease(runtime.clone())) + } else { + response = guest_error("denied", "a mediation session is already active"); + None + } + } else { + None + }; + let response_frame = encode_frame(&ResponseEnvelope { + request_id, + response, + }) + .map_err(|error| format!("encode mediation attach response: {error}"))?; + stream + .write_all(&response_frame) + .await + .map_err(|error| format!("write mediation attach response: {error}"))?; + stream + .flush() + .await + .map_err(|error| format!("flush mediation attach response: {error}"))?; + let Some(_lease) = lease else { + return Ok(()); + }; + let broker = runtime.network_accept_context()?; + run_boundary_mediation(stream, runtime, broker).await + } + + async fn run_boundary_mediation( + stream: tokio::io::DuplexStream, + runtime: Arc, + broker: NetworkBroker, + ) -> Result<(), String> { + let (mut reader, mut writer) = tokio::io::split(stream); + let (outbound_tx, mut outbound_rx) = + tokio::sync::mpsc::channel::(MEDIATION_EVENT_QUEUE); + let routes: BoundaryMediationRoutes = + Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())); + let writer_task = async { + while let Some(frame) = outbound_rx.recv().await { + mediation::write_frame(&mut writer, frame.kind, frame.stream_id, &frame.payload) + .await + .map_err(|error| format!("write persistent mediation frame: {error}"))?; + } + Ok::<(), String>(()) + }; + let reader_routes = routes.clone(); + let reader_task = async { + while let Some(frame) = mediation::read_frame(&mut reader) + .await + .map_err(|error| format!("read persistent mediation frame: {error}"))? + { + let route = reader_routes.lock().await.get(&frame.stream_id).cloned(); + if let Some(route) = route { + let _ = route.send(frame).await; + } + } + Ok::<(), String>(()) + }; + let accept_task = + run_boundary_accepts(runtime, broker, outbound_tx.clone(), routes.clone()); + tokio::pin!(writer_task); + tokio::pin!(reader_task); + tokio::pin!(accept_task); + let result = tokio::select! { + result = &mut writer_task => result, + result = &mut reader_task => result, + result = &mut accept_task => result, + }; + routes.lock().await.clear(); + result + } + + async fn run_boundary_accepts( + runtime: Arc, + broker: NetworkBroker, + outbound: tokio::sync::mpsc::Sender, + routes: BoundaryMediationRoutes, + ) -> Result<(), String> { + loop { + let pending = broker + .accept_dns() + .await + .map_err(|error| format!("accept sandbox DNS query: {error}"))?; + let stream_id = runtime + .next_mediation_stream_id + .fetch_add(1, Ordering::Relaxed); + let (route_tx, route_rx) = tokio::sync::mpsc::channel(MEDIATION_ROUTE_QUEUE); + routes.lock().await.insert(stream_id, route_tx); + tokio::spawn(run_boundary_dns_stream( + stream_id, + pending, + route_rx, + outbound.clone(), + routes.clone(), + )); + } + } + + async fn run_boundary_dns_stream( + stream_id: u64, + pending: crate::network_broker::PendingDnsQuery, + mut inbound: tokio::sync::mpsc::Receiver, + outbound: tokio::sync::mpsc::Sender, + routes: BoundaryMediationRoutes, + ) { + let query = DnsQueryWire { + request: pending.request.clone(), + transport: pending.transport, + identity: BinaryIdentityWire::from(pending.identity.clone()), + timing: MediationTimingWire { + notification_to_queue_us: duration_micros(pending.notification_to_queue), + queue_wait_us: duration_micros(pending.queued_at.elapsed()), + }, + }; + let Ok(payload) = mediation::encode_json(&query) else { + routes.lock().await.remove(&stream_id); + return; + }; + if outbound + .send(BoundaryOutboundFrame { + kind: MediationFrameKind::DnsQuery, + stream_id, + payload, + }) + .await + .is_err() + { + routes.lock().await.remove(&stream_id); + return; + } + let result = match inbound.recv().await { + Some(MediationFrame { + kind: MediationFrameKind::DnsResponse, + payload, + .. + }) => mediation::decode_json::(&payload) + .map_err(io::Error::other) + .and_then(|response| match response { + DnsQueryResultWire::Response(response) => Ok(response), + DnsQueryResultWire::Error(error) => Err(io::Error::other(error)), + }), + _ => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "mediation session ended before DNS response", + )), + }; + let _ = pending.complete(result); + routes.lock().await.remove(&stream_id); + } + + fn serve_one(mut stream: ControlStream, runtime: &BoundaryRuntime) -> Result<(), String> { + stream + .set_timeout(CONTROL_IO_TIMEOUT) + .map_err(|error| format!("set control timeout: {error}"))?; + let request: RequestEnvelope = + read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?; + if !runtime.authenticate(&request) { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control authentication failed"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + if request.validate_payload_digest().is_err() { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control request payload digest mismatch"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + match request.request.clone() { + Request::Exec { spec } => { + let started = + match runtime.start_exec(&request.request_id, &request.payload_digest, spec) { + Ok(started) => started, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write exec error response: {error}")); + } + }; + if let Err(error) = write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ExecStarted { + process_id: started.process_id.clone(), + pty: started.terminal, + }, + }, + ) { + return Err(format!("write exec start response: {error}")); + } + return runtime.stream_process(stream, started.attachment); + } + Request::AttachProcess { process_id } => { + let (attachment, terminal) = match runtime.attach_process(&process_id) { + Ok(attachment) => attachment, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write process attachment error: {error}")); + } + }; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ProcessAttached { terminal }, + }, + ) + .map_err(|error| format!("write process attachment response: {error}"))?; + return runtime.stream_process(stream, attachment); + } + Request::PortForward { host, port } => { + let target = match LoopbackTarget::new(host, port) + .map_err(|error| format!("validate port-forward target: {error}")) + .and_then(|target| { + runtime + .connect_port(target) + .map_err(|error| format!("connect boundary loopback port: {error}")) + }) { + Ok(target) => target, + Err(error) => { + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: guest_error("failed", error), + }, + ) + .map_err(|error| format!("write port-forward error response: {error}"))?; + return Ok(()); + } + }; + let mut target = target; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::PortConnected, + }, + ) + .map_err(|error| format!("write port-forward response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| format!("bridge boundary loopback stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptNetwork => { + let broker = runtime.network_accept_context()?; + let request_id = request.request_id; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + let mut disconnect_probe = [0_u8; 1]; + let pending = tokio::select! { + biased; + read = stream.read(&mut disconnect_probe) => { + match read { + Ok(0) => return Ok(()), + Ok(_) => return Err("control sent data before network mediation response".to_string()), + Err(error) => return Err(format!("watch network mediation control stream: {error}")), + } + } + pending = broker.accept() => pending + .map_err(|error| format!("accept sandbox network open: {error}"))?, + }; + let response = encode_frame(&ResponseEnvelope { + request_id, + response: Response::NetworkConnected { + identity: BinaryIdentityWire::from(pending.identity.clone()), + destination: pending.destination, + socket: pending.socket, + policy_generation: 0, + timing: MediationTimingWire { + notification_to_queue_us: duration_micros( + pending.notification_to_queue, + ), + queue_wait_us: duration_micros(pending.queued_at.elapsed()), + }, + }, + }) + .map_err(|error| format!("encode network mediation response: {error}"))?; + stream + .write_all(&response) + .await + .map_err(|error| format!("write network mediation response: {error}"))?; + let Some((channel, payload)) = read_stream_frame(&mut stream) + .await + .map_err(|error| format!("read network-open decision: {error}"))? + else { + return Err("control disconnected before network-open decision".to_string()); + }; + if channel != STREAM_NETWORK_DECISION { + return Err(format!( + "unexpected network-open decision channel {channel}" + )); + } + let decision = serde_json::from_slice(&payload) + .map_err(|error| format!("decode network-open decision: {error}"))?; + let Some(target) = pending + .complete(decision) + .await + .map_err(|error| format!("complete sandbox network open: {error}"))? + else { + return Ok(()); + }; + target + .set_nonblocking(true) + .map_err(|error| format!("set sandbox relay nonblocking: {error}"))?; + let mut target = tokio::net::TcpStream::from_std(target) + .map_err(|error| format!("register sandbox relay: {error}"))?; + openshell_core::net::set_tcp_nodelay_best_effort(&target); + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map(|_| ()) + .map_err(|error| format!("bridge sandbox network stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptDns => { + let broker = runtime.network_accept_context()?; + let request_id = request.request_id; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + let mut disconnect_probe = [0_u8; 1]; + let pending = tokio::select! { + biased; + read = stream.read(&mut disconnect_probe) => { + match read { + Ok(0) => return Ok(()), + Ok(_) => return Err("control sent data before DNS mediation response".to_string()), + Err(error) => return Err(format!("watch DNS mediation control stream: {error}")), + } + } + pending = broker.accept_dns() => pending + .map_err(|error| format!("accept sandbox DNS query: {error}"))?, + }; + let response = encode_frame(&ResponseEnvelope { + request_id, + response: Response::DnsQuery { + request: pending.request.clone(), + transport: pending.transport, + identity: BinaryIdentityWire::from(pending.identity.clone()), + timing: MediationTimingWire { + notification_to_queue_us: duration_micros( + pending.notification_to_queue, + ), + queue_wait_us: duration_micros(pending.queued_at.elapsed()), + }, + }, + }) + .map_err(|error| format!("encode DNS mediation response: {error}"))?; + stream + .write_all(&response) + .await + .map_err(|error| format!("write DNS mediation response: {error}"))?; + let Some((channel, payload)) = read_stream_frame(&mut stream) + .await + .map_err(|error| format!("read DNS mediation result: {error}"))? + else { + return Err("control disconnected before DNS response".to_string()); + }; + if channel != STREAM_DNS_RESPONSE { + return Err(format!("unexpected DNS response channel {channel}")); + } + let result: DnsQueryResultWire = serde_json::from_slice(&payload) + .map_err(|error| format!("decode DNS mediation result: {error}"))?; + let result = match result { + DnsQueryResultWire::Response(response) => Ok(response), + DnsQueryResultWire::Error(error) => Err(io::Error::other(error)), + }; + pending + .complete(result) + .map_err(|error| format!("complete sandbox DNS query: {error}"))?; + write_stream_frame(&mut stream, STREAM_DNS_ACK, &[]) + .await + .map_err(|error| format!("acknowledge sandbox DNS response: {error}")) + })?; + return Ok(()); + } + _ => {} + } + let response = ResponseEnvelope { + request_id: request.request_id.clone(), + response: runtime.dispatch(request), + }; + write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}"))?; + Ok(()) + } + + struct BoundaryRuntime { + config: BoundaryConfig, + process_runtime: tokio::runtime::Handle, + state: Mutex, + /// The wire policy bound at first attach, so an idempotent attach retry + /// carrying a different policy is denied instead of silently keeping + /// the first policy. + attached_policy: Mutex>, + /// The complete launch request accepted by the boundary. A replacement + /// control process may replay it after reconnecting, but may not change + /// any launch input or start a second workload. + started_agent: Mutex>, + next_exec_id: AtomicU64, + mediation_active: AtomicBool, + next_mediation_stream_id: AtomicU64, + exec_handles: Mutex>, + replay_ledger: Mutex, + network_broker: NetworkBroker, + workload_launcher: + openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + qualification: crate::RuntimeQualification, + } + + #[derive(Clone)] + struct ExecHandle { + request_id: String, + payload_digest: String, + process: Arc, + terminal: Option>, + session: Arc, + attached: Arc, + status: Arc>>, + } + + struct StartedExec { + process_id: String, + terminal: bool, + attachment: MainAttachment, + } + + #[derive(Clone)] + struct ReplayRecord { + payload_digest: String, + response: Response, + } + + #[derive(Default)] + struct ReplayLedger { + entries: std::collections::HashMap, + order: std::collections::VecDeque, + } + + impl ReplayLedger { + fn get(&self, request_id: &str) -> Option<&ReplayRecord> { + self.entries.get(request_id) + } + + fn insert(&mut self, request_id: String, record: ReplayRecord) { + if let Some(existing) = self.entries.get_mut(&request_id) { + *existing = record; + return; + } + while self.entries.len() >= MAX_REPLAY_LEDGER_ENTRIES { + let Some(oldest) = self.order.pop_front() else { + break; + }; + self.entries.remove(&oldest); + } + self.order.push_back(request_id.clone()); + self.entries.insert(request_id, record); + } + } + + #[derive(Clone, PartialEq, Eq)] + struct StartedAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + } + + impl StartedAgent { + /// Provider environment is mutable runtime state. A replacement + /// control must replay every immutable launch input exactly, then + /// reconcile the current provider snapshot through the CAS update. + fn matches_replay(&self, other: &Self) -> bool { + self.sandbox_id == other.sandbox_id + && self.spec == other.spec + && self.policy == other.policy + && self.ca_cert == other.ca_cert + && self.ca_bundle == other.ca_bundle + } + } + + struct MainAttachment { + session: Arc, + attached: Arc, + status: AttachmentStatus, + } + + enum AttachmentStatus { + Main(Arc), + Exec(Arc>>), + } + + impl MainAttachment { + fn exit_status(&self, fallback_code: i32) -> ExitStatusWire { + match &self.status { + AttachmentStatus::Main(process) => process + .exit_status() + .unwrap_or(ExitStatusWire::Exited(fallback_code)), + AttachmentStatus::Exec(status) => { + (*lock(status)).unwrap_or(ExitStatusWire::Exited(fallback_code)) + } + } + } + } + + impl Drop for MainAttachment { + fn drop(&mut self) { + self.attached.store(false, Ordering::Release); + } + } + + #[allow(clippy::result_large_err)] + fn acquire_exec_attachment(handle: &ExecHandle) -> Result { + acquire_attachment( + handle.session.clone(), + handle.attached.clone(), + AttachmentStatus::Exec(handle.status.clone()), + ) + } + + #[allow(clippy::result_large_err)] + fn acquire_attachment( + session: Arc, + attached: Arc, + status: AttachmentStatus, + ) -> Result { + if attached + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(guest_error( + "denied", + "process already has a control attachment", + )); + } + Ok(MainAttachment { + session, + attached, + status, + }) + } + + enum RuntimeState { + AwaitingAttach, + Bound(PreparedBoundary), + Ready(PreparedBoundary), + Running(Arc), + } + + #[derive(Clone)] + struct PreparedBoundary { + network_broker: NetworkBroker, + } + + impl BoundaryRuntime { + fn new( + config: BoundaryConfig, + process_runtime: tokio::runtime::Handle, + network_broker: NetworkBroker, + workload_launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + qualification: crate::RuntimeQualification, + ) -> Self { + Self { + config, + process_runtime, + state: Mutex::new(RuntimeState::AwaitingAttach), + attached_policy: Mutex::new(None), + started_agent: Mutex::new(None), + next_exec_id: AtomicU64::new(1), + mediation_active: AtomicBool::new(false), + next_mediation_stream_id: AtomicU64::new(1), + exec_handles: Mutex::new(std::collections::HashMap::new()), + replay_ledger: Mutex::new(ReplayLedger::default()), + network_broker, + workload_launcher, + qualification, + } + } + + fn shutdown(&self) { + let process = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + RuntimeState::AwaitingAttach + | RuntimeState::Bound(_) + | RuntimeState::Ready(_) => None, + } + }; + if let Some(process) = process { + process.boundary_runtime.deactivate(); + } + } + + fn dispatch(&self, envelope: RequestEnvelope) -> Response { + if !self.authenticate(&envelope) { + return guest_error("denied", "control authentication failed"); + } + if envelope.validate_payload_digest().is_err() { + return guest_error("denied", "control request payload digest mismatch"); + } + let replayable = envelope.request.is_replayable_mutation(); + let mut replay_ledger = replayable.then(|| lock(&self.replay_ledger)); + if let Some(record) = replay_ledger + .as_ref() + .and_then(|ledger| ledger.get(&envelope.request_id)) + { + return if record.payload_digest == envelope.payload_digest { + record.response.clone() + } else { + guest_error( + "denied", + "control request ID was reused with a different payload", + ) + }; + } + let request_id = envelope.request_id; + let payload_digest = envelope.payload_digest; + let response = match envelope.request { + Request::Attach { + policy, + resource_claims, + } => { + if resource_claims == self.config.resource_claims { + self.attach(*policy) + } else { + guest_error( + "denied", + "topology resource claims do not match the boundary configuration", + ) + } + } + Request::Confirm => self.confirm(), + Request::StartAgent { + sandbox_id, + spec, + policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + } => self.start_agent( + sandbox_id, + spec, + *policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + ), + Request::UpdateProviderEnvironment { + expected_revision, + revision, + provider_env, + } => self.update_provider_environment(expected_revision, revision, provider_env), + Request::Wait { process_id } => self.wait(&process_id), + Request::Signal { process_id, signal } => self.signal(&process_id, signal), + Request::Terminate { process_id } => self.terminate(&process_id), + Request::ExecSignal { process_id, signal } => self.signal_exec(&process_id, signal), + Request::Resize { + process_id, + cols, + rows, + } => self.resize_process(&process_id, cols, rows), + Request::OpenMediation => self.network_accept_context().map_or_else( + |error| guest_error("unavailable", error), + |_| Response::MediationReady, + ), + Request::Exec { .. } + | Request::AttachProcess { .. } + | Request::PortForward { .. } + | Request::AcceptNetwork + | Request::AcceptDns => { + guest_error("invalid", "streaming request used on control path") + } + }; + if let Some(ledger) = replay_ledger.as_mut() { + ledger.insert( + request_id, + ReplayRecord { + payload_digest, + response: response.clone(), + }, + ); + } + response + } + + fn authenticate(&self, envelope: &RequestEnvelope) -> bool { + constant_time_eq( + envelope.boundary_id.as_bytes(), + self.config.boundary_id.as_bytes(), + ) && constant_time_eq( + envelope.bootstrap_token.as_bytes(), + self.config.bootstrap_token.as_bytes(), + ) + } + + #[allow( + clippy::result_large_err, + reason = "protocol errors are returned directly as complete response frames" + )] + fn start_exec( + &self, + request_id: &str, + payload_digest: &str, + spec: ExecSpecWire, + ) -> Result { + let executor = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + process.boundary_exec() + }; + let mut handles = lock(&self.exec_handles); + if let Some((process_id, handle)) = handles + .iter() + .find(|(_, handle)| handle.request_id == request_id) + { + if handle.payload_digest != payload_digest { + return Err(guest_error( + "denied", + "exec request ID was reused with a different payload", + )); + } + if handle.attached.load(Ordering::Acquire) { + return Err(guest_error( + "unavailable", + "prior exec attachment is still being released", + )); + } + return Ok(StartedExec { + process_id: process_id.clone(), + terminal: handle.terminal.is_some(), + attachment: acquire_exec_attachment(handle)?, + }); + } + if handles.len() >= MAX_RETAINED_EXEC_PROCESSES { + let exited = handles + .iter() + .find(|(_, handle)| { + lock(&handle.status).is_some() && !handle.attached.load(Ordering::Acquire) + }) + .map(|(process_id, _)| process_id.clone()); + if let Some(process_id) = exited { + handles.remove(&process_id); + } else { + return Err(guest_error( + "unavailable", + "retained exec process limit reached", + )); + } + } + let session = self + .process_runtime + .block_on(executor.exec(spec.into())) + .map_err(|error| guest_error("failed", error.to_string()))?; + let process_id = format!( + "{}:exec:{}", + self.config.generation, + self.next_exec_id.fetch_add(1, Ordering::Relaxed) + ); + let ExecSession { + process, + stdin, + stdout, + stderr, + terminal, + } = session; + let Some(stdin) = stdin else { + return Err(guest_error( + "failed", + "exec process stdin pipe is unavailable", + )); + }; + let retained = { + let _runtime = self.process_runtime.enter(); + MainSession::from_boundary( + openshell_isolation_interface::contract::ProcessAttachment { + stdin, + stdout, + stderr, + terminal: terminal.clone(), + }, + process.clone(), + ) + }; + let status = Arc::new(Mutex::new(None)); + let wait_process = process.clone(); + let wait_session = retained.clone(); + let wait_status = status.clone(); + self.process_runtime.spawn(async move { + if let Ok(exit_status) = wait_process.wait().await { + *lock(&wait_status) = Some(ExitStatusWire::from(exit_status)); + let exit_code = match exit_status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited( + code, + ) => code, + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + ) => 128 + signal, + }; + let _ = wait_session.finish_remote(exit_code, false).await; + } + }); + let handle = ExecHandle { + request_id: request_id.to_string(), + payload_digest: payload_digest.to_string(), + process, + terminal, + session: retained, + attached: Arc::new(AtomicBool::new(false)), + status, + }; + let terminal = handle.terminal.is_some(); + let attachment = acquire_exec_attachment(&handle)?; + handles.insert(process_id.clone(), handle); + Ok(StartedExec { + process_id, + terminal, + attachment, + }) + } + + fn signal_exec(&self, process_id: &str, signal: SignalWire) -> Response { + let process = lock(&self.exec_handles) + .get(process_id) + .map(|handle| handle.process.clone()); + let Some(process) = process else { + return guest_error("invalid", "unknown exec process ID"); + }; + match self.process_runtime.block_on(process.signal(signal.into())) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn resize_process(&self, process_id: &str, cols: u16, rows: u16) -> Response { + if let Ok(process) = self.running_process(process_id) { + let session = process.main_session(); + if !session.terminal() { + return guest_error("invalid", "agent process has no terminal"); + } + self.process_runtime.block_on(session.resize( + u32::from(cols), + u32::from(rows), + 0, + 0, + )); + return Response::Resized; + } + let terminal = lock(&self.exec_handles) + .get(process_id) + .and_then(|handle| handle.terminal.clone()); + let Some(terminal) = terminal else { + return guest_error("invalid", "exec process has no terminal"); + }; + match self.process_runtime.block_on(terminal.resize(cols, rows)) { + Ok(()) => Response::Resized, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn connect_port( + &self, + target: LoopbackTarget, + ) -> Result { + let port_forward = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err("agent process has not been started".to_string()); + }; + process.port_forward() + }; + self.process_runtime + .block_on(port_forward.connect(target)) + .map_err(|error| error.to_string()) + } + + fn network_accept_context(&self) -> Result { + self.network_broker + .confirm_healthy() + .map_err(|error| format!("sandbox network broker unavailable: {error}"))?; + Ok(self.network_broker.clone()) + } + + #[allow( + clippy::result_large_err, + reason = "protocol errors are returned directly as complete response frames" + )] + fn attach_process(&self, process_id: &str) -> Result<(MainAttachment, bool), Response> { + if let Ok(process) = self.running_process(process_id) { + let session = process.main_session(); + let terminal = session.terminal(); + let attachment = acquire_attachment( + session, + process.attached.clone(), + AttachmentStatus::Main(process), + )?; + return Ok((attachment, terminal)); + } + let handles = lock(&self.exec_handles); + let handle = handles + .get(process_id) + .ok_or_else(|| guest_error("invalid", "unknown process ID"))?; + Ok((acquire_exec_attachment(handle)?, handle.terminal.is_some())) + } + + fn stream_process( + &self, + stream: ControlStream, + attachment: MainAttachment, + ) -> Result<(), String> { + self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_main_stream(stream, attachment).await + }) + } + + fn attach(&self, policy: SandboxPolicyWire) -> Response { + let mut state = lock(&self.state); + let accepted = match &*state { + RuntimeState::AwaitingAttach => { + let prepared = match PreparedBoundary::establish(self.network_broker.clone()) { + Ok(prepared) => prepared, + Err(error) => return guest_error("failed", error), + }; + *lock(&self.attached_policy) = Some(policy); + *state = RuntimeState::Bound(prepared); + true + } + RuntimeState::Bound(_) | RuntimeState::Ready(_) | RuntimeState::Running(_) => { + // Idempotent retry of the same attach; a different policy + // must not be silently coalesced onto the bound boundary. + lock(&self.attached_policy).as_ref() == Some(&policy) + } + }; + drop(state); + if accepted { + Response::Attached { + snapshot: self.session_snapshot(), + } + } else { + guest_error("denied", "attach policy does not match the bound boundary") + } + } + + fn session_snapshot(&self) -> SessionSnapshotWire { + let process = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + RuntimeState::AwaitingAttach + | RuntimeState::Bound(_) + | RuntimeState::Ready(_) => None, + } + }; + let mut processes = process + .into_iter() + .map(|process| { + let (first_sequence, next_sequence, truncated) = + process.main_session().output_window(); + ProcessSnapshotWire { + process_id: process.process_id(), + kind: ProcessKindWire::Main, + terminal: process.main_session().terminal(), + status: process.exit_status(), + retained_output: OutputWindowWire { + first_sequence, + next_sequence, + truncated, + }, + } + }) + .collect::>(); + processes.extend(lock(&self.exec_handles).iter().map(|(process_id, handle)| { + let (first_sequence, next_sequence, truncated) = handle.session.output_window(); + ProcessSnapshotWire { + process_id: process_id.clone(), + kind: ProcessKindWire::Exec, + terminal: handle.terminal.is_some(), + status: *lock(&handle.status), + retained_output: OutputWindowWire { + first_sequence, + next_sequence, + truncated, + }, + } + })); + processes.sort_by(|left, right| left.process_id.cmp(&right.process_id)); + SessionSnapshotWire { + generation: self.config.generation.clone(), + processes, + } + } + + fn confirm(&self) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::Bound(prepared) => { + if let Err(error) = prepared.confirm(&self.process_runtime) { + return guest_error("failed", error); + } + let evidence = match self.measure_confirmation_evidence() { + Ok(evidence) => evidence, + Err(error) => return guest_error("failed", error), + }; + *state = RuntimeState::Ready(prepared.clone()); + Response::Confirmed { + evidence: Box::new(evidence), + } + } + RuntimeState::Ready(_) | RuntimeState::Running(_) => { + self.measure_confirmation_evidence().map_or_else( + |error| guest_error("failed", error), + |evidence| Response::Confirmed { + evidence: Box::new(evidence), + }, + ) + } + RuntimeState::AwaitingAttach => { + guest_error("invalid", "boundary must be attached before confirm") + } + } + } + + fn measure_confirmation_evidence(&self) -> Result { + validate_running_identity(&self.config.workload_identity)?; + self.network_broker + .confirm_healthy() + .map_err(|error| format!("verify sandbox network broker: {error}"))?; + if !self.workload_launcher.is_alive() { + return Err("sandbox workload launcher is not running".to_string()); + } + let status = std::fs::read_to_string("/proc/self/status") + .map_err(|error| format!("read sandbox process status: {error}"))?; + let capabilities = CapabilityEvidence { + inheritable: parse_status_hex(&status, "CapInh")?, + permitted: parse_status_hex(&status, "CapPrm")?, + effective: parse_status_hex(&status, "CapEff")?, + bounding: parse_status_hex(&status, "CapBnd")?, + ambient: parse_status_hex(&status, "CapAmb")?, + }; + let no_new_privileges = parse_status_decimal(&status, "NoNewPrivs")? == 1; + // SAFETY: PR_GET_DUMPABLE reads one scalar process property. + let sandbox_dumpable = unsafe { libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0) } != 0; + let mut core_limit = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrlimit initializes the supplied output value on success. + if unsafe { libc::getrlimit(libc::RLIMIT_CORE, core_limit.as_mut_ptr()) } != 0 { + return Err(format!( + "read sandbox core limit: {}", + io::Error::last_os_error() + )); + } + // SAFETY: successful getrlimit initialized the value. + let core_limit = unsafe { core_limit.assume_init() }; + let (native_architecture, kernel_release) = uname_values()?; + Ok(SandboxConfirmEvidence { + generation: self.config.generation.clone(), + identity: self.config.workload_identity.clone(), + capabilities, + no_new_privileges, + sandbox_dumpable, + child_dumpable: true, + core_limit_zero: core_limit.rlim_cur == 0 && core_limit.rlim_max == 0, + native_architecture, + kernel_release, + seccomp: self.qualification.seccomp, + landlock_abi: self.qualification.landlock_abi, + landlock_allow_deny: self.qualification.landlock_allow_deny, + udp_dns_round_trip: self.qualification.udp_dns_round_trip, + tcp_dns_round_trip: self.qualification.tcp_dns_round_trip, + tcp_allow_round_trip: self.qualification.tcp_allow_round_trip, + tcp_deny_round_trip: self.qualification.tcp_deny_round_trip, + authenticated_supervisor: true, + session_epoch: self.config.session_epoch.clone(), + driver_fence: self.config.driver_fence.clone(), + resource_claims: self.config.resource_claims.clone(), + }) + } + + #[allow(clippy::too_many_arguments)] + fn start_agent( + &self, + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let spec = match resolve_agent_spec(spec) { + Ok(spec) => spec, + Err(error) => return guest_error("failed", error), + }; + let mut state = lock(&self.state); + let requested = StartedAgent { + sandbox_id: sandbox_id.clone(), + spec: spec.clone(), + policy: policy.clone(), + ca_cert: ca_cert.clone(), + ca_bundle: ca_bundle.clone(), + provider_env_revision, + provider_env: provider_env.clone(), + }; + if let RuntimeState::Running(process) = &*state { + return if lock(&self.started_agent) + .as_ref() + .is_some_and(|accepted| accepted.matches_replay(&requested)) + { + Response::Started { + process_id: process.process_id(), + provider_env_revision: process.provider_credentials.snapshot().revision, + } + } else { + guest_error( + "denied", + "start_agent inputs do not match the running boundary", + ) + }; + } + let RuntimeState::Ready(prepared) = &*state else { + return guest_error("invalid", "boundary must be confirmed before start_agent"); + }; + let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { + Ok(paths) => paths, + Err(error) => return guest_error("failed", error), + }; + let mut policy = policy.into(); + let driver_identity = DriverIdentity::Resolved { + uid: self.config.workload_identity.uid, + gid: self.config.workload_identity.gid, + }; + if let Err(error) = resolve_process_identity(&mut policy, &driver_identity) { + return guest_error("failed", error.to_string()); + } + let launch = ManagedProcessLaunch { + process_id: format!("{}:main:0", self.config.generation), + sandbox_id, + spec, + policy, + provider_env_revision, + provider_env, + ca_file_paths, + }; + let process = + match ManagedProcess::spawn(&self.process_runtime, launch, prepared.clone()) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; + let process_id = process.process_id(); + *lock(&self.started_agent) = Some(requested); + *state = RuntimeState::Running(process); + Response::Started { + process_id, + provider_env_revision, + } + } + + fn update_provider_environment( + &self, + expected_revision: u64, + revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let process = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return guest_error( + "invalid", + "agent process must be running before provider environment updates", + ); + }; + process.clone() + }; + let revision = process + .provider_credentials + .compare_and_install_child_env_snapshot(expected_revision, revision, provider_env); + Response::ProviderEnvironmentUpdated { revision } + } + + fn wait(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.wait() { + Ok(status) => Response::Exited { status }, + Err(error) => guest_error("failed", error), + } + } + + fn signal(&self, process_id: &str, signal: SignalWire) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(signal) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("terminated", error), + } + } + + fn terminate(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(SignalWire::Kill) { + Ok(()) => Response::Terminated, + Err(_) if process.has_exited() => Response::Terminated, + Err(error) => guest_error("failed", error), + } + } + + #[allow( + clippy::result_large_err, + reason = "protocol errors are returned directly as complete response frames" + )] + fn running_process(&self, process_id: &str) -> Result, Response> { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + if process.process_id() != process_id { + return Err(guest_error("invalid", "unknown process ID")); + } + Ok(process.clone()) + } + } + + fn parse_status_hex(status: &str, name: &str) -> Result { + let value = status + .lines() + .find_map(|line| { + line.strip_prefix(name) + .and_then(|value| value.strip_prefix(':')) + }) + .map(str::trim) + .ok_or_else(|| format!("sandbox process status omitted {name}"))?; + u64::from_str_radix(value, 16) + .map_err(|error| format!("parse sandbox process status {name}: {error}")) + } + + fn parse_status_decimal(status: &str, name: &str) -> Result { + let value = status + .lines() + .find_map(|line| { + line.strip_prefix(name) + .and_then(|value| value.strip_prefix(':')) + }) + .map(str::trim) + .ok_or_else(|| format!("sandbox process status omitted {name}"))?; + value + .parse::() + .map_err(|error| format!("parse sandbox process status {name}: {error}")) + } + + fn uname_values() -> Result<(String, String), String> { + let mut value = std::mem::MaybeUninit::::zeroed(); + // SAFETY: uname initializes the supplied utsname value on success. + if unsafe { libc::uname(value.as_mut_ptr()) } != 0 { + return Err(format!( + "measure sandbox kernel: {}", + io::Error::last_os_error() + )); + } + // SAFETY: successful uname initialized every fixed-size C string. + let value = unsafe { value.assume_init() }; + Ok((c_char_array(&value.machine), c_char_array(&value.release))) + } + + fn c_char_array(value: &[libc::c_char]) -> String { + let length = value + .iter() + .position(|byte| *byte == 0) + .unwrap_or(value.len()); + let bytes = value[..length] + .iter() + .map(|byte| byte.to_ne_bytes()[0]) + .collect::>(); + String::from_utf8_lossy(&bytes).into_owned() + } + + impl PreparedBoundary { + fn establish(network_broker: NetworkBroker) -> Result { + network_broker + .confirm_healthy() + .map_err(|error| format!("verify sandbox network broker: {error}"))?; + Ok(Self { network_broker }) + } + + fn confirm(&self, _runtime: &tokio::runtime::Handle) -> Result<(), String> { + self.network_broker + .confirm_healthy() + .map_err(|error| format!("verify sandbox network broker: {error}")) + } + } + + fn install_ca_material( + ca_cert: Option>, + ca_bundle: Option>, + ) -> Result, String> { + let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { + (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), + (None, None) => return Ok(None), + _ => { + return Err( + "boundary proxy CA certificate and bundle must be supplied together" + .to_string(), + ); + } + }; + install_ca_material_at(Path::new("/run/openshell-proxy-ca"), &ca_cert, &ca_bundle) + } + + fn install_ca_material_at( + directory: &Path, + ca_cert: &[u8], + ca_bundle: &[u8], + ) -> Result, String> { + use std::io::Write as _; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; + + let parent = directory + .parent() + .ok_or_else(|| "boundary proxy CA directory has no parent".to_string())?; + for path in [parent, directory] { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "boundary proxy CA directory component is a symlink: {}", + path.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!( + "boundary proxy CA directory component is not a directory: {}", + path.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => { + std::fs::create_dir(path).map_err(|error| { + format!( + "create boundary proxy CA directory {}: {error}", + path.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "inspect boundary proxy CA directory {}: {error}", + path.display() + )); + } + } + let current_mode = std::fs::metadata(path) + .map_err(|error| { + format!( + "inspect boundary proxy CA directory permissions {}: {error}", + path.display() + ) + })? + .permissions() + .mode(); + if path == directory { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).map_err( + |error| { + format!( + "set boundary proxy CA directory permissions {}: {error}", + path.display() + ) + }, + )?; + } else if current_mode & 0o111 != 0o111 { + return Err(format!( + "boundary proxy CA parent is not traversable by workload identities: {}", + path.display() + )); + } + } + let ca_path = directory.join("ca.crt"); + let bundle_path = directory.join("ca-bundle.crt"); + for (path, contents, label) in [ + (&ca_path, ca_cert, "boundary proxy CA"), + (&bundle_path, ca_bundle, "boundary proxy CA bundle"), + ] { + let temporary = path.with_extension("tmp"); + if let Ok(metadata) = std::fs::symlink_metadata(&temporary) { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "refusing unsafe temporary {label} path: {}", + temporary.display() + )); + } + std::fs::remove_file(&temporary) + .map_err(|error| format!("remove stale temporary {label}: {error}"))?; + } + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o444) + .custom_flags(libc::O_NOFOLLOW) + .open(&temporary) + .map_err(|error| format!("create temporary {label}: {error}"))?; + if let Err(error) = file + .write_all(contents) + .and_then(|()| file.sync_all()) + .and_then(|()| file.set_permissions(std::fs::Permissions::from_mode(0o444))) + .and_then(|()| std::fs::rename(&temporary, path)) + { + let _ = std::fs::remove_file(&temporary); + return Err(format!("install {label}: {error}")); + } + } + Ok(Some((ca_path, bundle_path))) + } + + type ProcessExit = Result; + type SharedProcessExit = Arc<(Mutex>, Condvar)>; + + struct ManagedProcess { + process_id: String, + signaler: AgentSignaler, + exit: SharedProcessExit, + boundary_exec: Arc, + port_forward: Arc, + main_session: Arc, + attached: Arc, + boundary_runtime: Arc, + provider_credentials: ProviderCredentialState, + } + + struct ManagedProcessLaunch { + process_id: String, + sandbox_id: String, + spec: AgentSpecWire, + policy: openshell_core::policy::SandboxPolicy, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + } + + fn resolve_agent_spec(mut spec: AgentSpecWire) -> Result { + if !spec.program.is_empty() { + return Ok(spec); + } + if !spec.args.is_empty() { + return Err("default agent command cannot include arguments".to_string()); + } + let shell = openshell_core::shell::detect_login_shell(); + if !openshell_core::shell::is_executable(&shell) { + return Err(format!( + "sandbox image does not provide an executable login shell at {shell}" + )); + } + spec.program = shell; + spec.args = vec!["-l".to_string()]; + Ok(spec) + } + + impl ManagedProcess { + fn spawn( + runtime: &tokio::runtime::Handle, + launch: ManagedProcessLaunch, + _prepared: PreparedBoundary, + ) -> Result { + let ManagedProcessLaunch { + process_id, + sandbox_id, + spec, + policy, + provider_env_revision, + provider_env, + ca_file_paths, + } = launch; + debug_assert!(!spec.program.is_empty()); + let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + provider_env_revision, + provider_env.clone(), + ); + let mut spawned = runtime + .block_on(spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + Some(&sandbox_id), + None, + None, + false, + &policy, + entrypoint_pid, + None, + provider_credentials.clone(), + provider_env, + ca_file_paths, + Some(boundary_runtime.clone()), + )) + .map_err(|error| format!("start process supervisor leaf: {error:?}"))?; + let signaler = spawned.signaler(); + let boundary_exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + let main_session = spawned.main_session(); + let exit = Arc::new((Mutex::new(None), Condvar::new())); + let reaper_exit = exit.clone(); + runtime.spawn(async move { + let result = spawned + .wait() + .await + .map(process_status) + .map_err(|error| format!("wait for process supervisor leaf: {error}")); + let (state, changed) = &*reaper_exit; + *lock(state) = Some(result); + changed.notify_all(); + }); + Ok(Self { + process_id, + signaler, + exit, + boundary_exec, + port_forward, + main_session, + attached: Arc::new(AtomicBool::new(false)), + boundary_runtime, + provider_credentials, + }) + } + + fn process_id(&self) -> String { + self.process_id.clone() + } + + fn wait(&self) -> ProcessExit { + let (state, changed) = &*self.exit; + let mut exit = lock(state); + while exit.is_none() { + exit = changed + .wait(exit) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + exit.as_ref().expect("exit checked above").clone() + } + + fn signal(&self, signal: SignalWire) -> Result<(), String> { + if self.has_exited() { + return Err("agent process has already exited".to_string()); + } + let result = match signal { + SignalWire::Term => self.signaler.term(), + SignalWire::Kill => self.signaler.kill(), + SignalWire::Int => self.signaler.interrupt(), + SignalWire::Hup => self.signaler.hangup(), + }; + result.map_err(|error| format!("signal process supervisor group: {error}")) + } + + fn has_exited(&self) -> bool { + let (state, _) = &*self.exit; + lock(state).is_some() + } + + fn exit_status(&self) -> Option { + let (state, _) = &*self.exit; + lock(state).as_ref().and_then(|result| result.clone().ok()) + } + + fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + fn main_session(&self) -> Arc { + self.main_session.clone() + } + } + + impl Drop for ManagedProcess { + fn drop(&mut self) { + self.boundary_runtime.deactivate(); + } + } + + async fn bridge_main_stream( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + attachment: MainAttachment, + ) -> Result<(), String> { + let session = attachment.session.clone(); + let (mut reader, writer) = tokio::io::split(stream); + let writer = Arc::new(tokio::sync::Mutex::new(writer)); + let input = session.acquire_input_if_open().map_err(str::to_string)?; + let owner = input.as_ref().map(|(owner, _)| *owner); + let mut output = session.subscribe(); + let input_session = session.clone(); + let mut input_task = tokio::spawn(async move { + let mut input = input.map(|(_, input)| input); + while let Some((channel, payload)) = read_stream_frame(&mut reader).await? { + match channel { + STREAM_STDIN => { + let Some(input) = input.as_ref() else { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "main process stdin already closed", + )); + }; + input.send(payload).await.map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "main process stdin closed") + })?; + } + // Keep reading after stdin closes so transport EOF still + // releases this control process's attachment lease. + STREAM_STDIN_CLOSED => { + input.take(); + if let Some(owner) = owner { + input_session.close_input(owner).await; + } + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-boundary main stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }); + let result = loop { + let output_message = tokio::select! { + input_result = &mut input_task => { + break match input_result { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(format!("read main process attachment: {error}")), + Err(error) => Err(format!("join main process input stream: {error}")), + }; + } + output_message = output.recv() => output_message, + }; + let (channel, payload) = match output_message { + Ok(MainOutput::Stdout(payload)) => (STREAM_STDOUT, payload.to_vec()), + Ok(MainOutput::Stderr(payload)) => (STREAM_STDERR, payload.to_vec()), + Ok(MainOutput::Exit(code)) => { + let status = serde_json::to_vec(&attachment.exit_status(code)) + .map_err(|error| format!("encode main process exit: {error}"))?; + break write_stream_frame(&mut *writer.lock().await, STREAM_EXIT, &status) + .await + .map_err(|error| format!("write main process exit: {error}")); + } + Err(error) => { + tracing::warn!( + skipped_chunks = error.skipped, + "main process attachment resumed after dropping retained output" + ); + continue; + } + }; + if let Err(error) = + write_stream_frame(&mut *writer.lock().await, channel, &payload).await + { + break Err(format!("write main process output: {error}")); + } + }; + input_task.abort(); + if let Some(owner) = owner { + session.release_input(owner); + } + result + } + + fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn process_status(status: ProcessStatus) -> ExitStatusWire { + status.signal().map_or_else( + || ExitStatusWire::Exited(status.code()), + ExitStatusWire::Signaled, + ) + } + + fn guest_error(kind: &str, message: impl Into) -> Response { + Response::Error { + kind: kind.to_string(), + message: message.into(), + } + } + + fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + let left_byte = left.get(index).copied().unwrap_or_default(); + let right_byte = right.get(index).copied().unwrap_or_default(); + difference |= usize::from(left_byte ^ right_byte); + } + difference == 0 + } + + enum ControlListener { + Vsock { + listener: OwnedFd, + server_config: Arc, + }, + Unix { + listener: std::os::unix::net::UnixListener, + server_config: Arc, + }, + Tcp { + listener: std::net::TcpListener, + server_config: Arc, + }, + } + + impl ControlListener { + fn bind(config: &BoundaryListenerConfig) -> io::Result { + match config { + BoundaryListenerConfig::Vsock { control_port, tls } => { + let listener = Self::bind_vsock(*control_port)?; + let server_config = Arc::new(load_tls_server_config(tls)?); + Ok(Self::Vsock { + listener, + server_config, + }) + } + BoundaryListenerConfig::Unix { socket_path, tls } => { + remove_owned_stale_control_socket(socket_path)?; + let listener = std::os::unix::net::UnixListener::bind(socket_path)?; + // Mutual TLS makes a same-UID pathname replacement a + // detectable denial of service rather than impersonation. + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o666))?; + listener.set_nonblocking(true)?; + let server_config = Arc::new(load_tls_server_config(tls)?); + Ok(Self::Unix { + listener, + server_config, + }) + } + BoundaryListenerConfig::TlsTcp { address, tls } => { + let listener = std::net::TcpListener::bind(address)?; + listener.set_nonblocking(true)?; + let server_config = Arc::new(load_tls_server_config(tls)?); + Ok(Self::Tcp { + listener, + server_config, + }) + } + } + } + + fn bind_vsock(port: u32) -> io::Result { + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t") + })?; + let address_length = libc::socklen_t::try_from(size_of::()) + .map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t") + })?; + let raw_fd = unsafe { + libc::socket( + libc::AF_VSOCK, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK, + 0, + ) + }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: port, + svm_cid: libc::VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let result = unsafe { + libc::bind( + fd.as_raw_fd(), + (&raw const address).cast::(), + address_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(fd) + } + + #[cfg(test)] + fn tcp_local_addr(&self) -> io::Result { + match self { + Self::Tcp { listener, .. } => listener.local_addr(), + Self::Unix { .. } | Self::Vsock { .. } => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control listener is not TCP", + )), + } + } + + fn accept(&self) -> io::Result { + match self { + Self::Vsock { + listener, + server_config, + } => { + let raw_fd = unsafe { + libc::accept4( + listener.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + if raw_fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ControlStream::PendingTls { + stream: PlainControlStream::Vsock(unsafe { File::from_raw_fd(raw_fd) }), + server_config: server_config.clone(), + }) + } + } + Self::Unix { + listener, + server_config, + } => { + let (stream, _) = listener.accept()?; + Ok(ControlStream::PendingTls { + stream: PlainControlStream::Unix(stream), + server_config: server_config.clone(), + }) + } + Self::Tcp { + listener, + server_config, + } => { + let (stream, _) = listener.accept()?; + if let Err(error) = stream.set_nodelay(true) { + tracing::debug!(%error, "Failed to set boundary TCP_NODELAY"); + } + Ok(ControlStream::PendingTls { + stream: PlainControlStream::Tcp(stream), + server_config: server_config.clone(), + }) + } + } + } + } + + fn remove_owned_stale_control_socket(socket_path: &Path) -> io::Result<()> { + let metadata = match std::fs::symlink_metadata(socket_path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + if !metadata.file_type().is_socket() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!( + "refusing to replace non-socket boundary control path {}", + socket_path.display() + ), + )); + } + // The private channel directory is driver-provisioned. Requiring the + // stale inode to have been created by this exact sandbox identity + // prevents a replacement run from unlinking another principal's path. + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "refusing to replace boundary control socket {} owned by UID {}", + socket_path.display(), + metadata.uid() + ), + )); + } + std::fs::remove_file(socket_path) + } + + fn load_tls_server_config( + tls: &openshell_isolation_interface::boundary_protocol::BoundaryServerTls, + ) -> io::Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certificate_bytes = std::fs::read(&tls.certificate_chain_path)?; + let certificates = rustls_pemfile::certs(&mut certificate_bytes.as_slice()) + .collect::, _>>()?; + if certificates.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS certificate chain contains no certificates", + )); + } + let private_key_bytes = std::fs::read(&tls.private_key_path)?; + let private_key = rustls_pemfile::private_key(&mut private_key_bytes.as_slice())? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS private-key file contains no private key", + ) + })?; + let client_ca_bytes = std::fs::read(&tls.client_ca_certificate_path)?; + let client_ca_certificates = rustls_pemfile::certs(&mut client_ca_bytes.as_slice()) + .collect::, _>>()?; + if client_ca_certificates.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS client CA contains no certificates", + )); + } + let mut client_roots = rustls::RootCertStore::empty(); + for certificate in client_ca_certificates { + client_roots + .add(certificate) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + } + let client_verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(client_roots)) + .build() + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + let config = rustls::ServerConfig::builder() + .with_client_cert_verifier(client_verifier) + .with_single_cert(certificates, private_key) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + for path in [ + &tls.certificate_chain_path, + &tls.private_key_path, + &tls.client_ca_certificate_path, + ] { + std::fs::remove_file(path)?; + } + Ok(config) + } + + enum PlainControlStream { + Vsock(File), + Unix(std::os::unix::net::UnixStream), + Tcp(std::net::TcpStream), + } + + impl PlainControlStream { + fn into_tokio( + self, + ) -> io::Result { + match self { + Self::Vsock(file) => { + let stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(file.into_raw_fd()) }; + stream.set_nonblocking(true)?; + Ok(Box::new(tokio::net::UnixStream::from_std(stream)?)) + } + Self::Unix(stream) => { + stream.set_nonblocking(true)?; + Ok(Box::new(tokio::net::UnixStream::from_std(stream)?)) + } + Self::Tcp(stream) => { + stream.set_nonblocking(true)?; + let stream = tokio::net::TcpStream::from_std(stream)?; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + Ok(Box::new(stream)) + } + } + } + } + + enum ControlStream { + PendingTls { + stream: PlainControlStream, + server_config: Arc, + }, + Tls { + stream: Option< + Box< + tokio_rustls::server::TlsStream< + openshell_isolation_interface::contract::BoundaryDuplexStream, + >, + >, + >, + runtime: tokio::runtime::Handle, + }, + Grpc { + stream: Option, + runtime: tokio::runtime::Handle, + }, + #[cfg(test)] + TestUnix(std::os::unix::net::UnixStream), + } + + impl ControlStream { + fn establish(self, runtime: &tokio::runtime::Handle) -> io::Result { + let Self::PendingTls { + stream, + server_config, + } = self + else { + return Ok(self); + }; + let stream = { + let _guard = runtime.enter(); + stream.into_tokio()? + }; + let acceptor = tokio_rustls::TlsAcceptor::from(server_config); + let stream = runtime.block_on(async { + tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out") + })? + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) + })?; + Ok(Self::Tls { + stream: Some(Box::new(stream)), + runtime: runtime.clone(), + }) + } + + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { + let _ = timeout; + if matches!(self, Self::Tls { .. } | Self::Grpc { .. }) { + return Ok(()); + } + if matches!(self, Self::PendingTls { .. }) { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )); + } + #[cfg(test)] + if let Self::TestUnix(stream) = self { + stream.set_read_timeout(Some(timeout))?; + return stream.set_write_timeout(Some(timeout)); + } + unreachable!("all established sandbox streams use mutual TLS") + } + + fn into_tokio( + self, + ) -> Result { + match self { + Self::Tls { mut stream, .. } => Ok(stream + .take() + .expect("boundary TLS stream can only be converted once")), + Self::PendingTls { .. } => { + Err("boundary TLS stream has not completed its handshake".to_string()) + } + Self::Grpc { mut stream, .. } => Ok(Box::new( + stream + .take() + .expect("gRPC boundary stream can only be converted once"), + )), + #[cfg(test)] + Self::TestUnix(stream) => { + stream + .set_nonblocking(true) + .map_err(|error| format!("set test Unix stream nonblocking: {error}"))?; + Ok(Box::new(tokio::net::UnixStream::from_std(stream).map_err( + |error| format!("register test Unix stream with Tokio: {error}"), + )?)) + } + } + } + } + + impl Read for ControlStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self { + Self::Tls { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .read(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS read timed out") + })? + }), + Self::PendingTls { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + Self::Grpc { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("gRPC boundary stream must be present") + .read(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary read timed out") + })? + }), + #[cfg(test)] + Self::TestUnix(stream) => stream.read(buffer), + } + } + } + + impl Write for ControlStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + match self { + Self::Tls { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .write(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS write timed out") + })? + }), + Self::PendingTls { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + Self::Grpc { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("gRPC boundary stream must be present") + .write(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary write timed out") + })? + }), + #[cfg(test)] + Self::TestUnix(stream) => stream.write(buffer), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Tls { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .flush(), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS flush timed out") + })? + }), + Self::PendingTls { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + Self::Grpc { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("gRPC boundary stream must be present") + .flush(), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary flush timed out") + })? + }), + #[cfg(test)] + Self::TestUnix(stream) => stream.flush(), + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryServerTls, generate_boundary_mutual_tls_material, + }; + + #[test] + fn replay_ledger_evicts_oldest_records_without_disabling_control() { + let mut ledger = ReplayLedger::default(); + for index in 0..=MAX_REPLAY_LEDGER_ENTRIES { + ledger.insert( + format!("request-{index}"), + ReplayRecord { + payload_digest: format!("digest-{index}"), + response: Response::Signaled, + }, + ); + } + assert!(ledger.get("request-0").is_none()); + assert!( + ledger + .get(&format!("request-{MAX_REPLAY_LEDGER_ENTRIES}")) + .is_some() + ); + assert_eq!(ledger.entries.len(), MAX_REPLAY_LEDGER_ENTRIES); + } + + #[test] + fn scratch_agent_command_resolves_inside_the_workload_filesystem() { + let resolved = resolve_agent_spec(AgentSpecWire { + program: String::new(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 0, + interactive: true, + }) + .expect("resolve scratch command"); + + assert!(openshell_core::shell::is_executable(&resolved.program)); + assert_eq!(resolved.args, vec!["-l".to_string()]); + assert_eq!(resolved.workdir.as_deref(), Some("/sandbox")); + assert!(resolved.interactive); + } + + fn placeholder_server_tls() -> BoundaryServerTls { + BoundaryServerTls { + certificate_chain_path: Path::new("/tmp/openshell-sandbox.crt").to_path_buf(), + private_key_path: Path::new("/tmp/openshell-sandbox.key").to_path_buf(), + client_ca_certificate_path: Path::new("/tmp/openshell-client-ca.crt").to_path_buf(), + } + } + + fn stage_test_tls( + directory: &Path, + prefix: &str, + ) -> (BoundaryServerTls, BoundaryClientTls) { + let material = generate_boundary_mutual_tls_material().expect("generate test TLS"); + let certificate_chain_path = directory.join(format!("{prefix}-sandbox.crt")); + let private_key_path = directory.join(format!("{prefix}-sandbox.key")); + let client_ca_certificate_path = directory.join(format!("{prefix}-client-ca.crt")); + std::fs::write(&certificate_chain_path, material.sandbox_certificate_pem) + .expect("write sandbox certificate"); + std::fs::write(&private_key_path, material.sandbox_private_key_pem) + .expect("write sandbox key"); + std::fs::write(&client_ca_certificate_path, &material.ca_certificate_pem) + .expect("write client CA"); + ( + BoundaryServerTls { + certificate_chain_path, + private_key_path, + client_ca_certificate_path, + }, + BoundaryClientTls { + server_name: material.server_name, + ca_certificate_pem: material.ca_certificate_pem, + certificate_chain_pem: material.supervisor_certificate_pem, + private_key_pem: material.supervisor_private_key_pem, + }, + ) + } + + fn test_client_config(tls: &BoundaryClientTls) -> rustls::ClientConfig { + let mut roots = rustls::RootCertStore::empty(); + for certificate in rustls_pemfile::certs(&mut tls.ca_certificate_pem.as_bytes()) { + roots + .add(certificate.expect("parse test CA")) + .expect("add test CA"); + } + let certificates = rustls_pemfile::certs(&mut tls.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .expect("parse test client certificate"); + let private_key = rustls_pemfile::private_key(&mut tls.private_key_pem.as_bytes()) + .expect("parse test client key") + .expect("test client key"); + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_client_auth_cert(certificates, private_key) + .expect("build test client config") + } + + #[test] + fn boundary_config_debug_redacts_token() { + let config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + multiplexed: false, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }; + let debug = format!("{config:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn installed_proxy_ca_is_readable_by_a_non_root_workload_identity() { + use std::os::unix::fs::PermissionsExt as _; + + let root = tempfile::tempdir().expect("temporary CA root"); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let directory = root.path().join("openshell-proxy-ca"); + let (ca_path, bundle_path) = install_ca_material_at( + &directory, + b"public test certificate", + b"public test bundle", + ) + .expect("install proxy CA") + .expect("CA paths"); + + assert_eq!( + std::fs::metadata(directory.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o111, + 0o111, + "non-root workload identities must be able to traverse the full path" + ); + assert_eq!( + std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, + 0o755, + "non-root workload identities must be able to traverse the CA directory" + ); + for (path, expected) in [ + (&ca_path, b"public test certificate".as_slice()), + (&bundle_path, b"public test bundle".as_slice()), + ] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o444, + "public CA material must be readable by the workload" + ); + assert_eq!(std::fs::read(path).unwrap(), expected); + } + } + + #[test] + fn proxy_ca_install_rejects_a_symlinked_directory() { + let root = tempfile::tempdir().expect("temporary CA root"); + let target = root.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let parent = root.path(); + std::os::unix::fs::symlink(&target, parent.join("openshell-proxy-ca")).unwrap(); + + let error = install_ca_material_at( + &parent.join("openshell-proxy-ca"), + b"certificate", + b"bundle", + ) + .expect_err("symlinked CA directory must fail closed"); + assert!(error.contains("symlink"), "unexpected error: {error}"); + } + + #[test] + fn constant_time_comparison_checks_length_and_content() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"different")); + assert!(!constant_time_eq(b"same", b"sam")); + } + + #[test] + fn supplementary_group_measurement_excludes_the_primary_group() { + assert_eq!( + normalized_supplementary_groups(vec![1002, 1001, 1000, 1001], 1000), + vec![1001, 1002] + ); + } + + #[test] + fn control_connection_slots_bound_unauthenticated_threads() { + let active = Arc::new(AtomicUsize::new(MAX_CONTROL_CONNECTIONS - 1)); + let slot = acquire_control_connection_slot(&active).expect("last available slot"); + assert!(acquire_control_connection_slot(&active).is_none()); + drop(slot); + assert_eq!(active.load(Ordering::Acquire), MAX_CONTROL_CONNECTIONS - 1); + } + + fn test_workload_identity() -> ResolvedWorkloadIdentity { + let mut supplementary_gids = nix::unistd::getgroups() + .unwrap() + .into_iter() + .map(nix::unistd::Gid::as_raw) + .collect::>(); + supplementary_gids.sort_unstable(); + supplementary_gids.dedup(); + ResolvedWorkloadIdentity::new( + nix::unistd::geteuid().as_raw(), + nix::unistd::getegid().as_raw(), + supplementary_gids, + "test".to_string(), + "a".repeat(64), + ) + .unwrap() + } + + fn test_driver_fence() -> openshell_isolation_interface::contract::DriverFenceEvidence { + openshell_isolation_interface::contract::DriverFenceEvidence::Vm { + generation: "generation-1".to_string(), + network_device_count: 0, + } + } + + fn test_runtime_qualification() -> crate::RuntimeQualification { + crate::RuntimeQualification { + seccomp: openshell_isolation_interface::contract::SeccompEvidence { + new_listener: true, + notification_round_trip: true, + id_validation: true, + addfd_send: true, + retained_socket_operation: true, + proc_fd_identity: true, + task_memory_read: true, + task_memory_write: true, + cancellation: true, + }, + landlock_abi: 6, + landlock_allow_deny: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + } + } + + fn test_network_broker() -> ( + NetworkBroker, + openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + ) { + let (launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start() + .expect("start test listener"); + crate::process::configure_workload_launcher(launcher.clone()) + .expect("configure test workload launcher"); + ( + NetworkBroker::start_for_test(listener).expect("start test network broker"), + launcher, + ) + } + + #[test] + fn unix_listener_allows_authenticated_cross_uid_control() { + use std::os::unix::fs::PermissionsExt as _; + + let directory = tempfile::tempdir().expect("temporary directory"); + let socket_path = directory.path().join("control.sock"); + let (tls, _) = stage_test_tls(directory.path(), "initial"); + let _listener = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + tls, + }) + .expect("bind Unix listener"); + let mode = socket_path + .metadata() + .expect("socket metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o666); + } + + #[test] + fn unix_listener_replaces_only_an_owned_stale_socket() { + let directory = tempfile::tempdir().expect("temporary directory"); + let socket_path = directory.path().join("control.sock"); + let (initial_tls, _) = stage_test_tls(directory.path(), "initial"); + drop( + ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + tls: initial_tls, + }) + .expect("bind initial Unix listener"), + ); + let (replacement_tls, _) = stage_test_tls(directory.path(), "replacement"); + let replacement = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + tls: replacement_tls, + }) + .expect("replace owned stale Unix listener"); + + drop(replacement); + std::fs::remove_file(&socket_path).expect("remove stale socket"); + std::fs::write(&socket_path, b"not a socket").expect("write collision"); + let (collision_tls, _) = stage_test_tls(directory.path(), "collision"); + let error = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path, + tls: collision_tls, + }) + .err() + .expect("regular-file collision must fail"); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + } + + #[test] + fn exact_workload_identity_is_required() { + let config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "a".repeat(64), + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + multiplexed: false, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }; + + validate_config(&config).unwrap(); + validate_running_identity(&config.workload_identity).unwrap(); + } + + #[test] + fn runtime_resource_claim_file_must_match_admitted_claim() { + let directory = tempfile::tempdir().expect("temporary directory"); + let pod_uid_path = directory.path().join("pod-uid"); + std::fs::write(&pod_uid_path, "pod-uid-a\n").expect("write runtime claim"); + let mut config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "a".repeat(64), + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + multiplexed: false, + resource_claims: std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + "pod-uid-a".to_string(), + )]), + resource_claim_files: std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + pod_uid_path, + )]), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }; + + validate_config(&config).expect("valid runtime claim configuration"); + validate_runtime_resource_claims(&config).expect("matching runtime claim"); + + config.resource_claims.insert( + "kubernetes.pod_uid".to_string(), + "replacement-pod-uid".to_string(), + ); + assert!(validate_runtime_resource_claims(&config).is_err()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn grpc_server_dispatches_authenticated_logical_streams() { + let (workload_launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start() + .expect("start multiplexed test listener"); + let network_broker = + NetworkBroker::start_for_test(listener).expect("start multiplexed test broker"); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "sandbox-multiplexed".to_string(), + generation: "generation-multiplexed".to_string(), + session_epoch: "session-multiplexed".to_string(), + bootstrap_token: "a".repeat(32), + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().expect("control address"), + tls: placeholder_server_tls(), + }, + multiplexed: true, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }, + tokio::runtime::Handle::current(), + network_broker, + workload_launcher, + test_runtime_qualification(), + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind gRPC test listener"); + let address = listener.local_addr().expect("gRPC test address"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept gRPC client"); + serve_grpc(Box::new(stream), boundary).await + }); + let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .expect("valid gRPC endpoint") + .connect() + .await + .expect("connect gRPC client"); + let policy = SandboxPolicyWire::from(openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }); + let request = RequestEnvelope::new( + "sandbox-multiplexed".to_string(), + "a".repeat(32), + Request::Attach { + policy: Box::new(policy), + resource_claims: std::collections::BTreeMap::new(), + }, + ) + .expect("encode attach request"); + let request_stream = tokio_stream::iter([BoundaryChunk { + data: encode_frame(&request).expect("encode logical request"), + }]); + let mut body = IsolationBoundaryClient::new(channel) + .exchange(request_stream) + .await + .expect("exchange logical request") + .into_inner(); + let mut frame = Vec::new(); + while let Some(chunk) = body.message().await.expect("read gRPC response") { + frame.extend_from_slice(&chunk.data); + } + let response: ResponseEnvelope = + openshell_isolation_interface::boundary_protocol::decode_frame(&frame) + .expect("decode logical response"); + assert!(matches!(response.response, Response::Attached { .. })); + server.abort(); + } + + #[test] + fn tls_listener_preserves_session_when_control_switches_to_async_streaming() { + let directory = tempfile::tempdir().expect("temporary directory"); + let (server_tls, client_tls) = stage_test_tls(directory.path(), "stream"); + let listener = ControlListener::bind(&BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:0".parse().expect("valid address"), + tls: server_tls, + }) + .expect("bind TLS listener"); + let address = listener.tcp_local_addr().expect("TLS listener address"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test runtime"); + let server_runtime = runtime.handle().clone(); + let server = std::thread::spawn(move || { + let mut stream = loop { + match listener.accept() { + Ok(stream) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::yield_now(); + } + Err(error) => panic!("accept TLS stream: {error}"), + } + } + .establish(&server_runtime) + .expect("establish TLS stream"); + let mut first = [0_u8; 4]; + Read::read_exact(&mut stream, &mut first).expect("read blocking TLS phase"); + assert_eq!(&first, b"sync"); + Write::write_all(&mut stream, b"ack1").expect("write blocking TLS phase"); + server_runtime.block_on(async move { + let mut stream = stream.into_tokio().expect("convert negotiated TLS stream"); + let mut second = [0_u8; 5]; + stream + .read_exact(&mut second) + .await + .expect("read async TLS phase"); + assert_eq!(&second, b"async"); + stream + .write_all(b"ack2") + .await + .expect("write async TLS phase"); + }); + }); + + runtime.block_on(async { + let client_config = test_client_config(&client_tls); + let stream = tokio::net::TcpStream::connect(address) + .await + .expect("connect TLS listener"); + let server_name = rustls::pki_types::ServerName::try_from(client_tls.server_name) + .expect("valid server name"); + let mut stream = tokio_rustls::TlsConnector::from(Arc::new(client_config)) + .connect(server_name, stream) + .await + .expect("verify TLS listener"); + stream.write_all(b"sync").await.expect("write first phase"); + let mut first_ack = [0_u8; 4]; + stream + .read_exact(&mut first_ack) + .await + .expect("read first acknowledgement"); + assert_eq!(&first_ack, b"ack1"); + stream + .write_all(b"async") + .await + .expect("write second phase"); + let mut second_ack = [0_u8; 4]; + stream + .read_exact(&mut second_ack) + .await + .expect("read second acknowledgement"); + assert_eq!(&second_ack, b"ack2"); + }); + server.join().expect("TLS boundary server thread"); + } + + #[test] + fn control_restart_replays_running_lifecycle_exactly_once() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_BOUNDARY_RECONNECT_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::control_restart_replays_running_lifecycle_exactly_once", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated reconnect test"); + assert!(status.success(), "isolated reconnect test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let (network_broker, workload_launcher) = test_network_broker(); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "sandbox-reconnect".to_string(), + generation: "generation-reconnect".to_string(), + session_epoch: "session-reconnect".to_string(), + bootstrap_token: "a".repeat(32), + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().expect("control address"), + tls: placeholder_server_tls(), + }, + multiplexed: false, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + network_broker, + workload_launcher, + test_runtime_qualification(), + )); + let policy = SandboxPolicyWire::from(openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }); + let spec = AgentSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + workdir: None, + timeout_secs: 60, + interactive: false, + }; + + assert!(matches!( + boundary.attach(policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + let start = || { + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec.clone(), + policy.clone(), + None, + None, + 0, + std::collections::HashMap::new(), + ) + }; + let Response::Started { + process_id, + provider_env_revision: 0, + } = start() + else { + panic!("initial start did not succeed"); + }; + + let update = RequestEnvelope::new( + "sandbox-reconnect".to_string(), + "a".repeat(32), + Request::UpdateProviderEnvironment { + expected_revision: 0, + revision: 7, + provider_env: std::collections::HashMap::from([( + "REPLAY_TEST".to_string(), + "set-once".to_string(), + )]), + }, + ) + .expect("build replayed update"); + assert_eq!( + boundary.dispatch(update.clone()), + Response::ProviderEnvironmentUpdated { revision: 7 } + ); + assert_eq!( + boundary.dispatch(update.clone()), + Response::ProviderEnvironmentUpdated { revision: 7 }, + "the same request ID and payload must replay its recorded response" + ); + let mut changed = RequestEnvelope::new( + "sandbox-reconnect".to_string(), + "a".repeat(32), + Request::Terminate { + process_id: process_id.clone(), + }, + ) + .expect("build changed request"); + changed.request_id = update.request_id; + assert!(matches!( + boundary.dispatch(changed), + Response::Error { kind, .. } if kind == "denied" + )); + + let (first_attachment, _) = boundary + .attach_process(&process_id) + .expect("initial main-process attachment"); + assert!(boundary.attach_process(&process_id).is_err()); + let (boundary_stream, control_stream) = + std::os::unix::net::UnixStream::pair().expect("main attachment socket pair"); + let stream_boundary = boundary.clone(); + let stream_thread = std::thread::spawn(move || { + stream_boundary + .stream_process(ControlStream::TestUnix(boundary_stream), first_attachment) + }); + drop(control_stream); + stream_thread + .join() + .expect("join disconnected main attachment") + .expect("transport EOF cleanly ends main attachment"); + let (replacement_attachment, _) = boundary + .attach_process(&process_id) + .expect("replacement main-process attachment after disconnect"); + drop(replacement_attachment); + + assert!(matches!( + boundary.attach(policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + assert_eq!( + start(), + Response::Started { + process_id: process_id.clone(), + provider_env_revision: 7, + } + ); + + let mut changed_policy = policy.clone(); + changed_policy.version += 1; + assert!(matches!( + boundary.attach(changed_policy.clone()), + Response::Error { kind, .. } if kind == "denied" + )); + assert!(matches!( + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec, + changed_policy, + None, + None, + 0, + std::collections::HashMap::new(), + ), + Response::Error { kind, .. } if kind == "denied" + )); + + let exec_spec = ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "printf reconnected".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }; + let exec_request = RequestEnvelope::new( + "sandbox-reconnect".to_string(), + "a".repeat(32), + Request::Exec { + spec: exec_spec.clone(), + }, + ) + .expect("build exec request"); + let exec = boundary + .start_exec( + &exec_request.request_id, + &exec_request.payload_digest, + exec_spec, + ) + .expect("exec after reconnect"); + let exec_id = exec.process_id.clone(); + let mut output = String::new(); + let mut cursor = exec.attachment.session.subscribe(); + process_runtime.block_on(async { + loop { + match cursor.recv().await.expect("retained exec output") { + MainOutput::Stdout(bytes) => { + output.push_str(std::str::from_utf8(&bytes).expect("UTF-8 output")); + } + MainOutput::Stderr(_) => {} + MainOutput::Exit(code) => { + assert_eq!(code, 0); + break; + } + } + } + }); + assert_eq!(output, "reconnected"); + drop(exec); + let Response::Attached { snapshot } = boundary.attach(policy.clone()) else { + panic!("reconnect attach did not return a session snapshot"); + }; + assert_eq!(snapshot.generation, "generation-reconnect"); + assert!(snapshot.processes.iter().any(|process| { + process.process_id == process_id && process.kind == ProcessKindWire::Main + })); + assert!(snapshot.processes.iter().any(|process| { + process.process_id == exec_id + && process.kind == ProcessKindWire::Exec + && process.status == Some(ExitStatusWire::Exited(0)) + && process.retained_output.next_sequence > 0 + })); + assert_eq!(boundary.terminate(&process_id), Response::Terminated); + } + + #[test] + fn canonical_exit_preserves_pending_network_accept_and_exec() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_RETAINED_BOUNDARY_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::canonical_exit_preserves_pending_network_accept_and_exec", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated retained-boundary test"); + assert!(status.success(), "isolated retained-boundary test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let policy = openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy { + mode: openshell_core::policy::NetworkMode::Proxy, + proxy: Some(openshell_core::policy::ProxyPolicy { + http_addr: Some("127.0.0.1:3128".parse().expect("proxy address")), + }), + }, + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }; + let (network_broker, workload_launcher) = test_network_broker(); + let prepared = PreparedBoundary { + network_broker: network_broker.clone(), + }; + let agent_spec = AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 5, + interactive: false, + }; + let wire_policy = SandboxPolicyWire::from(policy.clone()); + let process = Arc::new( + ManagedProcess::spawn( + process_runtime.handle(), + ManagedProcessLaunch { + process_id: "generation-retained:main:0".to_string(), + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy, + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + ca_file_paths: None, + }, + prepared, + ) + .expect("spawn canonical process"), + ); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "sandbox-retained".to_string(), + generation: "generation-retained".to_string(), + session_epoch: "session-retained".to_string(), + bootstrap_token: "a".repeat(32), + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().expect("control address"), + tls: placeholder_server_tls(), + }, + multiplexed: false, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + network_broker, + workload_launcher, + test_runtime_qualification(), + )); + *lock(&boundary.state) = RuntimeState::Running(process.clone()); + *lock(&boundary.attached_policy) = Some(wire_policy.clone()); + *lock(&boundary.started_agent) = Some(StartedAgent { + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy: wire_policy.clone(), + ca_cert: None, + ca_bundle: None, + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + }); + + // A replacement control process replays the durable lifecycle and + // receives the original process rather than spawning another one. + assert!(matches!( + boundary.attach(wire_policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec.clone(), + wire_policy.clone(), + None, + None, + 0, + std::collections::HashMap::new(), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 0, + } + ); + + assert_eq!( + boundary.update_provider_environment( + 0, + 2, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "refreshed".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment( + 0, + 1, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "stale".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment(2, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a numerically smaller opaque revision must revoke the environment" + ); + assert_eq!( + boundary.update_provider_environment( + 2, + 3, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "out-of-order".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a stale expected revision must not overwrite current state" + ); + assert_eq!( + boundary.update_provider_environment(1, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a duplicate update must be idempotent" + ); + + assert!(matches!( + boundary.attach(wire_policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec, + wire_policy, + None, + None, + 99, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "replacement-control-snapshot".to_string(), + )]), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 1, + }, + "a replacement control must resume from the boundary's current revision" + ); + + let sleep_spec = ExecSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }; + let sleep_request = RequestEnvelope::new( + "sandbox-retained".to_string(), + "a".repeat(32), + Request::Exec { + spec: sleep_spec.clone(), + }, + ) + .expect("build retained exec request"); + let started = boundary + .start_exec( + &sleep_request.request_id, + &sleep_request.payload_digest, + sleep_spec.clone(), + ) + .expect("start exec whose response is disconnected"); + let retained_id = started.process_id.clone(); + drop(started); + let replayed = boundary + .start_exec( + &sleep_request.request_id, + &sleep_request.payload_digest, + sleep_spec, + ) + .expect("reattach exec after response loss"); + assert_eq!(replayed.process_id, retained_id); + assert_eq!(lock(&boundary.exec_handles).len(), 1); + drop(replayed); + assert_eq!( + boundary.signal_exec(&retained_id, SignalWire::Kill), + Response::Signaled + ); + + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !process.has_exited() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(process.has_exited(), "canonical process did not exit"); + + let mut session = process_runtime + .block_on( + process.boundary_exec().exec( + ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec![ + "-c".to_string(), + "if [ -z \"${ROTATED_TOKEN+x}\" ]; then printf revoked; else printf 'unexpected:%s' \"$ROTATED_TOKEN\"; fi" + .to_string(), + ], + env: Vec::new(), + workdir: None, + pty: false, + } + .into(), + ), + ) + .expect("exec after canonical exit"); + let mut output = String::new(); + process_runtime + .block_on(session.stdout.read_to_string(&mut output)) + .expect("read retained exec output"); + assert_eq!( + output, "revoked", + "exec after canonical exit must use the latest reconciled provider snapshot" + ); + assert!(matches!( + process_runtime.block_on(session.process.wait()), + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) + )); + } + } +} + +#[cfg(target_os = "linux")] +pub use linux::run_boundary; + +#[cfg(not(target_os = "linux"))] +pub fn run_boundary( + _config_path: &Path, + _qualification: crate::RuntimeQualification, +) -> Result<(), String> { + Err("boundary mode is supported only on Linux".to_string()) +} diff --git a/crates/openshell-sandbox/src/child_env.rs b/crates/openshell-sandbox/src/child_env.rs new file mode 100644 index 0000000000..50549a7439 --- /dev/null +++ b/crates/openshell-sandbox/src/child_env.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +pub fn tls_env_vars( + ca_cert_path: &Path, + combined_bundle_path: &Path, +) -> [(&'static str, String); 6] { + let ca_cert_path = ca_cert_path.display().to_string(); + let combined_bundle_path = combined_bundle_path.display().to_string(); + [ + ("NODE_EXTRA_CA_CERTS", ca_cert_path.clone()), + ("DENO_CERT", ca_cert_path), + ("SSL_CERT_FILE", combined_bundle_path.clone()), + ("REQUESTS_CA_BUNDLE", combined_bundle_path.clone()), + ("CURL_CA_BUNDLE", combined_bundle_path.clone()), + // Ubuntu Noble's git links against libcurl-gnutls, which ignores SSL_CERT_FILE. + // git reads GIT_SSL_CAINFO (or http.sslCAInfo) to locate the CA bundle. + ("GIT_SSL_CAINFO", combined_bundle_path), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + use std::process::Stdio; + + #[test] + fn apply_tls_env_sets_node_and_bundle_paths() { + let mut cmd = Command::new("/usr/bin/env"); + cmd.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + + let ca_cert_path = Path::new("/etc/openshell-tls/openshell-ca.pem"); + let combined_bundle_path = Path::new("/etc/openshell-tls/ca-bundle.pem"); + for (key, value) in tls_env_vars(ca_cert_path, combined_bundle_path) { + cmd.env(key, value); + } + + let output = cmd.output().expect("spawn env"); + let stdout = String::from_utf8(output.stdout).expect("utf8"); + + assert!(stdout.contains("NODE_EXTRA_CA_CERTS=/etc/openshell-tls/openshell-ca.pem")); + assert!(stdout.contains("DENO_CERT=/etc/openshell-tls/openshell-ca.pem")); + assert!(stdout.contains("SSL_CERT_FILE=/etc/openshell-tls/ca-bundle.pem")); + assert!(stdout.contains("REQUESTS_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); + assert!(stdout.contains("CURL_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); + assert!(stdout.contains("GIT_SSL_CAINFO=/etc/openshell-tls/ca-bundle.pem")); + } +} diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs new file mode 100644 index 0000000000..d8c15535f0 --- /dev/null +++ b/crates/openshell-sandbox/src/delegated.rs @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process and access-plane assembly for the capability-free sandbox boundary. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::Duration; + +#[cfg(target_os = "linux")] +use miette::WrapErr as _; +use miette::{IntoDiagnostic as _, Result}; +use openshell_core::policy::SandboxPolicy; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; +use openshell_ocsf::{ + ActionId, ActivityId, DispositionId, LaunchTypeId, Process as OcsfProcess, + ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, +}; + +use crate::process::{ProcessHandle, ProcessStatus, ResolvedWorkspace}; + +fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { + openshell_ocsf::ctx::ctx() +} + +/// Spawn the admitted workload without placing the gateway or policy authority +/// inside its boundary. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn spawn_workload( + program: &str, + args: &[String], + workdir: Option<&str>, + timeout_secs: u64, + interactive: bool, + _sandbox_id: Option<&str>, + _openshell_endpoint: Option<&str>, + _ssh_socket_path: Option, + _shared_ssh_socket: bool, + policy: &SandboxPolicy, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + boundary_runtime: Option>, +) -> Result { + // Driver-selected workspaces are the sandbox identity's home. This keeps + // canonical and later exec processes consistent for image WorkingDir and + // the managed /sandbox fallback without consulting privileged account + // setup inside the capability-free boundary. + let workspace = ResolvedWorkspace::new(workdir.map(str::to_string), true); + + #[cfg(target_os = "linux")] + { + let mode = if std::env::var_os("OPENSHELL_REQUIRE_RUNTIME_PID_LIMIT").is_some() { + crate::process::RuntimePidLimitMode::Require + } else { + crate::process::RuntimePidLimitMode::Warn + }; + crate::process::check_runtime_pid_limit(mode).wrap_err("check runtime PID limit")?; + } + + let boundary_runtime = boundary_runtime + .unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new_exclusive_pid_namespace); + let mut user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + user_environment.retain(|key, _value| !crate::process::is_proxy_env_var(key)); + let port_forward: Arc = Arc::new( + crate::boundary_io::LocalPortForward::new(Some(boundary_runtime.clone())), + ); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + ca_file_paths.clone().map(Arc::new), + provider_credentials, + user_environment, + boundary_runtime.clone(), + )); + + #[cfg(target_os = "linux")] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + ca_file_paths.as_ref(), + &provider_env, + ) + .wrap_err("spawn delegated workload process")?; + #[cfg(not(target_os = "linux"))] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + ca_file_paths.as_ref(), + &provider_env, + )?; + + entrypoint_pid.store(handle.pid(), Ordering::Release); + if let Some(sender) = entrypoint_started_tx { + let _ = sender.send(handle.pid()); + } + let main_session = crate::main_session::MainSession::new(handle.take_io(), handle.pid()); + let (terminal, signal_lock) = handle.signaling_state(); + boundary_runtime + .register_process_group(handle.pid(), terminal.clone(), signal_lock.clone()) + .map_err(|error| miette::miette!(error.to_string()))?; + + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .launch_type(LaunchTypeId::Spawn) + .process(OcsfProcess::new(program, i64::from(handle.pid()))) + .message(format!("Process started: pid={}", handle.pid())) + .build() + ); + + Ok(SpawnedAgent { + handle, + timeout_secs, + terminal, + signal_lock, + main_session, + boundary_exec, + port_forward, + boundary_runtime, + }) +} + +/// Owned workload process and its live boundary capabilities. +pub struct SpawnedAgent { + handle: ProcessHandle, + timeout_secs: u64, + terminal: Arc, + signal_lock: Arc>, + main_session: Arc, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, +} + +impl SpawnedAgent { + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + AgentSignaler { + pid: self.handle.pid(), + terminal: self.terminal.clone(), + signal_lock: self.signal_lock.clone(), + } + } + + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + /// Retained canonical-process I/O owned by the boundary. + #[must_use] + pub fn main_session(&self) -> Arc { + self.main_session.clone() + } + + /// Wait for the canonical process to exit, enforcing its admitted + /// wall-clock timeout. Completion does not end the boundary: exec and + /// loopback forwarding remain available until the boundary owner tears + /// down the retained runtime. + pub async fn wait(&mut self) -> Result { + let signaler = self.signaler(); + let status = if self.timeout_secs == 0 { + self.handle.wait().await.into_diagnostic()? + } else if let Ok(status) = + tokio::time::timeout(Duration::from_secs(self.timeout_secs), self.handle.wait()).await + { + status.into_diagnostic()? + } else { + let _ = signaler.term(); + tokio::time::sleep(Duration::from_millis(100)).await; + let _ = signaler.kill(); + self.handle.wait().await.into_diagnostic()? + }; + self.boundary_runtime + .unregister_process_group(self.handle.pid(), &self.terminal); + let _ = self.main_session.finish(status.code(), false).await; + self.main_session.mark_terminal_reported(); + Ok(status) + } +} + +/// Lock-free process-group signal handle used while another task owns `wait`. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, signal: nix::sys::signal::Signal) -> Result<()> { + let _guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(nix::unistd::Pid::from_raw(pid), signal).into_diagnostic() + } + + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) + } +} diff --git a/crates/openshell-sandbox/src/google_cloud_metadata.rs b/crates/openshell-sandbox/src/google_cloud_metadata.rs deleted file mode 100644 index 9e1e179872..0000000000 --- a/crates/openshell-sandbox/src/google_cloud_metadata.rs +++ /dev/null @@ -1,536 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! GCE metadata server emulator for sandbox credential injection. -//! -//! Implements a subset of the GCE instance metadata API so that GCP client -//! libraries (Go, Python, Node.js) can obtain `OAuth2` tokens natively inside -//! sandboxes. Tokens are served from the existing `ProviderCredentialState` -//! store — no separate refresh mechanism is needed. -//! -//! The emulator runs as a loopback HTTP server inside the sandbox network -//! namespace (see [`metadata_server`](crate::metadata_server)). GCP SDKs -//! discover it via the `GCE_METADATA_HOST` environment variable, which is -//! set to the loopback address by `child_env_with_gcp_resolved()`. - -use miette::{IntoDiagnostic, Result}; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_core::secrets; -use openshell_ocsf::{ActivityId, HttpActivityBuilder, SeverityId, StatusId, ocsf_emit}; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; - -type MetadataResponse = (u16, &'static str, String); - -const PATH_SERVICE_ACCOUNTS: &str = "/computeMetadata/v1/instance/service-accounts"; -const PATH_SERVICE_ACCOUNT_DEFAULT: &str = "/computeMetadata/v1/instance/service-accounts/default"; -const PATH_TOKEN: &str = "/computeMetadata/v1/instance/service-accounts/default/token"; -const PATH_EMAIL: &str = "/computeMetadata/v1/instance/service-accounts/default/email"; -const PATH_SCOPES: &str = "/computeMetadata/v1/instance/service-accounts/default/scopes"; -const PATH_ALIASES: &str = "/computeMetadata/v1/instance/service-accounts/default/aliases"; -const PATH_PROJECT_ID: &str = "/computeMetadata/v1/project/project-id"; - -const ENV_GCP_PROJECT_ID: &str = openshell_core::google_cloud::PROJECT_ID_ENV_VARS[0]; -const ENV_GCP_SERVICE_ACCOUNT_EMAIL: &str = - openshell_core::google_cloud::SERVICE_ACCOUNT_EMAIL_ENV_VARS[0]; - -const METADATA_FLAVOR_HEADER: &str = "metadata-flavor"; -const METADATA_FLAVOR_VALUE: &str = "Google"; -const X_FORWARDED_FOR_HEADER: &str = "x-forwarded-for"; - -#[derive(Debug, Clone)] -pub struct MetadataContext { - credentials: ProviderCredentialState, -} - -impl MetadataContext { - pub fn new(credentials: ProviderCredentialState) -> Self { - Self { credentials } - } -} - -impl crate::metadata_server::MetadataHandler for MetadataContext { - async fn handle( - &self, - method: &str, - path: &str, - request: &[u8], - stream: &mut S, - ) -> Result<()> { - handle_forward_request(self, method, path, request, stream).await - } -} - -async fn handle_forward_request( - ctx: &MetadataContext, - method: &str, - path: &str, - initial_request: &[u8], - client: &mut S, -) -> Result<()> -where - S: AsyncRead + AsyncWrite + Unpin, -{ - let headers = parse_request_headers(initial_request); - let (status, content_type, body) = route_request(ctx, method, path, &headers); - write_metadata_response(client, status, content_type, &body).await -} - -fn route_request( - ctx: &MetadataContext, - method: &str, - path: &str, - headers: &[(String, String)], -) -> MetadataResponse { - if method != "GET" { - emit_metadata_event( - ActivityId::Refuse, - SeverityId::Low, - StatusId::Failure, - &format!("metadata: unsupported method {method}"), - ); - return (405, "text/html", "Method Not Allowed".to_string()); - } - - if let Err(resp) = validate_metadata_headers(headers) { - emit_metadata_event( - ActivityId::Refuse, - SeverityId::Medium, - StatusId::Failure, - &format!("metadata: header validation failed for {path}"), - ); - return resp; - } - - let (route, query) = path.split_once('?').map_or((path, ""), |(r, q)| (r, q)); - let route = route.strip_suffix('/').unwrap_or(route); - let recursive = query.split('&').any(|p| p == "recursive=true"); - - match route { - PATH_TOKEN => handle_token(ctx), - PATH_EMAIL => handle_env(ctx, ENV_GCP_SERVICE_ACCOUNT_EMAIL), - PATH_PROJECT_ID => handle_env(ctx, ENV_GCP_PROJECT_ID), - PATH_ALIASES => (200, "text/plain", "default\n".to_string()), - PATH_SCOPES => ( - 200, - "text/plain", - "https://www.googleapis.com/auth/cloud-platform".to_string(), - ), - PATH_SERVICE_ACCOUNT_DEFAULT => { - if recursive { - handle_service_account_recursive(ctx) - } else { - ( - 200, - "text/plain", - "aliases\nemail\nscopes\ntoken\n".to_string(), - ) - } - } - PATH_SERVICE_ACCOUNTS => (200, "text/plain", "default/\n".to_string()), - "" | "/" | "/computeMetadata" | "/computeMetadata/v1" => { - (200, "text/plain", "computeMetadata/\n".to_string()) - } - "/computeMetadata/v1/instance" => (200, "text/plain", "service-accounts/\n".to_string()), - _ => { - emit_metadata_event( - ActivityId::Refuse, - SeverityId::Low, - StatusId::Failure, - &format!("metadata: unknown path {route}"), - ); - ( - 404, - "application/json", - serde_json::json!({"error": "not_found"}).to_string(), - ) - } - } -} - -fn handle_token(ctx: &MetadataContext) -> MetadataResponse { - let Some((placeholder, expires_in)) = ctx.credentials.gcp_token_response() else { - let has_resolver = ctx.credentials.resolver().is_some(); - let (msg, error_key) = if has_resolver { - ( - "metadata: no GCP access token available or expired", - "token_unavailable", - ) - } else { - ( - "metadata: token request but no credentials configured", - "credentials_unavailable", - ) - }; - emit_metadata_event(ActivityId::Fail, SeverityId::Medium, StatusId::Failure, msg); - return ( - 503, - "application/json", - serde_json::json!({"error": error_key}).to_string(), - ); - }; - - emit_metadata_event( - ActivityId::Open, - SeverityId::Informational, - StatusId::Success, - "metadata: token placeholder served", - ); - - let body = serde_json::json!({ - "access_token": placeholder, - "expires_in": expires_in, - "token_type": "Bearer" - }); - (200, "application/json", body.to_string()) -} - -fn handle_service_account_recursive(ctx: &MetadataContext) -> MetadataResponse { - let resolver = ctx.credentials.resolver(); - let email = resolver - .as_ref() - .and_then(|r| { - let p = secrets::placeholder_for_env_key(ENV_GCP_SERVICE_ACCOUNT_EMAIL); - r.resolve_placeholder(&p).map(str::to_string) - }) - .unwrap_or_default(); - - let scopes = "https://www.googleapis.com/auth/cloud-platform"; - - let body = serde_json::json!({ - "aliases": ["default"], - "email": email, - "scopes": [scopes], - }); - (200, "application/json", body.to_string()) -} - -/// Serve a non-secret config value (project ID, SA email) as plain text. -/// -/// Unlike `handle_token` which serves placeholders, this resolves to the real -/// value. This matches real GCE metadata server behavior and is safe because -/// these values are non-secret configuration (project IDs, email addresses). -fn handle_env(ctx: &MetadataContext, env_key: &str) -> MetadataResponse { - let Some(resolver) = ctx.credentials.resolver() else { - emit_metadata_event( - ActivityId::Fail, - SeverityId::Medium, - StatusId::Failure, - &format!("metadata: {env_key} request but no credentials configured"), - ); - return (503, "text/plain", String::new()); - }; - - let placeholder = secrets::placeholder_for_env_key(env_key); - resolver.resolve_placeholder(&placeholder).map_or_else( - || { - emit_metadata_event( - ActivityId::Fail, - SeverityId::Low, - StatusId::Failure, - &format!("metadata: {env_key} not configured"), - ); - ( - 404, - "application/json", - serde_json::json!({"error": "not_found"}).to_string(), - ) - }, - |value| (200, "text/plain", value.to_string()), - ) -} - -fn validate_metadata_headers(headers: &[(String, String)]) -> Result<(), MetadataResponse> { - if headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(X_FORWARDED_FOR_HEADER)) - { - return Err((403, "text/html", "Forbidden".to_string())); - } - - let has_flavor = headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case(METADATA_FLAVOR_HEADER) - && value.trim().eq_ignore_ascii_case(METADATA_FLAVOR_VALUE) - }); - if !has_flavor { - return Err((403, "text/html", "Forbidden".to_string())); - } - - Ok(()) -} - -fn parse_request_headers(raw: &[u8]) -> Vec<(String, String)> { - let request = String::from_utf8_lossy(raw); - let mut headers = Vec::new(); - for line in request.split("\r\n").skip(1) { - if line.is_empty() { - break; - } - if let Some((name, value)) = line.split_once(':') { - headers.push((name.trim().to_string(), value.trim().to_string())); - } - } - headers -} - -fn status_text(status: u16) -> &'static str { - match status { - 403 => "Forbidden", - 404 => "Not Found", - 405 => "Method Not Allowed", - 503 => "Service Unavailable", - _ => "OK", - } -} - -async fn write_metadata_response( - client: &mut S, - status: u16, - content_type: &str, - body: &str, -) -> Result<()> -where - S: AsyncWrite + Unpin, -{ - let response = format!( - "HTTP/1.1 {status} {}\r\nContent-Type: {content_type}\r\nMetadata-Flavor: Google\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - status_text(status), - body.len(), - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - Ok(()) -} - -fn emit_metadata_event( - activity: ActivityId, - severity: SeverityId, - status: StatusId, - message: &str, -) { - let event = HttpActivityBuilder::new(crate::ocsf_ctx()) - .activity(activity) - .severity(severity) - .status(status) - .message(message.to_string()) - .build(); - ocsf_emit!(event); -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - fn make_context(env: HashMap) -> MetadataContext { - let state = - ProviderCredentialState::from_environment(0, env, HashMap::new(), HashMap::new()); - MetadataContext::new(state) - } - - fn make_context_with_expiry( - env: HashMap, - expires: HashMap, - ) -> MetadataContext { - let state = ProviderCredentialState::from_environment(0, env, expires, HashMap::new()); - MetadataContext::new(state) - } - - fn flavor_headers() -> Vec<(String, String)> { - vec![("Metadata-Flavor".to_string(), "Google".to_string())] - } - - #[test] - fn token_returns_placeholder_not_real_value() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.test-token".to_string(), - )])); - let (status, ct, body) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(ct, "application/json"); - let json: serde_json::Value = serde_json::from_str(&body).unwrap(); - let token = json["access_token"].as_str().unwrap(); - assert!( - token.starts_with("openshell:resolve:env:"), - "token should be a placeholder, got: {token}" - ); - assert!(!token.contains("ya29"), "real token must not be served"); - assert_eq!(json["token_type"], "Bearer"); - assert!(json["expires_in"].is_number()); - } - - #[test] - fn token_expires_in_computed_from_credential_expiry() { - let now_ms = openshell_core::time::now_ms(); - let expires_at = now_ms + 1_800_000; // 30 minutes from now - let ctx = make_context_with_expiry( - HashMap::from([("GCP_ADC_ACCESS_TOKEN".to_string(), "ya29.tok".to_string())]), - HashMap::from([("GCP_ADC_ACCESS_TOKEN".to_string(), expires_at)]), - ); - let (status, _, body) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 200); - let json: serde_json::Value = serde_json::from_str(&body).unwrap(); - let expires_in = json["expires_in"].as_i64().unwrap(); - assert!( - expires_in > 1700 && expires_in <= 1800, - "expires_in={expires_in}" - ); - } - - #[test] - fn token_no_expiry_defaults_to_3600() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let (_, _, body) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - let json: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert_eq!(json["expires_in"], 3600); - } - - #[test] - fn missing_metadata_flavor_header_403() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &[]); - assert_eq!(status, 403); - } - - #[test] - fn x_forwarded_for_header_403() { - let ctx = make_context(HashMap::new()); - let headers = vec![ - ("Metadata-Flavor".to_string(), "Google".to_string()), - ("X-Forwarded-For".to_string(), "10.0.0.1".to_string()), - ]; - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &headers); - assert_eq!(status, 403); - } - - #[test] - fn unknown_path_404() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request( - &ctx, - "GET", - "/computeMetadata/v1/unknown", - &flavor_headers(), - ); - assert_eq!(status, 404); - } - - #[test] - fn no_credentials_503() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 503); - } - - #[test] - fn post_method_405() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request(&ctx, "POST", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 405); - } - - #[test] - fn project_id_served_as_plain_text() { - let ctx = make_context(HashMap::from([( - "GCP_PROJECT_ID".to_string(), - "my-project-123".to_string(), - )])); - let (status, ct, body) = route_request(&ctx, "GET", PATH_PROJECT_ID, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(ct, "text/plain"); - assert_eq!(body, "my-project-123"); - } - - #[test] - fn email_served_as_plain_text() { - let ctx = make_context(HashMap::from([( - "GCP_SERVICE_ACCOUNT_EMAIL".to_string(), - "sa@project.iam.gserviceaccount.com".to_string(), - )])); - let (status, ct, body) = route_request(&ctx, "GET", PATH_EMAIL, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(ct, "text/plain"); - assert_eq!(body, "sa@project.iam.gserviceaccount.com"); - } - - #[test] - fn scopes_returns_cloud_platform() { - let ctx = make_context(HashMap::new()); - let (status, _, body) = route_request(&ctx, "GET", PATH_SCOPES, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(body, "https://www.googleapis.com/auth/cloud-platform"); - } - - #[test] - fn query_parameters_ignored_for_routing() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let path = format!("{PATH_TOKEN}?scopes=cloud-platform"); - let (status, _, _) = route_request(&ctx, "GET", &path, &flavor_headers()); - assert_eq!(status, 200); - } - - #[test] - fn metadata_flavor_case_insensitive() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let headers = vec![("metadata-FLAVOR".to_string(), "google".to_string())]; - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &headers); - assert_eq!(status, 200); - } - - #[test] - fn missing_env_var_returns_404() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - // project-id not set - let (status, _, _) = route_request(&ctx, "GET", PATH_PROJECT_ID, &flavor_headers()); - assert_eq!(status, 404); - } - - #[test] - fn trailing_slash_handled_for_service_account_default() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let with_slash = route_request( - &ctx, - "GET", - "/computeMetadata/v1/instance/service-accounts/default/", - &flavor_headers(), - ); - let without_slash = route_request( - &ctx, - "GET", - "/computeMetadata/v1/instance/service-accounts/default", - &flavor_headers(), - ); - assert_eq!(with_slash.0, 200); - assert_eq!(without_slash.0, 200); - assert_eq!(with_slash.2, without_slash.2); - } - - #[test] - fn parse_request_headers_extracts_correctly() { - let raw = b"GET /path HTTP/1.1\r\nHost: example.com\r\nMetadata-Flavor: Google\r\n\r\n"; - let headers = parse_request_headers(raw); - assert_eq!(headers.len(), 2); - assert_eq!(headers[0].0, "Host"); - assert_eq!(headers[0].1, "example.com"); - assert_eq!(headers[1].0, "Metadata-Flavor"); - assert_eq!(headers[1].1, "Google"); - } -} diff --git a/crates/openshell-sandbox/src/identity.rs b/crates/openshell-sandbox/src/identity.rs new file mode 100644 index 0000000000..df79a4137d --- /dev/null +++ b/crates/openshell-sandbox/src/identity.rs @@ -0,0 +1,833 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver identity normalization and OCI `USER` resolution. + +use crate::process::ResolvedProcessIdentity; +use miette::{IntoDiagnostic, Result}; +use openshell_core::policy::SandboxPolicy; +use std::fs::{File, OpenOptions}; +use std::io::Read; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; + +const PASSWD_PATH: &str = "/etc/passwd"; +const GROUP_PATH: &str = "/etc/group"; +const MAX_ACCOUNT_FILE_SIZE: u64 = 1024 * 1024; +const MAX_ACCOUNT_LINE_SIZE: usize = 8 * 1024; +const MAX_ACCOUNT_FIELD_SIZE: usize = 1024; + +/// Identity input selected by the active compute driver. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DriverIdentity { + /// Platform-selected identity used by Kubernetes and `OpenShift`. + Resolved { uid: u32, gid: u32 }, + /// Raw OCI `Config.User` selected by Docker and Podman. + OciUser { declaration: String }, + /// Drivers with no authoritative identity metadata. + None, +} + +impl DriverIdentity { + /// Normalize the protected driver environment into one identity variant. + pub fn from_env() -> Result { + let oci_user = optional_utf8_env(openshell_core::sandbox_env::OCI_IMAGE_USER)?; + let uid = optional_nonempty_utf8_env(openshell_core::sandbox_env::SANDBOX_UID)?; + let gid = optional_nonempty_utf8_env(openshell_core::sandbox_env::SANDBOX_GID)?; + Self::from_values(oci_user, uid, gid) + } + + fn from_values( + oci_user: Option, + uid: Option, + gid: Option, + ) -> Result { + // Resolved-identity drivers explicitly clear the OCI declaration so + // an image-baked or user-supplied value cannot select the OCI path. + // Preserve an empty declaration when no resolved pair is present: + // Docker and Podman use that state to reject images without USER. + let oci_user = if oci_user.as_deref() == Some("") && (uid.is_some() || gid.is_some()) { + None + } else { + oci_user + }; + + match (oci_user, uid, gid) { + (Some(declaration), None, None) => Ok(Self::OciUser { declaration }), + (None, Some(uid), Some(gid)) => { + let uid = uid.parse::().ok().filter(|uid| { + (openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) + .contains(uid) + }); + let gid = gid.parse::().ok().filter(|gid| { + (openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) + .contains(gid) + }); + let (Some(uid), Some(gid)) = (uid, gid) else { + return Err(miette::miette!( + "driver UID/GID must be numeric identities in range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + }; + Ok(Self::Resolved { uid, gid }) + } + (None, None, None) => Ok(Self::None), + (Some(_), _, _) => Err(miette::miette!( + "{} conflicts with non-empty {}/{} driver identity", + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SANDBOX_GID + )), + (None, _, _) => Err(miette::miette!( + "{} and {} must be supplied together", + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SANDBOX_GID + )), + } + } +} + +/// Apply a driver identity before any workload child becomes reachable. +pub fn resolve_process_identity( + policy: &mut SandboxPolicy, + driver_identity: &DriverIdentity, +) -> Result { + match driver_identity { + DriverIdentity::Resolved { uid, gid } => { + policy.process.run_as_user = Some(uid.to_string()); + policy.process.run_as_group = Some(gid.to_string()); + // Kubernetes/OpenShift already supply numeric policy values and + // retain their existing privilege-drop path. + Ok(ResolvedProcessIdentity::default()) + } + DriverIdentity::OciUser { declaration } => resolve_oci_process_identity_at( + policy, + declaration, + Path::new(PASSWD_PATH), + Path::new(GROUP_PATH), + ), + DriverIdentity::None => { + // VM/offline drivers retain the pre-OCI per-field fallback. A + // partial policy must never leave the omitted component at the + // root supervisor identity. + if policy + .process + .run_as_user + .as_deref() + .is_none_or(str::is_empty) + { + policy.process.run_as_user = Some("sandbox".into()); + } + if policy + .process + .run_as_group + .as_deref() + .is_none_or(str::is_empty) + { + policy.process.run_as_group = Some("sandbox".into()); + } + Ok(ResolvedProcessIdentity::default()) + } + } +} + +#[allow(clippy::similar_names)] +fn resolve_oci_process_identity_at( + policy: &mut SandboxPolicy, + declaration: &str, + passwd_path: &Path, + group_path: &Path, +) -> Result { + let explicit_user = policy + .process + .run_as_user + .as_deref() + .is_some_and(|value| !value.is_empty()); + let explicit_group = policy + .process + .run_as_group + .as_deref() + .is_some_and(|value| !value.is_empty()); + + if explicit_user && explicit_group { + return Ok(ResolvedProcessIdentity::default()); + } + + let (oci_user, oci_group) = split_oci_declaration(declaration); + let needs_primary_gid = !explicit_group && oci_group.is_none(); + let resolved_user = if !explicit_user || needs_primary_gid { + Some(resolve_required_oci_user( + oci_user, + passwd_path, + declaration, + needs_primary_gid, + )?) + } else { + None + }; + + let oci_uid = if explicit_user { + None + } else { + Some( + resolved_user + .as_ref() + .expect("omitted OCI user must have been resolved") + .0, + ) + }; + + if !explicit_user { + policy.process.run_as_user = Some(oci_user.to_string()); + } + + let oci_gid = if explicit_group { + None + } else { + let (group_value, gid) = match oci_group { + Some(group) if !group.is_empty() => { + let gid = validate_oci_group(group, group_path, declaration)?; + (group.to_string(), gid) + } + Some(_) => { + return Err(miette::miette!( + "OCI USER '{declaration}' has an empty group component" + )); + } + None => { + let gid = resolved_user + .and_then(|(_, primary_gid)| primary_gid) + .ok_or_else(|| { + miette::miette!( + "OCI USER '{declaration}' uses a numeric UID without an explicit group, \ + but /etc/passwd has no matching primary GID" + ) + })?; + (gid.to_string(), gid) + } + }; + policy.process.run_as_group = Some(group_value); + Some(gid) + }; + + Ok(ResolvedProcessIdentity::new(oci_uid, oci_gid)) +} + +fn split_oci_declaration(declaration: &str) -> (&str, Option<&str>) { + declaration + .split_once(':') + .map_or((declaration, None), |(user, group)| (user, Some(group))) +} + +fn resolve_required_oci_user( + user: &str, + passwd_path: &Path, + declaration: &str, + require_primary_gid: bool, +) -> Result<(u32, Option)> { + if user.is_empty() { + return Err(miette::miette!( + "OCI USER is required because run_as_user is omitted" + )); + } + validate_component(user, "OCI user")?; + if user == "root" { + return Err(miette::miette!("OCI USER '{declaration}' selects root")); + } + if let Ok(uid) = user.parse::() { + if uid == 0 { + return Err(miette::miette!("OCI USER '{declaration}' selects UID 0")); + } + let primary_gid = if require_primary_gid { + find_passwd_by_uid(passwd_path, uid)?.map(|entry| entry.gid) + } else { + None + }; + if primary_gid == Some(0) { + return Err(miette::miette!( + "OCI USER '{declaration}' resolves to prohibited primary GID 0" + )); + } + return Ok((uid, primary_gid)); + } + let entry = find_passwd_by_name(passwd_path, user)? + .ok_or_else(|| miette::miette!("OCI USER name '{user}' was not found in /etc/passwd"))?; + if entry.uid == 0 { + return Err(miette::miette!( + "OCI USER '{declaration}' resolves to prohibited UID 0" + )); + } + if require_primary_gid && entry.gid == 0 { + return Err(miette::miette!( + "OCI USER '{declaration}' resolves to prohibited primary GID 0" + )); + } + Ok((entry.uid, require_primary_gid.then_some(entry.gid))) +} + +fn validate_oci_group(value: &str, group_path: &Path, declaration: &str) -> Result { + validate_component(value, "OCI group")?; + if value == "root" { + return Err(miette::miette!( + "OCI USER '{declaration}' selects root group" + )); + } + let gid = if let Ok(gid) = value.parse::() { + gid + } else { + find_group_by_name(group_path, value)? + .ok_or_else(|| miette::miette!("OCI group '{value}' was not found in /etc/group"))? + .gid + }; + if gid == 0 { + return Err(miette::miette!( + "OCI USER '{declaration}' resolves to prohibited GID 0" + )); + } + Ok(gid) +} + +fn validate_component(value: &str, kind: &str) -> Result<()> { + if value.is_empty() + || value.len() > MAX_ACCOUNT_FIELD_SIZE + || value.trim() != value + || value.chars().any(|ch| ch.is_control() || ch == ':') + { + return Err(miette::miette!("{kind} component '{value}' is malformed")); + } + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PasswdEntry { + uid: u32, + gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct GroupEntry { + gid: u32, +} + +fn find_passwd_by_name(path: &Path, name: &str) -> Result> { + find_unique(path, |fields| { + (fields.first().copied() == Some(name)).then(|| parse_passwd(fields)) + }) +} + +fn find_passwd_by_uid(path: &Path, uid: u32) -> Result> { + find_unique(path, |fields| { + fields + .get(2) + .and_then(|value| value.parse::().ok()) + .filter(|candidate| *candidate == uid) + .map(|_| parse_passwd(fields)) + }) +} + +fn find_group_by_name(path: &Path, name: &str) -> Result> { + find_unique(path, |fields| { + (fields.first().copied() == Some(name)).then(|| parse_group(fields)) + }) +} + +/// Resolve supplementary groups declared for an OCI named user without +/// consulting NSS. Numeric OCI users have no trustworthy group-membership +/// name and therefore receive no supplementary groups. +pub fn resolve_oci_supplementary_gids(declaration: &str, primary_gid: u32) -> Result> { + resolve_oci_supplementary_gids_at(declaration, primary_gid, Path::new(GROUP_PATH)) +} + +fn resolve_oci_supplementary_gids_at( + declaration: &str, + primary_gid: u32, + group_path: &Path, +) -> Result> { + let (user, _) = split_oci_declaration(declaration); + validate_component(user, "OCI user")?; + if user.parse::().is_ok() { + return Ok(Vec::new()); + } + + let content = read_account_file(group_path)?; + let mut gids = vec![primary_gid]; + for line in content.lines() { + if line.is_empty() || line.starts_with('#') { + continue; + } + if line.len() > MAX_ACCOUNT_LINE_SIZE { + return Err(miette::miette!( + "account file '{}' contains an oversized line", + group_path.display() + )); + } + let fields = line.split(':').collect::>(); + if fields.len() != 4 + || fields + .iter() + .any(|field| field.len() > MAX_ACCOUNT_FIELD_SIZE) + { + return Err(miette::miette!( + "group membership entry in '{}' is malformed", + group_path.display() + )); + } + if !fields[3].split(',').any(|member| member == user) { + continue; + } + let gid = fields[2].parse::().map_err(|_| { + miette::miette!( + "group membership GID in '{}' is malformed", + group_path.display() + ) + })?; + if gid == 0 { + return Err(miette::miette!( + "OCI user '{user}' is a member of prohibited GID 0" + )); + } + gids.push(gid); + } + gids.sort_unstable(); + gids.dedup(); + Ok(gids) +} + +fn find_unique( + path: &Path, + mut select: impl FnMut(&[&str]) -> Option>, +) -> Result> { + let content = read_account_file(path)?; + let mut found = None; + for line in content.lines() { + if line.is_empty() || line.starts_with('#') { + continue; + } + if line.len() > MAX_ACCOUNT_LINE_SIZE { + return Err(miette::miette!( + "account file '{}' contains an oversized line", + path.display() + )); + } + let fields = line.split(':').collect::>(); + if fields + .iter() + .any(|field| field.len() > MAX_ACCOUNT_FIELD_SIZE) + { + return Err(miette::miette!( + "account file '{}' contains an oversized field", + path.display() + )); + } + let Some(candidate) = select(&fields) else { + continue; + }; + let candidate = candidate?; + if found.replace(candidate).is_some() { + return Err(miette::miette!( + "account identity is ambiguous in '{}'", + path.display() + )); + } + } + Ok(found) +} + +fn parse_passwd(fields: &[&str]) -> Result { + if fields.len() != 7 { + return Err(miette::miette!("matching /etc/passwd entry is malformed")); + } + Ok(PasswdEntry { + uid: fields[2] + .parse() + .map_err(|_| miette::miette!("matching /etc/passwd UID is malformed"))?, + gid: fields[3] + .parse() + .map_err(|_| miette::miette!("matching /etc/passwd GID is malformed"))?, + }) +} + +fn parse_group(fields: &[&str]) -> Result { + if fields.len() != 4 { + return Err(miette::miette!("matching /etc/group entry is malformed")); + } + Ok(GroupEntry { + gid: fields[2] + .parse() + .map_err(|_| miette::miette!("matching /etc/group GID is malformed"))?, + }) +} + +fn read_account_file(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + let mut file = options + .open(path) + .into_diagnostic() + .map_err(|error| miette::miette!("failed to open '{}': {error}", path.display()))?; + validate_account_file(&file, path)?; + + let mut bytes = Vec::new(); + file.by_ref() + .take(MAX_ACCOUNT_FILE_SIZE + 1) + .read_to_end(&mut bytes) + .into_diagnostic()?; + if bytes.len() as u64 > MAX_ACCOUNT_FILE_SIZE { + return Err(miette::miette!( + "account file '{}' exceeds {MAX_ACCOUNT_FILE_SIZE} bytes", + path.display() + )); + } + String::from_utf8(bytes) + .map_err(|_| miette::miette!("account file '{}' is not valid UTF-8", path.display())) +} + +fn validate_account_file(file: &File, path: &Path) -> Result<()> { + let metadata = file.metadata().into_diagnostic()?; + if !metadata.is_file() { + return Err(miette::miette!( + "account path '{}' is not a regular file", + path.display() + )); + } + if metadata.len() > MAX_ACCOUNT_FILE_SIZE { + return Err(miette::miette!( + "account file '{}' exceeds {MAX_ACCOUNT_FILE_SIZE} bytes", + path.display() + )); + } + Ok(()) +} + +fn optional_utf8_env(name: &str) -> Result> { + std::env::var_os(name) + .map(|value| { + value + .into_string() + .map_err(|_| miette::miette!("{name} is not valid UTF-8")) + }) + .transpose() +} + +fn optional_nonempty_utf8_env(name: &str) -> Result> { + Ok(optional_utf8_env(name)?.filter(|value| !value.is_empty())) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::SandboxPolicy; + use std::fs; + use tempfile::tempdir; + + fn account_files( + passwd: &str, + group: &str, + ) -> (tempfile::TempDir, std::path::PathBuf, std::path::PathBuf) { + let dir = tempdir().unwrap(); + let passwd_path = dir.path().join("passwd"); + let group_path = dir.path().join("group"); + fs::write(&passwd_path, passwd).unwrap(); + fs::write(&group_path, group).unwrap(); + (dir, passwd_path, group_path) + } + + fn policy(user: Option<&str>, group: Option<&str>) -> SandboxPolicy { + let mut policy = SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }; + policy.process.run_as_user = user.map(str::to_string); + policy.process.run_as_group = group.map(str::to_string); + policy + } + + #[test] + fn per_field_policy_precedence_resolves_complete_pair() { + let (_dir, passwd, group) = account_files( + "app:x:1234:1235::/home/app:/bin/sh\nsandbox:x:2000:2001::/sandbox:/bin/sh\n", + "staff:x:1235:\nsandbox:x:2001:\n", + ); + let cases = [ + ( + Some("2000"), + Some("2001"), + "root", + "2000", + "2001", + None, + None, + ), + ( + Some("2000"), + None, + "app:staff", + "2000", + "staff", + None, + Some(1235), + ), + ( + None, + Some("2001"), + "app:root", + "app", + "2001", + Some(1234), + None, + ), + ( + None, + None, + "app:staff", + "app", + "staff", + Some(1234), + Some(1235), + ), + (None, None, "app", "app", "1235", Some(1234), Some(1235)), + ]; + for ( + user, + group_name, + declaration, + expected_user, + expected_group, + resolved_uid, + resolved_gid, + ) in cases + { + let mut policy = policy(user, group_name); + let resolved = + resolve_oci_process_identity_at(&mut policy, declaration, &passwd, &group).unwrap(); + assert_eq!(policy.process.run_as_user.as_deref(), Some(expected_user)); + assert_eq!(policy.process.run_as_group.as_deref(), Some(expected_group)); + assert_eq!(resolved.uid(), resolved_uid); + assert_eq!(resolved.gid(), resolved_gid); + } + } + + #[test] + fn numeric_pair_does_not_require_account_entries() { + let dir = tempdir().unwrap(); + let passwd = dir.path().join("missing-passwd"); + let group = dir.path().join("missing-group"); + let mut policy = policy(None, None); + let resolved = + resolve_oci_process_identity_at(&mut policy, "1234:1235", &passwd, &group).unwrap(); + assert_eq!(policy.process.run_as_user.as_deref(), Some("1234")); + assert_eq!(policy.process.run_as_group.as_deref(), Some("1235")); + assert_eq!( + resolved, + ResolvedProcessIdentity::new(Some(1234), Some(1235)) + ); + } + + #[test] + fn explicit_identity_is_preserved_without_inspecting_oci_or_accounts() { + let dir = tempdir().unwrap(); + let mut policy = policy(Some("sandbox"), Some("sandbox")); + + let resolved = resolve_oci_process_identity_at( + &mut policy, + "root:root", + &dir.path().join("missing-passwd"), + &dir.path().join("missing-group"), + ) + .unwrap(); + + assert_eq!(policy.process.run_as_user.as_deref(), Some("sandbox")); + assert_eq!(policy.process.run_as_group.as_deref(), Some("sandbox")); + assert_eq!(resolved, ResolvedProcessIdentity::default()); + } + + #[test] + fn driver_identity_inputs_are_mutually_exclusive_and_complete() { + assert_eq!( + DriverIdentity::from_values(Some("app".into()), None, None).unwrap(), + DriverIdentity::OciUser { + declaration: "app".into() + } + ); + assert_eq!( + DriverIdentity::from_values(None, Some("1234".into()), Some("1235".into())).unwrap(), + DriverIdentity::Resolved { + uid: 1234, + gid: 1235 + } + ); + assert_eq!( + DriverIdentity::from_values(None, Some("500".into()), Some("30".into())).unwrap(), + DriverIdentity::Resolved { uid: 500, gid: 30 } + ); + assert_eq!( + DriverIdentity::from_values( + Some(String::new()), + Some("1234".into()), + Some("1235".into()) + ) + .unwrap(), + DriverIdentity::Resolved { + uid: 1234, + gid: 1235 + } + ); + assert_eq!( + DriverIdentity::from_values(Some(String::new()), None, None).unwrap(), + DriverIdentity::OciUser { + declaration: String::new() + } + ); + assert_eq!( + DriverIdentity::from_values(None, None, None).unwrap(), + DriverIdentity::None + ); + assert!( + DriverIdentity::from_values( + Some("app".into()), + Some("1234".into()), + Some("1235".into()) + ) + .is_err() + ); + assert!(DriverIdentity::from_values(None, Some("1234".into()), None).is_err()); + } + + #[test] + fn no_driver_identity_completes_partial_policy_with_sandbox() { + let cases = [ + (None, Some("staff"), "sandbox", "staff"), + (Some("app"), None, "app", "sandbox"), + (None, None, "sandbox", "sandbox"), + (Some("app"), Some("staff"), "app", "staff"), + ]; + + for (user, group, expected_user, expected_group) in cases { + let mut policy = policy(user, group); + let resolved = resolve_process_identity(&mut policy, &DriverIdentity::None).unwrap(); + + assert_eq!(policy.process.run_as_user.as_deref(), Some(expected_user)); + assert_eq!(policy.process.run_as_group.as_deref(), Some(expected_group)); + assert_eq!(resolved, ResolvedProcessIdentity::default()); + } + } + + #[test] + fn numeric_uid_uses_passwd_primary_gid() { + let (_dir, passwd, group) = account_files("app:x:1234:4321::/home/app:/bin/sh\n", ""); + let mut policy = policy(None, None); + let resolved = + resolve_oci_process_identity_at(&mut policy, "1234", &passwd, &group).unwrap(); + assert_eq!(policy.process.run_as_group.as_deref(), Some("4321")); + assert_eq!( + resolved, + ResolvedProcessIdentity::new(Some(1234), Some(4321)) + ); + } + + #[test] + fn named_oci_user_resolves_bounded_supplementary_groups() { + let (_dir, _passwd, group) = account_files( + "", + "primary:x:1235:\nvideo:x:44:app,other\naudio:x:63:other\nrender:x:107:app\n", + ); + + let gids = resolve_oci_supplementary_gids_at("app:primary", 1235, &group).unwrap(); + assert_eq!(gids, vec![44, 107, 1235]); + } + + #[test] + fn numeric_oci_user_has_no_named_supplementary_groups() { + let dir = tempdir().unwrap(); + let missing_group = dir.path().join("missing-group"); + + let gids = resolve_oci_supplementary_gids_at("1234:1235", 1235, &missing_group).unwrap(); + assert!(gids.is_empty()); + } + + #[test] + fn oci_supplementary_membership_rejects_root_group() { + let (_dir, _passwd, group) = account_files("", "root:x:0:app\n"); + + let error = + resolve_oci_supplementary_gids_at("app", 1235, &group).expect_err("GID 0 must fail"); + assert!(error.to_string().contains("prohibited GID 0")); + } + + #[test] + fn missing_unknown_ambiguous_and_root_identities_fail() { + let (_dir, passwd, group) = account_files( + "app:x:1234:1235::/home/app:/bin/sh\napp:x:2234:2235::/home/app2:/bin/sh\n", + "staff:x:1235:\nstaff:x:2235:\n", + ); + for declaration in ["", "unknown", "app", "9999", "0:1235", "1234:0"] { + let mut policy = policy(None, None); + assert!( + resolve_oci_process_identity_at(&mut policy, declaration, &passwd, &group).is_err(), + "{declaration:?} unexpectedly resolved" + ); + } + } + + #[test] + fn selected_component_is_validated_independently() { + let (_dir, passwd, group) = + account_files("app:x:1234:1235::/home/app:/bin/sh\n", "staff:x:1235:\n"); + + let mut explicit_user = policy(Some("1234"), None); + let resolved = + resolve_oci_process_identity_at(&mut explicit_user, "root:staff", &passwd, &group) + .unwrap(); + assert_eq!(explicit_user.process.run_as_user.as_deref(), Some("1234")); + assert_eq!(explicit_user.process.run_as_group.as_deref(), Some("staff")); + assert_eq!(resolved, ResolvedProcessIdentity::new(None, Some(1235))); + + let mut explicit_group = policy(None, Some("1235")); + let resolved = + resolve_oci_process_identity_at(&mut explicit_group, "app:root", &passwd, &group) + .unwrap(); + assert_eq!(explicit_group.process.run_as_user.as_deref(), Some("app")); + assert_eq!(explicit_group.process.run_as_group.as_deref(), Some("1235")); + assert_eq!(resolved, ResolvedProcessIdentity::new(Some(1234), None)); + } + + #[test] + fn named_oci_components_mapping_to_root_are_rejected() { + let (_dir, passwd, group) = account_files( + "root_alias:x:0:1235::/root:/bin/sh\napp:x:1234:1235::/home/app:/bin/sh\n", + "root_alias:x:0:\nstaff:x:1235:\n", + ); + + let mut root_user = policy(None, None); + assert!( + resolve_oci_process_identity_at(&mut root_user, "root_alias:staff", &passwd, &group) + .is_err() + ); + + let mut root_group = policy(None, None); + assert!( + resolve_oci_process_identity_at(&mut root_group, "app:root_alias", &passwd, &group) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn account_file_symlinks_are_rejected() { + use std::os::unix::fs::symlink; + + let (_dir, passwd, group) = + account_files("app:x:1234:1235::/home/app:/bin/sh\n", "staff:x:1235:\n"); + let link = passwd.with_file_name("passwd-link"); + symlink(&passwd, &link).unwrap(); + + let mut policy = policy(None, None); + assert!(resolve_oci_process_identity_at(&mut policy, "app:staff", &link, &group).is_err()); + } +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 7afae200b5..dea0ef1c77 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1,6308 +1,63 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Sandbox library. -//! -//! This crate provides process sandboxing and monitoring capabilities. +//! Capability-free in-workload sandbox boundary. -// `defaults-without-telemetry` is an alias for the default feature set minus -// `telemetry`, not a switch that turns telemetry off. Cargo cannot subtract a -// default feature, so adding it on top of the defaults would otherwise produce -// a telemetry-on build that reads as telemetry-free. Fail the build instead. -#[cfg(all(feature = "telemetry", feature = "defaults-without-telemetry"))] -compile_error!( - "features `telemetry` and `defaults-without-telemetry` are mutually exclusive; \ - build a telemetry-free supervisor with `--no-default-features --features defaults-without-telemetry`" -); - -mod activity_aggregator; -mod denial_aggregator; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod google_cloud_metadata; -mod mechanistic_mapper; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod metadata_server; -mod sidecar_control; - -use miette::{IntoDiagnostic, Result, WrapErr}; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; +pub mod boundary_exec; +pub mod boundary_io; +mod boundary_server; +pub mod child_env; #[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicBool, AtomicU32}; -use std::time::Duration; -use tracing::{debug, info, warn}; - -use openshell_core::PolicyValidationFailureMode; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, - DetectionFindingBuilder, DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, - StateId, StatusId, ocsf_emit, -}; - -// --------------------------------------------------------------------------- -// OCSF Context -// --------------------------------------------------------------------------- -// -// The following log sites intentionally remain as plain `tracing` macros -// and are NOT migrated to OCSF builders: -// -// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) -// - Transient "about to do X" events where the result is logged separately -// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") -// - Internal SSH channel warnings (unknown channel, PTY resize failures) -// - Denial flush telemetry (the individual denials are already OCSF events) -// - Status reporting failures (sync to gateway, non-actionable) -// - Route refresh interval validation warnings -// -// These are operational plumbing that don't represent security decisions, -// policy changes, or observable sandbox behavior worth structuring. -// --------------------------------------------------------------------------- - -/// Re-export the process-wide OCSF sandbox context getter. -/// -/// The singleton lives in `openshell-ocsf` so both supervisor leaves can -/// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. -pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; - -use openshell_core::denial::DenialEvent; -use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_supervisor_network::opa::OpaEngine; -use openshell_supervisor_network::proxy::ProxyHandle; -use openshell_supervisor_process::process::ProcessEnforcementMode; -pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; -use openshell_supervisor_process::skills; -use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] -use tokio::time::timeout; - -const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; -const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; -const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; - -#[cfg(any(test, target_os = "linux"))] -fn has_network_runtime_capability(capabilities: Option<&str>, required: &str) -> bool { - capabilities.is_some_and(|capabilities| { - capabilities - .split(',') - .any(|capability| capability.trim() == required) - }) -} -const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; -const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; - -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::implicit_hasher, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, - inference_routes: Option, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - network_enabled: bool, - process_enabled: bool, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, -) -> Result { - let (program, args) = command - .split_first() - .ok_or_else(|| miette::miette!("No command specified"))?; - - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; they are only significant for network - // events which happen after the netns is created. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(SandboxContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); - let process_uses_sidecar_control = - process_enabled && !network_enabled && sidecar_network_enforcement; - let mut process_control_connection = None; - let sidecar_bootstrap = if process_uses_sidecar_control { - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let (bootstrap, connection) = sidecar_control::connect_process_client( - &socket, - Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), - ) - .await?; - process_control_connection = Some(connection); - Some(bootstrap) - } else { - None - }; - - // Extension credentials are owned by this supervisor and shared by every - // gateway connection it opens, so the middleware registry's bearer slots - // and the policy poll loop that rotates them stay the same objects. - let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); - - // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); - let sandbox_name_for_agg = sandbox.clone(); - let ( - mut policy, - opa_engine, - retained_proto, - middleware_registry_status, - loaded_policy_origin, - initial_agent_proposals_enabled, - initial_extension_authentication_enabled, - ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let (policy, opa_engine, retained_proto, loaded_policy_origin) = - load_policy_from_sidecar_bootstrap(bootstrap)?; - ( - policy, - opa_engine, - retained_proto, - MiddlewareRegistryStatus::Synchronized, - loaded_policy_origin, - bootstrap.agent_proposals_enabled, - false, - ) - } else { - load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - &extension_credentials, - ) - .await? - }; - - // Normalize the active driver's identity contract once, while both the - // policy and launched image filesystem are available. Kubernetes and - // OpenShift retain their authoritative numeric pair; Docker fills only - // omitted policy fields from OCI Config.User. - #[cfg(unix)] - let (resolved_process_identity, workspace) = { - let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; - let use_workdir_as_home = matches!( - &driver_identity, - openshell_supervisor_process::identity::DriverIdentity::OciUser { .. } - ); - let resolved = openshell_supervisor_process::identity::resolve_process_identity( - &mut policy, - &driver_identity, - )?; - ( - resolved, - openshell_supervisor_process::process::ResolvedWorkspace::new( - workdir.clone(), - use_workdir_as_home, - ), - ) - }; - #[cfg(not(unix))] - let (resolved_process_identity, workspace) = ( - openshell_supervisor_process::process::ResolvedProcessIdentity::default(), - openshell_supervisor_process::process::ResolvedWorkspace::new(workdir.clone(), false), - ); - - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = if let Some(bootstrap) = - sidecar_bootstrap.as_ref() - { - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - bootstrap.provider_env_revision, - bootstrap.provider_child_env.clone(), - ); - (provider_credentials, bootstrap.provider_child_env.clone()) - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - static_credential_bindings, - non_secret_environment_keys, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - result.static_credential_bindings, - result.non_secret_environment_keys, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Failed to fetch provider environment; no provider credentials are active: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - Vec::new(), - ) - } - } - } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - Vec::new(), - ) - }; - - let dynamic_credentials_fallback = dynamic_credentials.clone(); - let provider_credentials = match ProviderCredentialState::from_bound_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - static_credential_bindings, - non_secret_environment_keys, - ) { - Ok(credentials) => credentials, - Err(error) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Rejected provider environment bindings; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" - )) - .build() - ); - ProviderCredentialState::from_environment( - provider_env_revision, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - dynamic_credentials_fallback, - ) - } - }; - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - - if credential_gating_unavailable( - &loaded_policy_origin, - provider_credentials.resolver().is_some(), - network_enabled, - ) { - report_credential_gating_unavailable(); - } - - // Canonical-process overrides are deliberately applied only to the main - // child. Keep the provider snapshot pristine because Kubernetes forwards - // it to the process sidecar for later exec/editor/SFTP children. - - // Shared agent-proposals feature flag. Seed from the same initial settings - // snapshot that produced the policy so networking and process setup agree - // before the poll loop starts reconciling later changes. - let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); - - let process_control_writer = process_control_connection - .as_ref() - .map(|connection| connection.writer.clone()); - let process_exit_ack = Arc::new(tokio::sync::Mutex::new(None)); - let initial_provider_env_generation = sidecar_bootstrap - .as_ref() - .map_or(0, |bootstrap| bootstrap.provider_env_generation); - let mut process_control_closed = None; - if let Some(connection) = process_control_connection { - process_control_closed = Some(connection.closed); - spawn_sidecar_control_update_watcher( - connection.updates, - provider_credentials.clone(), - agent_proposals.clone(), - Arc::clone(&process_exit_ack), - initial_provider_env_generation, - ); - } - - // Shared PID: set after process spawn so the proxy can look up - // the entrypoint process's /proc/net/tcp for identity binding. - let entrypoint_pid = Arc::new(AtomicU32::new(0)); - - // Create the workload's network namespace. It is shared infrastructure: - // the proxy binds to its host-side veth IP, the bypass monitor reads - // /dev/kmsg from inside it, and the workload child / SSH sessions enter - // it via setns(). The RAII handle lives in this frame for the duration - // of the sandbox. - #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { - openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? - } else { - None - }; - - #[cfg(target_os = "linux")] - let transparent_tcp_requested = opa_engine - .as_ref() - .map(|engine| engine.policy_dns_eligibility_snapshot()) - .transpose()? - .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); - #[cfg(target_os = "linux")] - let runtime_capabilities = - std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); - #[cfg(target_os = "linux")] - let transparent_tcp_capable = has_network_runtime_capability( - runtime_capabilities.as_deref(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, - ); - #[cfg(not(target_os = "linux"))] - let transparent_tcp_capable = false; - #[cfg(target_os = "linux")] - let transparent_runtime = if transparent_tcp_requested { - if !transparent_tcp_capable { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "unsupported_runtime") - .message( - "Policy DNS and transparent TCP unavailable: runtime capability is missing" - ) - .build() - ); - return Err(miette::miette!( - "policy contains protocol: tcp endpoints, but the selected runtime does not advertise policy DNS and transparent TCP support" - )); - } - if sidecar_network_enforcement { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "unsupported_topology") - .message("Policy DNS and transparent TCP unavailable: sidecar topology is unsupported") - .build() - ); - return Err(miette::miette!( - "policy DNS and transparent TCP are not yet supported by the sidecar topology" - )); - } - let namespace = netns.as_ref().ok_or_else(|| { - miette::miette!("policy DNS and transparent TCP require a workload network namespace") - })?; - let listeners = namespace - .bind_transparent_tcp_listeners() - .await - .into_diagnostic() - .wrap_err("failed to bind transparent TCP listeners")?; - let (dns_udp, dns_tcp) = namespace - .bind_policy_dns_sockets() - .await - .into_diagnostic() - .wrap_err("failed to bind policy DNS listeners")?; - let proxy_port = policy - .network - .proxy - .as_ref() - .and_then(|proxy| proxy.http_addr) - .map_or(3128, |address| address.port()); - let runtime = openshell_supervisor_network::run::TransparentRuntimeSetup::new( - listeners, - dns_udp, - dns_tcp, - sandbox_id.as_deref(), - )?; - let (ipv4_cidr, ipv6_cidr) = runtime.synthetic_cidrs(); - namespace.install_transparent_tcp_rules(proxy_port, &ipv4_cidr, &ipv6_cidr)?; - Some(runtime) - } else { - None - }; - #[cfg(target_os = "linux")] - let transparent_tcp_substrate_ready = transparent_runtime.is_some(); - #[cfg(not(target_os = "linux"))] - let transparent_tcp_substrate_ready = false; - // The denial channel is owned by the orchestrator: the proxy (in the - // networking leaf) and the bypass monitor (in the process leaf) both - // produce DenialEvents that the denial aggregator (orchestrator-side) - // consumes via the matching receiver. Both leaves are pure producers; - // the orchestrator owns the consumer task spawned below. - let (denial_tx, denial_rx, bypass_denial_tx): ( - Option>, - _, - Option>, - ) = if sandbox_id.is_some() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_denial_tx); - - // Anonymous activity channel: same orchestrator-owned pattern as the - // denial channel. The proxy and the bypass monitor both emit per-event - // activity records; the orchestrator-side aggregator drains, sanitizes, - // and flushes anonymous summaries to the gateway. - let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { - let (tx, rx) = - tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_activity_tx); - - // Workspace watch: the policy poll loop learns the workspace from - // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local - // API read the current value so proposals target the correct workspace. - let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - - let mut networking = if network_enabled { - #[cfg(target_os = "linux")] - let proxy_bind_ip = netns - .as_ref() - .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); - #[cfg(not(target_os = "linux"))] - let proxy_bind_ip: Option = None; - - Some( - openshell_supervisor_network::run::run_networking( - &policy, - proxy_bind_ip, - opa_engine.as_ref(), - retained_proto.as_ref(), - entrypoint_pid.clone(), - process_enabled, - &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), - inference_routes.as_deref(), - denial_tx, - activity_tx, - agent_proposals.clone(), - workspace_rx.clone(), - &upstream_proxy_args, - #[cfg(target_os = "linux")] - transparent_runtime, - ) - .await?, - ) - } else { - None - }; - - #[cfg(target_os = "linux")] - let sidecar_control_server = if network_enabled && sidecar_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" - )); - } - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let proto = retained_proto.as_ref().ok_or_else(|| { - miette::miette!( - "sidecar topology requires gateway policy data for the process supervisor" - ) - })?; - let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); - Some(sidecar_control::spawn_server( - &socket, - sidecar_control::BootstrapData { - policy_proto: proto.clone(), - provider_env_revision: provider_credentials.snapshot().revision, - provider_env_generation: 0, - provider_child_env: provider_env.clone(), - agent_proposals_enabled: agent_proposals.enabled(), - proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), - proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), - }, - sidecar_expected_peer()?, - )?) - } else { - None - }; - #[cfg(not(target_os = "linux"))] - let sidecar_control_server: Option = None; - - let sidecar_control_publisher = sidecar_control_server - .as_ref() - .map(sidecar_control::ServerHandle::publisher); - - #[cfg(target_os = "linux")] - let mut sidecar_control_task = None; - - #[cfg(target_os = "linux")] - if network_enabled - && sidecar_network_enforcement - && let Some(server) = sidecar_control_server - { - let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar network topology", - openshell_core::sandbox_env::SSH_SOCKET_PATH - ) - })?; - let (entrypoint_rx, connection_task) = server.into_runtime_parts(); - sidecar_control_task = Some(connection_task); - spawn_sidecar_entrypoint_handler( - entrypoint_rx, - SidecarEntrypointHandler { - entrypoint_pid: entrypoint_pid.clone(), - opa_engine: opa_engine.clone(), - retained_proto: retained_proto.clone(), - openshell_endpoint: openshell_endpoint.clone(), - sandbox_id: sandbox_id.clone(), - trusted_ssh_socket_path: std::path::PathBuf::from(trusted_ssh_socket_path), - control_publisher: sidecar_control_publisher.clone(), - }, - ); - } - - #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { - return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" - )); - } - - // Spawn the denial-aggregator flush task. The aggregator drains denial - // events from the proxy + bypass monitor, batches them, and ships - // summaries to the gateway via `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { - // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall - // back to the ID when the name isn't set. - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - - let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); - let denial_workspace_gate = workspace_rx.clone(); - let denial_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - |summaries| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = denial_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_proposals_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summaries, - ) - .await - { - warn!(error = %e, "Failed to flush denial summaries to gateway"); - } - } - }, - move || !denial_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn the activity-aggregator flush task. The aggregator drains - // anonymous activity events from the proxy, sanitizes deny groups, - // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( - std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") - .ok() - .as_deref(), - ); - - let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); - let activity_workspace_gate = workspace_rx.clone(); - let activity_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - move |summary| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = activity_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_activity_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summary, - ) - .await - { - warn!(error = %e, "Failed to flush activity summary to gateway"); - } - } - }, - move || !activity_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn background policy poll task (gRPC mode only). - if !process_uses_sidecar_control - && let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) - { - let poll_id = id.to_string(); - let poll_endpoint = endpoint.to_string(); - let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_ocsf_schema_version = ocsf_schema_version.clone(); - let poll_pid = entrypoint_pid.clone(); - let poll_provider_credentials = provider_credentials.clone(); - let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); - let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, - sandbox_id: poll_id, - opa_engine: poll_engine, - loaded_policy_origin, - entrypoint_pid: poll_pid, - interval_secs: poll_interval_secs, - ocsf_enabled: poll_ocsf_enabled, - ocsf_schema_version: poll_ocsf_schema_version, - provider_credentials: poll_provider_credentials, - policy_local_ctx: poll_policy_local, - agent_proposals: agent_proposals.clone(), - middleware_registry_status, - sidecar_control_publisher: sidecar_control_publisher.clone(), - workspace_tx, - extension_credentials: extension_credentials.clone(), - extension_authentication_enabled: initial_extension_authentication_enabled, - middleware_connector: default_middleware_connector(), - transparent_tcp: TransparentTcpReloadState { - capable: transparent_tcp_capable, - substrate_ready: transparent_tcp_substrate_ready, - }, - }; - - tokio::spawn(async move { - if let Err(e) = run_policy_poll_loop(poll_ctx).await { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .message(format!("Policy poll loop exited with error: {e}")) - .build() - ); - } - }); - } - - // Start GCE metadata loopback server inside the network namespace so - // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) - // can reach it via direct TCP. Must start before the process leaf so SSH - // sessions also see corrected env vars on bind failure. - #[cfg(target_os = "linux")] - if let Some(ns) = netns.as_ref() - && provider_credentials - .snapshot() - .child_env - .contains_key("GCE_METADATA_HOST") - { - let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); - let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); - match ns - .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) - .await - { - Ok(listener) => { - tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); - if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { - info!(addr = %addr, "GCE metadata loopback server ready"); - } else { - warn!("GCE metadata server failed to become ready, removing metadata env vars"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - Err(e) => { - warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - } - - let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let main_env = provider_env.clone(); - let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { - bootstrap - .proxy_ca_cert_path - .clone() - .zip(bootstrap.proxy_ca_bundle_path.clone()) - }); - - let proxy_exited: Pin + Send>> = if let Some(rx) = networking - .as_mut() - .and_then(|n| n.proxy.as_mut()) - .and_then(ProxyHandle::take_exit_receiver) - { - Box::pin(async { - let _ = rx.await; - }) - } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(proxy_exited); - - let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None - } - }); - - let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx { - Box::pin(async { - let _ = rx.await; - }) - } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(ssh_exited); - - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok((pid, instance_id)) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid, instance_id) - .await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); - } - } - }); - Some(tx) - } else { - None - }; - let sidecar_exit_tx = if process_uses_sidecar_control - && let Some(writer) = process_control_writer.clone() - { - let exit_ack = Arc::clone(&process_exit_ack); - let (tx, mut rx) = tokio::sync::mpsc::channel::< - openshell_supervisor_process::run::SidecarExitReport, - >(1); - tokio::spawn(async move { - while let Some(report) = rx.recv().await { - match report { - openshell_supervisor_process::run::SidecarExitReport::Exited { - instance_id, - exit_code, - ack, - } => { - let (durable_tx, durable_rx) = tokio::sync::oneshot::channel(); - *exit_ack.lock().await = Some((instance_id.clone(), durable_tx)); - let result = match sidecar_control::send_main_process_exited( - &writer, - instance_id, - exit_code, - ) - .await - { - Ok(()) => durable_rx.await.map_err(|_| { - "sidecar durable exit acknowledgement closed".to_string() - }), - Err(error) => Err(error.to_string()), - }; - let _ = ack.send(result); - } - openshell_supervisor_process::run::SidecarExitReport::Finalized { - instance_id, - ack, - } => { - let result = - sidecar_control::send_main_process_finalized(&writer, instance_id) - .await - .map_err(|error| error.to_string()); - let _ = ack.send(result); - } - } - } - }); - Some(tx) - } else { - None - }; - - let process = openshell_supervisor_process::run::run_process( - program, - args, - workspace, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - ssh_exit_tx, - &process_policy, - resolved_process_identity, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - sidecar_exit_tx, - provider_credentials, - main_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); - - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } - } else { - tokio::select! { - result = process => result?, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } - } - } else { - // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until shutdown. If the - // sole authenticated process-supervisor control connection closes, - // exit non-zero so Kubernetes restarts the network sidecar and creates - // a fresh one-client bootstrap listener for the restarted agent. - #[cfg(target_os = "linux")] - if let Some(control_task) = sidecar_control_task { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - result = control_task => { - warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); - 1 - } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } - } else { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } - } - #[cfg(not(target_os = "linux"))] - { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } - } - }; - - // Drop networking explicitly so the proxy + bypass monitor RAII - // handles tear down before we return. - drop(networking); - - Ok(exit_code) -} - -/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is -/// no entrypoint child whose lifetime drives the supervisor's exit. -async fn wait_for_shutdown_signal() { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(s) => s, - Err(e) => { - tracing::warn!( - error = %e, - "Failed to install SIGTERM handler; waiting on SIGINT only" - ); - let _ = tokio::signal::ctrl_c().await; - return; - } - }; - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Received SIGINT, shutting down network-only supervisor"); - } - _ = sigterm.recv() => { - info!("Received SIGTERM, shutting down network-only supervisor"); - } - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - info!("Received Ctrl-C, shutting down network-only supervisor"); - } -} - -fn sidecar_network_enforcement_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) - .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) -} - -fn process_enforcement_mode() -> ProcessEnforcementMode { - match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .ok() - .as_deref() - { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, - _ => ProcessEnforcementMode::Full, - } -} - -fn sidecar_control_socket() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) - .ok() - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn sidecar_expected_peer() -> Result { - fn required_numeric_env(name: &str) -> Result { - let value = std::env::var(name) - .into_diagnostic() - .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; - value.parse::().into_diagnostic().wrap_err_with(|| { - format!("{name} must be a numeric ID for sidecar control authentication") - }) - } - - Ok(sidecar_control::ExpectedPeer { - uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, - gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, - }) -} - -type LoadedPolicyBundle = ( - SandboxPolicy, - Option>, - Option, - LoadedPolicyOrigin, -); - -type MainProcessExitAckWaiter = - Arc)>>>; - -fn load_policy_from_sidecar_bootstrap( - bootstrap: &sidecar_control::BootstrapData, -) -> Result { - let proto = bootstrap.policy_proto.clone(); - let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); - let policy = SandboxPolicy::try_from(proto.clone())?; - info!("Loaded sidecar policy from control socket bootstrap"); - Ok(( - policy, - opa_engine, - Some(proto), - LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }, - )) -} - -fn spawn_sidecar_control_update_watcher( - mut updates: tokio::sync::mpsc::UnboundedReceiver, - provider_credentials: ProviderCredentialState, - agent_proposals: AgentProposals, - exit_ack: MainProcessExitAckWaiter, - mut provider_env_generation: u64, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - while let Some(update) = updates.recv().await { - match update { - sidecar_control::ControlUpdate::ProviderEnv { - revision, - generation, - provider_child_env, - } => { - if generation <= provider_env_generation { - continue; - } - let env_count = provider_credentials - .install_child_env_snapshot(revision, provider_child_env); - provider_env_generation = generation; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("provider_env_revision", serde_json::json!(revision)) - .unmapped("provider_env_generation", serde_json::json!(generation)) - .message(format!( - "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" - )) - .build() - ); - } - sidecar_control::ControlUpdate::Policy { - policy_proto, - policy_hash, - config_revision, - } => { - debug!( - version = policy_proto.version, - policy_hash, - config_revision, - "Received sidecar policy update for process supervisor" - ); - } - sidecar_control::ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - apply_agent_proposals_enabled( - &agent_proposals, - enabled, - "sidecar control", - Some(config_revision), - None, - skills::install_static_skills, - ); - } - sidecar_control::ControlUpdate::MainProcessExitAck { instance_id } => { - let mut waiter = exit_ack.lock().await; - if waiter - .as_ref() - .is_some_and(|(expected, _)| expected == &instance_id) - && let Some((_, ack)) = waiter.take() - { - let _ = ack.send(()); - } - } - } - } - }) -} - +pub(crate) mod delegated; +#[cfg(unix)] +pub mod identity; #[cfg(target_os = "linux")] -struct SidecarEntrypointHandler { - entrypoint_pid: Arc, - opa_engine: Option>, - retained_proto: Option, - openshell_endpoint: Option, - sandbox_id: Option, - trusted_ssh_socket_path: std::path::PathBuf, - control_publisher: Option, -} - +pub mod main_session; +pub mod managed_children; #[cfg(target_os = "linux")] -fn spawn_sidecar_entrypoint_handler( - mut entrypoint_rx: tokio::sync::mpsc::Receiver, - handler: SidecarEntrypointHandler, -) { - tokio::spawn(async move { - let SidecarEntrypointHandler { - entrypoint_pid, - opa_engine, - retained_proto, - openshell_endpoint, - sandbox_id, - trusted_ssh_socket_path, - control_publisher, - } = handler; - let mut session_started = false; - let mut session_task: Option> = None; - let mut trusted_supervisor_pid = None; - let terminating = Arc::new(AtomicBool::new(false)); - while let Some(started) = entrypoint_rx.recv().await { - if started.finalized { - if let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let mut delay = Duration::from_millis(250); - loop { - match openshell_supervisor_process::supervisor_session::finalize_main_process_exit( - endpoint, - id, - &started.instance_id, - ) - .await - { - Ok(()) => break, - Err(error) => { - warn!(%error, "sidecar main-process finalization failed; retrying"); - tokio::time::sleep(delay).await; - delay = (delay * 2).min(Duration::from_secs(2)); - } - } - } - } - terminating.store(true, Ordering::Release); - if let Some(task) = session_task.take() { - task.abort(); - } - break; - } - if let Some(exit_code) = started.exit_code { - if let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let mut delay = Duration::from_millis(250); - loop { - match openshell_supervisor_process::supervisor_session::report_main_process_exit( - endpoint, - id, - &started.instance_id, - exit_code, - ) - .await - { - Ok(()) => break, - Err(error) => { - warn!(%error, "sidecar main-process exit report failed; retrying"); - tokio::time::sleep(delay).await; - delay = (delay * 2).min(Duration::from_secs(2)); - } - } - } - if let Some(publisher) = control_publisher.as_ref() { - publisher.publish_main_process_exit_ack(started.instance_id.clone()); - } - } - continue; - } - entrypoint_pid.store(started.pid, Ordering::Release); - if started.start_session { - info!( - pid = started.pid, - ssh_socket = %trusted_ssh_socket_path.display(), - "Sidecar process supervisor reported entrypoint start" - ); - } else { - trusted_supervisor_pid = Some(started.pid); - info!( - pid = started.pid, - "Sidecar process supervisor reported initial process anchor" - ); - } - - if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { - match engine.reload_from_proto_with_pid(proto, started.pid) { - Ok(()) => info!( - pid = started.pid, - "Policy binary symlink resolution complete for sidecar process anchor" - ), - Err(err) => warn!( - error = %err, - pid = started.pid, - "Failed to rebuild OPA engine with sidecar process anchor PID" - ), - } - } - - if started.start_session - && !session_started - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let Some(supervisor_pid) = trusted_supervisor_pid else { - warn!( - pid = started.pid, - "Ignoring sidecar entrypoint event before authenticated supervisor anchor" - ); - continue; - }; - session_task = Some(openshell_supervisor_process::supervisor_session::spawn( - endpoint.clone(), - id.clone(), - trusted_ssh_socket_path.clone(), - None, - Some(supervisor_pid), - Arc::clone(&terminating), - started.instance_id.clone(), - )); - session_started = true; - info!("sidecar supervisor session task spawned"); - } - } - terminating.store(true, Ordering::Release); - }); -} - -fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) - .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); - let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); - let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); - (cert.exists() && bundle.exists()).then_some((cert, bundle)) -} - -fn process_policy_for_topology( - policy: &SandboxPolicy, - sidecar_network_enforcement: bool, -) -> Result { - let mut process_policy = policy.clone(); - if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { - let proxy = process_policy - .network - .proxy - .get_or_insert(ProxyPolicy { http_addr: None }); - if proxy.http_addr.is_none() { - proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); - } - } - Ok(process_policy) -} - -/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. -async fn flush_proposals_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summaries: Vec, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialSummary, L7RequestSample}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summaries: Vec = summaries - .into_iter() - .map(|s| DenialSummary { - sandbox_id: String::new(), - host: s.host, - port: u32::from(s.port), - binary: s.binary, - ancestors: s.ancestors, - deny_reason: s.deny_reason, - first_seen_ms: s.first_seen_ms, - last_seen_ms: s.last_seen_ms, - count: s.count, - suppressed_count: 0, - total_count: s.count, - sample_cmdlines: s.sample_cmdlines, - binary_sha256: String::new(), - persistent: false, - denial_stage: s.denial_stage, - l7_request_samples: s - .l7_samples - .into_iter() - .map(|l| L7RequestSample { - method: l.method, - path: l.path, - decision: "deny".to_string(), - count: l.count, - }) - .collect(), - l7_inspection_active: false, - }) - .collect(); - - // Run the mechanistic mapper sandbox-side to generate proposals. - // The gateway is a thin persistence + validation layer — it never - // generates proposals itself. - let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); - - info!( - sandbox_name = %sandbox_name, - summaries = proto_summaries.len(), - proposals = proposals.len(), - "Flushed denial analysis to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - proto_summaries, - proposals, - Vec::new(), - "mechanistic", - ) - .await?; - - Ok(()) -} - -/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. -async fn flush_activity_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summary: activity_aggregator::FlushableActivitySummary, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summary = NetworkActivitySummary { - network_activity_count: summary.network_activity_count, - denied_action_count: summary.denied_action_count, - denials_by_group: summary - .denials_by_group - .into_iter() - .map(|(group, count)| DenialGroupCount { - deny_group: group, - denied_count: count, - }) - .collect(), - }; - - info!( - sandbox_name = %sandbox_name, - network_activity_count = proto_summary.network_activity_count, - denied_action_count = proto_summary.denied_action_count, - "Flushed activity summary to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - Vec::new(), - Vec::new(), - vec![proto_summary], - "activity", - ) - .await?; - - Ok(()) -} - -// ============================================================================ -// Baseline filesystem path enrichment -// ============================================================================ - -/// Minimum read-only paths required for a proxy-mode sandbox child process to -/// function: dynamic linker, shared libraries, DNS resolution, CA certs, -/// Python venv, openshell logs, process info, and random bytes. -/// -/// `/proc` and `/dev/urandom` are included here for the same reasons they -/// appear in `restrictive_default_policy()`: virtually every process needs -/// them. Before the Landlock per-path fix (#677) these were effectively free -/// because a missing path silently disabled the entire ruleset; now they must -/// be explicit. -const PROXY_BASELINE_READ_ONLY: &[&str] = &[ - "/usr", - "/lib", - "/etc", - "/app", - "/var/log", - "/proc", - "/dev/urandom", -]; - -/// Minimum read-write paths required for a proxy-mode sandbox child process. -/// The active workspace is granted separately through `include_workdir`. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp"]; - -/// GPU read-only paths. -/// -/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced -/// socket at init time. If the directory exists but Landlock denies traversal -/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` -/// even though the daemon is optional. Only read/traversal access is needed. -/// -/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, -/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` -/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may -/// not be covered by the parent-directory Landlock rule when the mount crosses -/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal -/// is permitted regardless of Landlock's cross-mount behaviour. -const GPU_BASELINE_READ_ONLY: &[&str] = &[ - "/run/nvidia-persistenced", - "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory -]; - -/// GPU read-write paths (static). -/// -/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, -/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native -/// Linux. Landlock restricts `open(2)` on device files even when DAC allows -/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. -/// These devices do not exist on WSL2 and will be skipped by the existence -/// check in `enrich_proto_baseline_paths()`. -/// -/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver -/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects -/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux -/// and will be skipped there by the existence check. -/// -/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` -/// to set thread names. Without write access, `cuInit()` returns error 304. -/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to -/// inodes and child processes have different procfs inodes than the parent. -/// -/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by -/// `enumerate_gpu_device_nodes()` since the count varies. -const GPU_BASELINE_READ_WRITE: &[&str] = &[ - "/dev/nvidiactl", - "/dev/nvidia-uvm", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-modeset", - "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) - "/proc", -]; - -/// Returns true if GPU devices are present in the container. -/// -/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and -/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these -/// depending on the host kernel; the other will not exist. -fn has_gpu_devices() -> bool { - std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() -} - -/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). -fn enumerate_gpu_device_nodes() -> Vec { - let mut paths = Vec::new(); - if let Ok(entries) = std::fs::read_dir("/dev") { - for entry in entries.flatten() { - let name = entry.file_name(); - let name = name.to_string_lossy(); - if let Some(suffix) = name.strip_prefix("nvidia") { - if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { - continue; - } - paths.push(entry.path().to_string_lossy().into_owned()); - } - } - } - paths -} - -fn push_unique(paths: &mut Vec, path: String) { - if !paths.iter().any(|p| p == &path) { - paths.push(path); - } -} - -fn collect_baseline_enrichment_paths( - include_proxy: bool, - include_gpu: bool, - gpu_device_nodes: Vec, -) -> (Vec, Vec) { - let mut ro = Vec::new(); - let mut rw = Vec::new(); - - if include_proxy { - for &path in PROXY_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in PROXY_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - } - - if include_gpu { - for &path in GPU_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in GPU_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - for path in gpu_device_nodes { - push_unique(&mut rw, path); - } - } - - // A path promoted to read_write (e.g. /proc for GPU) should not also - // appear in read_only — Landlock handles the overlap correctly but the - // duplicate is confusing when inspecting the effective policy. - ro.retain(|p| !rw.contains(p)); - - (ro, rw) -} - -fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { - let include_gpu = has_gpu_devices(); - let gpu_device_nodes = if include_gpu { - enumerate_gpu_device_nodes() - } else { - Vec::new() - }; - collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) -} - -/// Collect all active baseline paths for tests and diagnostics. -/// Returns `(read_only, read_write)` as owned `String` vecs. -#[cfg(test)] -fn baseline_enrichment_paths() -> (Vec, Vec) { - active_baseline_enrichment_paths(true) -} - -fn enrich_proto_baseline_paths_with( - proto: &mut openshell_core::proto::SandboxPolicy, - ro: &[String], - rw: &[String], - path_exists: F, -) -> bool -where - F: Fn(&str) -> bool, -{ - if ro.is_empty() && rw.is_empty() { - return false; - } - - let fs = proto - .filesystem - .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { - include_workdir: true, - ..Default::default() - }); - - let mut modified = false; - for path in ro { - if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { - if !path_exists(path) { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - fs.read_only.push(path.clone()); - modified = true; - } - } - for path in rw { - if fs.read_write.iter().any(|p| p == path) { - continue; - } - if !path_exists(path) { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - if fs.read_only.iter().any(|p| p == path) { - if path == "/proc" { - info!( - path, - "Promoting /proc from read-only to read-write for GPU runtime compatibility" - ); - fs.read_only.retain(|p| p != path); - fs.read_write.push(path.clone()); - modified = true; - } - continue; - } - fs.read_write.push(path.clone()); - modified = true; - } - - modified -} - -/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths -/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if -/// missing; user-specified paths are never removed. -/// -/// Returns `true` if the policy was modified (caller may want to sync back). -fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); - - // Baseline paths are system-injected, not user-specified. Skip paths - // that do not exist in this container image to avoid noisy warnings from - // Landlock and, more critically, to prevent a single missing baseline - // path from abandoning the entire Landlock ruleset under best-effort - // mode (see issue #664). - let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { - std::path::Path::new(path).exists() - }); - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } - - modified -} - -fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - openshell_policy::strip_provider_rule_names(proto) -} - -fn proto_sync_payload_for_enriched_policy( - proto: &openshell_core::proto::SandboxPolicy, - enriched: bool, -) -> Option { - if !enriched { - return None; - } - - let mut sync_policy = proto.clone(); - strip_proto_provider_policy_entries(&mut sync_policy); - Some(sync_policy) -} - -/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem -/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the -/// local-file code path where no proto is available. -fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { - let (ro, rw) = - active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); - if ro.is_empty() && rw.is_empty() { - return; - } - - let mut modified = false; - for path in &ro { - let p = std::path::PathBuf::from(path); - if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { - if !p.exists() { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_only.push(p); - modified = true; - } - } - for path in &rw { - let p = std::path::PathBuf::from(path); - if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { - continue; - } - if !p.exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_write.push(p); - modified = true; - } - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } -} +mod network_broker; +#[cfg(target_os = "linux")] +pub mod process; +mod pty; +pub mod sandbox; -#[cfg(test)] +/// Results of actively qualifying the admitted workload runtime before the +/// sandbox consumes protected bootstrap material. +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug)] #[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." + clippy::struct_excessive_bools, + reason = "qualification preserves independently exercised security results" )] -mod baseline_tests { - use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; - use std::path::PathBuf; - - #[test] - fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { - // When GPU devices are present, /proc is promoted to read_write - // (CUDA needs to write /proc//task//comm). It should - // NOT also appear in read_only. - if !has_gpu_devices() { - // Can't test GPU dedup without GPU devices; skip silently. - return; - } - let (ro, rw) = baseline_enrichment_paths(); - assert!( - rw.contains(&"/proc".to_string()), - "/proc should be in read_write when GPU is present" - ); - assert!( - !ro.contains(&"/proc".to_string()), - "/proc should NOT be in read_only when it is already in read_write" - ); - } - - #[test] - fn proc_in_read_only_without_gpu() { - if has_gpu_devices() { - // On a GPU host we can't test the non-GPU path; skip silently. - return; - } - let (ro, _rw) = baseline_enrichment_paths(); - assert!( - ro.contains(&"/proc".to_string()), - "/proc should be in read_only when GPU is not present" - ); - } - - #[test] - fn baseline_read_write_does_not_hardcode_sandbox() { - let (_ro, rw) = baseline_enrichment_paths(); - assert!(rw.contains(&"/tmp".to_string())); - assert!(!rw.contains(&"/sandbox".to_string())); - } - - #[test] - fn enumerate_gpu_device_nodes_skips_bare_nvidia() { - // "nvidia" (without a trailing digit) is a valid /dev entry on some - // systems but is not a per-GPU device node. The enumerator must - // not match it. - let nodes = enumerate_gpu_device_nodes(); - assert!( - !nodes.contains(&"/dev/nvidia".to_string()), - "bare /dev/nvidia should not be enumerated: {nodes:?}" - ); - } - - #[test] - fn no_duplicate_paths_in_baseline() { - let (ro, rw) = baseline_enrichment_paths(); - // No path should appear in both lists. - for path in &ro { - assert!( - !rw.contains(path), - "path {path} appears in both read_only and read_write" - ); - } - } - - #[test] - fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { - read_only: vec!["/tmp".to_string()], - read_write: vec![], - include_workdir: false, - }); - policy.network_policies.insert( - "test".into(), - openshell_core::proto::NetworkPolicyRule { - name: "test-rule".into(), - endpoints: vec![openshell_core::proto::NetworkEndpoint { - host: "example.com".into(), - port: 443, - ..Default::default() - }], - ..Default::default() - }, - ); - - enrich_proto_baseline_paths(&mut policy); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - filesystem.read_only.contains(&"/tmp".to_string()), - "explicit read_only baseline path should be preserved" - ); - assert!( - !filesystem.read_write.contains(&"/tmp".to_string()), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - assert!(strip_proto_provider_policy_entries(&mut policy)); - assert!( - !policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(policy.network_policies.contains_key("sandbox_only")); - assert!(!strip_proto_provider_policy_entries(&mut policy)); - } - - #[test] - fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - - assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "provider-derived rules alone must not trigger sync or mutate runtime policy" - ); - } - - #[test] - fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - runtime_policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) - .expect("enrichment should create a sync payload"); - - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "runtime policy must retain provider-derived rules for OPA input" - ); - assert!( - !sync_policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(sync_policy.network_policies.contains_key("sandbox_only")); - } - - #[test] - fn proto_gpu_enrichment_promotes_proc_without_network_policy() { - let mut policy = openshell_policy::restrictive_default_policy(); - assert!( - policy.network_policies.is_empty(), - "regression setup must exercise the no-network default path" - ); - let (ro, rw) = - collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); - - let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { - matches!(path, "/proc" | "/dev/nvidia0") - }); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - enriched, - "GPU enrichment should not require network policies" - ); - assert!( - filesystem.read_write.contains(&"/dev/nvidia0".to_string()), - "GPU enrichment should add enumerated device nodes without network policies" - ); - assert!( - !filesystem.read_only.contains(&"/proc".to_string()), - "GPU enrichment should remove /proc from read_only" - ); - assert!( - filesystem.read_write.contains(&"/proc".to_string()), - "GPU enrichment should promote /proc to read_write" - ); - } - - #[test] - fn gpu_baseline_read_write_contains_dxg() { - // /dev/dxg must be present so WSL2 sandboxes get the Landlock - // read-write rule for the CDI-injected DXG device. The existence - // check in enrich_proto_baseline_paths() skips it on native Linux. - assert!( - GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), - "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" - ); - } - - #[test] - fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only: vec![PathBuf::from("/tmp")], - read_write: vec![], - include_workdir: false, - }, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - }; - - enrich_sandbox_baseline_paths(&mut policy); - - assert!( - policy.filesystem.read_only.contains(&PathBuf::from("/tmp")), - "explicit read_only baseline path should be preserved" - ); - assert!( - !policy - .filesystem - .read_write - .contains(&PathBuf::from("/tmp")), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn gpu_baseline_read_only_contains_usr_lib_wsl() { - // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library - // bind-mounts are accessible under Landlock. Skipped on native Linux. - assert!( - GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), - "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" - ); - } - - #[test] - fn has_gpu_devices_reflects_dxg_or_nvidiactl() { - // Verify the OR logic: result must match the manual disjunction of - // the two path checks. Passes in all environments. - let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); - let dxg = std::path::Path::new("/dev/dxg").exists(); - assert_eq!( - has_gpu_devices(), - nvidiactl || dxg, - "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" - ); - } -} - -/// Returns `true` if the error is transient and worth retrying. -/// -/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If -/// found, only the gRPC codes that represent transient failures are retryable. -/// If no `tonic::Status` is present (e.g. a raw connection error), assume the -/// failure is transient. -fn is_retryable_error(err: &miette::Report) -> bool { - let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); - while let Some(e) = source { - if let Some(status) = e.downcast_ref::() { - return matches!( - status.code(), - tonic::Code::Unavailable - | tonic::Code::DeadlineExceeded - | tonic::Code::ResourceExhausted - | tonic::Code::Aborted - | tonic::Code::Internal - | tonic::Code::Unknown - ); - } - source = e.source(); - } - true -} - -/// Retry a gRPC operation with exponential backoff (capped at 4 s). -/// -/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, -/// `PERMISSION_DENIED`) are returned immediately without retrying. -async fn grpc_retry(op_name: &str, f: F) -> Result -where - F: Fn() -> Fut, - Fut: Future>, -{ - let mut last_err = None; - for attempt in 1..=5u32 { - match f().await { - Ok(val) => return Ok(val), - Err(e) => { - if !is_retryable_error(&e) { - return Err(e); - } - if attempt < 5 { - warn!( - attempt, - max_attempts = 5, - error = %e, - "{op_name} failed, retrying" - ); - let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); - tokio::time::sleep(backoff).await; - } - last_err = Some(e); - } - } - } - Err(miette::miette!( - "{op_name} failed after 5 attempts: {}", - last_err.expect("loop executed at least once") - )) -} - -/// Load sandbox policy from local files or gRPC. -/// -/// Priority: -/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files -/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC -/// 3. If the server returns no policy, discover from disk or use restrictive default -/// 4. Otherwise, return an error -/// -/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto -/// policy. The proto is retained so the OPA engine can be rebuilt with symlink -/// resolution after the container entrypoint starts. -async fn load_policy( - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - extension_credentials: &openshell_extension_core::ExtensionCredentialStore, -) -> Result<( - SandboxPolicy, - Option>, - Option, - MiddlewareRegistryStatus, - LoadedPolicyOrigin, - bool, - bool, -)> { - // File mode: load OPA engine from rego rules + YAML data (dev override) - if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "loading") - .unmapped("policy_rules", serde_json::json!(policy_file)) - .unmapped("policy_data", serde_json::json!(data_file)) - .message(format!( - "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" - )) - .build()); - let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { - openshell_supervisor_middleware_builtins::validate_config(implementation, config) - .map_err(|error| error.to_string()) - }; - let engine = OpaEngine::from_files_with_middleware_config( - std::path::Path::new(policy_file), - std::path::Path::new(data_file), - Some(&validate_middleware_config), - )?; - let middleware_registry = - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - engine.replace_middleware_registry(middleware_registry)?; - let config = engine.query_sandbox_config()?; - let mut policy = SandboxPolicy { - version: 1, - filesystem: config.filesystem, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: config.landlock, - process: config.process, - }; - enrich_sandbox_baseline_paths(&mut policy); - // File mode has no operator-registered middleware to connect. - return Ok(( - policy, - Some(Arc::new(engine)), - None, - MiddlewareRegistryStatus::Synchronized, - LoadedPolicyOrigin::LocalOverride, - false, - false, - )); - } - - // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - info!( - sandbox_id = %id, - endpoint = %endpoint, - "Fetching sandbox policy via gRPC" - ); - let mut snapshot = grpc_retry("Policy fetch", || { - openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) - }) - .await?; - - let mut proto_policy = if let Some(p) = snapshot.policy.clone() { - p - } else { - // No policy configured on the server. Discover from disk or - // fall back to the restrictive default, then sync to the - // gateway so it becomes the authoritative baseline. - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "discovery") - .message("Server returned no policy; attempting local discovery") - .build() - ); - let mut discovered = discover_policy_from_disk_or_default(); - // Enrich before syncing so the gateway baseline includes - // baseline paths from the start. - enrich_proto_baseline_paths(&mut discovered); - strip_proto_provider_policy_entries(&mut discovered); - let sandbox = sandbox.as_deref().ok_or_else(|| { - miette::miette!( - "Cannot sync discovered policy: sandbox not available.\n\ - Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." - ) - })?; - - // Sync and re-fetch over a single connection to avoid extra - // TLS handshakes. - let ws = snapshot.workspace.clone(); - snapshot = grpc_retry("Policy discovery sync", || { - openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox, - &discovered, - &ws, - ) - }) - .await?; - snapshot.policy.clone().ok_or_else(|| { - miette::miette!("Server still returned no policy after sync — this is a bug") - })? - }; - - // True only while `snapshot` describes the exact policy that will be - // constructed below. If enrichment cannot be synced and re-fetched, - // the policy remains enforceable but cannot be acknowledged by - // inferred structural equality. - let mut policy_bound_to_snapshot = true; - - // Ensure baseline filesystem paths are present for proxy-mode - // sandboxes. If the policy was enriched, sync the updated version - // back to the gateway so users can see the effective policy. - let enriched = enrich_proto_baseline_paths(&mut proto_policy); - let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); - if let Some(sync_policy) = sync_policy { - if let Some(sandbox_name) = sandbox.as_deref() { - match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox_name, - &sync_policy, - &snapshot.workspace, - ) - .await - { - Ok(canonical) => { - if let Some(policy) = canonical.policy.clone() { - proto_policy = policy; - snapshot = canonical; - } else { - policy_bound_to_snapshot = false; - warn!( - "Gateway returned no policy after enrichment sync; initial revision will be reconciled" - ); - } - } - Err(e) => { - policy_bound_to_snapshot = false; - warn!( - error = %e, - "Failed to sync enriched policy back to gateway; initial revision will be reconciled" - ); - } - } - } else { - policy_bound_to_snapshot = false; - } - } - - let mut loaded_policy_revision = - policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); - - // Build OPA engine from baked-in rules + typed proto data. - // In cluster mode, proxy networking is always enabled so OPA is - // always required for allow/deny decisions. - // The initial load uses pid=0 (no symlink resolution) because the - // container hasn't started yet. After the entrypoint spawns, the - // engine is rebuilt with the real PID for symlink resolution. - info!("Creating OPA engine from proto policy data"); - let mut has_last_valid_policy = true; - let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => Arc::new(engine), - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - let validation_error = e.to_string(); - let candidate_version = snapshot.version; - let candidate_hash = snapshot.policy_hash.clone(); - // There is no in-memory last-known-good generation during - // startup, so both configured modes necessarily fail closed. - // Load the restrictive default atomically and keep the - // rejected revision unacknowledged for poll reconciliation. - has_last_valid_policy = false; - proto_policy = openshell_policy::restrictive_default_policy(); - let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); - let disposition = apply_policy_validation_failure( - &engine, - snapshot.policy_validation_failure_mode, - has_last_valid_policy, - candidate_version, - &validation_error, - )?; - emit_policy_validation_failure( - &disposition, - candidate_version, - &candidate_hash, - &validation_error, - ); - loaded_policy_revision = None; - engine - } - }; - - // Install the in-process catalog before any external connection can - // fail. A newly started sandbox must always be able to resolve built-in - // bindings, even while operator-run services are unavailable. - install_builtin_middleware_registry(&engine).await?; - - // Connect operator-registered middleware services. A connect/describe - // failure keeps the built-in registry active so each request's - // `on_error` policy governs matched traffic. The policy poll loop - // retries the install without waiting for a config change. - let middleware_services = snapshot.supervisor_middleware_services.clone(); - let middleware_registry_status = if middleware_services.is_empty() { - MiddlewareRegistryStatus::Synchronized - } else if let Err(error) = grpc_retry("Middleware connect", || { - let middleware_services = middleware_services.clone(); - let extension_credentials = extension_credentials.clone(); - let extension_authentication_enabled = snapshot.extension_authentication_enabled; - async move { - let credentials = if extension_authentication_enabled { - // Share the supervisor's store so the slots installed here - // are the ones the policy poll loop later rotates in place. - openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - endpoint, - extension_credentials, - ) - .await? - .refresh_extension_credentials(&middleware_services) - .await? - } else { - std::collections::HashMap::new() - }; - connect_middleware_registry( - &middleware_services, - &MiddlewareAuthentication { - credentials, - enabled: extension_authentication_enabled, - }, - ) - .await - } - }) - .await - .and_then(|registry| engine.replace_middleware_registry(registry)) - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(middleware_services.len()) - ) - .message(format!( - "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" - )) - .build() - ); - MiddlewareRegistryStatus::NeedsReconciliation - } else { - MiddlewareRegistryStatus::Synchronized - }; - let opa_engine = Some(engine); - - let policy = match SandboxPolicy::try_from(proto_policy.clone()) { - Ok(policy) => policy, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - return Ok(( - policy, - opa_engine, - Some(proto_policy), - middleware_registry_status, - LoadedPolicyOrigin::Gateway { - revision: loaded_policy_revision, - has_last_valid_policy, - }, - agent_proposals_enabled_from_settings(&snapshot.settings), - snapshot.extension_authentication_enabled, - )); - } - - // No policy source available - Err(miette::miette!( - "Sandbox policy required. Provide one of:\n\ - - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ - - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" - )) -} - -/// Try to discover a sandbox policy from the well-known disk path, falling -/// back to the legacy path, then to the hardcoded restrictive default. -fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { - let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); - if primary.exists() { - return discover_policy_from_path(primary); - } - let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); - if legacy.exists() { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "legacy_path", - serde_json::json!(legacy.display().to_string()) - ) - .unmapped("new_path", serde_json::json!(primary.display().to_string())) - .message(format!( - "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", - legacy.display(), - primary.display() - )) - .build() - ); - return discover_policy_from_path(legacy); - } - discover_policy_from_path(primary) -} - -/// Try to read a sandbox policy YAML from `path`, falling back to the -/// hardcoded restrictive default if the file is missing or invalid. -fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { - use openshell_policy::{ - parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, - }; - - let Ok(yaml) = std::fs::read_to_string(path) else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "default") - .message(format!( - "No policy file on disk, using restrictive default [path:{}]", - path.display() - )) - .build() - ); - return restrictive_default_policy(); - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Loaded sandbox policy from container disk [path:{}]", - path.display() - )) - .build() - ); - match parse_sandbox_policy(&yaml) { - Ok(policy) => { - // Validate the disk-loaded policy for safety. - if let Err(violations) = validate_sandbox_policy(&policy) { - let messages: Vec = violations.iter().map(ToString::to_string).collect(); - ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Medium) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .finding_info( - FindingInfo::new( - "unsafe-disk-policy", - "Unsafe Disk Policy Content", - ) - .with_desc(&format!( - "Disk policy at {} contains unsafe content: {}", - path.display(), - messages.join("; "), - )), - ) - .message(format!( - "Disk policy contains unsafe content, using restrictive default [path:{}]", - path.display() - )) - .build()); - return restrictive_default_policy(); - } - policy - } - Err(e) => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "fallback") - .message(format!( - "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", - path.display() - )) - .build()); - restrictive_default_policy() - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum MiddlewareRegistryStatus { - Synchronized, - NeedsReconciliation, -} - -#[derive(Debug)] -enum GatewayRuntimeReloadError { - PolicyValidation(miette::Report), - TransparentTcpPrerequisite(miette::Report), - MiddlewareRegistry(miette::Report), -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum GatewayRuntimeFailureClass { - PolicyValidation, - TransparentTcpPrerequisite, - MiddlewareRegistry, -} - -impl GatewayRuntimeReloadError { - fn class(&self) -> GatewayRuntimeFailureClass { - match self { - Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, - Self::TransparentTcpPrerequisite(_) => { - GatewayRuntimeFailureClass::TransparentTcpPrerequisite - } - Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, - } - } -} - -#[derive(Debug, PartialEq, Eq)] -struct FailedRuntimeRevision { - config_revision: u64, - policy_hash: String, - failure_class: GatewayRuntimeFailureClass, -} - -impl FailedRuntimeRevision { - fn new(config_revision: u64, policy_hash: &str, failure: &GatewayRuntimeReloadError) -> Self { - Self { - config_revision, - policy_hash: policy_hash.to_string(), - failure_class: failure.class(), - } - } -} - -struct MiddlewareReloadContext<'a> { - desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], - authentication: &'a MiddlewareAuthentication, - registry_changed: bool, - connector: &'a MiddlewareConnector, -} - -async fn reload_gateway_policy_runtime( - engine: &OpaEngine, - policy: Option<&openshell_core::proto::SandboxPolicy>, - entrypoint_pid: u32, - middleware: MiddlewareReloadContext<'_>, - transparent_tcp: TransparentTcpReloadState, -) -> std::result::Result<(), GatewayRuntimeReloadError> { - if let Some(policy) = policy - && policy_contains_explicit_tcp(policy) - { - if !transparent_tcp.capable { - return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( - miette::miette!( - "candidate policy introduces protocol: tcp, but the runtime does not advertise transparent TCP support; previous policy remains active" - ), - )); - } - if !transparent_tcp.substrate_ready { - return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( - miette::miette!( - "candidate policy introduces protocol: tcp, but this sandbox started without the transparent TCP substrate; recreate the sandbox to enable TCP; previous policy remains active" - ), - )); - } - } - match policy { - Some(policy) if middleware.registry_changed => { - let registry = (middleware.connector)( - middleware.desired_services.to_vec(), - middleware.authentication.clone(), - ) - .await - .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; - engine - .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) - .map_err(GatewayRuntimeReloadError::PolicyValidation) - } - // Policy-only change: the installed registry already matches the - // delivered service set, so swap the engine alone. This must not - // require middleware reachability. - Some(policy) => engine - .reload_from_proto_with_pid(policy, entrypoint_pid) - .map_err(GatewayRuntimeReloadError::PolicyValidation), - None => Err(GatewayRuntimeReloadError::PolicyValidation( - miette::miette!("runtime reload requires a policy payload but none was returned"), - )), - } -} - -fn policy_contains_explicit_tcp(policy: &openshell_core::proto::SandboxPolicy) -> bool { - policy.network_policies.values().any(|rule| { - rule.endpoints - .iter() - .any(|endpoint| endpoint.protocol.eq_ignore_ascii_case("tcp")) - }) -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct TransparentTcpReloadState { - capable: bool, - substrate_ready: bool, +pub struct RuntimeQualification { + pub seccomp: openshell_isolation_interface::contract::SeccompEvidence, + pub landlock_abi: u32, + pub landlock_allow_deny: bool, + pub udp_dns_round_trip: bool, + pub tcp_dns_round_trip: bool, + pub tcp_allow_round_trip: bool, + pub tcp_deny_round_trip: bool, } -/// True when the installed middleware registry no longer matches the desired -/// service set and must be rebuilt (reconnecting every delivered service). +/// Placeholder used when compiling the package on a non-Linux host. /// -/// A policy-only change never requires a rebuild: middleware configs were -/// validated at gateway admission and the installed registry's manifests -/// already cover the unchanged service set, so requiring the services to be -/// reachable would only let a middleware outage block the policy update. -fn middleware_registry_needs_rebuild( - registry_status: MiddlewareRegistryStatus, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> bool { - registry_status == MiddlewareRegistryStatus::NeedsReconciliation - || current_services != desired_services -} +/// The sandbox binary rejects execution on those hosts before constructing a +/// qualification, but retaining the type keeps the library API portable for +/// workspace-wide checks. +#[cfg(not(target_os = "linux"))] +#[derive(Clone, Copy, Debug)] +pub struct RuntimeQualification; -fn gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy: bool, - current_policy_hash: &str, - desired_policy_hash: &str, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - registry_status: MiddlewareRegistryStatus, -) -> bool { - reloads_gateway_policy - && (current_policy_hash != desired_policy_hash - || middleware_registry_needs_rebuild( - registry_status, - current_services, - desired_services, - )) -} - -/// Identity returned with the exact policy snapshot used to construct OPA. -#[derive(Clone, Debug, PartialEq, Eq)] -struct LoadedPolicyRevision { - version: u32, - policy_hash: String, - config_revision: u64, - policy_source: openshell_core::proto::PolicySource, -} - -/// Identifies where the policy currently loaded into OPA came from. +/// Run the authenticated boundary-local sandbox. /// -/// A missing gateway revision means the policy was loaded from the gateway but -/// could not be bound to an authoritative snapshot (for example, enrichment -/// sync failed). That state must reconcile on the first successful poll. A -/// local-file override is different: gateway policy revisions are observed for -/// settings/provider refreshes but must never replace the explicit local OPA -/// policy. -#[derive(Clone, Debug, PartialEq, Eq)] -enum LoadedPolicyOrigin { - LocalOverride, - Gateway { - revision: Option, - has_last_valid_policy: bool, - }, -} - -impl LoadedPolicyOrigin { - fn allows_gateway_policy_reload(&self) -> bool { - matches!(self, Self::Gateway { .. }) - } - - fn has_last_valid_policy(&self) -> bool { - match self { - Self::LocalOverride => true, - Self::Gateway { - has_last_valid_policy, - .. - } => *has_last_valid_policy, - } - } -} - -impl LoadedPolicyRevision { - fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { - Self { - version: snapshot.version, - policy_hash: snapshot.policy_hash.clone(), - config_revision: snapshot.config_revision, - policy_source: snapshot.policy_source, - } - } -} - -/// A sandbox-scoped policy revision that was constructed successfully at -/// startup and must be acknowledged to the gateway exactly once. -#[derive(Clone, Debug, PartialEq, Eq)] -struct InitialPolicyAck { - version: u32, - policy_hash: String, - config_revision: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PolicyStatusUpdate { - version: u32, - loaded: bool, - error: String, - success_event: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum PolicyStatusSuccessEvent { - InitialAcknowledgement { policy_hash: String }, - UnchangedAcknowledgement { policy_hash: String }, -} - -impl PolicyStatusUpdate { - fn initial_loaded(ack: &InitialPolicyAck) -> Self { - Self { - version: ack.version, - loaded: true, - error: String::new(), - success_event: Some(PolicyStatusSuccessEvent::InitialAcknowledgement { - policy_hash: ack.policy_hash.clone(), - }), - } - } - - fn loaded(version: u32) -> Self { - Self { - version, - loaded: true, - error: String::new(), - success_event: None, - } - } - - fn unchanged_loaded(version: u32, policy_hash: String) -> Self { - Self { - version, - loaded: true, - error: String::new(), - success_event: Some(PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash }), - } - } - - fn failed(version: u32, error: String) -> Self { - Self { - version, - loaded: false, - error, - success_event: None, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum InitialPollDisposition { - Acknowledge(InitialPolicyAck), - Reconcile, - TrackOnly, -} - -/// Determine whether the initially loaded policy corresponds to an -/// authoritative sandbox-scoped revision that must be acknowledged. +/// # Errors /// -/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose -/// captured gateway identity matches the current version and hash. Global -/// policies, local-file development policies, version zero, and changed -/// identities yield `None`, so those paths never emit a sandbox-revision -/// acknowledgement. -fn initial_policy_ack_candidate( - loaded: Option<&LoadedPolicyRevision>, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - let loaded = loaded?; - if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox - || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox - { - return None; - } - if loaded.version == 0 || canonical.version == 0 { - return None; - } - if loaded.version != canonical.version - || loaded.policy_hash != canonical.policy_hash - || canonical.config_revision < loaded.config_revision - { - return None; - } - Some(InitialPolicyAck { - version: loaded.version, - policy_hash: loaded.policy_hash.clone(), - config_revision: canonical.config_revision, - }) -} - -fn initial_poll_disposition( - origin: &LoadedPolicyOrigin, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> InitialPollDisposition { - match origin { - LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision, .. } => { - initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( - InitialPollDisposition::Reconcile, - InitialPollDisposition::Acknowledge, - ) - } - } -} - -fn unchanged_policy_revision_candidate( - reloads_gateway_policy: bool, - recovering_rejected_policy: bool, - current_policy_version: u32, - current_policy_hash: &str, - result: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - (reloads_gateway_policy - && !recovering_rejected_policy - && !current_policy_hash.is_empty() - && result.policy_source == openshell_core::proto::PolicySource::Sandbox - && result.version > current_policy_version - && result.policy_hash == current_policy_hash) - .then_some(result.version) -} - -fn unchanged_policy_revision_ready_to_ack( - candidate: Option, - policy_runtime_changed: bool, - policy_runtime_reconciled: bool, -) -> Option { - candidate.filter(|_| !policy_runtime_changed || policy_runtime_reconciled) -} - -/// Whether the credential-provenance gates cannot apply to the loaded policy. -/// -/// The gateway derives `provider_credentialed` and deliberately keeps it out of -/// the policy YAML schema, so a local-file policy never carries it and never -/// will: gateway revisions are observed for settings and providers but must not -/// replace the local OPA policy. Provider credentials still arrive from the -/// gateway on that path, so the raw-tunnel and WebSocket binary-frame refusals -/// have nothing to match on. The request-body backstop is unaffected because it -/// keys off the secret resolver rather than endpoint provenance. -fn credential_gating_unavailable( - origin: &LoadedPolicyOrigin, - has_resolver: bool, - network_enabled: bool, -) -> bool { - network_enabled && has_resolver && matches!(origin, LoadedPolicyOrigin::LocalOverride) -} - -/// Report that credential provenance is unavailable for the loaded policy. -/// -/// Carries no credential name, host, or value: the finding states which -/// controls are inactive, nothing about what they would have protected. -fn report_credential_gating_unavailable() { - ocsf_emit!( - DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::High) - .confidence(ConfidenceId::High) - .is_alert(true) - .finding_info( - FindingInfo::new( - "credential-gating-unavailable", - "Credential Provenance Unavailable", - ) - .with_desc( - "Provider credentials are injected, but the loaded policy comes from local \ - files and carries no gateway-derived credential provenance. Uninspected \ - credentialed tunnels and WebSocket binary frames are not refused. Load \ - policy from the gateway to enable these controls." - ), - ) - .evidence_pairs(&[ - ("policy_source", "local-override"), - ("uninspected_connect_gate", "inactive"), - ("websocket_binary_gate", "inactive"), - ("request_body_backstop", "active"), - ]) - .remediation( - "Remove the local policy override so the gateway-delivered effective policy \ - applies, or detach provider credentials from this sandbox." - ) - .message( - "Credential provenance unavailable for local-file policy; uninspected credential gates inactive" - ) - .build() - ); -} - -/// Deliver policy status updates independently from policy reconciliation. -/// -/// The channel is FIFO, so a delayed older status can never arrive after a -/// newer status and move the gateway's active version backward. Delivery uses -/// the existing bounded retry, but failures never delay policy enforcement. -#[tonic::async_trait] -trait PolicyGatewayClient: Clone + Send + Sync + 'static { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result; - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()>; - - async fn refresh_installed_extension_credentials(&self) -> Result<()> { - Ok(()) - } - - async fn extension_credentials_for( - &self, - _services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> { - Ok(std::collections::HashMap::new()) - } - - fn workspace(&self) -> String; -} - -#[tonic::async_trait] -impl PolicyGatewayClient for openshell_core::grpc_client::CachedOpenShellClient { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result { - self.poll_settings(sandbox_id).await - } - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.report_policy_status(sandbox_id, version, loaded, error) - .await - } - - async fn refresh_installed_extension_credentials(&self) -> Result<()> { - self.refresh_installed_extension_credentials().await - } - - async fn extension_credentials_for( - &self, - services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> { - self.extension_credentials_for(services).await - } - - fn workspace(&self) -> String { - self.workspace() - } -} - -async fn run_policy_status_reporter( - client: C, - sandbox_id: String, - mut updates: tokio::sync::mpsc::UnboundedReceiver, -) { - 'updates: while let Some(update) = updates.recv().await { - let operation = if matches!( - update.success_event, - Some(PolicyStatusSuccessEvent::InitialAcknowledgement { .. }) - ) { - "Initial policy acknowledgement" - } else { - "Policy status report" - }; - let mut attempt = 1_u32; - loop { - let sandbox_id = sandbox_id.clone(); - let error = update.error.clone(); - let client = client.clone(); - match client - .report_policy_status(&sandbox_id, update.version, update.loaded, &error) - .await - { - Ok(()) => break, - Err(error) if is_retryable_error(&error) => { - let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); - warn!( - %error, - attempt, - version = update.version, - loaded = update.loaded, - retry_in_secs = backoff.as_secs(), - "{operation} failed transiently; retaining ordered update" - ); - tokio::time::sleep(backoff).await; - attempt = attempt.saturating_add(1); - } - Err(error) => { - warn!( - %error, - version = update.version, - loaded = update.loaded, - "Discarding terminal policy status update" - ); - continue 'updates; - } - } - } - - if let Some(event) = update.success_event { - let (policy_hash, message) = match event { - PolicyStatusSuccessEvent::InitialAcknowledgement { policy_hash } => ( - policy_hash, - format!( - "Acknowledged initial policy revision as loaded [version:{}]", - update.version - ), - ), - PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash } => ( - policy_hash, - format!( - "Acknowledged unchanged policy revision as loaded [version:{}]", - update.version - ), - ), - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("version", serde_json::json!(update.version)) - .unmapped("policy_hash", serde_json::json!(policy_hash)) - .message(message) - .build() - ); - } - } -} - -fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { - let version = update.version; - if let Err(error) = sender.send(update) { - warn!( - %error, - version, - "Policy status reporter unavailable during shutdown" - ); - } -} - -/// Best-effort `FAILED` acknowledgement when initial policy construction or -/// conversion fails. -/// -/// Uses the revision identity captured with the policy that failed to build, -/// and preserves the original construction error as the reported message. A -/// delivery failure here is swallowed so it can never mask that error. -async fn report_initial_policy_failure( - endpoint: &str, - sandbox_id: &str, - revision: Option<&LoadedPolicyRevision>, - error: &miette::Report, -) { - let Some(revision) = revision.filter(|revision| { - revision.version > 0 - && revision.policy_source == openshell_core::proto::PolicySource::Sandbox - }) else { - return; - }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { - Ok(client) => client, - Err(e) => { - warn!(error = %e, "Failed to connect to report initial policy failure"); - return; - } - }; - let message = error.to_string(); - if let Err(e) = grpc_retry("Initial policy failure report", || { - let client = client.clone(); - let message = message.clone(); - async move { - client - .report_policy_status(sandbox_id, revision.version, false, &message) - .await - } - }) - .await - { - warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); - } -} - -/// Background loop that polls the server for policy updates. -/// -/// When a new version is detected, attempts to reload the OPA engine via -/// `reload_from_proto_with_pid()`. Reports load success/failure back to the -/// server. On failure, the previous engine is untouched (LKG behavior). -/// -/// When the entrypoint PID is available, policy reloads include symlink -/// resolution for binary paths via the container filesystem. -struct PolicyPollLoopContext { - endpoint: String, - sandbox_id: String, - opa_engine: Arc, - /// Source of the policy currently loaded into OPA. This distinguishes an - /// explicit local-file override from an unbound gateway revision so the - /// former is never replaced by policy polling. - loaded_policy_origin: LoadedPolicyOrigin, - entrypoint_pid: Arc, - interval_secs: u64, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - provider_credentials: ProviderCredentialState, - policy_local_ctx: Option>, - agent_proposals: AgentProposals, - middleware_registry_status: MiddlewareRegistryStatus, - sidecar_control_publisher: Option, - workspace_tx: tokio::sync::watch::Sender, - extension_credentials: openshell_extension_core::ExtensionCredentialStore, - extension_authentication_enabled: bool, - middleware_connector: MiddlewareConnector, - /// Immutable driver capability and startup substrate state. - transparent_tcp: TransparentTcpReloadState, -} - -type MiddlewareConnector = Arc< - dyn Fn( - Vec, - MiddlewareAuthentication, - ) -> Pin< - Box< - dyn std::future::Future< - Output = Result, - > + Send, - >, - > + Send - + Sync, ->; - -#[derive(Clone, Default)] -struct MiddlewareAuthentication { - credentials: std::collections::HashMap, - enabled: bool, -} - -fn default_middleware_connector() -> MiddlewareConnector { - Arc::new(|services, authentication| { - Box::pin(async move { connect_middleware_registry(&services, &authentication).await }) - }) -} - -async fn connect_middleware_registry( - services: &[openshell_core::proto::SupervisorMiddlewareService], - authentication: &MiddlewareAuthentication, -) -> Result { - if authentication.enabled { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services_authenticated( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - &authentication.credentials, - ) - .await - } else { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - ) - .await - } -} - -async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - opa_engine.replace_middleware_registry(registry) -} - -/// Wait the configured poll interval, but never past the point at which an -/// installed extension credential must be rotated. -fn next_poll_delay( - store: &openshell_extension_core::ExtensionCredentialStore, - interval: Duration, -) -> Duration { - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| { - i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) - }); - store.next_refresh_delay(interval, now_ms) -} - -/// Drop credentials for services no longer in the installed registry. -/// -/// Call only after a registry swap succeeds, so a failed candidate cannot -/// invalidate the last-known-good clients. -fn retain_extension_credentials( - store: &openshell_extension_core::ExtensionCredentialStore, - installed: &[openshell_core::proto::SupervisorMiddlewareService], - extension_authentication_enabled: bool, -) { - let retained = if extension_authentication_enabled { - installed - .iter() - .map(|service| service.name.as_str()) - .collect() - } else { - std::collections::HashSet::default() - }; - store.retain(&retained); -} - -struct MiddlewareRegistryReconciliation<'a> { - desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], - authentication: MiddlewareAuthentication, - registry_changed: bool, - extension_credentials: &'a openshell_extension_core::ExtensionCredentialStore, - current_services: &'a mut Vec, - status: &'a mut MiddlewareRegistryStatus, -} - -async fn reconcile_middleware_registry( - opa_engine: &OpaEngine, - middleware_connector: &MiddlewareConnector, - reconciliation: MiddlewareRegistryReconciliation<'_>, -) { - if !reconciliation.registry_changed { - return; - } - - match middleware_connector( - reconciliation.desired_services.to_vec(), - reconciliation.authentication.clone(), - ) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - retain_extension_credentials( - reconciliation.extension_credentials, - reconciliation.desired_services, - reconciliation.authentication.enabled, - ); - reconciliation.current_services.clear(); - reconciliation - .current_services - .extend_from_slice(reconciliation.desired_services); - *reconciliation.status = MiddlewareRegistryStatus::Synchronized; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(reconciliation.current_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - reconciliation.current_services.len() - )) - .build() - ); - } - Err(error) => { - // Emit only on the transition into the failed state to avoid - // repeating the same finding on every poll during an outage. - if *reconciliation.status == MiddlewareRegistryStatus::Synchronized { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - } - *reconciliation.status = MiddlewareRegistryStatus::NeedsReconciliation; - } - } -} - -#[derive(Debug, PartialEq, Eq)] -struct PolicyValidationFailureDisposition { - configured_mode: PolicyValidationFailureMode, - mode: PolicyValidationFailureMode, - previous_policy_active: bool, - active_generation: u64, -} - -struct RejectedPolicyGeneration { - version: u32, - policy_hash: String, - validation_error: String, - configured_mode: PolicyValidationFailureMode, -} - -enum GatewayRuntimeFailureDisposition { - PolicyRejected { - error: String, - disposition: PolicyValidationFailureDisposition, - }, - MiddlewareUnavailable { - error: String, - }, - TransparentTcpExpansionRejected { - error: String, - active_generation: u64, - }, -} - -fn apply_gateway_runtime_reload_failure( - engine: &OpaEngine, - failure: GatewayRuntimeReloadError, - configured_mode: PolicyValidationFailureMode, - has_last_valid_policy: bool, - version: u32, -) -> Result { - match failure { - GatewayRuntimeReloadError::PolicyValidation(error) => { - let error = error.to_string(); - let disposition = apply_policy_validation_failure( - engine, - configured_mode, - has_last_valid_policy, - version, - &error, - )?; - Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) - } - GatewayRuntimeReloadError::TransparentTcpPrerequisite(error) => Ok( - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - error: error.to_string(), - active_generation: engine.current_generation(), - }, - ), - GatewayRuntimeReloadError::MiddlewareRegistry(error) => { - Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { - error: error.to_string(), - }) - } - } -} - -fn emit_transparent_tcp_expansion_rejection( - version: u32, - policy_hash: &str, - active_generation: u64, - error: &str, -) { - let message = format!( - "Transparent TCP policy expansion rejected; previous policy IS active [version:{version} active_generation:{active_generation} error:{error}]" - ); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Enabled, "retained_previous_policy") - .unmapped("candidate_version", serde_json::json!(version)) - .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) - .unmapped("previous_policy_active", serde_json::json!(true)) - .unmapped("active_generation", serde_json::json!(active_generation)) - .unmapped("validation_error", serde_json::json!(error)) - .message(message) - .build() - ); -} - -fn apply_policy_validation_failure( - engine: &OpaEngine, - configured_mode: PolicyValidationFailureMode, - has_last_valid_policy: bool, - version: u32, - error: &str, -) -> Result { - let mode = if has_last_valid_policy { - configured_mode - } else { - PolicyValidationFailureMode::FailClosed - }; - match mode { - PolicyValidationFailureMode::FailClosed => { - let reason = format!( - "policy validation failed; fail-closed quarantine is active; candidate version {version} rejected: {error}" - ); - let active_generation = engine.enter_fail_closed(reason)?; - Ok(PolicyValidationFailureDisposition { - configured_mode, - mode, - previous_policy_active: false, - active_generation, - }) - } - PolicyValidationFailureMode::RetainLastValid => { - let active_generation = engine.exit_fail_closed()?; - Ok(PolicyValidationFailureDisposition { - configured_mode, - mode, - previous_policy_active: true, - active_generation, - }) - } - } -} - -fn policy_validation_failure_events( - disposition: &PolicyValidationFailureDisposition, - version: u32, - policy_hash: &str, - error: &str, -) -> [OcsfEvent; 2] { - let previous_policy_state = if disposition.previous_policy_active { - "IS active" - } else { - "IS NOT active" - }; - let state = if disposition.previous_policy_active { - (StateId::Enabled, "retained_last_valid") - } else { - (StateId::Disabled, "fail_closed") - }; - let message = format!( - "Policy validation failed; configured_mode={} effective_mode={}; previous policy {previous_policy_state} [version:{version} active_generation:{} error:{error}]", - disposition.configured_mode.as_str(), - disposition.mode.as_str(), - disposition.active_generation, - ); - let finding_uid = format!("policy-validation-failed-{version}"); - let version_string = version.to_string(); - let config = ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(state.0, state.1) - .unmapped("candidate_version", serde_json::json!(version)) - .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) - .unmapped( - "validation_failure_mode", - serde_json::json!(disposition.mode.as_str()), - ) - .unmapped( - "configured_validation_failure_mode", - serde_json::json!(disposition.configured_mode.as_str()), - ) - .unmapped( - "previous_policy_active", - serde_json::json!(disposition.previous_policy_active), - ) - .unmapped( - "active_generation", - serde_json::json!(disposition.active_generation), - ) - .unmapped("validation_error", serde_json::json!(error)) - .message(message.clone()) - .build(); - let finding = DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .is_alert(true) - .finding_info( - FindingInfo::new(&finding_uid, "Invalid policy generation rejected").with_desc(error), - ) - .evidence_pairs(&[ - ("candidate_version", &version_string), - ("candidate_policy_hash", policy_hash), - ("validation_failure_mode", disposition.mode.as_str()), - ( - "configured_validation_failure_mode", - disposition.configured_mode.as_str(), - ), - ( - "previous_policy_active", - if disposition.previous_policy_active { - "true" - } else { - "false" - }, - ), - ]) - .remediation("Submit a valid, unambiguous policy generation") - .message(message) - .build(); - [config, finding] -} - -fn emit_policy_validation_failure( - disposition: &PolicyValidationFailureDisposition, - version: u32, - policy_hash: &str, - error: &str, -) { - for event in policy_validation_failure_events(disposition, version, policy_hash, error) { - ocsf_emit!(event); - } -} - -async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { - let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - &ctx.endpoint, - ctx.extension_credentials.clone(), - ) - .await?; - run_policy_poll_loop_with_client(ctx, client).await -} - -async fn run_policy_poll_loop_with_client( - ctx: PolicyPollLoopContext, - client: C, -) -> Result<()> { - use openshell_core::proto::PolicySource; - use std::sync::atomic::Ordering; - - let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); - tokio::spawn(run_policy_status_reporter( - client.clone(), - ctx.sandbox_id.clone(), - status_receiver, - )); - - let mut current_config_revision: u64 = 0; - let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; - let mut current_policy_version: u32 = 0; - let mut current_policy_hash = String::new(); - let mut current_middleware_services = Vec::new(); - let mut current_extension_authentication_enabled = ctx.extension_authentication_enabled; - let mut middleware_registry_status = ctx.middleware_registry_status; - let mut current_settings: std::collections::HashMap< - String, - openshell_core::proto::EffectiveSetting, - > = std::collections::HashMap::new(); - let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); - let mut last_failed_runtime_revision: Option = None; - let mut rejected_policy_generation: Option = None; - let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); - - // A first poll that does not match the policy already loaded into OPA must - // pass through the normal reconciliation path immediately. It must never - // seed the applied-state trackers before OPA actually loads it. - let mut pending_result = None; - - // Initialize revision from the first poll and acknowledge the initial - // policy revision the supervisor actually loaded. A mismatched result is - // reconciled below instead of being recorded as already applied. - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { - InitialPollDisposition::Acknowledge(candidate) => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(candidate.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = candidate.config_revision; - current_policy_version = candidate.version; - current_policy_hash.clone_from(&candidate.policy_hash); - current_middleware_services = result.supervisor_middleware_services; - current_extension_authentication_enabled = - result.extension_authentication_enabled; - current_settings = result.settings; - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::initial_loaded(&candidate), - ); - debug!( - config_revision = current_config_revision, - "Settings poll: initial policy matches loaded revision" - ); - } - InitialPollDisposition::Reconcile => pending_result = Some(result), - InitialPollDisposition::TrackOnly => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash.clone(); - current_middleware_services = result.supervisor_middleware_services; - current_extension_authentication_enabled = - result.extension_authentication_enabled; - current_settings = result.settings; - debug!( - config_revision = current_config_revision, - "Settings poll: tracking gateway config while preserving local policy override" - ); - } - } - } - Err(e) => { - warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); - } - } - - let interval = Duration::from_secs(ctx.interval_secs); - loop { - let result = if let Some(result) = pending_result.take() { - result - } else { - tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await; - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - result - } - Err(e) => { - debug!(error = %e, "Settings poll: server unreachable, will retry"); - if current_extension_authentication_enabled - && let Err(refresh_error) = - client.refresh_installed_extension_credentials().await - { - warn!( - error = %refresh_error, - "Settings poll: extension credential refresh failed while configuration was unavailable" - ); - } - continue; - } - } - }; - - // Reuse installed per-service credentials, rotating only when one is - // missing or due. Rotation happens on the existing gateway channel and - // updates slots in place, so it is independent of config revision and - // registry equality. - let middleware_credentials = if result.extension_authentication_enabled { - match client - .extension_credentials_for(&result.supervisor_middleware_services) - .await - { - Ok(credentials) => credentials, - Err(error) => { - warn!(error = %error, "Settings poll: extension credential refresh failed"); - std::collections::HashMap::new() - } - } - } else { - std::collections::HashMap::new() - }; - - let config_changed = result.config_revision != current_config_revision; - let provider_env_changed = result.provider_env_revision != current_provider_env_revision; - let policy_changed = result.policy_hash != current_policy_hash; - let extension_authentication_changed = - current_extension_authentication_enabled != result.extension_authentication_enabled; - let middleware_registry_changed = extension_authentication_changed - || middleware_registry_needs_rebuild( - middleware_registry_status, - ¤t_middleware_services, - &result.supervisor_middleware_services, - ); - // A valid candidate may intentionally restore byte-for-byte policy - // content that was active before a rejected update. Its hash then - // equals `current_policy_hash`, but the runtime is still quarantined - // and must reload (or it would remain deny-all indefinitely). - let recovering_rejected_policy = reloads_gateway_policy - && rejected_policy_generation - .as_ref() - .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); - let policy_runtime_changed = recovering_rejected_policy - || extension_authentication_changed - || gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); - // Recovery already has its own acknowledgement path below. Giving it - // precedence here prevents a restored last-known-good policy from - // also being acknowledged as an ordinary same-hash revision. - let unchanged_policy_revision = unchanged_policy_revision_candidate( - reloads_gateway_policy, - recovering_rejected_policy, - current_policy_version, - ¤t_policy_hash, - &result, - ); - let mut policy_runtime_reconciled = false; - - // A local policy override is not coupled to the gateway policy - // snapshot, so its service registry can still be reconciled alone. - // Gateway policy snapshots, however, must install policy and registry - // as one generation below. - if !reloads_gateway_policy { - reconcile_middleware_registry( - &ctx.opa_engine, - &ctx.middleware_connector, - MiddlewareRegistryReconciliation { - desired_services: &result.supervisor_middleware_services, - authentication: MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - extension_credentials: &ctx.extension_credentials, - current_services: &mut current_middleware_services, - status: &mut middleware_registry_status, - }, - ) - .await; - if middleware_registry_status == MiddlewareRegistryStatus::Synchronized { - current_extension_authentication_enabled = result.extension_authentication_enabled; - } - } - - if !config_changed - && !provider_env_changed - && !policy_runtime_changed - && unchanged_policy_revision.is_none() - { - continue; - } - - if config_changed || provider_env_changed { - // Log which settings changed. - log_setting_changes(¤t_settings, &result.settings); - - // A posture change after a rejected update takes effect immediately. - // The compiled last-known-good engine remains available beneath a - // fail-closed quarantine, so an explicit retain_last_valid selection - // can reactivate it without accepting any part of the invalid policy. - if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { - let mode = result.policy_validation_failure_mode; - if mode != rejected.configured_mode { - let disposition = apply_policy_validation_failure( - &ctx.opa_engine, - mode, - has_last_valid_policy, - rejected.version, - &rejected.validation_error, - )?; - emit_policy_validation_failure( - &disposition, - rejected.version, - &rejected.policy_hash, - &rejected.validation_error, - ); - rejected.configured_mode = mode; - } - } - - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "detected") - .unmapped("old_config_revision", serde_json::json!(current_config_revision)) - .unmapped("new_config_revision", serde_json::json!(result.config_revision)) - .unmapped("policy_changed", serde_json::json!(policy_changed)) - .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) - .message(format!( - "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", - result.config_revision - )) - .build()); - } - - if provider_env_changed { - match openshell_core::grpc_client::fetch_provider_environment( - &ctx.endpoint, - &ctx.sandbox_id, - ) - .await - { - Ok(env_result) => { - let provider_env_revision = env_result.provider_env_revision; - let install_result = ctx.provider_credentials.install_bound_environment( - provider_env_revision, - env_result.environment, - env_result.credential_expires_at_ms, - env_result.dynamic_credentials, - env_result.static_credential_bindings, - env_result.non_secret_environment_keys, - ); - if let Err(error) = install_result { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Rejected provider environment refresh; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" - )) - .build() - ); - } else { - let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); - let env_count = child_env.len(); - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher - .publish_provider_env(provider_env_revision, child_env.clone()); - } - current_provider_env_revision = provider_env_revision; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "provider_env_revision", - serde_json::json!(provider_env_revision) - ) - .message(format!( - "Provider environment refreshed [revision:{provider_env_revision} env_count:{env_count}]" - )) - .build() - ); - } - } - Err(e) => { - ctx.provider_credentials - .revoke_static_provider_environment(result.provider_env_revision); - warn!( - error = %e, - provider_env_revision = result.provider_env_revision, - "Settings poll: failed to refresh provider environment; static provider credentials were revoked; previous dynamic token grants remain active" - ); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message( - "Provider environment refresh failed; static provider credentials were revoked; previous dynamic token grants remain active" - ) - .build() - ); - } - } - } - - if policy_runtime_changed { - let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = reload_gateway_policy_runtime( - &ctx.opa_engine, - result.policy.as_ref(), - pid, - MiddlewareReloadContext { - desired_services: &result.supervisor_middleware_services, - authentication: &MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - connector: &ctx.middleware_connector, - }, - ctx.transparent_tcp, - ) - .await; - - match runtime_result { - Ok(()) => { - policy_runtime_reconciled = true; - let policy = result - .policy - .as_ref() - .expect("successful runtime reload requires a policy payload"); - has_last_valid_policy = true; - rejected_policy_generation = None; - if policy_changed { - if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { - policy_local_ctx.set_current_policy(policy.clone()).await; - } - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_policy( - policy.clone(), - result.policy_hash.clone(), - result.config_revision, - ); - } - if result.global_policy_version > 0 { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .unmapped("global_version", serde_json::json!(result.global_policy_version)) - .message(format!( - "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", - result.policy_hash, - result.global_policy_version - )) - .build()); - } else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - } - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - current_policy_version = result.version; - } - } else if recovering_rejected_policy - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully and fail-closed quarantine cleared [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - current_policy_version = result.version; - } - - if middleware_registry_changed { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(result.supervisor_middleware_services.len()) - ) - .message(format!( - "Supervisor policy runtime reloaded atomically [service_count:{}]", - result.supervisor_middleware_services.len() - )) - .build()); - } - - current_policy_hash.clone_from(&result.policy_hash); - current_middleware_services.clone_from(&result.supervisor_middleware_services); - current_extension_authentication_enabled = - result.extension_authentication_enabled; - retain_extension_credentials( - &ctx.extension_credentials, - &result.supervisor_middleware_services, - result.extension_authentication_enabled, - ); - middleware_registry_status = MiddlewareRegistryStatus::Synchronized; - last_failed_runtime_revision = None; - } - Err(failure) => { - let failed_revision = FailedRuntimeRevision::new( - result.config_revision, - &result.policy_hash, - &failure, - ); - if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { - let failure_mode = result.policy_validation_failure_mode; - match apply_gateway_runtime_reload_failure( - &ctx.opa_engine, - failure, - failure_mode, - has_last_valid_policy, - result.version, - )? { - GatewayRuntimeFailureDisposition::PolicyRejected { - error, - disposition, - } => { - emit_policy_validation_failure( - &disposition, - result.version, - &result.policy_hash, - &error, - ); - rejected_policy_generation = Some(RejectedPolicyGeneration { - version: result.version, - policy_hash: result.policy_hash.clone(), - validation_error: error.clone(), - configured_mode: failure_mode, - }); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, error), - ); - } - } - GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error } => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(&error)) - .unmapped("previous_policy_active", serde_json::json!(true)) - .message(format!( - "Supervisor middleware registry unavailable, keeping last-known-good policy runtime active [version:{} error:{error}]", - result.version - )) - .build()); - } - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - error, - active_generation, - } => { - emit_transparent_tcp_expansion_rejection( - result.version, - &result.policy_hash, - active_generation, - &error, - ); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, error), - ); - } - } - } - } - last_failed_runtime_revision = Some(failed_revision); - // Nothing was installed, so the registry status still - // describes the live registry. The retry is driven by the - // persisting hash/service-set mismatch (or an existing - // NeedsReconciliation), not by degrading the status here. - } - } - } - - if let Some(version) = unchanged_policy_revision_ready_to_ack( - unchanged_policy_revision, - policy_runtime_changed, - policy_runtime_reconciled, - ) { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::unchanged_loaded(version, result.policy_hash.clone()), - ); - current_policy_version = version; - } - - // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); - - // Apply the agent-proposals feature toggle. On a false→true transition - // we lazily install the skill so a sandbox that started with the flag - // off picks up the surface without a recreate. We never uninstall on - // a true→false transition: stale skill content on disk is harmless - // because route_request and agent_next_steps both gate on the live - // shared flag, so the agent that reads the skill will see 404s and an - // empty `next_steps` array regardless. - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - - current_config_revision = result.config_revision; - if !reloads_gateway_policy { - current_policy_hash = result.policy_hash; - } - current_settings = result.settings; - } -} - -fn apply_ocsf_json_setting( - enabled: &AtomicBool, - settings: &std::collections::HashMap, -) { - use std::sync::atomic::Ordering; - - let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); - let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); - if new_ocsf != prev_ocsf { - info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); - } -} - -/// Extract a bool value from an effective setting, if present. -fn extract_bool_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) -} - -fn apply_ocsf_schema_version_setting( - version: &std::sync::Mutex, - settings: &std::collections::HashMap, -) { - let new_version = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default(); - if let Ok(mut current) = version.lock() - && *current != new_version - { - info!( - ocsf_schema_version = %new_version, - "OCSF schema version target changed" - ); - *current = new_version; - } -} - -fn extract_string_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::StringValue(s) => Some(s.clone()), - _ => None, - }) -} - -fn agent_proposals_enabled_from_settings( - settings: &std::collections::HashMap, -) -> bool { - extract_bool_setting( - settings, - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, - ) - .unwrap_or(false) -} - -fn apply_agent_proposals_enabled( - agent_proposals: &AgentProposals, - enabled: bool, - source: &'static str, - config_revision: Option, - sidecar_control_publisher: Option<&sidecar_control::Publisher>, - install_static_skills: impl FnOnce() -> Result, -) { - let previously_enabled = agent_proposals.swap_enabled(enabled); - if enabled == previously_enabled { - return; - } - - info!( - agent_policy_proposals_enabled = enabled, - source, config_revision, "agent-driven policy proposals toggled" - ); - - if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { - publisher.publish_agent_proposals(enabled, config_revision); - } - - if enabled && !previously_enabled { - match install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill on toggle-on" - ), - Err(error) => warn!( - error = %error, - "Failed to install sandbox agent skill on toggle-on" - ), - } - } -} - -/// Log individual setting changes between two snapshots. -fn log_setting_changes( - old: &std::collections::HashMap, - new: &std::collections::HashMap, -) { - for (key, new_es) in new { - let new_val = format_setting_value(new_es); - match old.get(key) { - Some(old_es) => { - let old_val = format_setting_value(old_es); - if old_val != new_val { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "updated") - .unmapped("key", serde_json::json!(key)) - .unmapped("old", serde_json::json!(old_val.clone())) - .unmapped("new", serde_json::json!(new_val.clone())) - .message(format!( - "Setting changed [key:{key} old:{old_val} new:{new_val}]" - )) - .build() - ); - } - } - None => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enabled") - .unmapped("key", serde_json::json!(key)) - .unmapped("value", serde_json::json!(new_val.clone())) - .message(format!("Setting added [key:{key} value:{new_val}]")) - .build() - ); - } - } - } - for key in old.keys() { - if !new.contains_key(key) { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Disabled, "disabled") - .unmapped("key", serde_json::json!(key)) - .message(format!("Setting removed [key:{key}]")) - .build() - ); - } - } -} - -/// Format an `EffectiveSetting` value for log display. -fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { - use openshell_core::proto::setting_value; - match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { - None => "".to_string(), - Some(setting_value::Value::StringValue(v)) => v.clone(), - Some(setting_value::Value::BoolValue(v)) => v.to_string(), - Some(setting_value::Value::IntValue(v)) => v.to_string(), - Some(setting_value::Value::BytesValue(_)) => "".to_string(), - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod tests { - use super::*; - - #[test] - fn transparent_tcp_capability_requires_exact_driver_marker() { - let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; - assert!(!has_network_runtime_capability(None, required)); - assert!(!has_network_runtime_capability(Some(""), required)); - assert!(!has_network_runtime_capability( - Some("policy-dns-transparent-tcp-extra"), - required - )); - assert!(has_network_runtime_capability( - Some("other, policy-dns-transparent-tcp"), - required - )); - } - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - fn proxy_policy(http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { - openshell_core::proto::EffectiveSetting { - value: Some(openshell_core::proto::SettingValue { - value: Some(openshell_core::proto::setting_value::Value::BoolValue( - value, - )), - }), - scope: openshell_core::proto::SettingScope::Global.into(), - } - } - - #[test] - fn sidecar_process_policy_sets_loopback_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, true).unwrap(); - - let http_addr = process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .expect("sidecar process policy should set proxy address"); - assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); - assert!( - policy - .network - .proxy - .as_ref() - .expect("original policy should keep proxy config") - .http_addr - .is_none(), - "process policy normalization must not mutate the network policy" - ); - } - - #[test] - fn non_sidecar_process_policy_preserves_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, false).unwrap(); - - assert!( - process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .is_none() - ); - } - - #[tokio::test] - async fn sidecar_control_provider_env_update_orders_by_generation() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - u64::MAX, - std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), - ); - let agent_proposals = AgentProposals::new(true); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials.clone(), - agent_proposals.clone(), - Arc::new(tokio::sync::Mutex::new(None)), - 10, - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 1, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "new".to_string(), - )]), - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 1 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 1); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "duplicate-generation".to_string(), - )]), - }) - .unwrap(); - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 1, - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - while agent_proposals.enabled() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - assert_eq!( - provider_credentials - .snapshot() - .child_env - .get("TOKEN") - .map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - generation: 12, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "newest".to_string(), - )]), - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 2 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: u64::MAX, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "stale".to_string(), - )]), - }) - .unwrap(); - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: true, - config_revision: 2, - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - while !agent_proposals.enabled() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 2); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("newest") - ); - handle.abort(); - } - - #[tokio::test] - async fn sidecar_control_agent_proposals_update_flips_shared_state() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = - ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); - let agent_proposals = AgentProposals::new(true); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials, - agent_proposals.clone(), - Arc::new(tokio::sync::Mutex::new(None)), - 0, - ); - - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 5, - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if !agent_proposals.enabled() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - handle.abort(); - } - - #[test] - fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { - let agent_proposals = AgentProposals::default(); - let installs = AtomicUsize::new(0); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(!agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - } - - #[test] - fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { - let enabled = AtomicBool::new(false); - let mut settings = std::collections::HashMap::new(); - settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(enabled.load(Ordering::Relaxed)); - } - - #[test] - fn apply_ocsf_json_setting_disables_when_setting_is_unset() { - let enabled = AtomicBool::new(true); - let settings = std::collections::HashMap::new(); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(!enabled.load(Ordering::Relaxed)); - } - - #[test] - fn agent_proposals_setting_enables_from_initial_settings_snapshot() { - let mut settings = std::collections::HashMap::new(); - settings.insert( - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), - effective_bool(true), - ); - - assert!(agent_proposals_enabled_from_settings(&settings)); - } - - #[test] - fn agent_proposals_setting_defaults_false_when_unset() { - let settings = std::collections::HashMap::new(); - - assert!(!agent_proposals_enabled_from_settings(&settings)); - } - - // ---- Policy disk discovery tests ---- - - #[test] - fn discover_policy_from_nonexistent_path_returns_restrictive_default() { - let path = std::path::Path::new("/nonexistent/policy.yaml"); - let policy = discover_policy_from_path(path); - // Restrictive default has no network policies. - assert!(policy.network_policies.is_empty()); - // It keeps filesystem restrictions while leaving identity to the - // active compute driver. - assert!(policy.filesystem.is_some()); - assert!(policy.process.is_none()); - } - - #[test] - fn discover_policy_from_valid_yaml_file() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr - read_write: - - /tmp -network_policies: - test: - name: test - endpoints: - - { host: example.com, port: 443 } - binaries: - - { path: /usr/bin/curl } -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - assert_eq!(policy.network_policies.len(), 1); - assert!(policy.network_policies.contains_key("test")); - let fs = policy.filesystem.unwrap(); - assert!(!fs.include_workdir); - } - - #[test] - fn discover_policy_from_invalid_yaml_returns_restrictive_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default. - assert!(policy.network_policies.is_empty()); - assert!(policy.filesystem.is_some()); - } - - #[test] - fn discover_policy_from_unsafe_yaml_falls_back_to_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -process: - run_as_user: root - run_as_group: root -filesystem_policy: - include_workdir: true - read_only: - - /usr - read_write: - - /tmp -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default because of root user. - assert!(policy.process.is_none()); - } - - #[test] - fn discover_policy_restrictive_default_blocks_network() { - // In cluster mode we keep proxy mode enabled so `inference.local` - // can always be routed through proxy/OPA controls. - let proto = openshell_policy::restrictive_default_policy(); - let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); - assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); - } - - // ---- Initial policy acknowledgement tests ---- - - fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::restrictive_default_policy() - } - - fn proto_tcp_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::parse_sandbox_policy( - r#" -version: 1 -network_policies: - redis: - name: redis - endpoints: - - host: redis.example.com - port: 6379 - protocol: tcp - binaries: - - path: /usr/bin/redis-cli -"#, - ) - .expect("parse TCP policy") - } - - fn settings_poll_result( - policy: Option, - version: u32, - source: openshell_core::proto::PolicySource, - ) -> openshell_core::grpc_client::SettingsPollResult { - openshell_core::grpc_client::SettingsPollResult { - policy, - version, - policy_hash: format!("hash-v{version}"), - config_revision: u64::from(version) * 100, - policy_source: source, - settings: std::collections::HashMap::new(), - global_policy_version: 0, - provider_env_revision: 0, - supervisor_middleware_services: Vec::new(), - workspace: String::new(), - policy_validation_failure_mode: PolicyValidationFailureMode::default(), - extension_authentication_enabled: false, - } - } - - #[derive(Clone)] - struct ScriptedPolicyGateway { - polls: Arc< - tokio::sync::Mutex< - tokio::sync::mpsc::UnboundedReceiver< - openshell_core::grpc_client::SettingsPollResult, - >, - >, - >, - reports: UnboundedSender<(u32, bool, String)>, - } - - #[tonic::async_trait] - impl PolicyGatewayClient for ScriptedPolicyGateway { - async fn poll_settings( - &self, - _sandbox_id: &str, - ) -> Result { - self.polls - .lock() - .await - .recv() - .await - .ok_or_else(|| miette::miette!("scripted policy poll channel closed")) - } - - async fn report_policy_status( - &self, - _sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.reports - .send((version, loaded, error.to_string())) - .map_err(|_| miette::miette!("scripted policy report channel closed")) - } - - fn workspace(&self) -> String { - "test-workspace".to_string() - } - } - - #[derive(Clone)] - struct CredentialRejectingPolicyGateway { - inner: ScriptedPolicyGateway, - credential_requests: Arc, - } - - #[tonic::async_trait] - impl PolicyGatewayClient for CredentialRejectingPolicyGateway { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result { - self.inner.poll_settings(sandbox_id).await - } - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.inner - .report_policy_status(sandbox_id, version, loaded, error) - .await - } - - async fn extension_credentials_for( - &self, - _services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> - { - self.credential_requests.fetch_add(1, Ordering::SeqCst); - Err(miette::miette!( - "gateway extension authentication is unavailable" - )) - } - - fn workspace(&self) -> String { - self.inner.workspace() - } - } - - fn scripted_policy_gateway() -> ( - ScriptedPolicyGateway, - UnboundedSender, - tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - ) { - let (poll_tx, poll_rx) = tokio::sync::mpsc::unbounded_channel(); - let (report_tx, report_rx) = tokio::sync::mpsc::unbounded_channel(); - ( - ScriptedPolicyGateway { - polls: Arc::new(tokio::sync::Mutex::new(poll_rx)), - reports: report_tx, - }, - poll_tx, - report_rx, - ) - } - - fn policy_poll_test_context( - opa_engine: Arc, - loaded_policy_origin: LoadedPolicyOrigin, - middleware_connector: MiddlewareConnector, - ) -> PolicyPollLoopContext { - let (workspace_tx, _workspace_rx) = tokio::sync::watch::channel(String::new()); - PolicyPollLoopContext { - endpoint: String::new(), - sandbox_id: "sandbox-test".to_string(), - opa_engine, - loaded_policy_origin, - entrypoint_pid: Arc::new(AtomicU32::new(0)), - interval_secs: 0, - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(std::sync::Mutex::new(String::new())), - provider_credentials: ProviderCredentialState::from_child_env_snapshot( - 0, - std::collections::HashMap::new(), - ), - policy_local_ctx: None, - agent_proposals: AgentProposals::default(), - middleware_registry_status: MiddlewareRegistryStatus::Synchronized, - sidecar_control_publisher: None, - workspace_tx, - extension_credentials: openshell_extension_core::ExtensionCredentialStore::new(), - extension_authentication_enabled: false, - middleware_connector, - transparent_tcp: TransparentTcpReloadState::default(), - } - } - - async fn expect_policy_report( - reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - version: u32, - ) { - let report = timeout(Duration::from_secs(1), reports.recv()) - .await - .expect("policy report timed out") - .expect("policy reporter stopped"); - assert_eq!(report, (version, true, String::new())); - } - - async fn expect_no_policy_report( - reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - ) { - assert!( - timeout(Duration::from_millis(50), reports.recv()) - .await - .is_err(), - "unexpected policy status report" - ); - } - - #[tokio::test] - async fn same_hash_poll_revision_is_acknowledged_once_without_opa_reload() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - - polls.send(v2.clone()).unwrap(); - expect_policy_report(&mut reports, 2).await; - polls.send(v2).unwrap(); - expect_no_policy_report(&mut reports).await; - - assert_eq!( - engine.current_generation(), - 0, - "same-hash acknowledgement must not reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { - let v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let v2 = settings_poll_result( - Some(proto_tcp_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let active_generation = engine.current_generation(); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let mut ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - ctx.transparent_tcp = TransparentTcpReloadState { - capable: true, - substrate_ready: false, - }; - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - let report = timeout(Duration::from_secs(1), reports.recv()) - .await - .expect("TCP rejection report timed out") - .expect("policy reporter stopped"); - - assert_eq!(report.0, 2); - assert!(!report.1); - assert!(report.2.contains("recreate the sandbox"), "{}", report.2); - assert!(report.2.contains("previous policy remains active")); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - handle.abort(); - } - - #[tokio::test] - async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "scripted-guard".to_string(), - grpc_endpoint: "http://scripted.invalid".to_string(), - ..Default::default() - }]; - - let connector_attempts = Arc::new(AtomicUsize::new(0)); - let (attempt_tx, mut attempt_rx) = tokio::sync::mpsc::unbounded_channel(); - let middleware_connector: MiddlewareConnector = { - let connector_attempts = connector_attempts.clone(); - Arc::new(move |_services, _authentication| { - let attempt = connector_attempts.fetch_add(1, Ordering::SeqCst) + 1; - attempt_tx.send(attempt).unwrap(); - Box::pin(async move { - if attempt == 1 { - Err(miette::miette!("scripted middleware connection failure")) - } else { - connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await - } - }) - }) - }; - - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - middleware_connector, - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - - polls.send(v2.clone()).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), attempt_rx.recv()) - .await - .unwrap(), - Some(1) - ); - expect_no_policy_report(&mut reports).await; - assert_eq!(engine.current_generation(), 0); - - polls.send(v2.clone()).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), attempt_rx.recv()) - .await - .unwrap(), - Some(2) - ); - expect_policy_report(&mut reports, 2).await; - assert_eq!(engine.current_generation(), 1); - - polls.send(v2).unwrap(); - expect_no_policy_report(&mut reports).await; - assert_eq!(connector_attempts.load(Ordering::SeqCst), 2); - handle.abort(); - } - - #[tokio::test] - async fn no_signer_capability_uses_legacy_middleware_connector_without_credentials() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "legacy-guard".to_string(), - grpc_endpoint: "http://legacy.invalid".to_string(), - ..Default::default() - }]; - assert!(!v2.extension_authentication_enabled); - - let (inner, polls, mut reports) = scripted_policy_gateway(); - let credential_requests = Arc::new(AtomicUsize::new(0)); - let client = CredentialRejectingPolicyGateway { - inner, - credential_requests: credential_requests.clone(), - }; - let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); - let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { - connector_tx - .send((authentication.credentials.len(), authentication.enabled)) - .unwrap(); - Box::pin(async move { - connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await - }) - }); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - connector, - ); - - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), connector_rx.recv()) - .await - .unwrap(), - Some((0, false)) - ); - expect_policy_report(&mut reports, 2).await; - assert_eq!(credential_requests.load(Ordering::SeqCst), 0); - handle.abort(); - } - - #[tokio::test] - async fn enabled_extension_authentication_keeps_credential_failure_fail_closed() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.extension_authentication_enabled = true; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "authenticated-guard".to_string(), - grpc_endpoint: "https://guard.invalid".to_string(), - ..Default::default() - }]; - - let (inner, polls, mut reports) = scripted_policy_gateway(); - let credential_requests = Arc::new(AtomicUsize::new(0)); - let client = CredentialRejectingPolicyGateway { - inner, - credential_requests: credential_requests.clone(), - }; - let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); - let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { - connector_tx - .send((authentication.credentials.len(), authentication.enabled)) - .unwrap(); - Box::pin(async move { - if authentication.enabled && authentication.credentials.is_empty() { - Err(miette::miette!( - "missing authenticated middleware credential" - )) - } else { - connect_middleware_registry(&[], &authentication).await - } - }) - }); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - connector, - ); - - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), connector_rx.recv()) - .await - .unwrap(), - Some((0, true)) - ); - expect_no_policy_report(&mut reports).await; - assert_eq!(credential_requests.load(Ordering::SeqCst), 1); - handle.abort(); - } - - async fn assert_poll_does_not_use_same_hash_acknowledgement( - initial: openshell_core::grpc_client::SettingsPollResult, - next: openshell_core::grpc_client::SettingsPollResult, - origin: LoadedPolicyOrigin, - initial_report: Option, - ) { - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let ctx = policy_poll_test_context(engine.clone(), origin, default_middleware_connector()); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(initial).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - - if let Some(version) = initial_report { - expect_policy_report(&mut reports, version).await; - } else { - expect_no_policy_report(&mut reports).await; - } - - polls.send(next).unwrap(); - expect_no_policy_report(&mut reports).await; - assert_eq!( - engine.current_generation(), - 0, - "negative same-hash scope must not reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn same_hash_ack_poll_loop_rejects_local_global_empty_equal_and_older_scopes() { - let mut sandbox_v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - sandbox_v1.policy_hash = "same-policy".to_string(); - let loaded_v1 = LoadedPolicyRevision::from_snapshot(&sandbox_v1); - let mut sandbox_v2 = sandbox_v1.clone(); - sandbox_v2.version = 2; - sandbox_v2.config_revision = 200; - - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - sandbox_v2.clone(), - LoadedPolicyOrigin::LocalOverride, - None, - ) - .await; - - let mut global_v2 = sandbox_v2.clone(); - global_v2.policy_source = openshell_core::proto::PolicySource::Global; - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - global_v2, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v1.clone()), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - let mut empty_v1 = sandbox_v1.clone(); - empty_v1.policy_hash.clear(); - let empty_loaded = LoadedPolicyRevision::from_snapshot(&empty_v1); - let mut empty_v2 = sandbox_v2.clone(); - empty_v2.policy_hash.clear(); - assert_poll_does_not_use_same_hash_acknowledgement( - empty_v1, - empty_v2, - LoadedPolicyOrigin::Gateway { - revision: Some(empty_loaded), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - sandbox_v1.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v1.clone()), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - let loaded_v2 = LoadedPolicyRevision::from_snapshot(&sandbox_v2); - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v2, - sandbox_v1, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v2), - has_last_valid_policy: true, - }, - Some(2), - ) - .await; - } - - #[tokio::test] - async fn changed_hash_poll_uses_normal_opa_reload_and_status_path() { - let v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let v2 = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - expect_policy_report(&mut reports, 2).await; - assert_eq!( - engine.current_generation(), - 1, - "changed policy content must still reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn failed_external_startup_registry_build_preserves_installed_builtins() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let builtins_generation = engine.current_generation(); - assert_eq!(builtins_generation, 1); - - let invalid_external = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_payload_bytes: 1024, - ..Default::default() - }; - connect_middleware_registry(&[invalid_external], &MiddlewareAuthentication::default()) - .await - .expect_err("unavailable external service must not replace built-ins"); - - assert_eq!(engine.current_generation(), builtins_generation); - } - - #[tokio::test] - async fn unavailable_middleware_reload_keeps_last_known_good_runtime_active() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let active_generation = engine.current_generation(); - let unavailable_service = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_payload_bytes: 1024, - ..Default::default() - }; - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[unavailable_service], - authentication: &MiddlewareAuthentication::default(), - registry_changed: true, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState::default(), - ) - .await - .expect_err("unavailable middleware must fail candidate preparation"); - let disposition = apply_gateway_runtime_reload_failure( - &engine, - failure, - PolicyValidationFailureMode::FailClosed, - true, - 2, - ) - .expect("middleware failure handling must succeed"); - - assert!(matches!( - disposition, - GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } - )); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[tokio::test] - async fn tcp_policy_reload_without_startup_substrate_is_rejected_and_keeps_previous_policy() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - let active_generation = engine.current_generation(); - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_tcp_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[], - authentication: &MiddlewareAuthentication::default(), - registry_changed: false, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState { - capable: true, - substrate_ready: false, - }, - ) - .await - .expect_err("TCP expansion must require startup substrate"); - let disposition = apply_gateway_runtime_reload_failure( - &engine, - failure, - PolicyValidationFailureMode::FailClosed, - true, - 2, - ) - .expect("runtime prerequisite failure handling must succeed"); - - assert!(matches!( - disposition, - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - active_generation: generation, - .. - } if generation == active_generation - )); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[tokio::test] - async fn tcp_policy_reload_on_unsupported_runtime_is_rejected() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_tcp_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[], - authentication: &MiddlewareAuthentication::default(), - registry_changed: false, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState::default(), - ) - .await - .expect_err("unsupported runtime must reject TCP expansion"); - - assert!(matches!( - failure, - GatewayRuntimeReloadError::TransparentTcpPrerequisite(_) - )); - assert_eq!(engine.current_generation(), 0); - } - - #[test] - fn policy_rejection_after_middleware_outage_is_not_deduplicated() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - let middleware_failure = GatewayRuntimeReloadError::MiddlewareRegistry(miette::miette!( - "middleware service unavailable" - )); - let first_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &middleware_failure); - let middleware_disposition = apply_gateway_runtime_reload_failure( - &engine, - middleware_failure, - PolicyValidationFailureMode::FailClosed, - true, - 7, - ) - .unwrap(); - - assert!(matches!( - middleware_disposition, - GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } - )); - assert!(engine.fail_closed_reason().is_none()); - - let policy_failure = GatewayRuntimeReloadError::PolicyValidation(miette::miette!( - "conflicting endpoint metadata" - )); - let second_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &policy_failure); - assert_ne!( - first_failure, second_failure, - "a changed failure class for the same candidate must be handled" - ); - - let policy_disposition = apply_gateway_runtime_reload_failure( - &engine, - policy_failure, - PolicyValidationFailureMode::FailClosed, - true, - 7, - ) - .unwrap(); - assert!(matches!( - policy_disposition, - GatewayRuntimeFailureDisposition::PolicyRejected { .. } - )); - assert!(engine.fail_closed_reason().is_some()); - } - - #[test] - fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { - let services = Vec::new(); - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - } - - #[test] - fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &no_services, - &no_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &no_services, - &desired_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - false, - "local-policy", - "hash-v2", - &no_services, - &desired_services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - } - - #[test] - fn policy_only_change_does_not_rebuild_middleware_registry() { - let services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - // The runtime must reconcile, but the registry (and therefore - // middleware reachability) is not part of that reconciliation. - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &services, - &services, - )); - } - - #[test] - fn registry_rebuild_requires_service_set_change_or_degraded_registry() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &no_services, - &desired_services, - )); - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::NeedsReconciliation, - &desired_services, - &desired_services, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &desired_services, - &desired_services, - )); - } - - #[test] - fn initial_ack_candidate_matches_sandbox_revision() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) - .expect("sandbox-sourced matching revision should be acknowledged"); - - assert_eq!(ack.version, 2); - assert_eq!(ack.policy_hash, "hash-v2"); - assert_eq!(ack.config_revision, 200); - } - - #[test] - fn initial_ack_candidate_ignores_global_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Global, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_version_zero() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 0, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_local_file_mode() { - // Local-file mode retains no proto policy, so there is nothing to - // acknowledge to the gateway. - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(None, &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_rejects_mismatched_identity() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let mut newer = proto_policy_fixture(); - newer.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule::default(), - ); - let canonical = - settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); - let canonical = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "hash-provider-change".to_string(), - config_revision: loaded.config_revision + 1, - ..canonical - }; - - assert_eq!( - initial_poll_disposition( - &LoadedPolicyOrigin::Gateway { - revision: Some(loaded), - has_last_valid_policy: true, - }, - &canonical, - ), - InitialPollDisposition::Reconcile - ); - } - - #[test] - fn initial_poll_tracks_local_override_without_reconciliation() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert_eq!( - initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), - InitialPollDisposition::TrackOnly - ); - assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); - } - - #[test] - fn initial_poll_reconciles_unbound_gateway_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let origin = LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }; - - assert_eq!( - initial_poll_disposition(&origin, &canonical), - InitialPollDisposition::Reconcile - ); - assert!(origin.allows_gateway_policy_reload()); - } - - #[test] - fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { - let sandbox_result = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "same-policy".to_string(), - ..settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ) - }; - - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "same-policy", &sandbox_result), - Some(2) - ); - assert_eq!( - unchanged_policy_revision_candidate(true, false, 2, "same-policy", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate( - true, - false, - 1, - "different-policy", - &sandbox_result, - ), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(false, false, 1, "same-policy", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(true, true, 1, "same-policy", &sandbox_result), - None - ); - - let global_result = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "same-policy".to_string(), - ..settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Global, - ) - }; - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "same-policy", &global_result), - None - ); - } - - #[test] - fn unchanged_policy_revision_waits_for_required_runtime_reconciliation() { - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), false, false), - Some(2), - "a same-hash revision needs no OPA reload" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), true, false), - None, - "failed runtime reconciliation must keep the revision pending" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), true, true), - Some(2), - "successful runtime reconciliation permits acknowledgement" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(None, false, true), - None, - "runtime success cannot manufacture a revision candidate" - ); - } - - #[test] - fn credential_gating_unavailable_for_local_override_with_credentials() { - assert!(credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - true, - true - )); - } - - #[test] - fn credential_gating_available_without_local_override_or_credentials() { - // A gateway policy is stamped with provenance, so the gates apply. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }, - true, - true - )); - // No provider credentials means there is nothing to leak. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - false, - true - )); - // Without networking the proxy never evaluates endpoint provenance. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - true, - false - )); - } - - #[test] - fn policy_status_outbox_preserves_all_revision_order() { - let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - for version in 1..=128 { - enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); - } - - for version in 1..=128 { - assert_eq!( - receiver.try_recv().unwrap(), - PolicyStatusUpdate::loaded(version) - ); - } - } - - #[test] - fn settings_snapshot_carries_workspace_for_policy_sync() { - let mut snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - snapshot.workspace = "beta".to_string(); - - let revision = LoadedPolicyRevision::from_snapshot(&snapshot); - assert_eq!(revision.version, 1); - assert_eq!( - snapshot.workspace, "beta", - "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" - ); - } - #[test] - fn fail_closed_validation_failure_deactivates_previous_generation() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - let previous_generation = engine.current_generation(); - - let disposition = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::FailClosed, - true, - 7, - "conflicting tls metadata", - ) - .unwrap(); - - assert!(!disposition.previous_policy_active); - assert!(disposition.active_generation > previous_generation); - assert!( - engine - .fail_closed_reason() - .expect("quarantine reason") - .contains("candidate version 7 rejected") - ); - } - - #[test] - fn retain_validation_failure_keeps_previous_generation_active() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - let previous_generation = engine.current_generation(); - - let quarantined = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::FailClosed, - true, - 6, - "conflicting tls metadata", - ) - .unwrap(); - assert!(!quarantined.previous_policy_active); - - let disposition = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::RetainLastValid, - true, - 7, - "conflicting tls metadata", - ) - .unwrap(); - - assert!(disposition.previous_policy_active); - assert!(disposition.active_generation > quarantined.active_generation); - assert!(disposition.active_generation > previous_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[test] - fn retain_validation_failure_without_last_valid_policy_stays_fail_closed() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - - let disposition = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::RetainLastValid, - false, - 1, - "conflicting tls metadata", - ) - .unwrap(); - - assert_eq!( - disposition.configured_mode, - PolicyValidationFailureMode::RetainLastValid - ); - assert_eq!(disposition.mode, PolicyValidationFailureMode::FailClosed); - assert!(!disposition.previous_policy_active); - assert!(engine.fail_closed_reason().is_some()); - - let [config, _] = policy_validation_failure_events( - &disposition, - 1, - "sha256:test", - "conflicting tls metadata", - ); - let config = config.to_json().unwrap(); - assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); - assert_eq!( - config["unmapped"]["configured_validation_failure_mode"], - "retain_last_valid" - ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("previous policy IS NOT active") - ); - } - - #[test] - fn validation_failure_ocsf_states_whether_previous_policy_is_active() { - let fail_closed = PolicyValidationFailureDisposition { - configured_mode: PolicyValidationFailureMode::FailClosed, - mode: PolicyValidationFailureMode::FailClosed, - previous_policy_active: false, - active_generation: 9, - }; - let [config, finding] = policy_validation_failure_events( - &fail_closed, - 8, - "sha256:test", - "conflicting tls metadata", - ); - let config = config.to_json().unwrap(); - assert_eq!(config["class_uid"], 5019); - assert_eq!(config["status"], "Failure"); - assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); - assert_eq!( - config["unmapped"]["configured_validation_failure_mode"], - "fail_closed" - ); - assert_eq!(config["unmapped"]["previous_policy_active"], false); - assert_eq!( - config["unmapped"]["validation_error"], - "conflicting tls metadata" - ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("previous policy IS NOT active") - ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("error:conflicting tls metadata") - ); - - let finding = finding.to_json().unwrap(); - assert_eq!(finding["class_uid"], 2004); - assert_eq!(finding["action"], "Denied"); - assert_eq!(finding["disposition"], "Blocked"); - - let retained = PolicyValidationFailureDisposition { - configured_mode: PolicyValidationFailureMode::RetainLastValid, - mode: PolicyValidationFailureMode::RetainLastValid, - previous_policy_active: true, - active_generation: 4, - }; - let [config, _] = policy_validation_failure_events( - &retained, - 8, - "sha256:test", - "conflicting tls metadata", - ); - let config = config.to_json().unwrap(); - assert_eq!(config["unmapped"]["previous_policy_active"], true); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("previous policy IS active") - ); - } +/// Returns an error when the protected bootstrap is invalid or the boundary +/// listener cannot be established. +pub fn run( + config_path: &std::path::Path, + qualification: RuntimeQualification, +) -> miette::Result<()> { + boundary_server::run_boundary(config_path, qualification) + .map_err(|error| miette::miette!(error)) } diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 1ad69e1070..192c9dec6e 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -1,288 +1,1525 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Sandbox - process sandbox and monitor. +//! `OpenShell` capability-free in-workload sandbox boundary. +#[cfg(target_os = "linux")] +use std::mem::size_of; use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; use clap::Parser; use miette::{IntoDiagnostic, Result}; -use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; -use tracing::{info, warn}; +#[cfg(target_os = "linux")] +use openshell_ocsf::OcsfShorthandLayer; +#[cfg(target_os = "linux")] use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::LevelFilter; +#[cfg(target_os = "linux")] use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; -use openshell_sandbox::run_sandbox; - -/// Subcommand name used to self-copy the supervisor binary into a shared volume. +/// Subcommand name used to self-copy the sandbox binary into a shared volume. /// /// Init containers invoke the binary directly instead of relying on `sh`/`cp` /// to copy the binary out. Invoking the binary itself with this argument /// performs the copy in pure Rust. const COPY_SELF_SUBCOMMAND: &str = "copy-self"; +const BOOTSTRAP_SUBCOMMAND: &str = "bootstrap"; +const SEED_WORKSPACE_SUBCOMMAND: &str = "seed-workspace"; +#[cfg(target_os = "linux")] +const BOOTSTRAP_INPUT_ROOT: &str = "/.openshell/bootstrap-input"; +#[cfg(target_os = "linux")] +const SANDBOX_RUNTIME_ROOT: &str = "/.openshell/runtime"; +#[cfg(target_os = "linux")] +const SANDBOX_STATE_ROOT: &str = "/.openshell/state"; -/// Subcommand for one-shot debug RPCs from inside a sandbox container. -/// -/// Reads the same token sources as the supervisor (env, file, K8s SA -/// bootstrap) and issues a single gRPC call against the gateway. Useful -/// for end-to-end verification: e.g. `docker exec` into a sandbox, then -/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` -/// to confirm the cross-sandbox IDOR guard fires. -const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; - -/// Default `--mode` value: run both supervisor leaves in a single binary. -const DEFAULT_MODE: &str = "network,process"; -const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; +const CAPABILITY_PROBE_SUBCOMMAND: &str = "capability-probe"; +const CAPABILITY_PROBE_LAUNCH_SUBCOMMAND: &str = "capability-probe-launch"; +const CAPABILITY_SOCKET_CHILD_SUBCOMMAND: &str = "capability-socket-child"; +const CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND: &str = "capability-landlock-child"; +const CAPABILITY_FREE_LAUNCH_SUBCOMMAND: &str = "launch-capability-free"; #[cfg(target_os = "linux")] -const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR; +const PROBE_DENIED_TCP_PEER: &str = "203.0.113.1:9"; #[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +const PROBE_SOCKADDR_IN_LEN: usize = size_of::(); #[cfg(target_os = "linux")] -const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +const LINUX_SIGNAL_LIMIT: i32 = 65; + +#[derive(Parser, Debug)] +#[command(name = "openshell-sandbox")] +#[command(version = openshell_core::VERSION)] +#[command(about = "OpenShell in-workload isolation boundary")] +struct BoundaryArgs { + /// Protected one-use bootstrap configuration staged by the driver. + #[arg(long)] + bootstrap: std::path::PathBuf, + + /// Log level (trace, debug, info, warn, error). + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, +} + +/// Internal one-shot command used by trusted driver bootstrap to validate an +/// image-provided workdir as the final sandbox identity. +#[derive(Parser, Debug)] +#[command(name = "validate-workspace", hide = true)] +struct ValidateWorkspaceArgs { + #[arg(long)] + workdir: String, + #[arg(long)] + expected_uid: u32, + #[arg(long)] + expected_gid: u32, +} + #[cfg(target_os = "linux")] -const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +fn validate_workspace(args: &[String]) -> Result<()> { + let args = ValidateWorkspaceArgs::try_parse_from( + std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), + ) + .into_diagnostic()?; + let actual = ( + nix::unistd::geteuid().as_raw(), + nix::unistd::getegid().as_raw(), + ); + if actual != (args.expected_uid, args.expected_gid) { + return Err(miette::miette!( + "workspace validator privilege drop failed: expected {}:{}, got {}:{}", + args.expected_uid, + args.expected_gid, + actual.0, + actual.1 + )); + } + openshell_sandbox::process::validate_oci_workspace_as_effective_identity(Path::new( + &args.workdir, + )) +} + +#[cfg(not(target_os = "linux"))] +fn validate_workspace(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "workspace validation is only supported on Unix" + )) +} + +/// Run the active Phase 0 probe inside the exact workload runtime profile. #[cfg(target_os = "linux")] -const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[allow(unsafe_code)] +fn run_capability_probe() -> Result<()> { + let (qualification, report) = qualify_runtime()?; + debug_assert!(qualification.seccomp.notification_round_trip); + println!("{report}"); + Ok(()) +} + +/// Actively qualify every kernel primitive used by the capability-free +/// sandbox. Callers decide whether to emit the resulting diagnostic report. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, serde_json::Value)> { + use miette::Context as _; + + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid == 0 || gid == 0 { + return Err(miette::miette!( + "capability-free sandbox probe requires non-root UID and GID, got {uid}:{gid}" + )); + } + let status = std::fs::read_to_string("/proc/self/status") + .into_diagnostic() + .wrap_err("read /proc/self/status")?; + for field in ["CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb"] { + let value = proc_status_hex(&status, field)?; + if value != 0 { + return Err(miette::miette!( + "capability-free sandbox probe found {field}=0x{value:x}" + )); + } + } + // SAFETY: PR_GET_NO_NEW_PRIVS reads one scalar process property. + let no_new_privileges = unsafe { libc::prctl(libc::PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) }; + if no_new_privileges != 1 { + return Err(miette::miette!( + "capability-free sandbox probe requires no_new_privs=1" + )); + } + + // The trusted sandbox must be nondumpable before it handles bootstrap or + // channel secrets. Perform the parent-to-child observation probe after + // tightening the parent; the synthetic child explicitly becomes dumpable. + // SAFETY: PR_SET_DUMPABLE accepts one scalar and only tightens this process. + if unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) } < 0 { + return Err(miette::miette!( + "set sandbox probe nondumpable: {}", + std::io::Error::last_os_error() + )); + } + openshell_isolation_interface::linux::task_memory::probe_child_access() + .into_diagnostic() + .wrap_err("same-UID task-memory probe")?; + probe_landlock_allow_deny().wrap_err("Landlock allow/deny probe")?; + let notification = + openshell_isolation_interface::linux::seccomp_notify::probe_notification_api() + .into_diagnostic() + .wrap_err("seccomp notification probe")?; + probe_socket_virtualization().wrap_err("socket virtualization probe")?; + probe_dns_relay_bind().wrap_err("DNS relay bind probe")?; + let landlock_abi = openshell_isolation_interface::linux::landlock::abi_version() + .into_diagnostic() + .wrap_err("Landlock ABI probe")?; + if landlock_abi == 0 { + return Err(miette::miette!("Landlock ABI version is zero")); + } + + let groups = nix::unistd::getgroups() + .into_diagnostic()? + .into_iter() + .map(nix::unistd::Gid::as_raw) + .collect::>(); + let report = serde_json::json!({ + "qualified": true, + "uid": uid, + "gid": gid, + "supplementary_groups": groups, + "capabilities_zero": true, + "no_new_privileges": true, + "sandbox_dumpable": false, + "child_dumpable": true, + "child_core_limit_zero": true, + "same_uid_self_protection": true, + "landlock_abi": landlock_abi, + "landlock_allow_deny": true, + "seccomp_notification": notification.notification_round_trip(), + "seccomp_addfd_send": notification.addfd_send(), + "task_memory_copy": notification.task_memory_copy(), + "connected_send_fast_path": notification.connected_send_fast_path(), + "socket_virtualization": true, + "dns_relay_bind": true, + "udp_dns_round_trip": true, + "tcp_dns_round_trip": true, + "tcp_allow_round_trip": true, + "tcp_deny_round_trip": true, + "wait_killable_recv": notification.wait_killable_recv, + }); + let qualification = openshell_sandbox::RuntimeQualification { + seccomp: openshell_isolation_interface::contract::SeccompEvidence { + new_listener: notification.notification_round_trip(), + notification_round_trip: notification.notification_round_trip(), + id_validation: notification.notification_round_trip(), + addfd_send: notification.addfd_send(), + retained_socket_operation: true, + proc_fd_identity: true, + task_memory_read: notification.task_memory_copy(), + task_memory_write: notification.task_memory_copy(), + cancellation: notification.wait_killable_recv, + }, + landlock_abi, + landlock_allow_deny: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + }; + Ok((qualification, report)) +} + #[cfg(target_os = "linux")] -const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +fn probe_dns_relay_bind() -> Result<()> { + use std::net::{TcpListener, UdpSocket}; + + let unprivileged_port_start = + std::fs::read_to_string("/proc/sys/net/ipv4/ip_unprivileged_port_start") + .into_diagnostic()? + .trim() + .parse::() + .into_diagnostic()?; + if unprivileged_port_start != 0 { + return Err(miette::miette!( + "DNS relay requires net.ipv4.ip_unprivileged_port_start=0, got {unprivileged_port_start}" + )); + } + let tcp = TcpListener::bind("127.0.0.53:53").into_diagnostic()?; + let udp = UdpSocket::bind("127.0.0.53:53").into_diagnostic()?; + drop((tcp, udp)); + Ok(()) +} + +/// Prove that the exact unprivileged runtime can install a hard Landlock +/// allow-list which admits one path and rejects an adjacent path. Landlock is +/// irreversible, so the restriction is exercised in a fresh trusted child. #[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +fn probe_landlock_allow_deny() -> Result<()> { + use miette::Context as _; + + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .into_diagnostic()? + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "openshell-landlock-probe-{}-{nonce}", + std::process::id() + )); + let allowed = root.join("allowed"); + let denied = root.join("denied"); + std::fs::create_dir(&root) + .into_diagnostic() + .wrap_err("create Landlock probe root")?; + let probe_result = (|| -> Result<()> { + std::fs::create_dir(&allowed).into_diagnostic()?; + std::fs::create_dir(&denied).into_diagnostic()?; + std::fs::write(allowed.join("sentinel"), b"allowed").into_diagnostic()?; + std::fs::write(denied.join("sentinel"), b"denied").into_diagnostic()?; + let status = std::process::Command::new(std::env::current_exe().into_diagnostic()?) + .arg(CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND) + .arg(&allowed) + .arg(&denied) + .env_clear() + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .status() + .into_diagnostic() + .wrap_err("run Landlock probe child")?; + if !status.success() { + return Err(miette::miette!( + "Landlock probe child exited with status {status}" + )); + } + Ok(()) + })(); + let cleanup_result = std::fs::remove_dir_all(&root).into_diagnostic(); + probe_result?; + cleanup_result.wrap_err("remove Landlock probe root") +} + #[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; +fn run_capability_landlock_child(args: &[String]) -> Result<()> { + use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkPolicy, ProcessPolicy, + SandboxPolicy, + }; + + let [allowed, denied] = args else { + return Err(miette::miette!( + "usage: {CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND} " + )); + }; + let allowed = Path::new(allowed); + let denied = Path::new(denied); + let policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![allowed.to_path_buf()], + read_write: Vec::new(), + include_workdir: false, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy { + compatibility: LandlockCompatibility::HardRequirement, + }, + process: ProcessPolicy::default(), + }; + let prepared = openshell_sandbox::sandbox::linux::prepare_capability_free(&policy, None)?; + openshell_sandbox::sandbox::linux::enforce(prepared)?; + if std::fs::read(allowed.join("sentinel")).into_diagnostic()? != b"allowed" { + return Err(miette::miette!("Landlock probe allowed-path mismatch")); + } + match std::fs::read(denied.join("sentinel")) { + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), + Err(error) => Err(error).into_diagnostic(), + Ok(_) => Err(miette::miette!( + "Landlock probe unexpectedly read the denied path" + )), + } +} + +#[cfg(not(target_os = "linux"))] +fn run_capability_landlock_child(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "Landlock qualification is supported only on Linux" + )) +} -/// Which supervisor leaves are enabled in this process. +/// Exercise the production listener inheritance and socket-time ADDFD shape. /// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. -#[derive(Clone, Copy, Debug)] -struct Mode { - network: bool, - process: bool, - network_init: bool, -} - -impl std::str::FromStr for Mode { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut mode = Self { - network: false, - process: false, - network_init: false, - }; - for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { - match part { - "network" => mode.network = true, - "process" => mode.process = true, - "network-init" => mode.network_init = true, - other => { - return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" - )); +/// One dedicated launcher thread installs the non-TSYNC listener, moves the +/// listener FD to this unfiltered broker through an in-process channel, then +/// execs the child. The child proves that the injected open-file description +/// survives dup and epoll registration before connect and that the broker can +/// return the original peer rather than the local relay endpoint. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_socket_virtualization() -> Result<()> { + use std::io::{Read as _, Write as _}; + use std::net::{Ipv4Addr, SocketAddr, TcpListener, UdpSocket}; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + use std::os::unix::process::CommandExt as _; + use std::sync::mpsc; + + use miette::Context as _; + use openshell_isolation_interface::linux::seccomp_notify::NotificationListener; + use openshell_isolation_interface::linux::socket_registry::{ + InetFamily, InetKind, SocketMetadata, SocketRegistry, SocketState, + }; + + let relay = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .into_diagnostic() + .wrap_err("bind socket probe relay")?; + let original_peer = relay.local_addr().into_diagnostic()?; + let relay_thread = std::thread::Builder::new() + .name("openshell-probe-relay".to_string()) + .spawn(move || -> std::io::Result<()> { + let (mut stream, _) = relay.accept()?; + stream.set_nodelay(true)?; + let mut request = [0_u8; 4]; + stream.read_exact(&mut request)?; + if &request != b"ping" { + return Err(std::io::Error::other("socket probe payload mismatch")); + } + stream.write_all(b"pong") + }) + .into_diagnostic()?; + let dns_relay_addr = "127.0.0.53:53" + .parse::() + .expect("fixed DNS relay address is valid"); + let dns_relay = UdpSocket::bind(dns_relay_addr) + .into_diagnostic() + .wrap_err("bind socket probe DNS relay")?; + let dns_thread = std::thread::Builder::new() + .name("openshell-probe-dns".to_string()) + .spawn(move || -> std::io::Result<()> { + let mut query = [0_u8; 512]; + let (length, peer) = dns_relay.recv_from(&mut query)?; + let response = build_probe_dns_response(&query[..length])?; + let sent = dns_relay.send_to(&response, peer)?; + if sent != response.len() { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "short DNS probe response", + )); + } + Ok(()) + }) + .into_diagnostic()?; + let dns_tcp_relay = TcpListener::bind(dns_relay_addr) + .into_diagnostic() + .wrap_err("bind socket probe TCP DNS relay")?; + let dns_tcp_thread = std::thread::Builder::new() + .name("openshell-probe-dns-tcp".to_string()) + .spawn(move || -> std::io::Result<()> { + let (mut stream, _) = dns_tcp_relay.accept()?; + let mut length = [0_u8; 2]; + stream.read_exact(&mut length)?; + let length = usize::from(u16::from_be_bytes(length)); + if length == 0 || length > 512 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid TCP DNS probe length", + )); + } + let mut query = vec![0_u8; length]; + stream.read_exact(&mut query)?; + let response = build_probe_dns_response(&query)?; + stream.write_all( + &u16::try_from(response.len()) + .expect("probe DNS response fits u16") + .to_be_bytes(), + )?; + stream.write_all(&response) + }) + .into_diagnostic()?; + + let executable = std::env::current_exe() + .into_diagnostic() + .wrap_err("resolve socket probe executable")?; + let sandbox_tgid = std::process::id(); + let mut child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(sandbox_tgid) + .into_diagnostic() + .wrap_err("prepare socket probe child hardening")?; + let (listener_tx, listener_rx) = mpsc::sync_channel::>(1); + let (child_tx, child_rx) = mpsc::sync_channel::>(1); + let launcher = std::thread::Builder::new() + .name("openshell-probe-launcher".to_string()) + .spawn(move || { + if let Err(error) = block_launcher_signals() { + let _ = listener_tx.send(Err(error)); + return; + } + let listener = + openshell_isolation_interface::linux::seccomp_notify::install_listener(&[ + libc::SYS_socket, + libc::SYS_connect, + libc::SYS_getpeername, + libc::SYS_sendto, + ]); + let Ok(listener) = listener else { + let _ = listener_tx.send(listener); + return; + }; + if listener_tx.send(Ok(listener)).is_err() { + return; + } + let mut command = std::process::Command::new(executable); + command + .arg(CAPABILITY_SOCKET_CHILD_SUBCOMMAND) + .arg(original_peer.to_string()) + .arg(sandbox_tgid.to_string()) + .env_clear() + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()); + // SAFETY: the hook uses only raw signal/process syscalls and the + // prebuilt, allocation-free seccomp installation path. + unsafe { + command.pre_exec(move || { + if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + set_child_core_limit()?; + reset_child_signal_dispositions()?; + child_hardening.install()?; + install_child_signal_mask() + }); + } + let child = command.spawn(); + let _ = child_tx.send(child); + }) + .into_diagnostic()?; + + let listener = listener_rx + .recv() + .into_diagnostic() + .wrap_err("socket probe launcher stopped before listener handoff")? + .into_diagnostic() + .wrap_err("install socket probe listener")?; + let mut child = child_rx + .recv() + .into_diagnostic() + .wrap_err("socket probe launcher stopped before child spawn")? + .into_diagnostic() + .wrap_err("spawn socket probe child")?; + let mut registry = SocketRegistry::new(1, 8).into_diagnostic()?; + let mut observed_tcp_sockets = 0_u8; + let mut observed_dns_socket = false; + let mut observed_connect = false; + let mut observed_dns_tcp_connect = false; + let mut observed_denied_connect = false; + let mut observed_peer = false; + let mut observed_dns_send = false; + + while !(observed_tcp_sockets == 3 + && observed_dns_socket + && observed_connect + && observed_dns_tcp_connect + && observed_denied_connect + && observed_peer + && observed_dns_send) + { + let notification = listener + .receive() + .into_diagnostic() + .wrap_err("receive socket probe notification")?; + match i64::from(notification.syscall) { + libc::SYS_socket => { + if notification.args[0] != u64::try_from(libc::AF_INET).unwrap() { + listener + .respond_errno(notification.id, libc::EPROTONOSUPPORT) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe request")); } + let requested_type = i32::try_from(notification.args[1]) + .map_err(|_| miette::miette!("socket type does not fit i32"))?; + let base_type = requested_type & !(libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK); + let protocol = i32::try_from(notification.args[2]) + .map_err(|_| miette::miette!("socket protocol does not fit i32"))?; + let (kind, canonical_protocol) = match (base_type, protocol) { + (libc::SOCK_STREAM, 0 | libc::IPPROTO_TCP) if observed_tcp_sockets < 3 => { + observed_tcp_sockets += 1; + (InetKind::Tcp, libc::IPPROTO_TCP) + } + (libc::SOCK_DGRAM, 0 | libc::IPPROTO_UDP) if !observed_dns_socket => { + observed_dns_socket = true; + (InetKind::DnsUdp, libc::IPPROTO_UDP) + } + _ => { + listener + .respond_errno(notification.id, libc::EPROTONOSUPPORT) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe request")); + } + }; + // SAFETY: scalar validated AF_INET/TCP arguments return one + // newly owned descriptor on success. + let source = + unsafe { libc::socket(libc::AF_INET, requested_type, canonical_protocol) }; + if source < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned FD. + let source = unsafe { OwnedFd::from_raw_fd(source) }; + let close_on_exec = requested_type & libc::SOCK_CLOEXEC != 0; + let tentative = registry + .stage( + source, + SocketMetadata { + family: InetFamily::V4, + kind, + close_on_exec, + nonblocking: requested_type & libc::SOCK_NONBLOCK != 0, + creator_generation: 1, + }, + ) + .into_diagnostic()?; + listener + .add_fd_and_send(notification.id, tentative.source_fd(), close_on_exec) + .into_diagnostic()?; + registry.commit(tentative).into_diagnostic()?; + } + libc::SYS_connect => { + let fd = i32::try_from(notification.args[0]) + .map_err(|_| miette::miette!("connect FD does not fit i32"))?; + let destination = read_probe_sockaddr( + notification.tid, + notification.args[1], + notification.args[2], + )?; + let denied_peer = PROBE_DENIED_TCP_PEER + .parse::() + .expect("fixed denied peer is valid"); + if destination == denied_peer && !observed_denied_connect { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + observed_denied_connect = true; + continue; + } + if destination != original_peer && destination != dns_relay_addr { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe destination")); + } + if (destination == original_peer && observed_connect) + || (destination == dns_relay_addr && observed_dns_tcp_connect) + { + listener + .respond_errno(notification.id, libc::EALREADY) + .into_diagnostic()?; + return Err(miette::miette!("duplicate socket probe connect")); + } + let entry = registry + .resolve_mut(notification.tid, fd) + .into_diagnostic()?; + entry.validate_retained_identity().into_diagnostic()?; + let (sockaddr, length) = encode_probe_sockaddr(destination)?; + // SAFETY: the retained FD is the registered injected socket; + // `sockaddr` is live for the declared IPv4 length. + let connected = unsafe { + libc::connect( + entry.retained_preconnect().into_diagnostic()?.as_raw_fd(), + sockaddr.as_ptr().cast(), + length, + ) + }; + if connected != 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + if destination == dns_relay_addr { + entry.set_state(SocketState::DnsTcp { + relay: dns_relay_addr, + }); + observed_dns_tcp_connect = true; + } else { + entry.set_state(SocketState::Connected { original_peer }); + observed_connect = true; + } + entry.release_preconnect(); + listener + .respond_value(notification.id, 0) + .into_diagnostic()?; + } + libc::SYS_getpeername => { + let fd = i32::try_from(notification.args[0]) + .map_err(|_| miette::miette!("peer FD does not fit i32"))?; + let entry = registry.resolve(notification.tid, fd).into_diagnostic()?; + let SocketState::Connected { original_peer } = entry.state() else { + listener + .respond_errno(notification.id, libc::ENOTCONN) + .into_diagnostic()?; + return Err(miette::miette!("peer query preceded mediated connect")); + }; + write_probe_sockaddr( + notification.tid, + notification.args[1], + notification.args[2], + *original_peer, + )?; + listener + .respond_value(notification.id, 0) + .into_diagnostic()?; + observed_peer = true; + } + libc::SYS_sendto => { + let fd = i32::try_from(notification.args[0]) + .map_err(|_| miette::miette!("sendto FD does not fit i32"))?; + let length = usize::try_from(notification.args[2]) + .map_err(|_| miette::miette!("DNS payload length does not fit usize"))?; + if length == 0 || length > 512 || notification.args[3] != 0 { + listener + .respond_errno(notification.id, libc::EMSGSIZE) + .into_diagnostic()?; + return Err(miette::miette!("unexpected DNS probe payload shape")); + } + let destination = read_probe_sockaddr( + notification.tid, + notification.args[4], + notification.args[5], + )?; + if observed_dns_send || destination != dns_relay_addr { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + return Err(miette::miette!("unexpected DNS probe destination")); + } + let mut payload = vec![0_u8; length]; + openshell_isolation_interface::linux::task_memory::read_exact( + notification.tid, + notification.args[1], + &mut payload, + ) + .into_diagnostic()?; + validate_probe_dns_query(&payload)?; + let entry = registry + .resolve_mut(notification.tid, fd) + .into_diagnostic()?; + if entry.metadata().kind != InetKind::DnsUdp + || !matches!(entry.state(), SocketState::Created) + { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + return Err(miette::miette!("DNS probe socket is not eligible")); + } + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(destination)?; + let retained = entry.retained_preconnect().into_diagnostic()?; + // SAFETY: the retained source is the exact injected OFD and + // both copied buffers remain live for their declared lengths. + if unsafe { + libc::connect( + retained.as_raw_fd(), + sockaddr.as_ptr().cast(), + sockaddr_length, + ) + } != 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let sent = unsafe { + libc::send( + retained.as_raw_fd(), + payload.as_ptr().cast(), + payload.len(), + libc::MSG_NOSIGNAL, + ) + }; + if sent != isize::try_from(payload.len()).expect("DNS payload fits isize") { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + entry.set_state(SocketState::DnsUdp { + relay: dns_relay_addr, + }); + entry.release_preconnect(); + listener + .respond_value( + notification.id, + i64::try_from(length).expect("length fits i64"), + ) + .into_diagnostic()?; + observed_dns_send = true; + } + _ => { + listener + .respond_errno(notification.id, libc::EPERM) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe syscall")); } } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); + } + + let status = child + .wait() + .into_diagnostic() + .wrap_err("wait for socket probe child")?; + launcher + .join() + .map_err(|_| miette::miette!("socket probe launcher panicked"))?; + relay_thread + .join() + .map_err(|_| miette::miette!("socket probe relay panicked"))? + .into_diagnostic()?; + dns_thread + .join() + .map_err(|_| miette::miette!("socket probe DNS relay panicked"))? + .into_diagnostic()?; + dns_tcp_thread + .join() + .map_err(|_| miette::miette!("socket probe TCP DNS relay panicked"))? + .into_diagnostic()?; + if !status.success() { + return Err(miette::miette!( + "socket probe child exited with status {status}" + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn validate_probe_dns_query(query: &[u8]) -> Result<()> { + const EXPECTED_QUESTION: &[u8] = b"\x05probe\x09openshell\x04test\x00\x00\x01\x00\x01"; + if query.len() != 12 + EXPECTED_QUESTION.len() + || query[2] & 0x80 != 0 + || query[4..6] != [0, 1] + || &query[12..] != EXPECTED_QUESTION + { + return Err(miette::miette!("DNS probe query is malformed")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn build_probe_dns_response(query: &[u8]) -> std::io::Result> { + validate_probe_dns_query(query).map_err(std::io::Error::other)?; + let mut response = query.to_vec(); + response[2..4].copy_from_slice(&[0x81, 0x80]); + response[6..8].copy_from_slice(&[0, 1]); + response.extend_from_slice(&[0xc0, 0x0c, 0, 1, 0, 1, 0, 0, 0, 30, 0, 4, 203, 0, 113, 7]); + Ok(response) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn block_launcher_signals() -> std::io::Result<()> { + let mut signals = std::mem::MaybeUninit::::uninit(); + // SAFETY: `signals` points to writable sigset storage and pthread_sigmask + // copies it during the call. + if unsafe { libc::sigfillset(signals.as_mut_ptr()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: sigfillset initialized the value above. + let signals = unsafe { signals.assume_init() }; + // SAFETY: changing the mask affects only the dedicated launcher thread. + let result = + unsafe { libc::pthread_sigmask(libc::SIG_BLOCK, &raw const signals, std::ptr::null_mut()) }; + if result != 0 { + return Err(std::io::Error::from_raw_os_error(result)); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +unsafe fn set_child_core_limit() -> std::io::Result<()> { + let limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `limit` is a live fixed-size rlimit and this child-only update + // permanently disables core dumps before any untrusted instruction. + if unsafe { libc::setrlimit(libc::RLIMIT_CORE, &raw const limit) } < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +unsafe fn reset_child_signal_dispositions() -> std::io::Result<()> { + // SAFETY: an all-zero sigaction is a valid base before the explicit + // default handler and empty mask are installed below. + let mut action = unsafe { std::mem::zeroed::() }; + action.sa_sigaction = libc::SIG_DFL; + // SAFETY: action.sa_mask points to writable sigset storage. + if unsafe { libc::sigemptyset(&raw mut action.sa_mask) } < 0 { + return Err(std::io::Error::last_os_error()); + } + for signal in 1..LINUX_SIGNAL_LIMIT { + if signal == libc::SIGKILL || signal == libc::SIGSTOP { + continue; } - if !mode.network && !mode.process && !mode.network_init { - return Err( - "--mode must enable at least one of: network, process, network-init".into(), - ); + // SAFETY: action contains the default disposition and the null output + // pointer requests no previous action. + if unsafe { libc::sigaction(signal, &raw const action, std::ptr::null_mut()) } < 0 { + let error = std::io::Error::last_os_error(); + // glibc reserves two real-time signals for its threading runtime; + // Linux rejects sigaction for those numbers with EINVAL. + if error.raw_os_error() != Some(libc::EINVAL) { + return Err(error); + } } - Ok(mode) } + Ok(()) } -/// `OpenShell` Sandbox - process isolation and monitoring. -// CLI flags are naturally boolean switches; grouping them into structs would -// only obscure the clap definition. -#[allow(clippy::struct_excessive_bools)] -#[derive(Parser, Debug)] -#[command(name = "openshell-sandbox")] -#[command(version = openshell_core::VERSION)] -#[command(about = "Process sandbox and monitor", long_about = None)] -struct Args { - /// Command to execute in the sandbox. - /// Defaults to a login shell if neither this nor the driver specification is - /// provided: `/bin/bash -l` when available, otherwise a shell detected in the - /// sandbox image (e.g. `/bin/sh` on Alpine). - #[arg(trailing_var_arg = true)] - command: Vec, - - /// Working directory for the sandboxed process. - #[arg(long, short)] - workdir: Option, - - /// Timeout in seconds (0 = no timeout). - #[arg(long, short, default_value = "0")] - timeout: u64, - - /// Run in interactive mode (inherit process group for terminal control). - #[arg(long, short = 'i')] - interactive: bool, - - /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. - /// Requires --openshell-endpoint to be set. - #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] - sandbox_id: Option, - - /// Sandbox (used for policy sync when the sandbox discovers policy - /// from disk or falls back to the restrictive default). - #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] - sandbox: Option, - - /// `OpenShell` server gRPC endpoint for fetching policy. - /// Required when using --sandbox-id. - #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] - openshell_endpoint: Option, - - /// Path to Rego policy file for OPA-based network access control. - /// Requires --policy-data to also be set. - #[arg(long, env = "OPENSHELL_POLICY_RULES")] - policy_rules: Option, - - /// Path to YAML data file containing network policies and sandbox config. - /// Requires --policy-rules to also be set. - #[arg(long, env = "OPENSHELL_POLICY_DATA")] - policy_data: Option, +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn install_child_signal_mask() -> std::io::Result<()> { + let mut signals = std::mem::MaybeUninit::::uninit(); + // SAFETY: `signals` points to writable sigset storage. + if unsafe { libc::sigemptyset(signals.as_mut_ptr()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: sigemptyset initialized the value above. + let signals = unsafe { signals.assume_init() }; + // SAFETY: this installs the declared empty target mask immediately before + // exec, after copied sandbox dispositions have been reset. + let result = unsafe { + libc::pthread_sigmask(libc::SIG_SETMASK, &raw const signals, std::ptr::null_mut()) + }; + if result != 0 { + return Err(std::io::Error::from_raw_os_error(result)); + } + Ok(()) +} - /// Log level (trace, debug, info, warn, error). - #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] - log_level: String, +#[cfg(target_os = "linux")] +fn read_probe_sockaddr(tid: u32, address: u64, length: u64) -> Result { + let length = + usize::try_from(length).map_err(|_| miette::miette!("sockaddr length too large"))?; + if length != PROBE_SOCKADDR_IN_LEN { + return Err(miette::miette!("socket probe requires an IPv4 sockaddr")); + } + let mut bytes = vec![0_u8; length]; + openshell_isolation_interface::linux::task_memory::read_exact(tid, address, &mut bytes) + .into_diagnostic()?; + decode_probe_sockaddr(&bytes) +} - /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning - /// with `@` selects an abstract socket in the network namespace. - /// The supervisor bridges `RelayStream` traffic from the gateway onto - /// this socket; nothing else should connect to it. - #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] - ssh_socket_path: Option, +#[cfg(target_os = "linux")] +fn encode_probe_sockaddr( + address: std::net::SocketAddr, +) -> Result<([u8; PROBE_SOCKADDR_IN_LEN], libc::socklen_t)> { + let std::net::SocketAddr::V4(address) = address else { + return Err(miette::miette!("socket probe requires IPv4")); + }; + let mut bytes = [0_u8; PROBE_SOCKADDR_IN_LEN]; + bytes[0..2].copy_from_slice( + &libc::sa_family_t::try_from(libc::AF_INET) + .expect("AF_INET fits sa_family_t") + .to_ne_bytes(), + ); + bytes[2..4].copy_from_slice(&address.port().to_be_bytes()); + bytes[4..8].copy_from_slice(&address.ip().octets()); + Ok(( + bytes, + libc::socklen_t::try_from(PROBE_SOCKADDR_IN_LEN) + .expect("sockaddr_in length fits socklen_t"), + )) +} - /// Path to YAML inference routes for standalone routing. - /// When set, inference routes are loaded from this file instead of - /// fetching a bundle from the gateway. - #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] - inference_routes: Option, +#[cfg(target_os = "linux")] +fn decode_probe_sockaddr(bytes: &[u8]) -> Result { + if bytes.len() != PROBE_SOCKADDR_IN_LEN { + return Err(miette::miette!("socket probe requires an IPv4 sockaddr")); + } + let family = libc::sa_family_t::from_ne_bytes([bytes[0], bytes[1]]); + if i32::from(family) != libc::AF_INET { + return Err(miette::miette!("socket probe sockaddr is not IPv4")); + } + Ok(std::net::SocketAddr::V4(std::net::SocketAddrV4::new( + std::net::Ipv4Addr::new(bytes[4], bytes[5], bytes[6], bytes[7]), + u16::from_be_bytes([bytes[2], bytes[3]]), + ))) +} - /// Enable health check endpoint. - #[arg(long)] - health_check: bool, - - /// Port for health check endpoint. - #[arg(long, default_value = "8080")] - health_port: u16, - - /// Which supervisor components to run. Comma-separated list of - /// "network" and/or "process". Defaults to both (single-binary - /// topology). Use --mode=network for a network-only sidecar, or - /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. Use --mode=network-init only in - /// the Kubernetes init container that prepares sidecar nftables. - #[arg(long, default_value = DEFAULT_MODE)] - mode: Mode, - - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] - proxy_uid: u32, - - /// GID assigned to shared sidecar state directories. Defaults to - /// `--proxy-uid` when omitted. - #[arg(long, env = "OPENSHELL_PROXY_GID")] - proxy_gid: Option, - - /// Shared state directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] - sidecar_state_dir: String, - - /// Shared TLS work directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] - sidecar_tls_dir: String, - - // Corporate upstream proxy. Operator-owned egress boundary: accepted - // only as command-line arguments (no `env =`), because the driver - // controls the supervisor's argv while a sandbox image could bake - // matching `ENV` values. - /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. - #[arg(long)] - upstream_proxy: Option, +#[cfg(target_os = "linux")] +fn write_probe_sockaddr( + tid: u32, + address: u64, + length_address: u64, + peer: std::net::SocketAddr, +) -> Result<()> { + use std::mem::size_of; + + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(peer)?; + let mut requested_length = [0_u8; size_of::()]; + openshell_isolation_interface::linux::task_memory::read_exact( + tid, + length_address, + &mut requested_length, + ) + .into_diagnostic()?; + let requested_length = libc::socklen_t::from_ne_bytes(requested_length); + if requested_length < sockaddr_length { + return Err(miette::miette!("peer sockaddr buffer is too small")); + } + openshell_isolation_interface::linux::task_memory::write_exact(tid, address, &sockaddr) + .into_diagnostic()?; + openshell_isolation_interface::linux::task_memory::write_exact( + tid, + length_address, + &sockaddr_length.to_ne_bytes(), + ) + .into_diagnostic()?; + Ok(()) +} - /// Comma-separated `NO_PROXY` list for the corporate proxy. - #[arg(long)] - upstream_no_proxy: Option, +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn run_capability_socket_child(args: &[String]) -> Result<()> { + use std::io::Read as _; + use std::net::SocketAddr; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + let [expected_peer, sandbox_tgid] = args else { + return Err(miette::miette!( + "usage: {CAPABILITY_SOCKET_CHILD_SUBCOMMAND} " + )); + }; + let expected_peer = expected_peer + .parse::() + .into_diagnostic() + .map_err(|error| miette::miette!("parse socket probe peer: {error}"))?; + let sandbox_tgid = sandbox_tgid + .parse::() + .into_diagnostic() + .map_err(|error| miette::miette!("parse sandbox TGID: {error}"))?; + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(expected_peer)?; + + // SAFETY: this call is intentionally intercepted and completed with one + // newly injected socket descriptor. + let socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + if socket < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(socket) }; + // SAFETY: dup creates an alias of the same open-file description. + let alias = unsafe { libc::dup(socket.as_raw_fd()) }; + if alias < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful dup returned one newly owned descriptor. + let alias = unsafe { OwnedFd::from_raw_fd(alias) }; + + // SAFETY: epoll_create1 returns one owned descriptor; epoll_ctl consumes + // only the live event value for this call. + let epoll = unsafe { libc::epoll_create1(libc::EPOLL_CLOEXEC) }; + if epoll < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful epoll_create1 returned one newly owned descriptor. + let epoll = unsafe { OwnedFd::from_raw_fd(epoll) }; + let mut event = libc::epoll_event { + events: u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).expect("epoll flags fit u32"), + u64: 1, + }; + // SAFETY: descriptors and event pointer are live for this call. + if unsafe { + libc::epoll_ctl( + epoll.as_raw_fd(), + libc::EPOLL_CTL_ADD, + socket.as_raw_fd(), + std::ptr::addr_of_mut!(event), + ) + } < 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + + // SAFETY: the sockaddr is live and the alias references the mediated OFD. + if unsafe { libc::connect(alias.as_raw_fd(), sockaddr.as_ptr().cast(), sockaddr_length) } != 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + + let mut observed_peer = [0_u8; PROBE_SOCKADDR_IN_LEN]; + let mut observed_length = + libc::socklen_t::try_from(PROBE_SOCKADDR_IN_LEN).expect("sockaddr length fits socklen_t"); + // SAFETY: the output objects are live for the full declared length. + if unsafe { + libc::getpeername( + socket.as_raw_fd(), + observed_peer.as_mut_ptr().cast(), + std::ptr::addr_of_mut!(observed_length), + ) + } != 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let observed = decode_probe_sockaddr(&observed_peer)?; + if observed != expected_peer { + return Err(miette::miette!( + "socket probe peer mismatch: expected {expected_peer}, got {observed}" + )); + } + + let request = b"ping"; + // SAFETY: null destination on a connected socket follows the cBPF fast + // path and reads only the live request buffer. + let sent = unsafe { + libc::sendto( + alias.as_raw_fd(), + request.as_ptr().cast(), + request.len(), + libc::MSG_NOSIGNAL, + std::ptr::null(), + 0, + ) + }; + if sent != isize::try_from(request.len()).expect("request length fits isize") { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let mut ready = libc::epoll_event { events: 0, u64: 0 }; + // SAFETY: event points to storage for one returned event. + if unsafe { libc::epoll_wait(epoll.as_raw_fd(), std::ptr::addr_of_mut!(ready), 1, 5_000) } <= 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let mut stream = std::net::TcpStream::from(socket); + let mut response = [0_u8; 4]; + stream.read_exact(&mut response).into_diagnostic()?; + if &response != b"pong" { + return Err(miette::miette!("socket probe response mismatch")); + } + probe_dns_socket_round_trip()?; + probe_tcp_dns_socket_round_trip()?; + probe_tcp_denial()?; + probe_child_self_protection(sandbox_tgid, alias.as_raw_fd())?; + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn run_capability_socket_child(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "socket qualification is supported only on Linux" + )) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_dns_socket_round_trip() -> Result<()> { + use std::net::SocketAddr; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + const DNS_QUERY: &[u8] = + b"\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x05probe\x09openshell\x04test\x00\x00\x01\x00\x01"; + let relay = "127.0.0.53:53" + .parse::() + .expect("fixed DNS relay address is valid"); + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(relay)?; + // SAFETY: the syscall is intercepted and completed with a DNS-only + // registered socket descriptor. + let socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_UDP, + ) + }; + if socket < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(socket) }; + // SAFETY: the destination and query buffers are live for the call. The + // broker copies and emulates this send before replying to the notification. + let sent = unsafe { + libc::sendto( + socket.as_raw_fd(), + DNS_QUERY.as_ptr().cast(), + DNS_QUERY.len(), + 0, + sockaddr.as_ptr().cast(), + sockaddr_length, + ) + }; + if sent != isize::try_from(DNS_QUERY.len()).expect("DNS query length fits isize") { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + + let mut response = [0_u8; 512]; + let mut source = [0_u8; PROBE_SOCKADDR_IN_LEN]; + let mut source_length = + libc::socklen_t::try_from(source.len()).expect("sockaddr length fits socklen_t"); + // SAFETY: all output buffers are live for their declared lengths. + let received = unsafe { + libc::recvfrom( + socket.as_raw_fd(), + response.as_mut_ptr().cast(), + response.len(), + 0, + source.as_mut_ptr().cast(), + std::ptr::addr_of_mut!(source_length), + ) + }; + if received < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let source = decode_probe_sockaddr(&source)?; + if source != relay { + return Err(miette::miette!( + "DNS response source mismatch: expected {relay}, got {source}" + )); + } + let received = usize::try_from(received).expect("positive recv length fits usize"); + if received < 16 + || response[0..2] != DNS_QUERY[0..2] + || response[2] & 0x80 == 0 + || response[received - 4..received] != [203, 0, 113, 7] + { + return Err(miette::miette!("DNS probe response is malformed")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn probe_tcp_dns_socket_round_trip() -> Result<()> { + use std::io::{Read as _, Write as _}; + + const DNS_QUERY: &[u8] = + b"\x56\x78\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x05probe\x09openshell\x04test\x00\x00\x01\x00\x01"; + let mut stream = std::net::TcpStream::connect("127.0.0.53:53").into_diagnostic()?; + stream.set_nodelay(true).into_diagnostic()?; + stream + .write_all( + &u16::try_from(DNS_QUERY.len()) + .expect("probe DNS query fits u16") + .to_be_bytes(), + ) + .into_diagnostic()?; + stream.write_all(DNS_QUERY).into_diagnostic()?; + let mut length = [0_u8; 2]; + stream.read_exact(&mut length).into_diagnostic()?; + let length = usize::from(u16::from_be_bytes(length)); + if length == 0 || length > 512 { + return Err(miette::miette!("TCP DNS probe response length is invalid")); + } + let mut response = vec![0_u8; length]; + stream.read_exact(&mut response).into_diagnostic()?; + if length < 16 + || response[0..2] != DNS_QUERY[0..2] + || response[2] & 0x80 == 0 + || response[length - 4..] != [203, 0, 113, 7] + { + return Err(miette::miette!("TCP DNS probe response is malformed")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_tcp_denial() -> Result<()> { + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + let peer = PROBE_DENIED_TCP_PEER + .parse::() + .expect("fixed denied peer is valid"); + let (sockaddr, length) = encode_probe_sockaddr(peer)?; + // SAFETY: socket creation is intercepted and returns one injected FD. + let socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + if socket < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(socket) }; + // SAFETY: both the injected FD and encoded sockaddr are live. + let result = unsafe { libc::connect(socket.as_raw_fd(), sockaddr.as_ptr().cast(), length) }; + require_probe_errno( + isize::try_from(result).expect("connect result fits isize"), + libc::EACCES, + "denied TCP connect", + ) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_child_self_protection(sandbox_tgid: libc::pid_t, socket: libc::c_int) -> Result<()> { + // SAFETY: PR_GET_DUMPABLE reads one scalar property. Normal exec of the + // trusted child image must make it observable to the same-UID sandbox. + if unsafe { libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0) } != 1 { + return Err(miette::miette!("workload child is not dumpable after exec")); + } + let mut core_limit = libc::rlimit { + rlim_cur: libc::rlim_t::MAX, + rlim_max: libc::rlim_t::MAX, + }; + // SAFETY: `core_limit` is writable storage for the current limit. + if unsafe { libc::getrlimit(libc::RLIMIT_CORE, &raw mut core_limit) } < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + if core_limit.rlim_cur != 0 || core_limit.rlim_max != 0 { + return Err(miette::miette!("workload child core limit is not zero")); + } + let mut local = 0_u8; + let remote = 0_u8; + let local_iov = libc::iovec { + iov_base: std::ptr::addr_of_mut!(local).cast(), + iov_len: 1, + }; + let remote_iov = libc::iovec { + iov_base: std::ptr::addr_of!(remote).cast_mut().cast(), + iov_len: 1, + }; + // SAFETY: live one-byte iovecs are supplied. The child filter must reject + // the operation before the kernel inspects the remote pointer. + let read = unsafe { + libc::process_vm_readv( + sandbox_tgid, + &raw const local_iov, + 1, + &raw const remote_iov, + 1, + 0, + ) + }; + require_probe_errno(read, libc::EPERM, "process_vm_readv sandbox")?; + // SAFETY: signal zero would only probe process existence if the filter did + // not reject the trusted sandbox target. + require_probe_errno( + isize::try_from(unsafe { libc::kill(sandbox_tgid, 0) }).expect("kill result fits isize"), + libc::EPERM, + "kill sandbox", + )?; + // SAFETY: scalar syscall arguments request a read-only resource query; + // the child filter rejects non-self targets. + require_probe_errno( + isize::try_from(unsafe { + libc::syscall(libc::SYS_prlimit64, sandbox_tgid, libc::RLIMIT_CORE, 0, 0) + }) + .expect("prlimit result fits isize"), + libc::EPERM, + "prlimit sandbox", + )?; + require_probe_errno( + isize::try_from(unsafe { libc::kill(-sandbox_tgid, 0) }).expect("kill result fits isize"), + libc::EPERM, + "process-group signal", + )?; + require_probe_errno( + isize::try_from(unsafe { libc::fcntl(socket, libc::F_SETOWN, sandbox_tgid) }) + .expect("fcntl result fits isize"), + libc::EPERM, + "fcntl F_SETOWN", + )?; + let mut owner = sandbox_tgid; + require_probe_errno( + isize::try_from(unsafe { + libc::syscall(libc::SYS_ioctl, socket, 0x8901_u32, &raw mut owner) + }) + .expect("ioctl result fits isize"), + libc::EPERM, + "ioctl FIOSETOWN", + )?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn require_probe_errno(result: isize, expected: i32, operation: &str) -> Result<()> { + if result == -1 && std::io::Error::last_os_error().raw_os_error() == Some(expected) { + Ok(()) + } else { + Err(miette::miette!( + "{operation} was not rejected with errno {expected}" + )) + } +} + +/// Exercise the trusted VM bootstrap transition before running the Phase 0 +/// capability probe. This command is intentionally hidden: it exists so the +/// VM conformance lane can prove that a privileged guest init can hand off to +/// a non-root, capability-free sandbox without relying on a shell utility. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn enter_capability_free_identity(uid: u32, gid: u32) -> Result<()> { + use miette::{Context as _, IntoDiagnostic as _}; + + #[repr(C)] + struct CapabilityHeader { + version: u32, + pid: i32, + } + #[repr(C)] + #[derive(Clone, Copy)] + struct CapabilityData { + effective: u32, + permitted: u32, + inheritable: u32, + } + + if uid == 0 || gid == 0 { + return Err(miette::miette!( + "capability-free launch requires non-root UID and GID" + )); + } + if nix::unistd::geteuid().as_raw() != 0 { + return Err(miette::miette!("capability-free launch must start as root")); + } + + let cap_last_cap = std::fs::read_to_string("/proc/sys/kernel/cap_last_cap") + .into_diagnostic() + .wrap_err("read cap_last_cap")? + .trim() + .parse::() + .into_diagnostic() + .wrap_err("parse cap_last_cap")?; + for capability in 0..=cap_last_cap { + // SAFETY: PR_CAPBSET_DROP only removes one capability from the current + // process' bounding set. The loop runs while guest init still has the + // authority required to perform the transition. + if unsafe { libc::prctl(libc::PR_CAPBSET_DROP, capability, 0, 0, 0) } < 0 { + return Err(miette::miette!( + "drop capability {capability} from bounding set: {}", + std::io::Error::last_os_error() + )); + } + } + + // SAFETY: the process is single-threaded at this pre-clap bootstrap path; + // the null pointer is valid for a zero-length supplementary group list. + if unsafe { libc::setgroups(0, std::ptr::null()) } < 0 { + return Err(miette::miette!( + "clear supplementary groups: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: scalar credential transition to the operator-selected guest + // identity. All saved IDs are changed so the process cannot regain root. + if unsafe { libc::setresgid(gid, gid, gid) } < 0 { + return Err(miette::miette!( + "set guest GID {gid}: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: see setresgid above. + if unsafe { libc::setresuid(uid, uid, uid) } < 0 { + return Err(miette::miette!( + "set guest UID {uid}: {}", + std::io::Error::last_os_error() + )); + } - /// Path to the root-only file holding corporate proxy credentials (`user:pass`). - #[arg(long)] - upstream_proxy_auth_file: Option, + let mut header = CapabilityHeader { + version: 0x2008_0522, + pid: 0, + }; + let data = [CapabilityData { + effective: 0, + permitted: 0, + inheritable: 0, + }; 2]; + // SAFETY: capset reads the fixed-size header and two zeroed V3 data words. + if unsafe { libc::syscall(libc::SYS_capset, &raw mut header, data.as_ptr()) } < 0 { + return Err(miette::miette!( + "clear process capability sets: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: clears any ambient capabilities, then permanently forbids + // privilege gain across the following exec. + if unsafe { + libc::prctl( + libc::PR_CAP_AMBIENT, + libc::PR_CAP_AMBIENT_CLEAR_ALL, + 0, + 0, + 0, + ) + } < 0 + { + return Err(miette::miette!( + "clear ambient capabilities: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: PR_SET_NO_NEW_PRIVS is a one-way process hardening transition. + if unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) } < 0 { + return Err(miette::miette!( + "set no_new_privs: {}", + std::io::Error::last_os_error() + )); + } - /// Acknowledge that proxy credentials travel as cleartext Basic auth over - /// the plain-TCP connection to the `http://` proxy. - #[arg(long)] - upstream_proxy_auth_allow_insecure: bool, + Ok(()) +} - /// Send the destination hostname in CONNECT instead of a validated IP - /// (for proxies whose ACLs filter on hostnames). - #[arg(long)] - upstream_proxy_connect_by_hostname: bool, +#[cfg(target_os = "linux")] +fn launch_capability_probe(args: &[String]) -> Result<()> { + use miette::{Context as _, IntoDiagnostic as _}; - /// Path to a PEM CA bundle trusted for the corporate proxy: the TLS - /// handshake with an `https://` proxy and, for TLS-intercepting proxies, - /// re-signed upstream certificates and the sandbox trust bundle. - #[arg(long)] - upstream_proxy_ca_bundle: Option, + let [uid, gid] = args else { + return Err(miette::miette!( + "usage: openshell-sandbox {CAPABILITY_PROBE_LAUNCH_SUBCOMMAND} " + )); + }; + let uid = uid.parse::().into_diagnostic().wrap_err("parse UID")?; + let gid = gid.parse::().into_diagnostic().wrap_err("parse GID")?; + enter_capability_free_identity(uid, gid)?; + run_capability_probe() } -/// Internal one-shot command used by the privileged supervisor to validate an -/// image-provided workdir as the final sandbox identity. -#[derive(Parser, Debug)] -#[command(name = "validate-workspace", hide = true)] -struct ValidateWorkspaceArgs { - #[arg(long)] - workdir: String, - #[arg(long)] - expected_uid: u32, - #[arg(long)] - expected_gid: u32, +#[cfg(not(target_os = "linux"))] +fn launch_capability_probe(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "capability probe launch is supported only on Linux" + )) } #[cfg(target_os = "linux")] -fn validate_workspace(args: &[String]) -> Result<()> { - let args = ValidateWorkspaceArgs::try_parse_from( - std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), - ) - .into_diagnostic()?; - let actual = ( - nix::unistd::geteuid().as_raw(), - nix::unistd::getegid().as_raw(), - ); - if actual != (args.expected_uid, args.expected_gid) { +fn launch_capability_free(args: &[String]) -> Result<()> { + use miette::{Context as _, IntoDiagnostic as _}; + + let [uid, gid, bootstrap] = args else { return Err(miette::miette!( - "workspace validator privilege drop failed: expected {}:{}, got {}:{}", - args.expected_uid, - args.expected_gid, - actual.0, - actual.1 + "usage: openshell-sandbox {CAPABILITY_FREE_LAUNCH_SUBCOMMAND} " )); - } - openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new( - &args.workdir, + }; + let uid = uid.parse::().into_diagnostic().wrap_err("parse UID")?; + let gid = gid.parse::().into_diagnostic().wrap_err("parse GID")?; + enter_capability_free_identity(uid, gid)?; + let log_level = std::env::var(openshell_core::sandbox_env::LOG_LEVEL) + .unwrap_or_else(|_| "warn".to_string()); + run_boundary(Path::new(bootstrap), &log_level) +} + +#[cfg(not(target_os = "linux"))] +fn launch_capability_free(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "capability-free launch is only supported on Linux" )) } #[cfg(not(target_os = "linux"))] -fn validate_workspace(_args: &[String]) -> Result<()> { +fn run_capability_probe() -> Result<()> { Err(miette::miette!( - "workspace validation is only supported on Unix" + "capability-free sandbox probe is supported only on Linux" )) } +#[cfg(target_os = "linux")] +fn proc_status_hex(status: &str, field: &str) -> Result { + let value = status + .lines() + .find_map(|line| line.strip_prefix(&format!("{field}:"))) + .map(str::trim) + .ok_or_else(|| miette::miette!("/proc/self/status is missing {field}"))?; + u64::from_str_radix(value, 16) + .map_err(|error| miette::miette!("invalid {field} value {value:?}: {error}")) +} + /// Copy the running executable to `dest`, creating parent directories as /// needed and ensuring the result is executable (mode `0755`). /// @@ -323,431 +1560,402 @@ fn copy_self(dest: &str) -> Result<()> { Ok(()) } +/// Stage the immutable Kubernetes bootstrap Secret into private writable +/// memory-backed volumes. The projected Secret remains mounted only in this +/// trusted init container; the long-lived sandbox consumes and unlinks the +/// staged configuration before it starts workload code. #[cfg(target_os = "linux")] -fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {uid}:{gid}", - path.display() - ) - })?; - Ok(()) +fn stage_kubernetes_bootstrap() -> Result<()> { + stage_kubernetes_bootstrap_at( + Path::new(BOOTSTRAP_INPUT_ROOT), + Path::new(SANDBOX_RUNTIME_ROOT), + Path::new(SANDBOX_STATE_ROOT), + ) } +/// Stage protected state without performing a duplicate runtime probe. The +/// long-lived sandbox actively qualifies its own exact admitted profile before +/// consuming this material. #[cfg(target_os = "linux")] -fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; +fn run_kubernetes_bootstrap() -> Result<()> { + stage_kubernetes_bootstrap() +} - let uid = Uid::current(); - let gid = Gid::current(); - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - chown(path, Some(uid), Some(gid)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {}:{}", - path.display(), - uid.as_raw(), - gid.as_raw() - ) - })?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; +#[cfg(not(target_os = "linux"))] +fn run_kubernetes_bootstrap() -> Result<()> { + Err(miette::miette!( + "Kubernetes sandbox bootstrap requires Linux" + )) +} + +#[cfg(any(target_os = "linux", test))] +fn stage_kubernetes_bootstrap_at(source: &Path, runtime: &Path, state: &Path) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::os::unix::fs::PermissionsExt as _; + + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid == 0 || gid == 0 { + return Err(miette::miette!( + "Kubernetes bootstrap staging requires a non-root UID and GID, got {uid}:{gid}" + )); + } + + fs::create_dir_all(runtime).into_diagnostic()?; + fs::create_dir_all(state).into_diagnostic()?; + + let nonce = format!( + "{}.{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .into_diagnostic()? + .as_nanos() + ); + let runtime_tmp = runtime.join(format!(".openshell-sandbox.{nonce}")); + let runtime_final = runtime.join("openshell-sandbox"); + let executable = std::env::current_exe().into_diagnostic()?; + copy_regular_file(&executable, &runtime_tmp, 0o500)?; + fs::rename(&runtime_tmp, &runtime_final).into_diagnostic()?; + + let bundle_tmp = state.join(format!(".bootstrap.{nonce}")); + let bundle_final = state.join("bootstrap"); + fs::create_dir(&bundle_tmp).into_diagnostic()?; + fs::set_permissions(&bundle_tmp, fs::Permissions::from_mode(0o700)).into_diagnostic()?; + for name in ["boundary.json", "tls.crt", "tls.key", "client-ca.crt"] { + copy_projected_secret_file(source, name, &bundle_tmp.join(name), 0o600)?; + } + fs::rename(&bundle_tmp, &bundle_final).into_diagnostic()?; + + // Flush the two directory entries before the init container exits. Both + // targets are tmpfs in production, but keeping the staging operation + // durable also makes the helper safe in local conformance tests. + OpenOptions::new() + .read(true) + .open(runtime) + .into_diagnostic()? + .sync_all() + .into_diagnostic()?; + OpenOptions::new() + .read(true) + .open(state) + .into_diagnostic()? + .sync_all() + .into_diagnostic()?; Ok(()) } -#[cfg(target_os = "linux")] -fn copy_sidecar_client_tls_if_present( - source_dir: &Path, - sidecar_tls_dir: &Path, - uid: u32, - gid: u32, +#[cfg(any(target_os = "linux", test))] +fn copy_projected_secret_file( + source_root: &Path, + name: &str, + destination: &Path, + mode: u32, ) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; + let canonical_root = std::fs::canonicalize(source_root).into_diagnostic()?; + let canonical_source = std::fs::canonicalize(source_root.join(name)).into_diagnostic()?; + if !canonical_source.starts_with(&canonical_root) { + return Err(miette::miette!( + "projected bootstrap input escapes its mounted Secret: {}", + source_root.join(name).display() + )); + } + copy_regular_file(&canonical_source, destination, mode) +} - if !source_dir.exists() { - return Ok(()); +#[cfg(any(target_os = "linux", test))] +fn copy_regular_file(source: &Path, destination: &Path, mode: u32) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::OpenOptionsExt as _; + + let metadata = fs::symlink_metadata(source).into_diagnostic()?; + if !metadata.file_type().is_file() || metadata.len() == 0 { + return Err(miette::miette!( + "bootstrap input must be a non-empty regular file: {}", + source.display() + )); + } + let mut input = OpenOptions::new() + .read(true) + .open(source) + .into_diagnostic()?; + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(mode) + .open(destination) + .into_diagnostic()?; + let mut buffer = [0_u8; 16 * 1024]; + loop { + let length = input.read(&mut buffer).into_diagnostic()?; + if length == 0 { + break; + } + output.write_all(&buffer[..length]).into_diagnostic()?; } + output.sync_all().into_diagnostic()?; + Ok(()) +} + +/// Seed the persistent workspace from the agent image as the final workload +/// identity. This replaces the former root shell/tar init container. +fn seed_kubernetes_workspace() -> Result<()> { + seed_kubernetes_workspace_at(Path::new("/sandbox"), Path::new("/mnt/openshell-workspace")) +} - let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); - prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; - for file_name in CLIENT_TLS_FILES { - let source = source_dir.join(file_name); - if !source.exists() { +fn copy_workspace_tree(source: &Path, destination: &Path) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _, symlink}; + + for entry in fs::read_dir(source).into_diagnostic()? { + let entry = entry.into_diagnostic()?; + let source_path = entry.path(); + let destination_path = destination.join(entry.file_name()); + let metadata = fs::symlink_metadata(&source_path).into_diagnostic()?; + if metadata.file_type().is_symlink() { + symlink( + fs::read_link(&source_path).into_diagnostic()?, + &destination_path, + ) + .into_diagnostic()?; + } else if metadata.is_dir() { + fs::create_dir(&destination_path).into_diagnostic()?; + fs::set_permissions(&destination_path, fs::Permissions::from_mode(0o700)) + .into_diagnostic()?; + copy_workspace_tree(&source_path, &destination_path)?; + } else if metadata.is_file() { + let mut input = OpenOptions::new() + .read(true) + .open(&source_path) + .into_diagnostic()?; + let mode = 0o600 | (metadata.permissions().mode() & 0o100); + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(mode) + .open(&destination_path) + .into_diagnostic()?; + let mut buffer = [0_u8; 16 * 1024]; + loop { + let length = input.read(&mut buffer).into_diagnostic()?; + if length == 0 { + break; + } + output.write_all(&buffer[..length]).into_diagnostic()?; + } + output.sync_all().into_diagnostic()?; + } else { return Err(miette::miette!( - "client TLS source file is missing: {}", - source.display() + "workspace seed contains unsupported file type: {}", + source_path.display() )); } - let dest = dest_dir.join(file_name); - if dest.exists() { - std::fs::remove_file(&dest) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to remove stale client TLS file {}", dest.display()) - })?; - } - std::fs::copy(&source, &dest) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to copy client TLS file {} to {}", - source.display(), - dest.display() - ) - })?; - let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); - perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); - std::fs::set_permissions(&dest, perms) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to chmod copied client TLS file {}", dest.display()) - })?; - chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown copied client TLS file {} to {uid}:{gid}", - dest.display() - ) - })?; } - - prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; - Ok(()) } -#[cfg(target_os = "linux")] -fn run_network_init( - proxy_user_id: u32, - proxy_primary_group_id: u32, - sidecar_state_dir: &str, - sidecar_tls_dir: &str, -) -> Result<()> { - validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; - - let sidecar_state_dir = Path::new(sidecar_state_dir); - let sidecar_tls_dir = Path::new(sidecar_tls_dir); - prepare_sidecar_directory( - sidecar_state_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_STATE_DIR_MODE, - )?; - // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the - // TLS work directory owned by the init user until the client cert copy is - // complete, then hand it to the long-running proxy UID. - prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; - copy_sidecar_client_tls_if_present( - Path::new(CLIENT_TLS_DIR), - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - )?; - prepare_sidecar_directory( - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_TLS_DIR_MODE, - )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) -} +fn seed_kubernetes_workspace_at(source: &Path, destination: &Path) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; -#[cfg(target_os = "linux")] -fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { - if proxy_user_id != 0 - && !(openshell_policy::MIN_SANDBOX_PROXY_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&proxy_user_id) - { - return Err(miette::miette!( - "--proxy-uid must be 0 or in range [{}, {}]", - openshell_policy::MIN_SANDBOX_PROXY_UID, - openshell_policy::MAX_SANDBOX_UID, - )); + let sentinel = destination.join(".openshell-initialized"); + if sentinel.try_exists().into_diagnostic()? { + return Ok(()); } - if !(openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&proxy_primary_group_id) - { + let destination_metadata = fs::symlink_metadata(destination).into_diagnostic()?; + if !destination_metadata.is_dir() || destination_metadata.file_type().is_symlink() { return Err(miette::miette!( - "--proxy-gid must be in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, + "workspace target must be a real directory: {}", + destination.display() )); } + + if source.try_exists().into_diagnostic()? { + let metadata = fs::symlink_metadata(source).into_diagnostic()?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(miette::miette!( + "image workspace must be a real directory: {}", + source.display() + )); + } + copy_workspace_tree(source, destination)?; + } + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&sentinel) + .into_diagnostic()?; + file.write_all(b"initialized\n").into_diagnostic()?; + file.sync_all().into_diagnostic()?; + OpenOptions::new() + .read(true) + .open(destination) + .into_diagnostic()? + .sync_all() + .into_diagnostic()?; Ok(()) } +#[cfg(target_os = "linux")] +fn run_boundary(bootstrap: &Path, log_level: &str) -> Result<()> { + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(log_level)); + let _ = tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .try_init(); + let (qualification, _) = qualify_runtime()?; + openshell_sandbox::run(bootstrap, qualification) +} + #[cfg(not(target_os = "linux"))] -fn run_network_init( - _proxy_uid: u32, - _proxy_gid: u32, - _sidecar_state_dir: &str, - _sidecar_tls_dir: &str, -) -> Result<()> { - Err(miette::miette!( - "--mode=network-init is only supported on Linux" - )) +fn run_boundary(_bootstrap: &Path, _log_level: &str) -> Result<()> { + Err(miette::miette!("openshell-sandbox requires Linux")) } fn main() -> Result<()> { - // Handle `copy-self ` before clap so it works without any of the - // sandbox flags. Kubernetes init containers invoke this path to seed an - // emptyDir volume that the agent container then executes from. - let raw_args: Vec = std::env::args().collect(); + let raw_args = std::env::args().collect::>(); if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { let dest = raw_args.get(2).ok_or_else(|| { miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") })?; return copy_self(dest); } - - // Handle `debug-rpc [args]` before clap. Uses a small - // dedicated runtime so we don't pay the supervisor's full startup cost. - if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .into_diagnostic()?; - return runtime.block_on(async move { - let _ = rustls::crypto::ring::default_provider().install_default(); - let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; - std::process::exit(exit); - }); + if raw_args.get(1).map(String::as_str) == Some(BOOTSTRAP_SUBCOMMAND) { + if raw_args.len() != 2 { + return Err(miette::miette!( + "usage: openshell-sandbox {BOOTSTRAP_SUBCOMMAND}" + )); + } + return run_kubernetes_bootstrap(); + } + if raw_args.get(1).map(String::as_str) == Some(SEED_WORKSPACE_SUBCOMMAND) { + if raw_args.len() != 2 { + return Err(miette::miette!( + "usage: openshell-sandbox {SEED_WORKSPACE_SUBCOMMAND}" + )); + } + return seed_kubernetes_workspace(); } if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { return validate_workspace(&raw_args[2..]); } - - let args = Args::parse(); - - if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); - return run_network_init( - args.proxy_uid, - proxy_gid, - &args.sidecar_state_dir, - &args.sidecar_tls_dir, - ); + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_PROBE_SUBCOMMAND) { + return run_capability_probe(); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_PROBE_LAUNCH_SUBCOMMAND) { + return launch_capability_probe(&raw_args[2..]); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_SOCKET_CHILD_SUBCOMMAND) { + return run_capability_socket_child(&raw_args[2..]); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND) { + return run_capability_landlock_child(&raw_args[2..]); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_FREE_LAUNCH_SUBCOMMAND) { + return launch_capability_free(&raw_args[2..]); } - // Try to open a rolling log file; fall back to stderr-only logging if it fails - // (e.g., /var/log is not writable in custom workload images). - // Rotates daily, keeps the 3 most recent files to bound disk usage. - let file_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - (writer, guard) - }); - - let console_filter = - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .into_diagnostic()?; + let args = BoundaryArgs::parse(); + run_boundary(&args.bootstrap, &args.log_level) +} - let exit_code = runtime.block_on(async move { - // Install rustls crypto provider before any TLS connections (including log push). - let _ = rustls::crypto::ring::default_provider().install_default(); +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; - // Set up optional log push layer (gRPC mode only). - let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = - (&args.sandbox_id, &args.openshell_endpoint) - { - let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), - sandbox_id.clone(), + #[test] + fn kubernetes_bootstrap_stages_private_memory_bundle() { + if nix::unistd::geteuid().is_root() || nix::unistd::getegid().as_raw() == 0 { + return; + } + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("input"); + let runtime = root.path().join("runtime"); + let state = root.path().join("state"); + std::fs::create_dir(&source).unwrap(); + let revision = source.join("..2026_09_04_00_00_00"); + std::fs::create_dir(&revision).unwrap(); + for (name, contents) in [ + ("boundary.json", b"{}".as_slice()), + ("tls.crt", b"certificate".as_slice()), + ("tls.key", b"private-key".as_slice()), + ("client-ca.crt", b"client-ca".as_slice()), + ] { + std::fs::write(revision.join(name), contents).unwrap(); + std::os::unix::fs::symlink(format!("..data/{name}"), source.join(name)).unwrap(); + } + std::os::unix::fs::symlink(revision.file_name().unwrap(), source.join("..data")).unwrap(); + + stage_kubernetes_bootstrap_at(&source, &runtime, &state).unwrap(); + + assert!(runtime.join("openshell-sandbox").is_file()); + assert_eq!( + std::fs::metadata(runtime.join("openshell-sandbox")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o500 + ); + for name in ["boundary.json", "tls.crt", "tls.key", "client-ca.crt"] { + let staged = state.join("bootstrap").join(name); + assert_eq!( + std::fs::read(&staged).unwrap(), + std::fs::read(source.join(name)).unwrap() ); - let layer = - openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); - Some((layer, handle)) - } else { - None - }; - let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); - let _log_push_handle = log_push_state.map(|(_, handle)| handle); - - // Shared flag: the sandbox poll loop toggles this when the - // `ocsf_json_enabled` setting changes. The JSONL layer checks it - // on each event and short-circuits when false. - let ocsf_enabled = Arc::new(AtomicBool::new(false)); - let ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new())); - - // Keep guards alive for the entire process. When a guard is dropped the - // non-blocking writer flushes remaining logs. - let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { - let file_filter = EnvFilter::new("info"); - - // OCSF JSONL file: rolling appender matching the main log file - // (daily rotation, 3 files max). Created eagerly but gated by the - // enabled flag — no JSONL is written until ocsf_json_enabled is set. - let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell-ocsf") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer) - .with_enabled_flag(ocsf_enabled.clone()) - .with_target_version(ocsf_schema_version.clone()); - (layer, guard) - }); - let (jsonl_layer, jsonl_guard) = match jsonl_logging { - Some((layer, guard)) => (Some(layer), Some(guard)), - None => (None, None), - }; - - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with( - OcsfShorthandLayer::new(file_writer) - .with_non_ocsf(true) - .with_filter(file_filter), - ) - .with(jsonl_layer.with_filter(LevelFilter::INFO)) - .with(push_layer.clone()) - .init(); - (Some(file_guard), jsonl_guard) - } else { - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with(push_layer) - .init(); - // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stderr-only logging"); - (None, None) - }; - - // Resolve an exact canonical process. Explicit offline/test argv wins; - // drivers otherwise provide a versioned JSON transport so argument - // boundaries are never reconstructed with shell parsing. - let workdir = args.workdir.clone(); - let (command, interactive, await_main_process_attachment) = if !args.command.is_empty() { - (args.command, args.interactive, false) - } else if let Ok(json) = std::env::var(openshell_core::sandbox_env::MAIN_PROCESS_SPEC) { - let config = openshell_core::sandbox_env::MainProcessConfig::decode(&json) - .map_err(|error| miette::miette!("{error}"))?; - ( - config.command, - config.tty, - config.await_main_process_attachment, - ) - } else { - let config = openshell_core::sandbox_env::MainProcessConfig::scratch(); - ( - config.command, - config.tty, - config.await_main_process_attachment, - ) - }; - - // An omitted command (the gateway leaves the default empty rather than - // baking a shell it cannot verify) is resolved to a login shell here, in - // the supervisor, so it matches the sandbox image: bash when present, - // otherwise /bin/sh (e.g. Alpine). An explicit command is used verbatim. - let command = resolve_default_command(command); - - info!(command = ?command, "Starting sandbox"); - // Note: "Starting sandbox" stays as plain info!() since the OCSF context - // is not yet initialized at this point (run_sandbox hasn't been called). - // The shorthand layer will render it in fallback format. - - let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { - https_proxy: args.upstream_proxy, - no_proxy: args.upstream_no_proxy, - proxy_auth_file: args.upstream_proxy_auth_file, - proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, - proxy_ca_bundle: args.upstream_proxy_ca_bundle, - }; + assert_eq!( + std::fs::metadata(staged).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + } - run_sandbox( - command, - workdir, - args.timeout, - interactive, - await_main_process_attachment, - args.sandbox_id, - args.sandbox, - args.openshell_endpoint, - args.policy_rules, - args.policy_data, - args.ssh_socket_path, - args.health_check, - args.health_port, - args.inference_routes, - ocsf_enabled, - ocsf_schema_version, - args.mode.network, - args.mode.process, - upstream_proxy_args, - ) - .await - })?; + #[test] + fn kubernetes_workspace_seed_preserves_files_and_symlinks_without_root() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source"); + let destination = root.path().join("destination"); + std::fs::create_dir_all(source.join("bin")).unwrap(); + std::fs::create_dir(&destination).unwrap(); + std::fs::write(source.join("README"), b"workspace").unwrap(); + std::fs::write(source.join("bin/tool"), b"tool").unwrap(); + let mut executable = std::fs::metadata(source.join("bin/tool")) + .unwrap() + .permissions(); + executable.set_mode(0o755); + std::fs::set_permissions(source.join("bin/tool"), executable).unwrap(); + std::os::unix::fs::symlink("README", source.join("latest")).unwrap(); - std::process::exit(exit_code); -} + seed_kubernetes_workspace_at(&source, &destination).unwrap(); + seed_kubernetes_workspace_at(&source, &destination).unwrap(); -/// Resolve an omitted canonical command to a login shell that exists in this -/// sandbox image. Empty means "use the default": the gateway leaves an omitted -/// command empty rather than persisting a shell it cannot verify, so the -/// supervisor picks one here against the real sandbox filesystem (bash when -/// present, otherwise `/bin/sh`). An explicit command is returned unchanged. -fn resolve_default_command(command: Vec) -> Vec { - if !command.is_empty() { - return command; + assert_eq!( + std::fs::read(destination.join("README")).unwrap(), + b"workspace" + ); + assert_eq!( + std::fs::read_link(destination.join("latest")).unwrap(), + Path::new("README") + ); + assert_ne!( + std::fs::metadata(destination.join("bin/tool")) + .unwrap() + .permissions() + .mode() + & 0o100, + 0 + ); + assert!(destination.join(".openshell-initialized").is_file()); } - let shell = openshell_core::shell::detect_login_shell(); - info!(shell = %shell, "no command specified; resolved default login shell"); - vec![shell, "-l".to_string()] -} - -#[cfg(test)] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; #[cfg(target_os = "linux")] #[test] @@ -826,56 +2034,4 @@ mod tests { let final_path = dest_dir.join("openshell-sandbox"); assert!(final_path.exists(), "binary should land inside dest dir"); } - - #[test] - fn mode_parses_network_init_standalone() { - let mode = "network-init".parse::().unwrap(); - assert!(mode.network_init); - assert!(!mode.network); - assert!(!mode.process); - } - - #[test] - fn mode_rejects_combined_network_init() { - let err = "network-init,network".parse::().unwrap_err(); - assert!(err.contains("cannot be combined")); - } - - #[test] - fn mode_rejects_empty_value() { - let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); - } - - #[cfg(target_os = "linux")] - #[test] - fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { - assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); - assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); - assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); - assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { - validate_network_init_ids(0, 30).unwrap(); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_still_rejects_low_non_root_proxy_uid_and_root_gid() { - let uid_err = - validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); - assert!(uid_err.to_string().contains("--proxy-uid")); - - let gid_err = validate_network_init_ids(0, 0).unwrap_err(); - assert!(gid_err.to_string().contains("--proxy-gid")); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_non_root_system_proxy_group() { - validate_network_init_ids(openshell_policy::MIN_SANDBOX_PROXY_UID, 30).unwrap(); - } } diff --git a/crates/openshell-sandbox/src/main_session.rs b/crates/openshell-sandbox/src/main_session.rs new file mode 100644 index 0000000000..898cfeaed3 --- /dev/null +++ b/crates/openshell-sandbox/src/main_session.rs @@ -0,0 +1,1067 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Retained I/O multiplexer for the canonical sandbox process. + +use std::collections::VecDeque; +use std::io::{Read, Write}; +use std::os::fd::AsRawFd; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use nix::fcntl::{FcntlArg, OFlag, fcntl}; +use nix::pty::Winsize; +use tokio::io::unix::AsyncFd; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Notify; +use tokio::sync::watch; + +use openshell_isolation_interface::contract::{ + BoundaryProcess, BoundarySignal, BoundaryTerminal, ProcessAttachment, +}; + +use crate::process::ProcessIo; + +const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Debug)] +pub enum MainOutput { + Stdout(Bytes), + Stderr(Bytes), + Exit(i32), +} + +impl MainOutput { + fn len(&self) -> usize { + match self { + Self::Stdout(data) | Self::Stderr(data) => data.len(), + Self::Exit(_) => 0, + } + } +} + +#[derive(Clone, Debug)] +struct SequencedOutput { + sequence: u64, + event: MainOutput, +} + +#[derive(Debug)] +struct OutputLogState { + events: VecDeque, + retained_bytes: usize, + next_sequence: u64, +} + +#[derive(Debug)] +struct OutputLog { + state: Mutex, + version: watch::Sender, + terminal_reported: AtomicBool, + terminal_reported_notify: Notify, +} + +impl OutputLog { + fn new() -> Arc { + let (version, _) = watch::channel(0); + Arc::new(Self { + state: Mutex::new(OutputLogState { + events: VecDeque::new(), + retained_bytes: 0, + next_sequence: 0, + }), + version, + terminal_reported: AtomicBool::new(false), + terminal_reported_notify: Notify::new(), + }) + } + + fn publish(&self, event: MainOutput) { + let version = { + let mut state = self.state.lock().expect("main output log lock poisoned"); + let sequence = state.next_sequence; + state.next_sequence = state + .next_sequence + .checked_add(1) + .expect("main output sequence exhausted"); + state.retained_bytes += event.len(); + state.events.push_back(SequencedOutput { sequence, event }); + while state.retained_bytes > OUTPUT_BUFFER_BYTES { + let Some(removed) = state.events.pop_front() else { + break; + }; + state.retained_bytes = state.retained_bytes.saturating_sub(removed.event.len()); + } + state.next_sequence + }; + self.version.send_replace(version); + } + + fn subscribe(self: &Arc) -> MainOutputCursor { + let version = self.version.subscribe(); + let state = self.state.lock().expect("main output log lock poisoned"); + let next_sequence = state + .events + .front() + .map_or(state.next_sequence, |retained| retained.sequence); + drop(state); + MainOutputCursor { + output: Arc::clone(self), + next_sequence, + version, + } + } +} + +#[derive(Debug)] +struct TerminalAttachmentState { + active: usize, + process_finished: bool, + expectation: AttachmentExpectation, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AttachmentExpectation { + None, + Pending, + Satisfied, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MainOutputLagged { + pub skipped: u64, +} + +pub struct MainOutputCursor { + output: Arc, + next_sequence: u64, + version: watch::Receiver, +} + +impl MainOutputCursor { + pub async fn recv(&mut self) -> Result { + loop { + let next = { + let state = self + .output + .state + .lock() + .expect("main output log lock poisoned"); + let oldest = state + .events + .front() + .map_or(state.next_sequence, |event| event.sequence); + if self.next_sequence < oldest { + let skipped = oldest - self.next_sequence; + self.next_sequence = oldest; + return Err(MainOutputLagged { skipped }); + } + if self.next_sequence >= state.next_sequence { + None + } else { + let offset = usize::try_from(self.next_sequence - oldest) + .expect("main output cursor offset exceeds usize"); + let event = state + .events + .get(offset) + .expect("main output cursor references retained event") + .event + .clone(); + self.next_sequence += 1; + Some(event) + } + }; + if let Some(event) = next { + return Ok(event); + } + // The log owns a sender for the cursor lifetime, so closure is not + // expected. A changed version means there is another event to read. + let _ = self.version.changed().await; + } + } +} + +enum MainInput { + Data(Vec), + Close, +} + +#[derive(Clone)] +pub(crate) struct MainInputSender { + sender: tokio::sync::mpsc::Sender, +} + +impl MainInputSender { + pub(crate) async fn send(&self, data: Vec) -> Result<(), &'static str> { + self.sender + .send(MainInput::Data(data)) + .await + .map_err(|_| "canonical process stdin closed") + } + + async fn close(&self) { + let _ = self.sender.send(MainInput::Close).await; + } +} + +pub struct MainSession { + pid: u32, + terminal: bool, + input: MainInputSender, + output: Arc, + input_owner: Mutex>, + input_closed: AtomicBool, + next_owner: AtomicU64, + pty_master: Option>, + boundary_process: Option>, + boundary_terminal: Option>, + readers_remaining: AtomicUsize, + readers_done: Notify, + finished: AtomicBool, + terminal_attachments: Mutex, + terminal_attachments_done: Notify, +} + +impl MainSession { + const REMOTE_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + #[cfg(test)] + pub fn inert() -> Arc { + let (input, _input_rx) = tokio::sync::mpsc::channel(64); + Arc::new(Self { + pid: 1, + terminal: false, + input: MainInputSender { sender: input }, + output: OutputLog::new(), + input_owner: Mutex::new(None), + input_closed: AtomicBool::new(false), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: None, + boundary_terminal: None, + readers_remaining: AtomicUsize::new(0), + readers_done: Notify::new(), + finished: AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }) + } + + #[cfg(test)] + pub fn terminal_for_test() -> (Arc, std::fs::File) { + let pty = nix::pty::openpty(None, None).expect("open test PTY"); + let slave = std::fs::File::from(pty.slave); + ( + Self::new(ProcessIo::Pty(std::fs::File::from(pty.master)), 1), + slave, + ) + } + + #[cfg(test)] + #[allow(unsafe_code)] + pub fn terminal_size_for_test(&self) -> (u16, u16) { + let master = self.pty_master.as_ref().expect("terminal PTY master"); + let mut winsize: libc::winsize = unsafe { std::mem::zeroed() }; + let result = unsafe { libc::ioctl(master.as_raw_fd(), libc::TIOCGWINSZ, &mut winsize) }; + assert_eq!(result, 0, "read terminal dimensions"); + (winsize.ws_col, winsize.ws_row) + } + + #[must_use] + pub fn new(io: ProcessIo, pid: u32) -> Arc { + let terminal = matches!(io, ProcessIo::Pty(_)); + let (input, input_rx) = tokio::sync::mpsc::channel(64); + let pty_master = match &io { + ProcessIo::Pty(master) => { + set_nonblocking(master).expect("set canonical PTY master nonblocking"); + master.try_clone().ok().map(Arc::new) + } + ProcessIo::Pipes { .. } => None, + }; + let session = Arc::new(Self { + pid, + terminal, + input: MainInputSender { sender: input }, + output: OutputLog::new(), + input_owner: Mutex::new(None), + input_closed: AtomicBool::new(false), + next_owner: AtomicU64::new(1), + pty_master, + boundary_process: None, + boundary_terminal: None, + readers_remaining: AtomicUsize::new(if terminal { 1 } else { 2 }), + readers_done: Notify::new(), + finished: AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + Self::start_io(&session, io, input_rx); + session + } + + /// Build the control-side multiplexer around a boundary-owned admitted + /// process. Process lifecycle and PTY operations remain delegated to the + /// boundary process handle. + #[must_use] + pub fn from_boundary( + attachment: ProcessAttachment, + process: Arc, + ) -> Arc { + let ProcessAttachment { + stdin, + stdout, + stderr, + terminal, + } = attachment; + let terminal_mode = terminal.is_some(); + let (input, mut input_rx) = tokio::sync::mpsc::channel(64); + let session = Arc::new(Self { + pid: 0, + terminal: terminal_mode, + input: MainInputSender { sender: input }, + output: OutputLog::new(), + input_owner: Mutex::new(None), + input_closed: AtomicBool::new(false), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: Some(process), + boundary_terminal: terminal, + readers_remaining: AtomicUsize::new(if terminal_mode { 1 } else { 2 }), + readers_done: Notify::new(), + finished: AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + let stdout_session = Arc::clone(&session); + tokio::spawn(async move { + let mut stdout = stdout; + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stdout_session.reader_finished(); + }); + if let Some(mut stderr) = stderr { + let stderr_session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stderr_session + .publish(MainOutput::Stderr(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stderr_session.reader_finished(); + }); + } + tokio::spawn(async move { + let mut stdin = stdin; + while let Some(input) = input_rx.recv().await { + match input { + MainInput::Data(data) => { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + MainInput::Close => break, + } + } + }); + session + } + + fn start_io( + this: &Arc, + io: ProcessIo, + mut input_rx: tokio::sync::mpsc::Receiver, + ) { + match io { + ProcessIo::Pty(master) => { + let master = Arc::new(AsyncFd::new(master).expect("register canonical PTY master")); + let reader = Arc::clone(&master); + let output = Arc::clone(this); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + let Ok(mut ready) = reader.readable().await else { + break; + }; + match ready.try_io(|inner| { + let mut file = inner.get_ref(); + file.read(&mut buffer) + }) { + Ok(Ok(0) | Err(_)) => break, + Ok(Ok(read)) => output.publish(MainOutput::Stdout( + Bytes::copy_from_slice(&buffer[..read]), + )), + Err(_would_block) => {} + } + } + output.reader_finished(); + }); + tokio::spawn(async move { + while let Some(input) = input_rx.recv().await { + let MainInput::Data(data) = input else { + return; + }; + let mut remaining = data.as_slice(); + while !remaining.is_empty() { + let Ok(mut ready) = master.writable().await else { + return; + }; + match ready.try_io(|inner| { + let mut file = inner.get_ref(); + file.write(remaining) + }) { + Ok(Ok(0) | Err(_)) => return, + Ok(Ok(written)) => remaining = &remaining[written..], + Err(_would_block) => {} + } + } + } + }); + } + ProcessIo::Pipes { + mut stdin, + mut stdout, + mut stderr, + } => { + let stdout_session = Arc::clone(this); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => { + stdout_session.publish(MainOutput::Stdout(Bytes::copy_from_slice( + &buffer[..read], + ))); + } + } + } + stdout_session.reader_finished(); + }); + let stderr_session = Arc::clone(this); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => { + stderr_session.publish(MainOutput::Stderr(Bytes::copy_from_slice( + &buffer[..read], + ))); + } + } + } + stderr_session.reader_finished(); + }); + tokio::spawn(async move { + while let Some(input) = input_rx.recv().await { + match input { + MainInput::Data(data) => { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + MainInput::Close => break, + } + } + }); + } + } + } + + fn publish(&self, event: MainOutput) { + self.output.publish(event); + } + + fn reader_finished(&self) { + if self.readers_remaining.fetch_sub(1, Ordering::AcqRel) == 1 { + self.readers_done.notify_waiters(); + } + } + + /// Publish the terminal event and retain the transport only when a real + /// foreground attachment exists or the creating client declared one. + /// + /// Returns whether terminal delivery must complete before shutdown. + pub async fn finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.wait_for_output_readers().await; + self.complete_finish(exit_code, attachment_expected) + } + + /// Finish a remotely owned process without allowing descendants that keep + /// inherited output descriptors open to block terminal publication forever. + pub async fn finish_remote(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.finish_remote_with_timeout( + exit_code, + attachment_expected, + Self::REMOTE_OUTPUT_DRAIN_TIMEOUT, + ) + .await + } + + async fn finish_remote_with_timeout( + &self, + exit_code: i32, + attachment_expected: bool, + timeout: std::time::Duration, + ) -> bool { + let _ = tokio::time::timeout(timeout, self.wait_for_output_readers()).await; + self.complete_finish(exit_code, attachment_expected) + } + + async fn wait_for_output_readers(&self) { + let notified = self.readers_done.notified(); + if self.readers_remaining.load(Ordering::Acquire) != 0 { + notified.await; + } + } + + fn complete_finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + let delivery_pending = { + let mut state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + state.process_finished = true; + state.expectation = if attachment_expected { + if state.active == 0 && state.expectation != AttachmentExpectation::Satisfied { + AttachmentExpectation::Pending + } else { + AttachmentExpectation::Satisfied + } + } else { + AttachmentExpectation::None + }; + attachment_expected || state.active != 0 + }; + self.finished.store(true, Ordering::Release); + self.publish(MainOutput::Exit(exit_code)); + delivery_pending + } + + pub fn subscribe(&self) -> MainOutputCursor { + self.output.subscribe() + } + + /// Return the bounded output sequence range currently retained for a + /// replacement supervisor. A nonzero first sequence is an explicit + /// truncation watermark rather than silent data loss. + #[must_use] + pub fn output_window(&self) -> (u64, u64, bool) { + let state = self + .output + .state + .lock() + .expect("main output log lock poisoned"); + let first_sequence = state + .events + .front() + .map_or(state.next_sequence, |event| event.sequence); + (first_sequence, state.next_sequence, first_sequence != 0) + } + + /// Wait until the gateway durably acknowledges the main-process result. + pub async fn wait_for_terminal_reported(&self) { + let notified = self.output.terminal_reported_notify.notified(); + if self.output.terminal_reported.load(Ordering::Acquire) { + return; + } + notified.await; + } + + /// Release attached clients to receive their SSH exit status after the + /// durable sandbox phase and exit code have been recorded. + pub fn mark_terminal_reported(&self) { + self.output.terminal_reported.store(true, Ordering::Release); + self.output.terminal_reported_notify.notify_waiters(); + } + + /// Register a foreground main attachment while the process is live. + pub fn begin_terminal_attachment(&self) -> Result<(), &'static str> { + let mut state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + if state.process_finished && state.expectation != AttachmentExpectation::Pending { + return Err("canonical main process already finished"); + } + state.active = state + .active + .checked_add(1) + .expect("terminal attachment count exhausted"); + state.expectation = AttachmentExpectation::Satisfied; + self.terminal_attachments_done.notify_waiters(); + Ok(()) + } + + /// Release a foreground main attachment after its SSH channel closes. + pub fn end_terminal_attachment(&self) { + let completed = { + let mut state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + debug_assert!(state.active != 0, "terminal attachment count underflow"); + if state.active == 0 { + return; + } + state.active -= 1; + state.active == 0 + }; + if completed { + self.terminal_attachments_done.notify_waiters(); + } + } + + /// Wait for the declared foreground attachment to start, then for every + /// accepted attachment to close naturally. + pub async fn wait_for_terminal_attachments(&self) { + loop { + let notified = self.terminal_attachments_done.notified(); + let complete = { + let state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + state.active == 0 && state.expectation != AttachmentExpectation::Pending + }; + if complete { + return; + } + notified.await; + } + } + + pub(crate) fn acquire_input(&self) -> Result<(u64, MainInputSender), &'static str> { + if self.input_closed.load(Ordering::Acquire) { + return Err("canonical process stdin closed"); + } + let mut owner = self.input_owner.lock().expect("main input lock poisoned"); + if owner.is_some() { + return Err("canonical main process already has an input owner"); + } + let id = self.next_owner.fetch_add(1, Ordering::Relaxed); + *owner = Some(id); + Ok((id, self.input.clone())) + } + + /// Acquire canonical input when it remains open. A replacement control may + /// still attach output after a prior control intentionally closed stdin. + pub(crate) fn acquire_input_if_open( + &self, + ) -> Result, &'static str> { + match self.acquire_input() { + Ok(input) => Ok(Some(input)), + Err(_) if self.input_closed.load(Ordering::Acquire) => Ok(None), + Err(error) => Err(error), + } + } + + pub(crate) fn release_input(&self, id: u64) { + let mut owner = self.input_owner.lock().expect("main input lock poisoned"); + if *owner == Some(id) { + *owner = None; + } + } + + pub(crate) async fn close_input(&self, id: u64) { + let owns_input = { + let mut owner = self.input_owner.lock().expect("main input lock poisoned"); + if *owner == Some(id) { + *owner = None; + true + } else { + false + } + }; + if owns_input && !self.input_closed.swap(true, Ordering::AcqRel) { + self.input.close().await; + } + } + + pub async fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + if let Some(terminal) = self.boundary_terminal.as_ref() { + let _ = terminal + .resize( + u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ) + .await; + return; + } + let Some(master) = self.pty_master.as_ref() else { + return; + }; + let winsize = Winsize { + ws_row: u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ws_col: u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + ws_xpixel: u16::try_from(pixel_width).unwrap_or(u16::MAX), + ws_ypixel: u16::try_from(pixel_height).unwrap_or(u16::MAX), + }; + #[allow(unsafe_code)] + unsafe { + libc::ioctl(master.as_raw_fd(), libc::TIOCSWINSZ, &winsize); + } + } + + pub async fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), String> { + if let Some(process) = self.boundary_process.as_ref() { + let signal = match signal { + nix::sys::signal::Signal::SIGHUP => BoundarySignal::Hup, + nix::sys::signal::Signal::SIGINT => BoundarySignal::Int, + nix::sys::signal::Signal::SIGKILL => BoundarySignal::Kill, + nix::sys::signal::Signal::SIGTERM => BoundarySignal::Term, + other => return Err(format!("boundary signal {other:?} is unsupported")), + }; + return process + .signal(signal) + .await + .map_err(|error| error.to_string()); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::kill(nix::unistd::Pid::from_raw(-pid), signal) + .map_err(|error| error.to_string()) + } + + #[must_use] + pub const fn terminal(&self) -> bool { + self.terminal + } + + #[must_use] + pub fn finished(&self) -> bool { + self.finished.load(Ordering::Acquire) + } +} + +fn set_nonblocking(file: &std::fs::File) -> Result<(), nix::errno::Errno> { + let flags = fcntl(file.as_raw_fd(), FcntlArg::F_GETFL)?; + let flags = OFlag::from_bits_truncate(flags); + fcntl( + file.as_raw_fd(), + FcntlArg::F_SETFL(flags | OFlag::O_NONBLOCK), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_isolation_interface::contract::{ + BackendError, BoundaryExitStatus, BoundaryInput, BoundaryOutput, + }; + + struct TestBoundaryProcess { + signals: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryProcess for TestBoundaryProcess { + async fn wait(&self) -> Result { + Ok(BoundaryExitStatus::Exited(0)) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.signals.lock().unwrap().push(signal); + Ok(()) + } + + async fn terminate(&self) -> Result<(), BackendError> { + Ok(()) + } + } + + struct TestBoundaryTerminal { + size: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryTerminal for TestBoundaryTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } + } + + #[tokio::test] + async fn boundary_attachment_drives_main_io_signal_and_terminal() { + let (stdin, mut stdin_peer) = tokio::io::duplex(1024); + let (stdout, mut stdout_peer) = tokio::io::duplex(1024); + let process = Arc::new(TestBoundaryProcess { + signals: Mutex::new(Vec::new()), + }); + let terminal = Arc::new(TestBoundaryTerminal { + size: Mutex::new(None), + }); + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let attachment = ProcessAttachment { + stdin, + stdout, + stderr: None, + terminal: Some(terminal.clone()), + }; + let session = MainSession::from_boundary(attachment, process.clone()); + let mut output = session.subscribe(); + + stdout_peer.write_all(b"ready\n").await.unwrap(); + assert!(matches!( + output.recv().await.unwrap(), + MainOutput::Stdout(data) if data == b"ready\n"[..] + )); + + let (owner, input) = session.acquire_input().unwrap(); + input.send(b"hello\n".to_vec()).await.unwrap(); + let mut received = [0_u8; 6]; + stdin_peer.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"hello\n"); + session.close_input(owner).await; + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + stdin_peer.read(&mut received) + ) + .await + .expect("boundary stdin close timed out") + .expect("read boundary stdin EOF"), + 0 + ); + assert!(session.acquire_input().is_err()); + assert!(session.acquire_input_if_open().unwrap().is_none()); + + session.resize(120, 40, 0, 0).await; + assert_eq!(*terminal.size.lock().unwrap(), Some((120, 40))); + session + .signal_group(nix::sys::signal::Signal::SIGINT) + .await + .unwrap(); + assert_eq!(*process.signals.lock().unwrap(), vec![BoundarySignal::Int]); + } + + #[test] + fn input_lease_has_one_owner_and_can_be_reacquired() { + let session = MainSession::inert(); + let (first, _) = session.acquire_input().expect("first owner"); + assert!(session.acquire_input().is_err()); + + session.release_input(first); + let (second, _) = session.acquire_input().expect("replacement owner"); + assert_ne!(first, second); + } + + #[tokio::test] + async fn subscribers_receive_replay_then_live_output() { + let session = MainSession::inert(); + session.publish(MainOutput::Stdout(Bytes::from_static(b"before"))); + + let mut output = session.subscribe(); + assert!(matches!( + output.recv().await.expect("replayed output"), + MainOutput::Stdout(data) if data == b"before"[..] + )); + + session.publish(MainOutput::Stderr(Bytes::from_static(b"after"))); + assert!(matches!( + output.recv().await.expect("live output"), + MainOutput::Stderr(data) if data == b"after"[..] + )); + } + + #[tokio::test] + async fn finish_without_attachment_does_not_defer_shutdown() { + let session = MainSession::inert(); + assert!(!session.finish(0, false).await); + assert!(session.finished()); + assert!(session.begin_terminal_attachment().is_err()); + } + + #[tokio::test] + async fn terminal_report_acknowledgement_is_independent_from_delivery() { + let session = MainSession::inert(); + + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + session.wait_for_terminal_reported(), + ) + .await + .is_err(), + "draining output must not imply durable gateway persistence" + ); + + session.mark_terminal_reported(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + session.wait_for_terminal_reported(), + ) + .await + .expect("durable report acknowledgement should wake waiter"); + } + + #[tokio::test] + async fn finish_waits_for_an_active_attachment_to_close_naturally() { + let session = MainSession::inert(); + session + .begin_terminal_attachment() + .expect("begin terminal attachment"); + assert!(session.finish(0, false).await); + + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + session.wait_for_terminal_attachments(), + ) + .await + .is_err(), + "an active attachment must keep terminal delivery open" + ); + + session.end_terminal_attachment(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + session.wait_for_terminal_attachments(), + ) + .await + .expect("closing the attachment should wake the waiter"); + } + + #[tokio::test] + async fn remote_finish_bounds_output_drain_before_publishing_exit() { + let mut session = MainSession::inert(); + Arc::get_mut(&mut session) + .expect("sole test session reference") + .readers_remaining = AtomicUsize::new(1); + let mut output = session.subscribe(); + + session + .finish_remote_with_timeout(19, false, std::time::Duration::from_millis(10)) + .await; + + assert!(matches!( + output + .recv() + .await + .expect("terminal status after bounded drain"), + MainOutput::Exit(19) + )); + } + + #[tokio::test] + async fn declared_attachment_waits_for_connection_then_natural_close() { + let session = MainSession::inert(); + assert!(session.finish(0, true).await); + + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + session.wait_for_terminal_attachments(), + ) + .await + .is_err(), + "declared attachment must connect before delivery is complete" + ); + + session + .begin_terminal_attachment() + .expect("declared post-exit attachment"); + session.end_terminal_attachment(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + session.wait_for_terminal_attachments(), + ) + .await + .expect("natural attachment close should complete delivery"); + } + + #[tokio::test] + async fn exit_is_retained_in_the_output_log() { + let session = MainSession::inert(); + let _ = session.finish(0, false).await; + + let mut output = session.subscribe(); + assert!(matches!( + output.recv().await.expect("replayed exit"), + MainOutput::Exit(0) + )); + } + + #[tokio::test] + async fn slow_subscriber_reports_evicted_events_then_resumes() { + let session = MainSession::inert(); + let mut output = session.subscribe(); + let chunk = Bytes::from(vec![0; 4096]); + for _ in 0..=(OUTPUT_BUFFER_BYTES / chunk.len()) { + session.publish(MainOutput::Stdout(chunk.clone())); + } + + let lag = output.recv().await.expect_err("oldest event was evicted"); + assert_eq!(lag.skipped, 1); + assert!(matches!( + output.recv().await.expect("resume at oldest retained event"), + MainOutput::Stdout(data) if data.len() == chunk.len() + )); + } + + #[tokio::test] + async fn terminal_pump_reads_output_and_writes_input() { + let (session, mut slave) = MainSession::terminal_for_test(); + set_nonblocking(&slave).expect("set test PTY slave nonblocking"); + let mut output = session.subscribe(); + + slave + .write_all(b"process output") + .expect("write PTY output"); + let event = tokio::time::timeout(std::time::Duration::from_secs(1), output.recv()) + .await + .expect("PTY output timed out") + .expect("PTY output was retained"); + assert!(matches!( + event, + MainOutput::Stdout(data) if data == b"process output"[..] + )); + + let (owner, input) = session.acquire_input().expect("acquire PTY input"); + input + .send(b"client input\n".to_vec()) + .await + .expect("queue PTY input"); + let mut received = [0; 64]; + let read = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match slave.read(&mut received) { + Ok(read) => break read, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + tokio::task::yield_now().await; + } + Err(error) => panic!("read PTY input: {error}"), + } + } + }) + .await + .expect("PTY input timed out"); + assert_eq!(&received[..read], b"client input\n"); + session.release_input(owner); + } +} diff --git a/crates/openshell-sandbox/src/managed_children.rs b/crates/openshell-sandbox/src/managed_children.rs new file mode 100644 index 0000000000..64b444e1f8 --- /dev/null +++ b/crates/openshell-sandbox/src/managed_children.rs @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process-wide tracker for sandbox-managed child PIDs. +//! +//! The supervisor spawns several long-lived children (the entrypoint, SSH +//! sessions). Each registers its PID here on spawn and removes it on exit so +//! the orchestrator's `SIGCHLD` reaper can distinguish supervised processes +//! from incidental zombies. + +#![cfg(target_os = "linux")] + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{LazyLock, Mutex, MutexGuard}; + +static MANAGED_CHILDREN: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); +static NEXT_GENERATION: AtomicU64 = AtomicU64::new(1); + +/// Identity of one registry entry. The generation prevents an old waiter from +/// removing a newer child that reused the same numeric PID after reap. +#[derive(Clone, Copy)] +pub struct ManagedChild { + pid: i32, + generation: u64, +} + +/// Exclusive access to the managed-child registry. +/// +/// A process spawner holds this guard from immediately before `spawn` or +/// `fork` until the returned PID is registered. The orphan reaper holds the +/// same guard while deciding whether to reap an exited child. This closes the +/// otherwise unavoidable window in which a fast-exiting managed child exists +/// but its PID has not yet been published. +pub struct RegistryGuard(MutexGuard<'static, HashMap>); + +impl RegistryGuard { + /// Add a newly spawned managed child. + pub fn register(&mut self, pid: u32) -> Option { + let Ok(pid) = i32::try_from(pid) else { + return None; + }; + if pid <= 0 { + return None; + } + let generation = NEXT_GENERATION.fetch_add(1, Ordering::Relaxed); + self.0.insert(pid, generation); + Some(ManagedChild { pid, generation }) + } + + /// Return whether the PID belongs to an explicit waiter. + #[must_use] + pub fn contains(&self, pid: i32) -> bool { + self.0.contains_key(&pid) + } +} + +/// Lock the registry for an atomic spawn-and-register or inspect-and-reap +/// operation. +pub fn lock() -> RegistryGuard { + RegistryGuard( + MANAGED_CHILDREN + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ) +} + +/// Register a child and return the generation-bearing removal token. +pub fn register(pid: u32) -> Option { + lock().register(pid) +} + +/// Remove exactly this supervised-child registration. A newer registration +/// for a reused PID is preserved. +pub fn unregister(child: ManagedChild) { + if let Ok(mut children) = MANAGED_CHILDREN.lock() + && children.get(&child.pid) == Some(&child.generation) + { + children.remove(&child.pid); + } +} + +/// Return `true` if `pid` is currently in the supervised-child set. +#[must_use] +pub fn is_managed(pid: i32) -> bool { + lock().contains(pid) +} + +/// Wait until a managed child is terminal without reaping it. +/// +/// Keeping the child as a zombie prevents PID/process-group reuse until the +/// owner publishes terminal state and performs the final wait. +pub fn wait_until_terminal(pid: u32) -> std::io::Result<()> { + use nix::sys::wait::{Id, WaitPidFlag, waitid}; + let pid = i32::try_from(pid) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "PID out of range"))?; + waitid( + Id::Pid(nix::unistd::Pid::from_raw(pid)), + WaitPidFlag::WEXITED | WaitPidFlag::WNOWAIT, + ) + .map(|_| ()) + .map_err(std::io::Error::other) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stale_unregister_preserves_reused_pid_registration() { + let pid = i32::MAX as u32; + let first = lock().register(pid).expect("first registration"); + let second = lock().register(pid).expect("replacement registration"); + + unregister(first); + assert!(is_managed(i32::try_from(pid).expect("test pid"))); + + unregister(second); + assert!(!is_managed(i32::try_from(pid).expect("test pid"))); + } +} diff --git a/crates/openshell-sandbox/src/metadata_server.rs b/crates/openshell-sandbox/src/metadata_server.rs deleted file mode 100644 index dcfe3e439a..0000000000 --- a/crates/openshell-sandbox/src/metadata_server.rs +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Loopback HTTP server for cloud metadata emulators. -//! -//! Binds a TCP listener inside the sandbox network namespace so that -//! cloud SDKs that bypass `HTTP_PROXY` (e.g. Go's -//! `cloud.google.com/go/compute/metadata`) can reach the emulator via -//! direct TCP. -//! -//! The server is generic over [`MetadataHandler`] — any cloud provider -//! that needs an instance metadata emulator can implement the trait. - -use miette::Result; -use openshell_core::net::set_tcp_nodelay_best_effort; -use std::future::Future; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::net::TcpListener; -use tokio::sync::{Semaphore, oneshot}; -use tracing::{debug, warn}; - -const MAX_REQUEST_BYTES: usize = 4096; -const MAX_CONCURRENT_CONNECTIONS: usize = 32; - -/// Handler for cloud metadata HTTP requests. -/// -/// Implementors receive the parsed HTTP method, path, raw request bytes, -/// and a bidirectional stream to write the response. The handler owns the -/// response format (status, headers, body) — the server only does TCP -/// accept and HTTP request-line parsing. -pub trait MetadataHandler: Send + Sync + 'static { - fn handle( - &self, - method: &str, - path: &str, - request: &[u8], - stream: &mut S, - ) -> impl Future> + Send; -} - -/// Bind a TCP listener inside the sandbox network namespace. -/// -/// Run the metadata server accept loop. -/// -/// Signals `ready_tx` with the bound address before entering the loop. -/// Returns when the listener encounters a fatal error or the runtime shuts down. -pub async fn run( - listener: TcpListener, - handler: H, - ready_tx: oneshot::Sender, -) { - let local_addr = match listener.local_addr() { - Ok(addr) => addr, - Err(e) => { - warn!("metadata server failed to get local address: {e}"); - return; - } - }; - - let _ = ready_tx.send(local_addr); - - let handler = Arc::new(handler); - let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_CONNECTIONS)); - - loop { - let Ok(permit) = semaphore.clone().acquire_owned().await else { - break; - }; - - match listener.accept().await { - Ok((stream, _addr)) => { - // Small-request IMDS-style endpoint an agent polls for - // credentials/identity — disable Nagle to avoid delayed-ACK stalls. - set_tcp_nodelay_best_effort(&stream); - let handler = handler.clone(); - tokio::spawn(async move { - if let Err(e) = handle_connection(handler.as_ref(), stream).await { - debug!("metadata server connection error: {e}"); - } - drop(permit); - }); - } - Err(e) => { - warn!("metadata server accept error: {e}"); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - } - } -} - -const READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -async fn handle_connection( - handler: &H, - mut stream: tokio::net::TcpStream, -) -> Result<()> { - let mut buf = vec![0u8; MAX_REQUEST_BYTES]; - let mut used = 0; - let deadline = tokio::time::sleep(READ_TIMEOUT); - tokio::pin!(deadline); - loop { - tokio::select! { - result = stream.read(&mut buf[used..]) => { - let n = result.map_err(|e| miette::miette!("{e}"))?; - if n == 0 { - return Ok(()); - } - used += n; - if buf[..used].windows(4).any(|w| w == b"\r\n\r\n") { - break; - } - if used >= buf.len() { - let _ = stream - .write_all(b"HTTP/1.1 413 Request Entity Too Large\r\nContent-Length: 0\r\n\r\n") - .await; - return Ok(()); - } - } - () = &mut deadline => { - return Ok(()); - } - } - } - let request = String::from_utf8_lossy(&buf[..used]); - let request_line = request.split("\r\n").next().unwrap_or(""); - let mut parts = request_line.split_whitespace(); - let method = parts.next().unwrap_or(""); - let path = parts.next().unwrap_or("/"); - - tokio::time::timeout( - READ_TIMEOUT, - handler.handle(method, path, &buf[..used], &mut stream), - ) - .await - .unwrap_or_else(|_| { - debug!(method, path, "metadata handler timed out"); - Ok(()) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::sync::mpsc; - - struct RecordingHandler { - requests: mpsc::UnboundedSender<(String, String)>, - } - - impl MetadataHandler for RecordingHandler { - async fn handle( - &self, - method: &str, - path: &str, - _request: &[u8], - stream: &mut S, - ) -> Result<()> { - self.requests - .send((method.to_string(), path.to_string())) - .unwrap(); - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await - .map_err(|error| miette::miette!("{error}"))?; - Ok(()) - } - } - - async fn connection_pair() -> (tokio::net::TcpStream, tokio::net::TcpStream) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let client = tokio::net::TcpStream::connect(listener.local_addr().unwrap()) - .await - .unwrap(); - let (server, _) = listener.accept().await.unwrap(); - (client, server) - } - - #[tokio::test] - async fn metadata_loopback_dispatches_method_path_and_response() { - let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); - let handler = RecordingHandler { - requests: requests_tx, - }; - let (mut client, server) = connection_pair().await; - let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); - - client - .write_all(b"GET /computeMetadata/v1/instance HTTP/1.1\r\nHost: metadata\r\n\r\n") - .await - .unwrap(); - let mut response = Vec::new(); - client.read_to_end(&mut response).await.unwrap(); - server_task.await.unwrap().unwrap(); - - assert_eq!( - requests_rx.try_recv().unwrap(), - ( - "GET".to_string(), - "/computeMetadata/v1/instance".to_string() - ) - ); - assert_eq!(response, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); - } - - #[tokio::test] - async fn metadata_loopback_rejects_oversized_headers_before_handler() { - let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); - let handler = RecordingHandler { - requests: requests_tx, - }; - let (mut client, server) = connection_pair().await; - let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); - - client - .write_all(&vec![b'x'; MAX_REQUEST_BYTES]) - .await - .unwrap(); - let mut response = Vec::new(); - client.read_to_end(&mut response).await.unwrap(); - server_task.await.unwrap().unwrap(); - - assert_eq!( - response, - b"HTTP/1.1 413 Request Entity Too Large\r\nContent-Length: 0\r\n\r\n" - ); - assert!(requests_rx.try_recv().is_err()); - } -} diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs new file mode 100644 index 0000000000..049ba5a432 --- /dev/null +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -0,0 +1,2038 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Seccomp-notification broker owned by the in-workload sandbox. + +#![allow(unsafe_code)] + +use std::collections::HashMap; +use std::io; +use std::mem::size_of; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd}; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use openshell_binary_identity::ProcfsIdentityResolver; +use openshell_isolation_interface::contract::{ + BinaryIdentity, DnsTransport, NetworkOpenResult, NetworkSocketMetadata, ResolveError, +}; +use openshell_isolation_interface::linux::seccomp_notify::{Notification, NotificationListener}; +use openshell_isolation_interface::linux::socket_registry::{ + InetFamily, InetKind, SocketMetadata, SocketRegistry, SocketState, +}; +use openshell_isolation_interface::linux::task_memory; +use tokio::sync::{mpsc, oneshot}; + +const SOCKET_CAPACITY: usize = 4_096; +const OPEN_QUEUE_CAPACITY: usize = 256; +const ACCEPT_WORKER_CAPACITY: usize = 64; +const DNS_QUEUE_CAPACITY: usize = 256; +const DNS_WORKER_CAPACITY: usize = 256; +const DNS_QUERY_TIMEOUT: Duration = Duration::from_secs(10); +const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(250); +const DNS_RELAY_ADDRESS: SocketAddr = SocketAddr::V4(std::net::SocketAddrV4::new( + Ipv4Addr::new(127, 0, 0, 53), + 53, +)); +const RELAY_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug)] +struct PendingOpenSlot(Arc); + +impl Drop for PendingOpenSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Debug)] +struct PendingDnsSlot(Arc); + +impl Drop for PendingDnsSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +fn acquire_pending_dns_slot(active: &Arc) -> io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < DNS_WORKER_CAPACITY).then_some(current + 1) + }) + .map(|_| PendingDnsSlot(Arc::clone(active))) + .map_err(|_| io::Error::from_raw_os_error(libc::EAGAIN)) +} + +struct PendingAcceptSlot { + active: Arc, +} + +impl Drop for PendingAcceptSlot { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +fn acquire_pending_accept_slot(active: &Arc) -> io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < ACCEPT_WORKER_CAPACITY).then_some(current + 1) + }) + .map_err(|_| io::Error::from_raw_os_error(libc::EAGAIN))?; + Ok(PendingAcceptSlot { + active: Arc::clone(active), + }) +} + +fn acquire_pending_open_slot(active: &Arc) -> io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < OPEN_QUEUE_CAPACITY).then_some(current + 1) + }) + .map(|_| PendingOpenSlot(Arc::clone(active))) + .map_err(|_| io::Error::from_raw_os_error(libc::EAGAIN)) +} + +/// One external TCP open blocked in `connect(2)` until the supervisor decides. +pub struct PendingTcpOpen { + pub(crate) destination: SocketAddr, + pub(crate) identity: Result, + pub(crate) socket: NetworkSocketMetadata, + pub(crate) notification_to_queue: Duration, + pub(crate) queued_at: Instant, + decision: std::sync::mpsc::SyncSender, + relay: oneshot::Receiver>, + _slot: PendingOpenSlot, +} + +impl PendingTcpOpen { + pub(crate) async fn complete( + self, + decision: NetworkOpenResult, + ) -> io::Result> { + self.decision + .send(decision) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "network broker stopped"))?; + if matches!(decision, NetworkOpenResult::Denied { .. }) { + return Ok(None); + } + self.relay + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "network relay setup was cancelled", + ) + })? + .map(Some) + } +} + +/// One DNS exchange received by the exact sandbox-local resolver endpoint. +pub struct PendingDnsQuery { + pub(crate) request: Vec, + pub(crate) transport: DnsTransport, + pub(crate) identity: Result, + pub(crate) notification_to_queue: Duration, + pub(crate) queued_at: Instant, + response: std::sync::mpsc::SyncSender>>, +} + +impl PendingDnsQuery { + pub(crate) fn complete(self, response: io::Result>) -> io::Result<()> { + self.response + .send(response) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "DNS relay stopped")) + } +} + +#[derive(Clone)] +struct DnsRelay { + address: SocketAddr, + udp_attribution: Arc>>>, + tcp_attribution: Arc>>>, +} + +#[derive(Clone)] +struct NotificationQueues { + pending: mpsc::Sender, + dns_relay: DnsRelay, + active_opens: Arc, + active_accepts: Arc, +} + +/// Live broker handle retained by the sandbox boundary. +#[derive(Clone)] +pub struct NetworkBroker { + pending: Arc>>, + pending_dns: Arc>>, + dns_address: SocketAddr, + healthy: Arc, +} + +impl NetworkBroker { + pub(crate) fn start(listener: NotificationListener) -> io::Result { + Self::start_with_dns_address(listener, DNS_RELAY_ADDRESS) + } + + #[cfg(any(test, feature = "perf-harness"))] + pub(crate) fn start_for_test(listener: NotificationListener) -> io::Result { + Self::start_with_dns_address( + listener, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + ) + } + + fn start_with_dns_address( + listener: NotificationListener, + dns_address: SocketAddr, + ) -> io::Result { + let listener = Arc::new(listener); + let (pending_tx, pending_rx) = mpsc::channel(OPEN_QUEUE_CAPACITY); + let (pending_dns_tx, pending_dns_rx) = mpsc::channel(DNS_QUEUE_CAPACITY); + let registry = Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)); + let active_opens = Arc::new(AtomicUsize::new(0)); + let active_accepts = Arc::new(AtomicUsize::new(0)); + let dns_relay = start_dns_relay(dns_address, pending_dns_tx)?; + let dns_address = dns_relay.address; + let queues = NotificationQueues { + pending: pending_tx, + dns_relay, + active_opens, + active_accepts, + }; + let healthy = Arc::new(AtomicBool::new(true)); + let broker_healthy = healthy.clone(); + std::thread::Builder::new() + .name("openshell-network-broker".to_string()) + .spawn(move || { + while broker_healthy.load(Ordering::Acquire) { + let notification = match listener.receive() { + Ok(notification) => notification, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => { + tracing::error!(%error, "sandbox network broker listener failed"); + broker_healthy.store(false, Ordering::Release); + break; + } + }; + if let Err(error) = dispatch_notification( + Arc::clone(®istry), + Arc::clone(&listener), + notification, + queues.clone(), + ) { + tracing::warn!( + tid = notification.tid, + syscall = notification.syscall, + %error, + "sandbox network notification denied (tid={}, syscall={}): {error}", + notification.tid, + notification.syscall + ); + let _ = listener.respond_errno(notification.id, error_to_errno(&error)); + } + } + }) + .map_err(|error| io::Error::other(format!("start network broker: {error}")))?; + Ok(Self { + pending: Arc::new(tokio::sync::Mutex::new(pending_rx)), + pending_dns: Arc::new(tokio::sync::Mutex::new(pending_dns_rx)), + dns_address, + healthy, + }) + } + + pub(crate) async fn accept(&self) -> io::Result { + self.pending + .lock() + .await + .recv() + .await + .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "network broker queue closed")) + } + + pub(crate) async fn accept_dns(&self) -> io::Result { + self.pending_dns + .lock() + .await + .recv() + .await + .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "DNS broker queue closed")) + } + + #[cfg(any(test, feature = "perf-harness"))] + pub(crate) fn dns_address(&self) -> SocketAddr { + self.dns_address + } + + pub(crate) fn confirm_healthy(&self) -> io::Result<()> { + if self.healthy.load(Ordering::Acquire) && self.dns_address.port() != 0 { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "network broker is not running", + )) + } + } +} + +fn start_dns_relay( + address: SocketAddr, + pending: mpsc::Sender, +) -> io::Result { + let (udp, tcp, address) = bind_dns_relay_sockets(address)?; + let udp_attribution = Arc::new(Mutex::new(HashMap::new())); + let tcp_attribution = Arc::new(Mutex::new(HashMap::new())); + let active_workers = Arc::new(AtomicUsize::new(0)); + let relay = DnsRelay { + address, + udp_attribution: Arc::clone(&udp_attribution), + tcp_attribution: Arc::clone(&tcp_attribution), + }; + + let udp_active_workers = Arc::clone(&active_workers); + let udp_pending = pending.clone(); + std::thread::Builder::new() + .name("openshell-dns-udp".to_string()) + .spawn(move || { + let mut request = vec![0_u8; u16::MAX as usize]; + while let Ok((length, peer)) = udp.recv_from(&mut request) { + let Some(identity) = lock(&udp_attribution).get(&peer).cloned() else { + tracing::warn!(%peer, "dropping DNS datagram from unattributed socket"); + continue; + }; + let Ok(worker_slot) = acquire_pending_dns_slot(&udp_active_workers) else { + tracing::warn!(%peer, "dropping DNS datagram because the worker quota is full"); + continue; + }; + let (response_tx, response_rx) = std::sync::mpsc::sync_channel(1); + let query = PendingDnsQuery { + request: request[..length].to_vec(), + transport: DnsTransport::Udp, + identity, + notification_to_queue: Duration::ZERO, + queued_at: Instant::now(), + response: response_tx, + }; + if pending_try_send(&udp_pending, query).is_err() { + continue; + } + let Ok(udp_response) = udp.try_clone() else { + continue; + }; + let _ = std::thread::Builder::new() + .name("openshell-dns-udp-query".to_string()) + .spawn(move || { + let _worker_slot = worker_slot; + if let Ok(Ok(response)) = response_rx.recv_timeout(DNS_QUERY_TIMEOUT) { + let _ = udp_response.send_to(&response, peer); + } + }); + } + }) + .map_err(|error| io::Error::other(format!("start UDP DNS relay: {error}")))?; + + let tcp_active_workers = active_workers; + std::thread::Builder::new() + .name("openshell-dns-tcp".to_string()) + .spawn(move || { + for accepted in tcp.incoming() { + let Ok((stream, peer)) = accepted.and_then(|stream| { + let peer = stream.peer_addr()?; + Ok((stream, peer)) + }) else { + break; + }; + let identity = lock(&tcp_attribution).get(&peer).cloned(); + let Some(identity) = identity else { + tracing::warn!(%peer, "dropping DNS stream from unattributed socket"); + continue; + }; + let Ok(worker_slot) = acquire_pending_dns_slot(&tcp_active_workers) else { + tracing::warn!(%peer, "dropping DNS stream because the worker quota is full"); + continue; + }; + let tcp_pending = pending.clone(); + let _ = std::thread::Builder::new() + .name("openshell-dns-tcp-query".to_string()) + .spawn(move || { + let _worker_slot = worker_slot; + serve_dns_tcp(stream, identity, tcp_pending); + }); + } + }) + .map_err(|error| io::Error::other(format!("start TCP DNS relay: {error}")))?; + Ok(relay) +} + +fn bind_dns_relay_sockets(address: SocketAddr) -> io::Result<(UdpSocket, TcpListener, SocketAddr)> { + const EPHEMERAL_BIND_ATTEMPTS: usize = 32; + + if address.port() != 0 { + let udp = UdpSocket::bind(address)?; + let tcp = TcpListener::bind(address)?; + return Ok((udp, tcp, address)); + } + + // TCP and UDP have independent ephemeral-port allocators. The port picked + // by the first bind can therefore already be occupied by the other + // protocol, especially while the test suite starts several brokers in + // parallel. Retry the pair rather than treating that collision as an + // unavailable network broker. + for _ in 0..EPHEMERAL_BIND_ATTEMPTS { + let udp = UdpSocket::bind(address)?; + let selected = udp.local_addr()?; + match TcpListener::bind(selected) { + Ok(tcp) => return Ok((udp, tcp, selected)), + Err(error) if error.kind() == io::ErrorKind::AddrInUse => {} + Err(error) => return Err(error), + } + } + + Err(io::Error::new( + io::ErrorKind::AddrInUse, + "could not reserve a shared ephemeral TCP/UDP DNS relay port", + )) +} + +fn pending_try_send( + pending: &mpsc::Sender, + query: PendingDnsQuery, +) -> Result<(), ()> { + pending.try_send(query).map_err(|error| { + tracing::warn!(%error, "dropping DNS query because mediation queue is unavailable"); + }) +} + +fn serve_dns_tcp( + mut stream: TcpStream, + identity: Result, + pending: mpsc::Sender, +) { + use std::io::{Read as _, Write as _}; + + let _ = stream.set_read_timeout(Some(DNS_QUERY_TIMEOUT)); + let _ = stream.set_write_timeout(Some(DNS_QUERY_TIMEOUT)); + loop { + let mut length = [0_u8; 2]; + if stream.read_exact(&mut length).is_err() { + return; + } + let message_length = usize::from(u16::from_be_bytes(length)); + let mut request = Vec::with_capacity(message_length + 2); + request.extend_from_slice(&length); + request.resize(message_length + 2, 0); + if stream.read_exact(&mut request[2..]).is_err() { + return; + } + let (response_tx, response_rx) = std::sync::mpsc::sync_channel(1); + let query = PendingDnsQuery { + request, + transport: DnsTransport::Tcp, + identity: identity.clone(), + notification_to_queue: Duration::ZERO, + queued_at: Instant::now(), + response: response_tx, + }; + if pending_try_send(&pending, query).is_err() { + return; + } + let Ok(Ok(response)) = response_rx.recv_timeout(DNS_QUERY_TIMEOUT) else { + return; + }; + if stream.write_all(&response).is_err() { + return; + } + } +} + +fn dispatch_notification( + registry: Arc>, + listener: Arc, + notification: Notification, + queues: NotificationQueues, +) -> io::Result<()> { + let syscall = i64::from(notification.syscall); + if syscall == libc::SYS_socket { + return create_socket(®istry, &listener, notification); + } + if syscall == libc::SYS_connect { + return connect_socket( + registry, + listener, + notification, + queues.pending, + &queues.dns_relay, + queues.active_opens, + ); + } + if syscall == libc::SYS_bind { + return bind_socket(®istry, &listener, notification); + } + if syscall == libc::SYS_listen { + return listen_socket(®istry, &listener, notification); + } + if matches!(syscall, libc::SYS_accept | libc::SYS_accept4) { + return accept_socket(registry, listener, notification, queues.active_accepts); + } + if matches!( + syscall, + libc::SYS_sendto | libc::SYS_sendmsg | libc::SYS_sendmmsg + ) { + return classify_send(®istry, &listener, notification, &queues.dns_relay); + } + if syscall == libc::SYS_getpeername { + return get_peer_name(®istry, &listener, notification); + } + if syscall == libc::SYS_setsockopt { + let level = i32::try_from(notification.args[1]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + let option = i32::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if (level == libc::IPPROTO_TCP && option == libc::TCP_FASTOPEN_CONNECT) + || (level == libc::IPPROTO_IPV6 && option == libc::IPV6_ADDRFORM) + { + return Err(io::Error::from_raw_os_error(libc::EPERM)); + } + return listener.respond_continue(notification.id); + } + Err(io::Error::from_raw_os_error(libc::EPERM)) +} + +fn create_socket( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let domain = i32::try_from(notification.args[0]) + .map_err(|_| io::Error::from_raw_os_error(libc::EAFNOSUPPORT))?; + if !matches!(domain, libc::AF_INET | libc::AF_INET6) { + return listener.respond_continue(notification.id); + } + let raw_kind = i32::try_from(notification.args[1]) + .map_err(|_| io::Error::from_raw_os_error(libc::EPROTONOSUPPORT))?; + let protocol = i32::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EPROTONOSUPPORT))?; + let base_kind = raw_kind & !(libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK); + let kind = match (base_kind, protocol) { + (libc::SOCK_STREAM, 0 | libc::IPPROTO_TCP) => InetKind::Tcp, + (libc::SOCK_DGRAM, 0 | libc::IPPROTO_UDP) => InetKind::DnsUdp, + _ => return Err(io::Error::from_raw_os_error(libc::EPROTONOSUPPORT)), + }; + let family = if domain == libc::AF_INET { + InetFamily::V4 + } else { + InetFamily::V6 + }; + // SAFETY: arguments were reduced to the supported native INET matrix. A + // successful call returns one newly owned descriptor. + let mut source = unsafe { libc::socket(domain, raw_kind, protocol) }; + if source < 0 && io::Error::last_os_error().raw_os_error() == Some(libc::EMFILE) { + collect_closed_socket_entries(registry)?; + // SAFETY: same validated native INET socket creation after reclaiming + // broker-held descriptors for closed workload sockets. + source = unsafe { libc::socket(domain, raw_kind, protocol) }; + } + if source < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful socket returned one owned descriptor. + let source = unsafe { OwnedFd::from_raw_fd(source) }; + let metadata = SocketMetadata { + family, + kind, + close_on_exec: raw_kind & libc::SOCK_CLOEXEC != 0, + nonblocking: raw_kind & libc::SOCK_NONBLOCK != 0, + creator_generation: u64::from(notification.tid), + }; + let mut registry = lock(registry); + if registry.is_full() { + collect_closed_socket_entries_locked(&mut registry)?; + } + let tentative = registry.stage(source, metadata)?; + listener.add_fd_and_send( + notification.id, + tentative.source_fd(), + metadata.close_on_exec, + )?; + registry.commit(tentative)?; + Ok(()) +} + +fn connect_socket( + registry: Arc>, + listener: Arc, + notification: Notification, + pending: mpsc::Sender, + dns_relay: &DnsRelay, + active_opens: Arc, +) -> io::Result<()> { + let notification_started = Instant::now(); + let fd = raw_fd(notification.args[0])?; + let address_family = + read_socket_family(notification.tid, notification.args[1], notification.args[2])?; + if !matches!(address_family, libc::AF_INET | libc::AF_INET6) { + if address_family == libc::AF_UNSPEC { + let mut registry = lock(®istry); + if let Ok(entry) = registry.resolve_mut(notification.tid, fd) + && entry.metadata().kind == InetKind::DnsUdp + && matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) + { + // Address-selection implementations disconnect a temporary + // UDP route-probe socket with AF_UNSPEC before trying the next + // candidate. The probe below never connects the real OFD, so + // this is an idempotent no-op rather than a kernel CONTINUE. + return listener.respond_value(notification.id, 0); + } + } + if lock(®istry).resolve(notification.tid, fd).is_ok() { + // Every registered descriptor is an injected INET socket. Never + // CONTINUE based on a mutable workload sockaddr for such an FD. + return Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)); + } + // Native non-INET descriptors remain kernel-driven. + return listener.respond_continue(notification.id); + } + let destination = + read_socket_addr(notification.tid, notification.args[1], notification.args[2])?; + let (kind, socket_cookie, nonblocking) = { + let registry = lock(®istry); + let entry = registry.resolve(notification.tid, fd)?; + ( + entry.metadata().kind, + entry.identity().cookie, + entry.metadata().nonblocking, + ) + }; + if kind == InetKind::DnsUdp && destination.port() == 0 { + let mut registry = lock(®istry); + let entry = registry.resolve_mut(notification.tid, fd)?; + if !matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) { + return Err(io::Error::from_raw_os_error(libc::EISCONN)); + } + let destination_family = match destination { + SocketAddr::V4(_) => InetFamily::V4, + SocketAddr::V6(_) => InetFamily::V6, + }; + if entry.metadata().family != destination_family { + return Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)); + } + // glibc and uv use UDP connect(..., port 0), getsockname(), and an + // AF_UNSPEC disconnect to rank resolved addresses. Bind only to the + // matching loopback family and report success; never connect the + // kernel socket to the external candidate. write(2) therefore remains + // EDESTADDRREQ and destination-bearing sends remain broker-denied. + let local = ensure_dns_source_bound( + entry.retained_preconnect()?.as_raw_fd(), + entry.metadata().family, + )?; + entry.set_state(SocketState::Bound { local }); + return listener.respond_value(notification.id, 0); + } + if destination == dns_relay.address { + let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let mut registry = lock(®istry); + let entry = registry.resolve_mut(notification.tid, fd)?; + if !matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) { + return Err(io::Error::from_raw_os_error(libc::EISCONN)); + } + let source_fd = entry.retained_preconnect()?.as_raw_fd(); + let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; + let attribution = match kind { + InetKind::Tcp => &dns_relay.tcp_attribution, + InetKind::DnsUdp => &dns_relay.udp_attribution, + }; + lock(attribution).insert(peer, identity); + if let Err(error) = connect_exact(source_fd, destination) { + lock(attribution).remove(&peer); + return Err(error); + } + entry.set_state(match kind { + InetKind::Tcp => SocketState::DnsTcp { relay: destination }, + InetKind::DnsUdp => SocketState::DnsUdp { relay: destination }, + }); + entry.release_preconnect(); + return listener.respond_value(notification.id, 0); + } + if destination.ip().is_loopback() { + let mut registry = lock(®istry); + let entry = registry.resolve_mut(notification.tid, fd)?; + connect_exact(entry.retained_preconnect()?.as_raw_fd(), destination)?; + entry.set_state(SocketState::Local { peer: destination }); + entry.release_preconnect(); + return listener.respond_value(notification.id, 0); + } + if kind != InetKind::Tcp { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + + let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let (decision_tx, decision_rx) = std::sync::mpsc::sync_channel(1); + let (relay_tx, relay_rx) = oneshot::channel(); + let slot = acquire_pending_open_slot(&active_opens)?; + pending + .try_send(PendingTcpOpen { + destination, + identity, + socket: NetworkSocketMetadata { + socket_cookie, + nonblocking, + process_generation: u64::from(notification.tid), + }, + notification_to_queue: notification_started.elapsed(), + queued_at: Instant::now(), + decision: decision_tx, + relay: relay_rx, + _slot: slot, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => io::Error::from_raw_os_error(libc::EAGAIN), + mpsc::error::TrySendError::Closed(_) => { + io::Error::new(io::ErrorKind::BrokenPipe, "network-open queue closed") + } + })?; + let worker_listener = Arc::clone(&listener); + std::thread::Builder::new() + .name("openshell-network-open".to_string()) + .spawn(move || { + let result = decision_rx.recv().unwrap_or(NetworkOpenResult::Denied { + errno: libc::ECANCELED, + }); + match result { + NetworkOpenResult::Denied { errno } => { + let _ = worker_listener.respond_errno(notification.id, errno); + } + NetworkOpenResult::RelayReady => { + match establish_relay(®istry, notification.tid, fd, destination) { + Ok(stream) => { + let result = worker_listener + .respond_value(notification.id, 0) + .map(|()| stream); + let _ = relay_tx.send(result); + } + Err(error) => { + let _ = worker_listener + .respond_errno(notification.id, error_to_errno(&error)); + let _ = relay_tx.send(Err(error)); + } + } + } + } + }) + .map_err(|error| io::Error::other(format!("start network-open worker: {error}")))?; + Ok(()) +} + +fn ensure_dns_source_bound(fd: RawFd, family: InetFamily) -> io::Result { + let mut address = socket_local_addr(fd)?; + let loopback = match family { + InetFamily::V4 => IpAddr::V4(Ipv4Addr::LOCALHOST), + InetFamily::V6 => IpAddr::V6(Ipv6Addr::LOCALHOST), + }; + if address.port() == 0 { + bind_exact(fd, SocketAddr::new(loopback, 0))?; + address = socket_local_addr(fd)?; + } + // Async resolvers commonly bind an unspecified address before sendto(2). + // A loopback destination makes the kernel select loopback as the actual + // source, so key attribution by that effective peer rather than by the + // wildcard returned before connect/send. Otherwise the relay observes + // 127.0.0.1: (or ::1:) and drops a valid query registered as + // 0.0.0.0: (or [::]:). + if address.ip().is_unspecified() { + address.set_ip(loopback); + } + Ok(address) +} + +fn establish_relay( + registry: &Mutex, + tid: u32, + fd: RawFd, + destination: SocketAddr, +) -> io::Result { + let relay = TcpListener::bind(match destination { + SocketAddr::V4(_) => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + SocketAddr::V6(_) => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + })?; + relay.set_nonblocking(false)?; + let relay_address = relay.local_addr()?; + let expected_peer = { + let mut registry = lock(registry); + let entry = registry.resolve_mut(tid, fd)?; + connect_exact(entry.retained_preconnect()?.as_raw_fd(), relay_address)?; + let expected_peer = socket_local_addr(entry.retained_preconnect()?.as_raw_fd())?; + entry.set_state(SocketState::Connected { + original_peer: destination, + }); + entry.release_preconnect(); + expected_peer + }; + relay.set_nonblocking(true)?; + let deadline = Instant::now() + RELAY_CONNECT_TIMEOUT; + let stream = loop { + let now = Instant::now(); + if now >= deadline { + return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); + } + let timeout = deadline.saturating_duration_since(now); + let mut poll = libc::pollfd { + fd: relay.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let timeout = i32::try_from(timeout.as_millis()).unwrap_or(i32::MAX); + // SAFETY: poll points to one live descriptor record. + if unsafe { libc::poll(&raw mut poll, 1, timeout) } <= 0 { + return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); + } + match relay.accept() { + Ok((stream, peer)) if peer == expected_peer => break stream, + Ok((_stream, peer)) => { + tracing::warn!(%peer, %expected_peer, "rejected unexpected sandbox relay peer"); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error), + } + }; + stream.set_nodelay(true)?; + Ok(stream) +} + +fn bind_socket( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + if !socket_address_is_inet(notification.tid, notification.args[1], notification.args[2])? { + if lock(registry).resolve(notification.tid, fd).is_ok() { + return Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)); + } + return listener.respond_continue(notification.id); + } + let local = read_socket_addr(notification.tid, notification.args[1], notification.args[2])?; + if !local.ip().is_loopback() && !local.ip().is_unspecified() { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + let bind_result = { + let mut registry = lock(registry); + let entry = registry.resolve_mut(notification.tid, fd)?; + bind_exact(entry.retained_preconnect()?.as_raw_fd(), local) + }; + if bind_result + .as_ref() + .is_err_and(|error| error.raw_os_error() == Some(libc::EADDRINUSE)) + { + collect_closed_socket_entries(registry)?; + let mut registry = lock(registry); + let entry = registry.resolve_mut(notification.tid, fd)?; + bind_exact(entry.retained_preconnect()?.as_raw_fd(), local)?; + entry.set_state(SocketState::Bound { local }); + } else { + bind_result?; + lock(registry) + .resolve_mut(notification.tid, fd)? + .set_state(SocketState::Bound { local }); + } + listener.respond_value(notification.id, 0) +} + +fn collect_closed_socket_entries(registry: &Mutex) -> io::Result<()> { + let mut registry = lock(registry); + collect_closed_socket_entries_locked(&mut registry) +} + +fn collect_closed_socket_entries_locked(registry: &mut SocketRegistry) -> io::Result<()> { + let installed = + openshell_isolation_interface::linux::proc_fd::installed_socket_inodes_excluding( + std::process::id(), + )?; + registry.retain_installed(&installed); + Ok(()) +} + +fn listen_socket( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let backlog = i32::try_from(notification.args[1]).unwrap_or(i32::MAX); + let mut registry = lock(registry); + let Ok(entry) = registry.resolve_mut(notification.tid, fd) else { + return listener.respond_continue(notification.id); + }; + // SAFETY: retained descriptor is the exact registered socket OFD. + if unsafe { libc::listen(entry.retained_preconnect()?.as_raw_fd(), backlog) } < 0 { + return Err(io::Error::last_os_error()); + } + let local = socket_local_addr(entry.retained_preconnect()?.as_raw_fd())?; + entry.set_state(SocketState::Listening { local }); + listener.respond_value(notification.id, 0) +} + +fn accept_socket( + registry: Arc>, + listener: Arc, + notification: Notification, + active_accepts: Arc, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let flags = if i64::from(notification.syscall) == libc::SYS_accept4 { + i32::try_from(notification.args[3]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))? + } else { + 0 + }; + if flags & !(libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK) != 0 { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + if (notification.args[1] == 0) != (notification.args[2] == 0) { + return Err(io::Error::from_raw_os_error(libc::EFAULT)); + } + let (listener_inode, metadata, source) = { + let registry = lock(®istry); + let Ok(entry) = registry.resolve(notification.tid, fd) else { + return listener.respond_continue(notification.id); + }; + if !matches!(entry.state(), SocketState::Listening { .. }) + || entry.metadata().kind != InetKind::Tcp + { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + let source = duplicate_close_on_exec(entry.retained_preconnect()?.as_raw_fd())?; + (entry.identity().inode, entry.metadata(), source) + }; + let slot = acquire_pending_accept_slot(&active_accepts)?; + let worker_listener = Arc::clone(&listener); + std::thread::Builder::new() + .name("openshell-local-accept".to_string()) + .spawn(move || { + let _slot = slot; + if let Err(error) = accept_and_inject( + ®istry, + &worker_listener, + notification, + flags, + listener_inode, + metadata, + source, + ) { + let _ = worker_listener.respond_errno(notification.id, error_to_errno(&error)); + } + }) + .map_err(|error| io::Error::other(format!("start local-accept worker: {error}")))?; + Ok(()) +} + +fn accept_and_inject( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, + flags: i32, + listener_inode: u64, + metadata: SocketMetadata, + source: OwnedFd, +) -> io::Result<()> { + let mut poll = libc::pollfd { + fd: source.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: F_GETFL reads the live listener OFD flags. + let current_flags = unsafe { libc::fcntl(source.as_raw_fd(), libc::F_GETFL) }; + if current_flags < 0 { + return Err(io::Error::last_os_error()); + } + let nonblocking = current_flags & libc::O_NONBLOCK != 0; + let timeout = if nonblocking { + 0 + } else { + i32::try_from(ACCEPT_POLL_INTERVAL.as_millis()).expect("accept poll interval fits i32") + }; + loop { + listener.validate_id(notification.id)?; + // SAFETY: poll references one live pollfd for this call. + let ready = unsafe { libc::poll(&raw mut poll, 1, timeout) }; + if ready < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if ready == 0 { + if nonblocking { + return Err(io::Error::from_raw_os_error(libc::EAGAIN)); + } + continue; + } + break; + } + + let mut storage = std::mem::MaybeUninit::::zeroed(); + let mut length = libc::socklen_t::try_from(size_of::()) + .expect("sockaddr storage size fits"); + // Always keep the broker-side descriptor close-on-exec. ADDFD separately + // applies the workload's requested descriptor flag. + let accepted_flags = flags | libc::SOCK_CLOEXEC; + // SAFETY: storage and length are live outputs and source is a listening + // socket proven by the registry. + let accepted = unsafe { + libc::accept4( + source.as_raw_fd(), + storage.as_mut_ptr().cast(), + &raw mut length, + accepted_flags, + ) + }; + if accepted < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful accept4 returned one newly owned descriptor. + let accepted = unsafe { OwnedFd::from_raw_fd(accepted) }; + // SAFETY: accept4 initialized the reported prefix of storage. + let peer = decode_sockaddr( + unsafe { storage.assume_init() }, + usize::try_from(length).unwrap_or(0), + )?; + if !peer.ip().is_loopback() { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + if notification.args[1] != 0 { + write_socket_addr( + listener, + notification.id, + notification.tid, + notification.args[1], + notification.args[2], + peer, + )?; + } + + let accepted_metadata = SocketMetadata { + family: metadata.family, + kind: InetKind::Tcp, + close_on_exec: flags & libc::SOCK_CLOEXEC != 0, + nonblocking: flags & libc::SOCK_NONBLOCK != 0, + creator_generation: u64::from(notification.tid), + }; + let mut registry = lock(registry); + let notifying_fd = raw_fd(notification.args[0])?; + if registry + .resolve(notification.tid, notifying_fd)? + .identity() + .inode + != listener_inode + { + return Err(io::Error::from_raw_os_error(libc::EBADF)); + } + if registry.is_full() { + collect_closed_socket_entries_locked(&mut registry)?; + } + let tentative = registry.stage(accepted, accepted_metadata)?; + listener.add_fd_and_send( + notification.id, + tentative.source_fd(), + accepted_metadata.close_on_exec, + )?; + registry.commit_with_state(tentative, SocketState::AcceptedLocal { peer })?; + Ok(()) +} + +fn duplicate_close_on_exec(fd: RawFd) -> io::Result { + // SAFETY: F_DUPFD_CLOEXEC returns an independent owned descriptor for the + // same open-file description. + let duplicate = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 3) }; + if duplicate < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful fcntl returned one newly owned descriptor. + Ok(unsafe { OwnedFd::from_raw_fd(duplicate) }) +} + +fn classify_send( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, + dns_relay: &DnsRelay, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let syscall = i64::from(notification.syscall); + let (state, metadata) = { + let registry = lock(registry); + let Ok(entry) = registry.resolve(notification.tid, fd) else { + // Non-INET sockets are never injected into the registry. Leave + // their native sendmsg/control-message semantics to the kernel. + return listener.respond_continue(notification.id); + }; + (entry.state().clone(), entry.metadata()) + }; + if matches!( + &state, + SocketState::Connected { .. } | SocketState::AcceptedLocal { .. } + ) || (metadata.kind == InetKind::Tcp && matches!(&state, SocketState::Local { .. })) + { + return listener.respond_continue(notification.id); + } + let messages = match syscall { + libc::SYS_sendto => vec![read_sendto_message(notification)?], + libc::SYS_sendmsg => vec![read_sendmsg_message( + notification.tid, + notification.args[1], + i32::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?, + None, + )?], + libc::SYS_sendmmsg => read_sendmmsg_messages(notification)?, + _ => return Err(io::Error::from_raw_os_error(libc::ENOSYS)), + }; + + let mut registry = lock(registry); + let resolution = registry.resolve(notification.tid, fd); + match resolution { + Ok(entry) + if entry.metadata().kind == InetKind::DnsUdp + && matches!(entry.state(), SocketState::Local { .. }) => + { + if messages.iter().all(|message| message.destination.is_none()) { + listener.respond_continue(notification.id) + } else { + Err(io::Error::from_raw_os_error(libc::EACCES)) + } + } + Ok(entry) if matches!(entry.state(), SocketState::DnsUdp { .. }) => { + let SocketState::DnsUdp { relay } = entry.state() else { + unreachable!("guard requires DNS UDP state"); + }; + // musl-based resolvers, including the statically linked `uv` + // client, send A and AAAA as separate destination-bearing + // datagrams on one socket. The first send pins the socket to the + // private relay; permit later sends only when their copied + // destination is absent or names that same relay. The mandatory + // outer network fence remains the fail-closed backstop for the + // sibling-thread pointer race inherent in seccomp CONTINUE. + if messages.iter().all(|message| { + message + .destination + .is_none_or(|destination| destination == *relay) + }) { + listener.respond_continue(notification.id) + } else { + Err(io::Error::from_raw_os_error(libc::EACCES)) + } + } + Ok(entry) + if entry.metadata().kind == InetKind::DnsUdp + && matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) + && messages.iter().all(|message| { + message + .destination + .is_some_and(|value| value == dns_relay.address) + }) => + { + let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let entry = registry.resolve_mut(notification.tid, fd)?; + let source_fd = entry.retained_preconnect()?.as_raw_fd(); + let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; + lock(&dns_relay.udp_attribution).insert(peer, identity); + if let Err(error) = connect_exact(source_fd, dns_relay.address) { + lock(&dns_relay.udp_attribution).remove(&peer); + return Err(error); + } + for message in &messages { + send_dns_message(source_fd, message)?; + if let Some(length_address) = message.result_length_address { + let length = u32::try_from(message.data.len()) + .map_err(|_| io::Error::from_raw_os_error(libc::EMSGSIZE))?; + listener.validate_id(notification.id)?; + task_memory::write_exact( + notification.tid, + length_address, + &length.to_ne_bytes(), + )?; + } + } + entry.set_state(SocketState::DnsUdp { + relay: dns_relay.address, + }); + entry.release_preconnect(); + let result = if syscall == libc::SYS_sendmmsg { + i64::try_from(messages.len()).unwrap_or(i64::MAX) + } else { + i64::try_from(messages[0].data.len()).unwrap_or(i64::MAX) + }; + listener.respond_value(notification.id, result) + } + Ok(_) => Err(io::Error::from_raw_os_error(libc::EDESTADDRREQ)), + // Non-INET sockets and accepted local sockets were never registered. + // The mandatory outer fence still prevents an external kernel route. + Err(_) => listener.respond_continue(notification.id), + } +} + +struct SendMessage { + data: Vec, + destination: Option, + flags: i32, + result_length_address: Option, +} + +fn read_sendto_message(notification: Notification) -> io::Result { + let length = usize::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EMSGSIZE))?; + if u16::try_from(length).is_err() { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let mut data = vec![0_u8; length]; + task_memory::read_exact(notification.tid, notification.args[1], &mut data)?; + let destination = if notification.args[4] == 0 { + None + } else { + Some(read_socket_addr( + notification.tid, + notification.args[4], + notification.args[5], + )?) + }; + Ok(SendMessage { + data, + destination, + flags: i32::try_from(notification.args[3]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?, + result_length_address: None, + }) +} + +fn read_sendmsg_message( + tid: u32, + address: u64, + flags: i32, + result_length_address: Option, +) -> io::Result { + let header = read_task_value::(tid, address)?; + if header.msg_controllen != 0 { + return Err(io::Error::from_raw_os_error(libc::EOPNOTSUPP)); + } + let destination = if header.msg_name.is_null() { + None + } else { + Some(read_socket_addr( + tid, + header.msg_name as u64, + u64::from(header.msg_namelen), + )?) + }; + #[cfg(target_env = "musl")] + let iov_count = usize::try_from(header.msg_iovlen) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + #[cfg(not(target_env = "musl"))] + let iov_count = header.msg_iovlen; + if iov_count > 32 { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let mut data = Vec::new(); + for index in 0..iov_count { + let offset = index + .checked_mul(size_of::()) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?; + let iov = read_task_value::( + tid, + (header.msg_iov as u64) + .checked_add(u64::try_from(offset).unwrap_or(u64::MAX)) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?, + )?; + let start = data.len(); + let end = start + .checked_add(iov.iov_len) + .filter(|length| u16::try_from(*length).is_ok()) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EMSGSIZE))?; + data.resize(end, 0); + task_memory::read_exact(tid, iov.iov_base as u64, &mut data[start..end])?; + } + Ok(SendMessage { + data, + destination, + flags, + result_length_address, + }) +} + +fn read_sendmmsg_messages(notification: Notification) -> io::Result> { + let count = usize::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if count == 0 || count > 32 { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let flags = i32::try_from(notification.args[3]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + (0..count) + .map(|index| { + let offset = index + .checked_mul(size_of::()) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?; + let base = notification.args[1] + .checked_add(u64::try_from(offset).unwrap_or(u64::MAX)) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?; + read_sendmsg_message( + notification.tid, + base, + flags, + Some( + base.checked_add( + u64::try_from(std::mem::offset_of!(libc::mmsghdr, msg_len)) + .unwrap_or(u64::MAX), + ) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?, + ), + ) + }) + .collect() +} + +fn read_task_value(tid: u32, address: u64) -> io::Result { + let mut bytes = vec![0_u8; size_of::()]; + task_memory::read_exact(tid, address, &mut bytes)?; + // SAFETY: `bytes` contains exactly one copied native value; unaligned read + // avoids imposing alignment on the task-memory scratch allocation. + Ok(unsafe { std::ptr::read_unaligned(bytes.as_ptr().cast::()) }) +} + +fn send_dns_message(fd: RawFd, message: &SendMessage) -> io::Result<()> { + // SAFETY: `fd` is the retained exact UDP socket and the buffer remains + // valid for the duration of the syscall. + let sent = unsafe { + libc::send( + fd, + message.data.as_ptr().cast(), + message.data.len(), + message.flags, + ) + }; + if sent < 0 { + return Err(io::Error::last_os_error()); + } + if usize::try_from(sent).ok() == Some(message.data.len()) { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(libc::EIO)) + } +} + +fn get_peer_name( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let registry = lock(registry); + let Ok(entry) = registry.resolve(notification.tid, fd) else { + return listener.respond_continue(notification.id); + }; + let peer = match entry.state() { + SocketState::Connected { original_peer } => *original_peer, + SocketState::Local { peer } | SocketState::AcceptedLocal { peer } => *peer, + _ => return Err(io::Error::from_raw_os_error(libc::ENOTCONN)), + }; + write_socket_addr( + listener, + notification.id, + notification.tid, + notification.args[1], + notification.args[2], + peer, + )?; + listener.respond_value(notification.id, 0) +} + +fn connect_exact(fd: RawFd, address: SocketAddr) -> io::Result<()> { + // Never let a blocking connect pin the single notification dispatcher. + // O_NONBLOCK is an OFD flag, so restore the workload's original setting + // after the bounded connect attempt completes. + // SAFETY: F_GETFL/F_SETFL operate on the live retained socket descriptor. + let original_flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if original_flags < 0 { + return Err(io::Error::last_os_error()); + } + let changed_flags = original_flags & libc::O_NONBLOCK == 0; + if changed_flags + && unsafe { libc::fcntl(fd, libc::F_SETFL, original_flags | libc::O_NONBLOCK) } < 0 + { + return Err(io::Error::last_os_error()); + } + let result = with_sockaddr(address, |pointer, length| { + // SAFETY: pointer/length describe a live native sockaddr and `fd` is + // the retained exact socket OFD. + let result = unsafe { libc::connect(fd, pointer, length) }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::EINPROGRESS) { + return Err(error); + } + let mut poll = libc::pollfd { + fd, + events: libc::POLLOUT, + revents: 0, + }; + // SAFETY: poll points to one live pollfd. + let timeout = i32::try_from(RELAY_CONNECT_TIMEOUT.as_millis()) + .expect("relay timeout fits poll milliseconds"); + if unsafe { libc::poll(&raw mut poll, 1, timeout) } <= 0 { + return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); + } + let mut socket_error = 0_i32; + let mut size = libc::socklen_t::try_from(size_of::()).expect("SO_ERROR size fits"); + // SAFETY: getsockopt writes one i32 into live storage. + if unsafe { + libc::getsockopt( + fd, + libc::SOL_SOCKET, + libc::SO_ERROR, + (&raw mut socket_error).cast(), + &raw mut size, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + if socket_error == 0 { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(socket_error)) + } + }); + let restore = if changed_flags && unsafe { libc::fcntl(fd, libc::F_SETFL, original_flags) } < 0 + { + Err(io::Error::last_os_error()) + } else { + Ok(()) + }; + result.and(restore) +} + +fn bind_exact(fd: RawFd, address: SocketAddr) -> io::Result<()> { + with_sockaddr(address, |pointer, length| { + // SAFETY: pointer/length describe a live native sockaddr. + if unsafe { libc::bind(fd, pointer, length) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + }) +} + +fn socket_local_addr(fd: RawFd) -> io::Result { + let mut storage = std::mem::MaybeUninit::::zeroed(); + let mut length = libc::socklen_t::try_from(size_of::()) + .expect("sockaddr storage size fits"); + // SAFETY: storage and length are live output buffers. + if unsafe { libc::getsockname(fd, storage.as_mut_ptr().cast(), &raw mut length) } < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: getsockname initialized `length` bytes, including the family. + decode_sockaddr( + unsafe { storage.assume_init() }, + usize::try_from(length).unwrap_or(0), + ) +} + +fn read_socket_addr(tid: u32, address: u64, length: u64) -> io::Result { + let length = usize::try_from(length).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if length < size_of::() || length > size_of::() { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + let mut bytes = vec![0_u8; length]; + task_memory::read_exact(tid, address, &mut bytes)?; + let mut storage = std::mem::MaybeUninit::::zeroed(); + // SAFETY: destination spans sockaddr_storage and `length` was bounded. + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), storage.as_mut_ptr().cast(), length); + decode_sockaddr(storage.assume_init(), length) + } +} + +fn socket_address_is_inet(tid: u32, address: u64, length: u64) -> io::Result { + Ok(matches!( + read_socket_family(tid, address, length)?, + libc::AF_INET | libc::AF_INET6 + )) +} + +fn read_socket_family(tid: u32, address: u64, length: u64) -> io::Result { + let length = usize::try_from(length).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if address == 0 || length < size_of::() { + return Err(io::Error::from_raw_os_error(libc::EFAULT)); + } + let mut family = [0_u8; size_of::()]; + task_memory::read_exact(tid, address, &mut family)?; + Ok(i32::from(libc::sa_family_t::from_ne_bytes(family))) +} + +fn decode_sockaddr(storage: libc::sockaddr_storage, length: usize) -> io::Result { + match i32::from(storage.ss_family) { + libc::AF_INET if length >= size_of::() => { + // SAFETY: family and length establish sockaddr_in layout. + let address = unsafe { *(&raw const storage).cast::() }; + Ok(SocketAddr::new( + IpAddr::V4(Ipv4Addr::from(address.sin_addr.s_addr.to_ne_bytes())), + u16::from_be(address.sin_port), + )) + } + libc::AF_INET6 if length >= size_of::() => { + // SAFETY: family and length establish sockaddr_in6 layout. + let address = unsafe { *(&raw const storage).cast::() }; + Ok(SocketAddr::new( + IpAddr::V6(Ipv6Addr::from(address.sin6_addr.s6_addr)), + u16::from_be(address.sin6_port), + )) + } + _ => Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)), + } +} + +fn write_socket_addr( + listener: &NotificationListener, + notification_id: u64, + tid: u32, + address: u64, + length_address: u64, + value: SocketAddr, +) -> io::Result<()> { + let mut supplied_length = [0_u8; size_of::()]; + task_memory::read_exact(tid, length_address, &mut supplied_length)?; + let supplied_length = libc::socklen_t::from_ne_bytes(supplied_length); + let (bytes, actual_length) = sockaddr_bytes(value); + let copied = usize::try_from(supplied_length) + .unwrap_or(0) + .min(bytes.len()); + listener.validate_id(notification_id)?; + if copied != 0 { + task_memory::write_exact(tid, address, &bytes[..copied])?; + } + listener.validate_id(notification_id)?; + task_memory::write_exact(tid, length_address, &actual_length.to_ne_bytes()) +} + +fn sockaddr_bytes(address: SocketAddr) -> (Vec, libc::socklen_t) { + match address { + SocketAddr::V4(address) => { + let native = libc::sockaddr_in { + sin_family: libc::sa_family_t::try_from(libc::AF_INET) + .expect("AF_INET fits sa_family_t"), + sin_port: address.port().to_be(), + sin_addr: libc::in_addr { + s_addr: u32::from_ne_bytes(address.ip().octets()), + }, + sin_zero: [0; 8], + }; + // SAFETY: native is plain initialized storage. + let bytes = unsafe { + std::slice::from_raw_parts( + (&raw const native).cast::(), + size_of::(), + ) + }; + ( + bytes.to_vec(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in size fits socklen_t"), + ) + } + SocketAddr::V6(address) => { + let native = libc::sockaddr_in6 { + sin6_family: libc::sa_family_t::try_from(libc::AF_INET6) + .expect("AF_INET6 fits sa_family_t"), + sin6_port: address.port().to_be(), + sin6_flowinfo: address.flowinfo(), + sin6_addr: libc::in6_addr { + s6_addr: address.ip().octets(), + }, + sin6_scope_id: address.scope_id(), + }; + // SAFETY: native is plain initialized storage. + let bytes = unsafe { + std::slice::from_raw_parts( + (&raw const native).cast::(), + size_of::(), + ) + }; + ( + bytes.to_vec(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in6 size fits socklen_t"), + ) + } + } +} + +fn with_sockaddr( + address: SocketAddr, + operation: impl FnOnce(*const libc::sockaddr, libc::socklen_t) -> io::Result, +) -> io::Result { + match address { + SocketAddr::V4(address) => { + let native = libc::sockaddr_in { + sin_family: libc::sa_family_t::try_from(libc::AF_INET) + .expect("AF_INET fits sa_family_t"), + sin_port: address.port().to_be(), + sin_addr: libc::in_addr { + s_addr: u32::from_ne_bytes(address.ip().octets()), + }, + sin_zero: [0; 8], + }; + operation( + (&raw const native).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in size fits socklen_t"), + ) + } + SocketAddr::V6(address) => { + let native = libc::sockaddr_in6 { + sin6_family: libc::sa_family_t::try_from(libc::AF_INET6) + .expect("AF_INET6 fits sa_family_t"), + sin6_port: address.port().to_be(), + sin6_flowinfo: address.flowinfo(), + sin6_addr: libc::in6_addr { + s6_addr: address.ip().octets(), + }, + sin6_scope_id: address.scope_id(), + }; + operation( + (&raw const native).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in6 size fits socklen_t"), + ) + } + } +} + +fn raw_fd(value: u64) -> io::Result { + RawFd::try_from(value).map_err(|_| io::Error::from_raw_os_error(libc::EBADF)) +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn error_to_errno(error: &io::Error) -> i32 { + error.raw_os_error().unwrap_or(libc::EACCES).max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read as _, Write as _}; + use std::os::unix::net::{UnixListener, UnixStream}; + + #[test] + fn pending_external_open_slots_are_bounded_and_reusable() { + let active = Arc::new(AtomicUsize::new(OPEN_QUEUE_CAPACITY - 1)); + let last = acquire_pending_open_slot(&active).expect("last available slot"); + assert_eq!( + acquire_pending_open_slot(&active) + .expect_err("open limit must fail closed") + .raw_os_error(), + Some(libc::EAGAIN) + ); + drop(last); + let reused = acquire_pending_open_slot(&active).expect("released slot"); + drop(reused); + assert_eq!(active.load(Ordering::Acquire), OPEN_QUEUE_CAPACITY - 1); + } + + #[test] + fn dns_worker_slots_are_bounded_and_reusable() { + let active = Arc::new(AtomicUsize::new(DNS_WORKER_CAPACITY - 1)); + let last = acquire_pending_dns_slot(&active).expect("last available slot"); + assert_eq!( + acquire_pending_dns_slot(&active) + .expect_err("DNS worker limit must fail closed") + .raw_os_error(), + Some(libc::EAGAIN) + ); + drop(last); + let reused = acquire_pending_dns_slot(&active).expect("released slot"); + drop(reused); + assert_eq!(active.load(Ordering::Acquire), DNS_WORKER_CAPACITY - 1); + } + + #[test] + fn unix_connect_remains_kernel_driven() { + let directory = tempfile::tempdir().expect("temporary Unix socket directory"); + let path = directory.path().join("service.sock"); + let service = UnixListener::bind(&path).expect("bind Unix service"); + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result<()> { + let mut stream = UnixStream::connect(path)?; + stream.write_all(b"unix") + }) + .expect("launcher result") + }); + let (mut stream, _) = service.accept().expect("accept Unix client"); + let mut payload = [0_u8; 4]; + stream.read_exact(&mut payload).expect("read Unix payload"); + assert_eq!(&payload, b"unix"); + client.join().expect("join client").expect("Unix client"); + } + + #[test] + fn accepted_loopback_stream_is_registered_for_notified_operations() { + let reservation = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + let address = reservation.local_addr().expect("reserved address"); + drop(reservation); + + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + let workload = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result { + let listener = TcpListener::bind(address)?; + ready_tx + .send(()) + .map_err(|_| io::Error::other("test client disappeared"))?; + let (stream, _) = listener.accept()?; + let peer = stream.peer_addr()?; + let payload = b"accepted"; + let iov = libc::iovec { + iov_base: payload.as_ptr().cast_mut().cast(), + iov_len: payload.len(), + }; + let message = libc::msghdr { + msg_name: std::ptr::null_mut(), + msg_namelen: 0, + msg_iov: (&raw const iov).cast_mut(), + msg_iovlen: 1, + msg_control: std::ptr::null_mut(), + msg_controllen: 0, + msg_flags: 0, + }; + // SAFETY: message references one live immutable payload; + // the accepted stream remains open for the call. + let sent = unsafe { libc::sendmsg(stream.as_raw_fd(), &raw const message, 0) }; + if sent != isize::try_from(payload.len()).expect("payload fits isize") { + return Err(io::Error::last_os_error()); + } + Ok(peer) + }) + .expect("launcher result") + }); + + ready_rx.recv().expect("listener ready"); + let mut client = TcpStream::connect(address).expect("connect loopback client"); + let mut payload = [0_u8; 8]; + client + .read_exact(&mut payload) + .expect("read accepted stream"); + assert_eq!(&payload, b"accepted"); + assert!( + workload + .join() + .expect("join workload") + .expect("accepted workload") + .ip() + .is_loopback() + ); + } + + #[test] + fn external_connect_waits_for_explicit_relay_decision() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let client = std::thread::spawn(move || { + launcher + .execute(|| -> io::Result<()> { + let mut stream = TcpStream::connect("203.0.113.7:443")?; + stream.write_all(b"request")?; + let mut response = [0_u8; 8]; + stream.read_exact(&mut response)?; + if &response != b"response" { + return Err(io::Error::other("relay returned wrong response")); + } + Ok(()) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let pending = runtime.block_on(broker.accept()).expect("pending TCP open"); + assert_eq!(pending.destination, "203.0.113.7:443".parse().unwrap()); + assert!(pending.socket.socket_cookie != 0); + let mut relay = runtime + .block_on(pending.complete(NetworkOpenResult::RelayReady)) + .expect("complete relay") + .expect("authorized relay stream"); + let mut request = [0_u8; 7]; + relay + .read_exact(&mut request) + .expect("read relayed request"); + assert_eq!(&request, b"request"); + relay.write_all(b"response").expect("write relay response"); + client.join().expect("join client").expect("client relay"); + } + + #[test] + fn denied_external_connect_keeps_socket_unconnected() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let client = std::thread::spawn(move || { + launcher + .execute(|| TcpStream::connect("198.51.100.9:80")) + .expect("launcher result") + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let pending = runtime.block_on(broker.accept()).expect("pending TCP open"); + assert!( + runtime + .block_on(pending.complete(NetworkOpenResult::Denied { + errno: libc::EACCES, + })) + .expect("complete denial") + .is_none() + ); + assert_eq!( + client + .join() + .expect("join client") + .expect_err("connect must be denied") + .raw_os_error(), + Some(libc::EACCES) + ); + } + + #[test] + fn udp_dns_normalizes_wildcard_source_for_relay_attribution() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let dns_address = broker.dns_address(); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result { + // Tokio/Hickory-style resolvers bind a wildcard source + // before sending to the configured nameserver. + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_read_timeout(Some(Duration::from_secs(5)))?; + socket.send_to(b"dns-query", dns_address)?; + let mut response = [0_u8; 32]; + let (length, source) = socket.recv_from(&mut response)?; + if &response[..length] != b"dns-response" { + return Err(io::Error::other("wrong DNS response")); + } + Ok(source) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let query = runtime.block_on(broker.accept_dns()).expect("DNS query"); + assert_eq!(query.transport, DnsTransport::Udp); + assert_eq!(query.request, b"dns-query"); + query.complete(Ok(b"dns-response".to_vec())).unwrap(); + assert_eq!( + client.join().expect("join client").expect("DNS client"), + dns_address + ); + } + + #[test] + fn udp_dns_allows_repeated_destination_sends_to_the_pinned_relay() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let dns_address = broker.dns_address(); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result>> { + // Static musl clients send A and AAAA with two sendto(2) + // calls on the same initially-unconnected socket. + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_read_timeout(Some(Duration::from_secs(5)))?; + socket.send_to(b"dns-query-a", dns_address)?; + socket.send_to(b"dns-query-aaaa", dns_address)?; + let mut responses = Vec::new(); + for _ in 0..2 { + let mut response = [0_u8; 32]; + let (length, source) = socket.recv_from(&mut response)?; + if source != dns_address { + return Err(io::Error::other("wrong DNS response source")); + } + responses.push(response[..length].to_vec()); + } + responses.sort(); + Ok(responses) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for _ in 0..2 { + let query = runtime.block_on(broker.accept_dns()).expect("DNS query"); + let response = if query.request == b"dns-query-a" { + b"dns-response-a".to_vec() + } else if query.request == b"dns-query-aaaa" { + b"dns-response-aaaa".to_vec() + } else { + panic!("unexpected DNS query: {:?}", query.request); + }; + query.complete(Ok(response)).unwrap(); + } + assert_eq!( + client.join().expect("join client").expect("DNS client"), + vec![b"dns-response-a".to_vec(), b"dns-response-aaaa".to_vec()] + ); + } + + #[test] + fn udp_port_zero_route_probes_are_local_and_reusable() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + launcher + .execute(|| -> io::Result<()> { + // Address-selection probes create an unbound datagram socket; + // binding to INADDR_ANY first would intentionally preserve an + // unspecified local address and would not model that path. + // SAFETY: the return value is checked before ownership moves + // into UdpSocket. + let raw_socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_UDP, + ) + }; + if raw_socket < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: raw_socket is a new, owned socket descriptor. + let socket = unsafe { UdpSocket::from_raw_fd(raw_socket) }; + socket.connect("198.51.100.7:0")?; + let local = socket.local_addr()?; + if !local.ip().is_loopback() || local.port() == 0 { + return Err(io::Error::other(format!( + "route probe did not expose a local source: {local}" + ))); + } + + let unspecified = libc::sockaddr { + sa_family: libc::sa_family_t::try_from(libc::AF_UNSPEC) + .expect("AF_UNSPEC fits sa_family_t"), + sa_data: [0; 14], + }; + // SAFETY: unspecified is a live native sockaddr used for the + // conventional UDP disconnect operation. + let disconnected = unsafe { + libc::connect( + socket.as_raw_fd(), + (&raw const unspecified).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr size fits socklen_t"), + ) + }; + if disconnected != 0 { + return Err(io::Error::last_os_error()); + } + socket.connect("203.0.113.9:0")?; + + // The route probe never commits an external UDP peer. A + // destination-free send must therefore remain kernel-denied. + // SAFETY: payload is live for the duration of this syscall. + let sent = unsafe { + libc::send( + socket.as_raw_fd(), + b"blocked".as_ptr().cast(), + b"blocked".len(), + 0, + ) + }; + if sent >= 0 { + return Err(io::Error::other("route probe became a data path")); + } + let error = io::Error::last_os_error(); + if !matches!( + error.raw_os_error(), + Some(libc::EDESTADDRREQ | libc::ENOTCONN) + ) { + return Err(error); + } + Ok(()) + }) + .expect("launcher result") + .expect("route-probe workload"); + } + + #[test] + fn tcp_dns_preserves_length_framing() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let dns_address = broker.dns_address(); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result> { + let mut stream = TcpStream::connect(dns_address)?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.write_all(&[0, 3, 1, 2, 3])?; + let mut response = vec![0_u8; 5]; + stream.read_exact(&mut response)?; + Ok(response) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let query = runtime.block_on(broker.accept_dns()).expect("DNS query"); + assert_eq!(query.transport, DnsTransport::Tcp); + assert_eq!(query.request, [0, 3, 1, 2, 3]); + query.complete(Ok(vec![0, 3, 4, 5, 6])).unwrap(); + assert_eq!( + client.join().expect("join client").expect("DNS client"), + [0, 3, 4, 5, 6] + ); + } +} diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs new file mode 100644 index 0000000000..18e6d9dfe8 --- /dev/null +++ b/crates/openshell-sandbox/src/process.rs @@ -0,0 +1,3844 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process management and signal handling. + +use crate::child_env; +#[cfg(target_os = "linux")] +use crate::managed_children; +use crate::sandbox; +#[cfg(target_os = "linux")] +use miette::WrapErr; +use miette::{IntoDiagnostic, Result}; +use nix::sys::signal::{self, Signal}; +use nix::unistd::{Gid, Group, Pid, Uid, User}; +use openshell_core::policy::SandboxPolicy; +use std::collections::HashMap; +use std::ffi::CString; +#[cfg(unix)] +use std::os::fd::AsRawFd; +#[cfg(unix)] +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +#[cfg(any(test, unix))] +use std::path::Path; +use std::path::PathBuf; +use std::process::Stdio; +use std::sync::Arc; +#[cfg(target_os = "linux")] +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; +use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; +use tracing::{debug, info}; + +// `libc::TIOCSCTTY` and the request parameter accepted by `ioctl` vary across +// glibc, musl, and BSD targets. The conversion is a no-op on some targets but +// is required on others. +#[cfg(unix)] +#[allow(unsafe_code, clippy::useless_conversion)] +fn set_controlling_tty(fd: libc::c_int) -> std::io::Result<()> { + if unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) } < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +/// Numeric identity components resolved once from driver-owned metadata. +/// +/// A component is `None` when the corresponding policy field was explicit and +/// must continue through the existing policy identity path. OCI-derived +/// components are carried numerically so later filesystem setup and direct/SSH +/// privilege drops cannot resolve them differently through NSS. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ResolvedProcessIdentity { + uid: Option, + gid: Option, +} + +impl ResolvedProcessIdentity { + #[must_use] + pub const fn new(uid: Option, gid: Option) -> Self { + Self { uid, gid } + } + + #[must_use] + pub const fn uid(self) -> Option { + self.uid + } + + #[must_use] + pub const fn gid(self) -> Option { + self.gid + } + + /// Whether at least one process identity component came from OCI `USER`. + /// + /// Platform-resolved identities are written directly into the policy and + /// return the default value, so this is specific to Docker/Podman OCI + /// fallback without adding another driver contract. + #[must_use] + pub const fn uses_oci_user_fallback(self) -> bool { + self.uid.is_some() || self.gid.is_some() + } +} + +/// Resolved process workspace and its child-environment semantics. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResolvedWorkspace { + root: Option, + use_as_home: bool, +} + +impl ResolvedWorkspace { + #[must_use] + pub fn new(root: Option, use_as_home: bool) -> Self { + Self { root, use_as_home } + } + + #[must_use] + pub fn root(&self) -> Option<&str> { + self.root.as_deref() + } + + #[must_use] + pub fn owned_root(&self) -> Option { + self.root.clone() + } + + #[must_use] + pub fn home(&self) -> Option<&str> { + self.use_as_home.then(|| self.root()).flatten() + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn prepare_child_sandbox( + policy: &SandboxPolicy, + workdir: Option<&str>, + runtime_read_only: &[PathBuf], +) -> Result> { + let effective_policy = policy_with_runtime_read_only(policy, runtime_read_only); + let prepared = sandbox::linux::prepare_capability_free(&effective_policy, workdir)?; + Ok(Some(prepared)) +} + +#[cfg(target_os = "linux")] +fn policy_with_runtime_read_only( + policy: &SandboxPolicy, + runtime_read_only: &[PathBuf], +) -> SandboxPolicy { + let mut effective_policy = policy.clone(); + for path in runtime_read_only { + if !effective_policy.filesystem.read_only.contains(path) { + effective_policy.filesystem.read_only.push(path.clone()); + } + } + effective_policy +} + +#[cfg(target_os = "linux")] +pub(crate) fn ca_runtime_read_only_paths(ca_paths: Option<&(PathBuf, PathBuf)>) -> Vec { + let Some((certificate, bundle)) = ca_paths else { + return Vec::new(); + }; + let mut paths = Vec::with_capacity(3); + if let Some(directory) = certificate.parent() { + paths.push(directory.to_path_buf()); + } + paths.push(certificate.clone()); + paths.push(bundle.clone()); + paths +} + +const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SANDBOX_GID, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, +]; + +const PROXY_ENV_VARS: &[&str] = &[ + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + "no_proxy", + "grpc_proxy", + "NODE_USE_ENV_PROXY", +]; + +pub fn is_supervisor_only_env_var(key: &str) -> bool { + SUPERVISOR_ONLY_ENV_VARS.contains(&key) +} + +fn strip_supervisor_only_env(cmd: &mut Command) { + for key in SUPERVISOR_ONLY_ENV_VARS { + cmd.env_remove(key); + } +} + +/// Remove ambient proxy routing from a transparently mediated child. +pub fn strip_proxy_env(cmd: &mut Command) { + for key in PROXY_ENV_VARS { + cmd.env_remove(key); + } +} + +/// [`strip_proxy_env`] for synchronous exec commands. +pub fn strip_proxy_env_std(cmd: &mut std::process::Command) { + for key in PROXY_ENV_VARS { + cmd.env_remove(key); + } +} + +/// Whether an environment key can redirect a child around transparent +/// network mediation. +#[must_use] +pub fn is_proxy_env_var(key: &str) -> bool { + PROXY_ENV_VARS.contains(&key) +} + +fn inject_provider_env(cmd: &mut Command, provider_env: &HashMap) { + for (key, value) in provider_env { + if is_supervisor_only_env_var(key) { + continue; + } + cmd.env(key, value); + } +} + +/// Derive the child USER and HOME from the policy's sandbox identity. +/// +/// Name-based identities use their passwd entry. Numeric identities have no +/// reliable passwd entry, so their workspace remains the portable fallback. +pub(crate) fn session_user_and_home( + policy: &SandboxPolicy, + workdir_home: Option<&str>, +) -> (String, String) { + let (user, default_home) = match policy.process.run_as_user.as_deref() { + Some(user) if !user.is_empty() => { + if user.parse::().is_ok() { + (user.to_string(), "/sandbox".to_string()) + } else { + let home = User::from_name(user).ok().flatten().map_or_else( + || format!("/home/{user}"), + |entry| entry.dir.to_string_lossy().into_owned(), + ); + (user.to_string(), home) + } + } + _ => ("sandbox".to_string(), "/sandbox".to_string()), + }; + let home = workdir_home.map_or(default_home, str::to_string); + (user, home) +} + +fn apply_canonical_process_environment( + cmd: &mut Command, + policy: &SandboxPolicy, + workspace: &ResolvedWorkspace, + interactive: bool, + user_environment: &HashMap, +) { + let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + // Resolve a shell present in the sandbox image (minimal images such as + // Alpine ship only `/bin/sh`, not bash). Runs in the supervisor. + let shell = openshell_core::shell::detect_login_shell(); + + for (key, value) in [ + ("HOME", session_home.as_str()), + ("USER", session_user.as_str()), + ("SHELL", shell.as_str()), + ( + "TERM", + if interactive { + "xterm-256color" + } else { + "dumb" + }, + ), + ] { + if !user_environment.contains_key(key) { + cmd.env(key, value); + } + } +} + +fn configured_user_environment() -> HashMap { + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default() +} + +#[cfg(unix)] +pub fn harden_child_process() -> Result<()> { + use rustix::process::{Resource, Rlimit, setrlimit}; + + setrlimit( + Resource::Core, + Rlimit { + current: Some(0), + maximum: Some(0), + }, + ) + .map_err(|e| miette::miette!("Failed to disable core dumps: {e}"))?; + + #[cfg(target_os = "linux")] + { + use rustix::process::{DumpableBehavior, set_dumpable_behavior}; + set_dumpable_behavior(DumpableBehavior::NotDumpable) + .map_err(|e| miette::miette!("Failed to set PR_SET_DUMPABLE=0: {e}"))?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +const CGROUP_PIDS_MAX_PATH: &str = "/sys/fs/cgroup/pids.max"; + +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RuntimePidLimitStatus { + Limited(u64), + Unlimited, + Unavailable(String), +} + +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimePidLimitMode { + Warn, + Require, +} + +#[cfg(target_os = "linux")] +pub fn check_runtime_pid_limit(mode: RuntimePidLimitMode) -> Result<()> { + check_runtime_pid_limit_status(runtime_pid_limit_status(), mode) +} + +#[cfg(target_os = "linux")] +fn check_runtime_pid_limit_status( + status: RuntimePidLimitStatus, + mode: RuntimePidLimitMode, +) -> Result<()> { + match status { + RuntimePidLimitStatus::Limited(limit) => { + debug!(pids_max = limit, "runtime PID limit detected"); + Ok(()) + } + RuntimePidLimitStatus::Unlimited => { + let message = "runtime cgroup pids.max is unlimited; configure the compute driver or container runtime to enforce a PID limit"; + if matches!(mode, RuntimePidLimitMode::Require) { + Err(miette::miette!(message)) + } else { + tracing::warn!("{message}"); + Ok(()) + } + } + RuntimePidLimitStatus::Unavailable(reason) => { + let message = format!( + "runtime cgroup pids.max is unavailable ({reason}); configure the compute driver or container runtime to enforce a PID limit" + ); + if matches!(mode, RuntimePidLimitMode::Require) { + Err(miette::miette!(message)) + } else { + tracing::warn!("{message}"); + Ok(()) + } + } + } +} + +#[cfg(target_os = "linux")] +fn runtime_pid_limit_status() -> RuntimePidLimitStatus { + match std::fs::read_to_string(CGROUP_PIDS_MAX_PATH) { + Ok(contents) => parse_pids_max(&contents), + Err(err) => RuntimePidLimitStatus::Unavailable(err.to_string()), + } +} + +#[cfg(target_os = "linux")] +fn parse_pids_max(contents: &str) -> RuntimePidLimitStatus { + let raw = contents.trim(); + if raw.eq_ignore_ascii_case("max") { + return RuntimePidLimitStatus::Unlimited; + } + match raw.parse::() { + Ok(limit) => RuntimePidLimitStatus::Limited(limit), + Err(err) => { + RuntimePidLimitStatus::Unavailable(format!("invalid pids.max value {raw:?}: {err}")) + } + } +} + +#[cfg(target_os = "linux")] +fn drop_capability_bounding_set() -> Result<()> { + let clear_result = capctl::caps::bounding::clear(); + let remaining = capctl::caps::bounding::probe(); + + validate_capability_bounding_set_clear( + clear_result, + remaining, + capctl::caps::bounding::clear_unknown, + ) +} + +#[cfg(target_os = "linux")] +fn validate_capability_bounding_set_clear( + clear_result: capctl::Result<()>, + remaining: capctl::caps::CapSet, + clear_unknown: impl FnOnce() -> capctl::Result<()>, +) -> Result<()> { + match clear_result { + Ok(()) if remaining.is_empty() => Ok(()), + Ok(()) => Err(miette::miette!( + "Failed to clear child capability bounding set: capabilities remain raised: {remaining:?}" + )), + Err(err) if err.code() == libc::EPERM && remaining.is_empty() => match clear_unknown() { + Ok(()) => { + debug!( + "CAP_SETPCAP is unavailable, but the child capability bounding set is already empty" + ); + Ok(()) + } + Err(unknown_err) => Err(miette::miette!( + "Failed to clear unknown child capability bounding set entries: {unknown_err}" + )), + }, + Err(err) => Err(miette::miette!( + "Failed to clear child capability bounding set: {err}" + )), + } +} + +#[cfg(target_os = "linux")] +static WORKLOAD_LAUNCHER: OnceLock< + openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, +> = OnceLock::new(); + +/// Install the sandbox-owned launcher that every later workload spawn must +/// traverse. A second launcher would create a second listener generation and +/// is therefore rejected. +#[cfg(target_os = "linux")] +pub fn configure_workload_launcher( + launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, +) -> std::io::Result<()> { + WORKLOAD_LAUNCHER.set(launcher).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "workload launcher was already configured", + ) + }) +} + +#[cfg(target_os = "linux")] +pub fn spawn_command_with_workload_launcher(mut cmd: Command) -> std::io::Result { + let launcher = WORKLOAD_LAUNCHER.get().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "sandbox workload launcher is not configured", + ) + })?; + let runtime = tokio::runtime::Handle::current(); + launcher.execute(move || { + let _guard = runtime.enter(); + cmd.spawn() + })? +} + +#[cfg(target_os = "linux")] +pub fn spawn_std_command_with_workload_launcher( + mut cmd: std::process::Command, +) -> std::io::Result { + let launcher = WORKLOAD_LAUNCHER.get().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "sandbox workload launcher is not configured", + ) + })?; + launcher.execute(move || cmd.spawn())? +} + +/// Handle to a running process. +pub struct ProcessHandle { + child: Child, + pid: u32, + io: Option, + terminal: Arc, + signal_lock: Arc>, + #[cfg(target_os = "linux")] + managed_child: Option, +} + +/// Supervisor-owned canonical-process I/O. These handles outlive individual +/// SSH attachments and are consumed by the main-session multiplexer. +pub enum ProcessIo { + Pty(std::fs::File), + Pipes { + stdin: ChildStdin, + stdout: ChildStdout, + stderr: ChildStderr, + }, +} + +impl ProcessHandle { + /// Spawn a new process. + /// + /// # Errors + /// + /// Returns an error if the process fails to start. + #[cfg(target_os = "linux")] + #[allow(clippy::too_many_arguments)] + pub fn spawn( + program: &str, + args: &[String], + workspace: &ResolvedWorkspace, + interactive: bool, + policy: &SandboxPolicy, + ca_paths: Option<&(PathBuf, PathBuf)>, + provider_env: &HashMap, + ) -> Result { + Self::spawn_impl( + program, + args, + workspace, + interactive, + policy, + ca_paths, + provider_env, + ) + } + + /// Spawn a new process (non-Linux platforms). + /// + /// # Errors + /// + /// Returns an error if the process fails to start. + #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] + pub fn spawn( + program: &str, + args: &[String], + workspace: &ResolvedWorkspace, + interactive: bool, + policy: &SandboxPolicy, + ca_paths: Option<&(PathBuf, PathBuf)>, + provider_env: &HashMap, + ) -> Result { + Self::spawn_impl( + program, + args, + workspace, + interactive, + policy, + ca_paths, + provider_env, + ) + } + + #[cfg(target_os = "linux")] + #[allow(clippy::too_many_arguments)] + fn spawn_impl( + program: &str, + args: &[String], + workspace: &ResolvedWorkspace, + interactive: bool, + policy: &SandboxPolicy, + ca_paths: Option<&(PathBuf, PathBuf)>, + provider_env: &HashMap, + ) -> Result { + let mut cmd = Command::new(program); + cmd.args(args) + .kill_on_drop(true) + .env(openshell_core::sandbox_env::SANDBOX, "1"); + + let mut pty_master = None; + let mut terminal_slave_fd = None; + if interactive { + let winsize = nix::pty::Winsize { + ws_row: 24, + ws_col: 80, + ws_xpixel: 0, + ws_ypixel: 0, + }; + let pty = nix::pty::openpty(Some(&winsize), None).into_diagnostic()?; + let master = std::fs::File::from(pty.master); + let slave = std::fs::File::from(pty.slave); + terminal_slave_fd = Some(slave.as_raw_fd()); + cmd.stdin(slave.try_clone().into_diagnostic()?) + .stdout(slave.try_clone().into_diagnostic()?) + .stderr(slave); + pty_master = Some(master); + } else { + cmd.stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + } + + // Strip supervisor-only identity material from the entrypoint's + // inherited environment. The entrypoint drops to the sandbox user + // before `exec`; without this strip, sandbox code could recover + // supervisor credentials from its inherited environment. + strip_supervisor_only_env(&mut cmd); + + inject_provider_env(&mut cmd, provider_env); + apply_canonical_process_environment( + &mut cmd, + policy, + workspace, + interactive, + &configured_user_environment(), + ); + + if let Some(dir) = workspace.root() { + cmd.current_dir(dir); + } + + strip_proxy_env(&mut cmd); + + // Set TLS trust store env vars so sandbox processes trust the ephemeral CA + if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { + for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { + cmd.env(key, value); + } + } + + // Probe Landlock availability and emit OCSF logs from the parent + // process where the tracing subscriber is functional. The child's + // pre_exec context cannot reliably emit structured logs. + #[cfg(target_os = "linux")] + sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + + // Prepare the Landlock ruleset as the workload UID. Inaccessible paths + // are already unavailable to the child and remain omitted. + #[cfg(target_os = "linux")] + let runtime_read_only = ca_runtime_read_only_paths(ca_paths); + let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), &runtime_read_only) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; + #[cfg(target_os = "linux")] + let mut child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| { + miette::miette!("prepare child self-protection filter: {error}") + })?; + // Set up process group for signal handling (non-interactive mode only). + // In interactive mode, we inherit the parent's process group to maintain + // proper terminal control for shells and interactive programs. + // SAFETY: pre_exec runs after fork but before exec in the child process. + // setpgid and setns are async-signal-safe and safe to call in this context. + { + // Wrap in Option so we can .take() it out of the FnMut closure. + // pre_exec is only called once (after fork, before exec). + #[cfg(target_os = "linux")] + let mut prepared_sandbox = prepared_sandbox; + #[allow(unsafe_code)] + unsafe { + cmd.pre_exec(move || { + if let Some(slave_fd) = terminal_slave_fd { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + set_controlling_tty(slave_fd)?; + } else if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + + harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; + + // Phase 2 (as unprivileged user): Enforce the prepared + // Landlock ruleset via restrict_self() + apply seccomp. + // restrict_self() does not require root. + #[cfg(target_os = "linux")] + if let Some(prepared) = prepared_sandbox.take() { + sandbox::linux::enforce_capability_free(prepared, &mut child_hardening) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } + + Ok(()) + }); + } + } + + // Name the program in the error: a bare "No such file or directory" + // here is otherwise indistinguishable from a missing working directory + // or interpreter, and is a common failure on images that lack the + // requested shell/binary (e.g. bash on Alpine). + #[cfg(target_os = "linux")] + let mut child = spawn_command_with_workload_launcher(cmd) + .into_diagnostic() + .wrap_err_with(|| format!("failed to spawn sandbox entrypoint process '{program}'"))?; + #[cfg(not(target_os = "linux"))] + let mut child = cmd + .spawn() + .into_diagnostic() + .wrap_err_with(|| format!("failed to spawn sandbox entrypoint process '{program}'"))?; + let pid = child.id().unwrap_or(0); + let managed_child = managed_children::register(pid); + + let io = if let Some(master) = pty_master { + ProcessIo::Pty(master) + } else { + ProcessIo::Pipes { + stdin: child.stdin.take().expect("canonical stdin must be piped"), + stdout: child.stdout.take().expect("canonical stdout must be piped"), + stderr: child.stderr.take().expect("canonical stderr must be piped"), + } + }; + + debug!(pid, program, "Process spawned"); + + Ok(Self { + child, + pid, + io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), + #[cfg(target_os = "linux")] + managed_child, + }) + } + + #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] + fn spawn_impl( + program: &str, + args: &[String], + workspace: &ResolvedWorkspace, + interactive: bool, + policy: &SandboxPolicy, + ca_paths: Option<&(PathBuf, PathBuf)>, + provider_env: &HashMap, + ) -> Result { + let mut cmd = Command::new(program); + cmd.args(args) + .kill_on_drop(true) + .env(openshell_core::sandbox_env::SANDBOX, "1"); + + let mut pty_master = None; + let mut terminal_slave_fd = None; + #[cfg(unix)] + if interactive { + let winsize = nix::pty::Winsize { + ws_row: 24, + ws_col: 80, + ws_xpixel: 0, + ws_ypixel: 0, + }; + let pty = nix::pty::openpty(Some(&winsize), None).into_diagnostic()?; + let master = std::fs::File::from(pty.master); + let slave = std::fs::File::from(pty.slave); + terminal_slave_fd = Some(slave.as_raw_fd()); + cmd.stdin(slave.try_clone().into_diagnostic()?) + .stdout(slave.try_clone().into_diagnostic()?) + .stderr(slave); + pty_master = Some(master); + } + if !interactive { + cmd.stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + } + + // Strip supervisor-only identity material from the entrypoint's + // inherited environment. + strip_supervisor_only_env(&mut cmd); + + inject_provider_env(&mut cmd, provider_env); + apply_canonical_process_environment( + &mut cmd, + policy, + workspace, + interactive, + &configured_user_environment(), + ); + + if let Some(dir) = workspace.root() { + cmd.current_dir(dir); + } + + strip_proxy_env(&mut cmd); + + // Set TLS trust store env vars so sandbox processes trust the ephemeral CA + if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { + for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { + cmd.env(key, value); + } + } + + // Create a dedicated session for PTY children and a dedicated process + // group for pipe children so attachment signals target only the + // canonical workload tree. + // SAFETY: pre_exec runs after fork but before exec in the child process. + // setpgid is async-signal-safe and safe to call in this context. + #[cfg(unix)] + { + let policy = policy.clone(); + let workdir = workspace.owned_root(); + #[allow(unsafe_code)] + unsafe { + cmd.pre_exec(move || { + if let Some(slave_fd) = terminal_slave_fd { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + set_controlling_tty(slave_fd)?; + } else if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + + harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; + sandbox::apply(&policy, workdir.as_deref()) + .map_err(|err| std::io::Error::other(err.to_string()))?; + + Ok(()) + }); + } + } + + let mut child = cmd.spawn().into_diagnostic()?; + let pid = child.id().unwrap_or(0); + #[cfg(target_os = "linux")] + managed_children::register(pid); + + debug!(pid, program, "Process spawned"); + + let io = if let Some(master) = pty_master { + ProcessIo::Pty(master) + } else { + ProcessIo::Pipes { + stdin: child.stdin.take().expect("canonical stdin must be piped"), + stdout: child.stdout.take().expect("canonical stdout must be piped"), + stderr: child.stderr.take().expect("canonical stderr must be piped"), + } + }; + + Ok(Self { + child, + pid, + io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), + }) + } + + /// Get the process ID. + #[must_use] + pub const fn pid(&self) -> u32 { + self.pid + } + + /// Transfer retained stdio to the main-session multiplexer. + pub fn take_io(&mut self) -> ProcessIo { + self.io.take().expect("canonical process I/O already taken") + } + + /// Shared state used by an independent boundary signal handle. + #[must_use] + pub fn signaling_state(&self) -> (Arc, Arc>) { + (self.terminal.clone(), self.signal_lock.clone()) + } + + /// Wait for the process to exit. + /// + /// # Errors + /// + /// Returns an error if waiting fails. + pub async fn wait(&mut self) -> std::io::Result { + let status = self.child.wait().await; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); + #[cfg(target_os = "linux")] + if let Some(child) = self.managed_child.take() { + managed_children::unregister(child); + } + let status = status?; + Ok(ProcessStatus::from(status)) + } + + /// Observe an already-terminated child without blocking. + pub fn try_wait(&mut self) -> std::io::Result> { + let status = self.child.try_wait()?; + if status.is_some() { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); + #[cfg(target_os = "linux")] + if let Some(child) = self.managed_child.take() { + managed_children::unregister(child); + } + } + Ok(status.map(ProcessStatus::from)) + } + + /// Send a signal to the process. + /// + /// # Errors + /// + /// Returns an error if the signal cannot be sent. + pub fn signal(&self, sig: Signal) -> Result<()> { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("process has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + signal::kill(Pid::from_raw(pid), sig).into_diagnostic() + } + + /// Kill the process. + /// + /// # Errors + /// + /// Returns an error if the process cannot be killed. + pub fn kill(&mut self) -> Result<()> { + // First try SIGTERM + if let Err(e) = self.signal(Signal::SIGTERM) { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Close) + .severity(openshell_ocsf::SeverityId::Medium) + .status(openshell_ocsf::StatusId::Failure) + .message(format!("Failed to send SIGTERM: {e}")) + .build() + ); + } + + // Give the process a moment to terminate gracefully + std::thread::sleep(std::time::Duration::from_millis(100)); + + // Force kill if still running + if let Some(id) = self.child.id() { + debug!(pid = id, "Sending SIGKILL"); + let pid = i32::try_from(id).unwrap_or(i32::MAX); + let _ = signal::kill(Pid::from_raw(pid), Signal::SIGKILL); + } + + Ok(()) + } +} + +impl Drop for ProcessHandle { + fn drop(&mut self) { + #[cfg(target_os = "linux")] + if let Some(child) = self.managed_child.take() { + managed_children::unregister(child); + } + } +} + +/// Validate the configured process user. +/// +/// Numeric identities do not require a passwd entry. The legacy explicit +/// `"sandbox"` identity and other names must resolve in `/etc/passwd`. +#[cfg(unix)] +pub fn validate_sandbox_user(policy: &SandboxPolicy) -> Result<()> { + let identity = policy.process.run_as_user.as_deref().unwrap_or("sandbox"); + + if let Ok(uid) = identity.parse::() { + if !(MIN_SANDBOX_UID..=MAX_SANDBOX_UID).contains(&uid) { + return Err(miette::miette!( + "process user UID must be in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" + )); + } + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "validated") + .message(format!( + "Accepted numeric UID {identity} (no passwd entry required)" + )) + .build() + ); + return Ok(()); + } + + // Legacy explicit "sandbox" name — must exist in /etc/passwd. + if identity == "sandbox" { + match User::from_name("sandbox") { + Ok(Some(_)) => { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "validated") + .message("Validated 'sandbox' user exists in image") + .build() + ); + } + Ok(None) => { + return Err(miette::miette!( + "explicit process user 'sandbox' was not found in the image" + )); + } + Err(e) => { + return Err(miette::miette!("failed to look up 'sandbox' user: {e}")); + } + } + } else if !identity.is_empty() { + // Other names are supported by local/offline policy paths and must + // resolve before privilege dropping. + match User::from_name(identity) { + Ok(Some(_)) => { + tracing::warn!(identity, "named process user accepted via passwd entry"); + } + Ok(None) => { + return Err(miette::miette!( + "unrecognized sandbox identity '{identity}'; \ + expected 'sandbox' or a numeric UID in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" + )); + } + Err(e) => { + return Err(miette::miette!( + "failed to look up identity '{identity}': {e}" + )); + } + } + } + + Ok(()) +} + +/// Validate that the configured sandbox group identity is acceptable. +/// +/// Mirrors [`validate_sandbox_user`] for the group dimension. +#[cfg(unix)] +pub fn validate_sandbox_group(policy: &SandboxPolicy) -> Result<()> { + let identity = policy.process.run_as_group.as_deref().unwrap_or("sandbox"); + + if let Ok(gid) = identity.parse::() { + if !(MIN_SANDBOX_UID..=MAX_SANDBOX_UID).contains(&gid) { + return Err(miette::miette!( + "process group GID must be in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" + )); + } + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "validated") + .message(format!( + "Accepted numeric GID {identity} (no group entry required)" + )) + .build() + ); + return Ok(()); + } + + if identity == "sandbox" { + match Group::from_name("sandbox") { + Ok(Some(_)) => { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "validated") + .message("Validated 'sandbox' group exists in image") + .build() + ); + } + Ok(None) => { + return Err(miette::miette!( + "explicit process group 'sandbox' was not found in the image" + )); + } + Err(e) => { + return Err(miette::miette!("failed to look up 'sandbox' group: {e}")); + } + } + } else if !identity.is_empty() { + match Group::from_name(identity) { + Ok(Some(_)) => { + tracing::warn!(identity, "named process group accepted via group entry"); + } + Ok(None) => { + return Err(miette::miette!( + "unrecognized sandbox group identity '{identity}'; \ + expected 'sandbox' or a numeric GID in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" + )); + } + Err(e) => { + return Err(miette::miette!( + "failed to look up group identity '{identity}': {e}" + )); + } + } + } + + Ok(()) +} + +#[cfg(unix)] +pub fn validate_sandbox_user_with_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result<()> { + let Some(uid) = resolved_identity.uid() else { + return validate_sandbox_user(policy); + }; + if uid == 0 { + return Err(miette::miette!("process user must not select UID 0")); + } + Ok(()) +} + +#[cfg(unix)] +pub fn validate_sandbox_group_with_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result<()> { + let Some(gid) = resolved_identity.gid() else { + return validate_sandbox_group(policy); + }; + if gid == 0 { + return Err(miette::miette!("process group must not select GID 0")); + } + Ok(()) +} + +pub use openshell_policy::{MAX_SANDBOX_UID, MIN_SANDBOX_UID}; + +/// Prepare a `read_write` path for the sandboxed process. +/// +/// Returns `true` when the path was created by the supervisor and therefore +/// still needs to be chowned to the sandbox user/group. Existing paths keep +/// their image-defined ownership. +#[cfg(unix)] +fn prepare_read_write_path(path: &Path) -> Result { + // SECURITY: use symlink_metadata (lstat) to inspect each path *before* + // calling chown. chown follows symlinks, so a malicious container image + // could place a symlink (e.g. /sandbox -> /etc/shadow) to trick the + // root supervisor into transferring ownership of arbitrary files. + // The TOCTOU window between lstat and chown is not exploitable because + // no untrusted process is running yet (the child has not been forked). + if let Ok(meta) = std::fs::symlink_metadata(path) { + if meta.file_type().is_symlink() { + return Err(miette::miette!( + "read_write path '{}' is a symlink — refusing to chown (potential privilege escalation)", + path.display() + )); + } + + debug!( + path = %path.display(), + "Preserving ownership for existing read_write path" + ); + Ok(false) + } else { + debug!(path = %path.display(), "Creating read_write directory"); + std::fs::create_dir_all(path).into_diagnostic()?; + Ok(true) + } +} + +/// Update `/etc/passwd` and `/etc/group` so the "sandbox" user/group entries +/// match the driver-injected UID/GID from environment variables. +/// +/// When `OPENSHELL_SANDBOX_UID` is set, the image-baked "sandbox" entry may +/// have a different UID. Updating the files ensures `whoami`, `id`, `ls -l`, +/// SSH sessions, and `initgroups` resolve the sandbox identity correctly. +/// If no "sandbox" entry exists, one is appended. +#[cfg(unix)] +pub fn update_sandbox_passwd_entries() -> Result<()> { + let uid_str = match std::env::var(openshell_core::sandbox_env::SANDBOX_UID) { + Ok(v) if !v.is_empty() => v, + _ => return Ok(()), + }; + let gid_str = match std::env::var(openshell_core::sandbox_env::SANDBOX_GID) { + Ok(v) if !v.is_empty() => v, + _ => uid_str.clone(), + }; + + let _: u32 = uid_str + .parse() + .map_err(|e| miette::miette!("invalid OPENSHELL_SANDBOX_UID '{uid_str}': {e}"))?; + let _: u32 = gid_str + .parse() + .map_err(|e| miette::miette!("invalid OPENSHELL_SANDBOX_GID '{gid_str}': {e}"))?; + + update_passwd_file(&uid_str, &gid_str)?; + update_group_file(&gid_str)?; + + info!( + uid = %uid_str, + gid = %gid_str, + "Updated /etc/passwd and /etc/group for sandbox identity" + ); + Ok(()) +} + +/// Rewrite the `sandbox` line in `/etc/passwd` with the given UID/GID, +/// or append a new entry if none exists. +#[cfg(unix)] +fn update_passwd_file(uid: &str, gid: &str) -> Result<()> { + rewrite_passwd_at(Path::new("/etc/passwd"), uid, gid) +} + +/// Rewrite the `sandbox` line in `/etc/group` with the given GID, +/// or append a new entry if none exists. +#[cfg(unix)] +fn update_group_file(gid: &str) -> Result<()> { + rewrite_group_at(Path::new("/etc/group"), gid) +} + +#[cfg(unix)] +fn rewrite_passwd_at(path: &Path, uid: &str, gid: &str) -> Result<()> { + let content = std::fs::read_to_string(path).into_diagnostic()?; + + let mut found = false; + let mut lines: Vec = content + .lines() + .map(|line| { + if line.starts_with("sandbox:") { + found = true; + let fields: Vec<&str> = line.split(':').collect(); + if let [name, pass, _, _, gecos, home, shell, ..] = fields.as_slice() { + format!("{name}:{pass}:{uid}:{gid}:{gecos}:{home}:{shell}") + } else { + line.to_string() + } + } else { + line.to_string() + } + }) + .collect(); + + if !found { + lines.push(format!("sandbox:x:{uid}:{gid}::/sandbox:/bin/sh")); + } + + let mut output = lines.join("\n"); + if content.ends_with('\n') || !found { + output.push('\n'); + } + + std::fs::write(path, output).into_diagnostic()?; + Ok(()) +} + +#[cfg(unix)] +fn rewrite_group_at(path: &Path, gid: &str) -> Result<()> { + let content = std::fs::read_to_string(path).into_diagnostic()?; + + let mut found = false; + let mut lines: Vec = content + .lines() + .map(|line| { + if line.starts_with("sandbox:") { + found = true; + let fields: Vec<&str> = line.split(':').collect(); + if let [name, pass, _, members, ..] = fields.as_slice() { + format!("{name}:{pass}:{gid}:{members}") + } else { + line.to_string() + } + } else { + line.to_string() + } + }) + .collect(); + + if !found { + lines.push(format!("sandbox:x:{gid}:")); + } + + let mut output = lines.join("\n"); + if content.ends_with('\n') || !found { + output.push('\n'); + } + + std::fs::write(path, output).into_diagnostic()?; + Ok(()) +} + +/// Recursively chown a directory tree to the given UID/GID. +/// +/// This retains the Kubernetes/OpenShift workspace reconciliation from before +/// OCI image identity fallback. Symlinks are skipped, and read-only nested +/// mounts are not traversed. +#[cfg(unix)] +fn chown_sandbox_home(root: &Path, uid: Option, gid: Option) -> Result<()> { + let meta = std::fs::symlink_metadata(root).into_diagnostic()?; + if meta.file_type().is_symlink() { + return Err(miette::miette!( + "path '{}' is a symlink — refusing to chown (potential privilege escalation)", + root.display() + )); + } + + nix::unistd::chown(root, uid, gid).into_diagnostic()?; + + if meta.is_dir() { + chown_children(root, uid, gid, &nix::unistd::chown)?; + } + + Ok(()) +} + +#[cfg(unix)] +fn prepare_oci_workspace( + root: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> Result<()> { + prepare_oci_workspace_with(root, uid, gid, supplementary_gids, &nix::unistd::chown) +} + +/// Validate that selecting an image-provided OCI workdir does not grant the +/// sandbox identity any filesystem authority it lacked in the immutable image. +/// +/// Every path component must be a real directory (never a symlink), every +/// parent must already be traversable, and the final directory must already be +/// writable and traversable. No ownership or mode bits are changed. +#[cfg(unix)] +pub fn validate_oci_workspace( + root: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> Result<()> { + let components = validated_workspace_components(root, false)?; + let mut current = PathBuf::from("/"); + validate_workspace_component(¤t, uid, gid, supplementary_gids, false)?; + let last_component = components.len().saturating_sub(1); + for (index, component) in components.into_iter().enumerate() { + current.push(component); + validate_workspace_component( + ¤t, + uid, + gid, + supplementary_gids, + index == last_component, + )?; + } + Ok(()) +} + +/// Validate an image-provided workdir in a clean copy of the supervisor so the +/// main process retains the root authority needed for subsequent setup. +#[cfg(target_os = "linux")] +fn validate_oci_workspace_in_subprocess( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + workdir: &Path, +) -> Result<()> { + use std::os::unix::process::CommandExt; + + let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; + let uid = uid.ok_or_else(|| miette::miette!("workspace validator UID is unresolved"))?; + let gid = gid.ok_or_else(|| miette::miette!("workspace validator GID is unresolved"))?; + let groups = supplementary_gids + .iter() + .map(|group| group.as_raw()) + .collect::>(); + let executable = std::env::current_exe().into_diagnostic()?; + let mut command = std::process::Command::new(executable); + command + .arg("validate-workspace") + .arg("--workdir") + .arg(workdir) + .arg("--expected-uid") + .arg(uid.to_string()) + .arg("--expected-gid") + .arg(gid.to_string()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + + // `pre_exec` runs after fork and before exec. These direct credential + // syscalls are async-signal-safe and affect only the one-shot child. + #[allow(unsafe_code)] + unsafe { + command.pre_exec(move || { + if libc::setgroups(groups.len(), groups.as_ptr()) != 0 + || libc::setgid(gid.as_raw()) != 0 + || libc::setuid(uid.as_raw()) != 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + + let output = command.output().into_diagnostic()?; + if output.status.success() { + return Ok(()); + } + + let diagnostic = String::from_utf8_lossy(&output.stderr); + let diagnostic = diagnostic.trim(); + if diagnostic.is_empty() { + return Err(miette::miette!( + "image workspace validation failed with status {}", + output.status + )); + } + Err(miette::miette!( + "image workspace validation failed: {diagnostic}" + )) +} + +#[cfg(unix)] +fn validate_workspace_component( + path: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], + is_workspace: bool, +) -> Result<()> { + let metadata = std::fs::symlink_metadata(path).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + miette::miette!( + "image workspace path component '{}' does not exist", + path.display() + ) + } else { + miette::miette!( + "failed to inspect image workspace path component '{}': {error}", + path.display() + ) + } + })?; + if metadata.file_type().is_symlink() { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + path.display() + )); + } + if !metadata.is_dir() { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + path.display() + )); + } + let required = if is_workspace { 0o3 } else { 0o1 }; + if !identity_has_permissions(&metadata, uid, gid, supplementary_gids, required) { + let requirement = if is_workspace { + "writable and traversable" + } else { + "traversable" + }; + return Err(miette::miette!( + "workspace path component '{}' is not {requirement} by the sandbox identity in the image", + path.display() + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +pub fn validate_oci_workspace_as_effective_identity(root: &Path) -> Result<()> { + use rustix::fs::{Access, AtFlags, FileType, Mode, OFlags}; + + let components = validated_workspace_components(root, false)?; + let open_flags = OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let mut current_path = PathBuf::from("/"); + let mut current_fd = rustix::fs::open("/", open_flags, Mode::empty()).into_diagnostic()?; + rustix::fs::accessat( + ¤t_fd, + ".", + Access::EXEC_OK, + AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(|error| { + miette::miette!( + "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", + current_path.display() + ) + })?; + + let last_component = components.len().saturating_sub(1); + for (index, component) in components.into_iter().enumerate() { + current_path.push(&component); + let stat = rustix::fs::statat(¤t_fd, &component, AtFlags::SYMLINK_NOFOLLOW).map_err( + |error| { + if error == rustix::io::Errno::NOENT { + miette::miette!( + "image workspace path component '{}' does not exist", + current_path.display() + ) + } else { + miette::miette!( + "failed to inspect image workspace path component '{}': {error}", + current_path.display() + ) + } + }, + )?; + let file_type = FileType::from_raw_mode(stat.st_mode); + if file_type.is_symlink() { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + current_path.display() + )); + } + if !file_type.is_dir() { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + current_path.display() + )); + } + + let is_workspace = index == last_component; + rustix::fs::accessat( + ¤t_fd, + &component, + Access::EXEC_OK, + AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(|error| { + miette::miette!( + "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", + current_path.display() + ) + })?; + + let next_fd = rustix::fs::openat(¤t_fd, &component, open_flags, Mode::empty()) + .map_err(|error| { + miette::miette!( + "failed to open image workspace path component '{}': {error}", + current_path.display() + ) + })?; + if is_workspace { + validate_effective_workspace_write(&next_fd, ¤t_path)?; + } + current_fd = next_fd; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn validate_effective_workspace_write(fd: &impl std::os::fd::AsFd, path: &Path) -> Result<()> { + use rustix::fs::{AtFlags, Mode, OFlags}; + + let mode = Mode::RUSR | Mode::WUSR; + let tmpfile_flags = OFlags::TMPFILE | OFlags::WRONLY | OFlags::CLOEXEC; + match rustix::fs::openat(fd, ".", tmpfile_flags, mode) { + Ok(_probe) => return Ok(()), + Err(rustix::io::Errno::INVAL | rustix::io::Errno::ISDIR | rustix::io::Errno::NOTSUP) => {} + Err(error) => { + return Err(miette::miette!( + "workspace path component '{}' is not writable by the sandbox identity in the image: {error}", + path.display() + )); + } + } + + // Some filesystems do not implement O_TMPFILE. Fall back to a short-lived, + // no-follow entry. A collision fails closed after bounded retries. + let create_flags = + OFlags::CREATE | OFlags::EXCL | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + for attempt in 0..16 { + let name = format!(".openshell-workdir-probe-{}-{attempt}", std::process::id()); + match rustix::fs::openat(fd, &name, create_flags, mode) { + Ok(_probe) => { + rustix::fs::unlinkat(fd, &name, AtFlags::empty()).map_err(|error| { + miette::miette!( + "workspace write probe cleanup failed for '{}': {error}", + path.display() + ) + })?; + return Ok(()); + } + Err(rustix::io::Errno::EXIST) => {} + Err(error) => { + return Err(miette::miette!( + "workspace path component '{}' is not writable by the sandbox identity in the image: {error}", + path.display() + )); + } + } + } + + Err(miette::miette!( + "workspace write probe could not allocate a unique entry in '{}'", + path.display() + )) +} + +/// Prepare only the resolved `OpenShell` workspace directory itself. +/// +/// Image-provided children retain their declared ownership. This avoids +/// crossing symlinks or user-provided nested mounts. +#[cfg(unix)] +fn prepare_oci_workspace_with( + root: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], + do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, +) -> Result<()> { + let components = validated_workspace_components(root, true)?; + + let last_component = components.len().saturating_sub(1); + let mut current = PathBuf::from("/"); + for (index, component) in components.into_iter().enumerate() { + current.push(component); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + current.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + current.display() + )); + } + Ok(metadata) => { + if index != last_component + && !identity_can_traverse(&metadata, uid, gid, supplementary_gids) + { + return Err(miette::miette!( + "workspace parent '{}' is not traversable by the sandbox identity", + current.display() + )); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(¤t).into_diagnostic()?; + std::fs::set_permissions(¤t, std::fs::Permissions::from_mode(0o755)) + .into_diagnostic()?; + } + Err(error) => return Err(error).into_diagnostic(), + } + } + + do_chown(root, uid, gid).into_diagnostic()?; + + let metadata = std::fs::symlink_metadata(root).into_diagnostic()?; + let mode = metadata.permissions().mode() & 0o7777; + if mode & 0o300 != 0o300 { + std::fs::set_permissions(root, std::fs::Permissions::from_mode(mode | 0o300)) + .into_diagnostic()?; + } + Ok(()) +} + +#[cfg(unix)] +fn validated_workspace_components( + root: &Path, + allow_managed_fallback: bool, +) -> Result> { + let root_str = root + .to_str() + .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; + let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) + .map_err(|error| miette::miette!(error))?; + if Path::new(&validated_root) != root + || (!allow_managed_fallback + && validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) + { + return Err(miette::miette!( + "workspace path '{}' must be a normalized absolute {}path", + root.display(), + if allow_managed_fallback { + "non-root " + } else { + "non-fallback " + } + )); + } + + root.components() + .skip(1) + .map(|component| match component { + std::path::Component::Normal(component) => Ok(component.to_os_string()), + _ => Err(miette::miette!( + "workspace path '{}' must be normalized", + root.display() + )), + }) + .collect() +} + +#[cfg(unix)] +fn identity_can_traverse( + metadata: &std::fs::Metadata, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> bool { + identity_has_permissions(metadata, uid, gid, supplementary_gids, 0o1) +} + +#[cfg(unix)] +fn identity_has_permissions( + metadata: &std::fs::Metadata, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], + required: u32, +) -> bool { + let user_id = uid.unwrap_or_else(nix::unistd::geteuid).as_raw(); + if user_id == 0 { + return true; + } + + let group_id = gid.unwrap_or_else(nix::unistd::getegid).as_raw(); + let mode = metadata.permissions().mode(); + if metadata.uid() == user_id { + mode & (required << 6) == required << 6 + } else if metadata.gid() == group_id + || supplementary_gids + .iter() + .any(|supplementary_gid| supplementary_gid.as_raw() == metadata.gid()) + { + mode & (required << 3) == required << 3 + } else { + mode & required == required + } +} + +#[cfg(not(any( + target_os = "aix", + target_os = "haiku", + target_os = "illumos", + target_os = "ios", + target_os = "macos", + target_os = "redox", + target_os = "solaris" +)))] +fn named_user_supplementary_groups(user_name: &str, primary_gid: Gid) -> Result> { + let user_name = CString::new(user_name).map_err(|_| miette::miette!("Invalid user name"))?; + nix::unistd::getgrouplist(user_name.as_c_str(), primary_gid).into_diagnostic() +} + +#[cfg(any( + target_os = "aix", + target_os = "haiku", + target_os = "illumos", + target_os = "ios", + target_os = "macos", + target_os = "redox", + target_os = "solaris" +))] +#[allow(clippy::unnecessary_wraps)] +fn named_user_supplementary_groups(_user_name: &str, _primary_gid: Gid) -> Result> { + // Privilege dropping does not call initgroups on these targets. + Ok(Vec::new()) +} + +#[cfg(unix)] +fn chown_children( + dir: &Path, + uid: Option, + gid: Option, + do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, +) -> Result<()> { + match std::fs::read_dir(dir) { + Ok(entries) => { + for entry in entries { + let entry = entry.into_diagnostic()?; + chown_recursive(&entry.path(), uid, gid, do_chown)?; + } + } + Err(error) => { + debug!( + path = %dir.display(), + %error, + "Cannot list directory during sandbox home chown" + ); + } + } + Ok(()) +} + +#[cfg(unix)] +fn chown_recursive( + path: &Path, + uid: Option, + gid: Option, + do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, +) -> Result<()> { + let meta = std::fs::symlink_metadata(path).into_diagnostic()?; + if meta.file_type().is_symlink() { + debug!(path = %path.display(), "Skipping symlink during sandbox home chown"); + return Ok(()); + } + + if let Err(error) = do_chown(path, uid, gid) { + if error == nix::errno::Errno::EROFS { + debug!(path = %path.display(), "Skipping read-only path during sandbox home chown"); + return Ok(()); + } + return Err(error).into_diagnostic(); + } + + if meta.is_dir() { + chown_children(path, uid, gid, do_chown)?; + } + + Ok(()) +} + +/// Prepare filesystem for the sandboxed process. +/// +/// Creates `read_write` directories if they don't exist and sets ownership +/// on newly-created paths to the configured sandbox user/group. This runs as +/// the supervisor (root) before forking the child process. +/// +/// Accepts both name-based identities (resolved via `/etc/passwd`) and numeric +/// UIDs/GIDs (passed directly to `chown` without a passwd lookup). +#[cfg(unix)] +pub fn prepare_filesystem(policy: &SandboxPolicy) -> Result<()> { + prepare_filesystem_with_identity(policy, ResolvedProcessIdentity::default(), None, false) +} + +#[cfg(unix)] +pub fn prepare_filesystem_with_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + workdir: Option<&str>, + prepare_workspace: bool, +) -> Result<()> { + use nix::unistd::chown; + + // If no user/group configured, nothing to do + if policy + .process + .run_as_user + .as_deref() + .is_none_or(str::is_empty) + && policy + .process + .run_as_group + .as_deref() + .is_none_or(str::is_empty) + { + return Ok(()); + } + + let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; + + // Docker owns workspace resolution and must make the selected root usable + // by the final effective identity, including when both policy identity + // fields were explicit. Validate it before processing any user-authored + // read-write paths so an unsafe image path fails first. Other drivers + // retain their preparation. + if prepare_workspace { + let workspace = workdir.ok_or_else(|| { + miette::miette!("local container driver did not supply a workspace workdir") + })?; + let workspace = Path::new(workspace); + if workspace == Path::new(openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) { + info!(path = %workspace.display(), ?uid, ?gid, "Preparing managed workspace"); + prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; + } else { + info!(path = %workspace.display(), ?uid, ?gid, "Validating image workspace authority"); + #[cfg(target_os = "linux")] + validate_oci_workspace_in_subprocess(policy, resolved_identity, workspace)?; + #[cfg(not(target_os = "linux"))] + validate_oci_workspace(workspace, uid, gid, &supplementary_gids)?; + } + } + + // Create missing read_write paths and only chown the ones we created. + for path in &policy.filesystem.read_write { + if prepare_read_write_path(path)? { + debug!( + path = %path.display(), + ?uid, + ?gid, + "Setting ownership on newly created read_write path" + ); + chown(path, uid, gid).into_diagnostic()?; + } + } + + // Retain the existing Kubernetes/OpenShift behavior for driver-injected + // numeric identities. Docker clears this variable and does not receive + // identity-specific workspace preparation. + if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { + let sandbox_home = Path::new("/sandbox"); + if sandbox_home.exists() { + info!(?uid, ?gid, "Chowning /sandbox for driver-injected UID/GID"); + chown_sandbox_home(sandbox_home, uid, gid)?; + } + } + + Ok(()) +} + +#[cfg(unix)] +fn resolve_filesystem_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result<(Option, Option, Vec)> { + let user_name = policy + .process + .run_as_user + .as_deref() + .filter(|name| !name.is_empty()); + let group_name = policy + .process + .run_as_group + .as_deref() + .filter(|name| !name.is_empty()); + + let uid = match resolved_identity.uid() { + Some(uid) => Some(Uid::from_raw(uid)), + None => match user_name { + Some(name) if name.parse::().is_ok() => { + Some(Uid::from_raw(name.parse().into_diagnostic()?)) + } + Some(name) => User::from_name(name).into_diagnostic()?.map(|u| u.uid), + _ => None, + }, + }; + + // Resolve GID: numeric values are passed directly; names resolve via group. + let gid = match resolved_identity.gid() { + Some(gid) => Some(Gid::from_raw(gid)), + None => match group_name { + Some(name) if name.parse::().is_ok() => { + Some(Gid::from_raw(name.parse().into_diagnostic()?)) + } + Some(name) => Group::from_name(name).into_diagnostic()?.map(|g| g.gid), + _ => None, + }, + }; + + let supplementary_gids = match user_name { + Some(name) if name.parse::().is_err() => { + let primary_gid = if let Some(gid) = gid { + gid + } else { + let uid = + uid.ok_or_else(|| miette::miette!("Failed to resolve sandbox user '{name}'"))?; + User::from_uid(uid) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("Failed to resolve user from UID {uid}"))? + .gid + }; + if resolved_identity.uid().is_some() { + crate::identity::resolve_oci_supplementary_gids(name, primary_gid.as_raw())? + .into_iter() + .map(Gid::from_raw) + .collect() + } else { + named_user_supplementary_groups(name, primary_gid)? + } + } + _ => Vec::new(), + }; + + Ok((uid, gid, supplementary_gids)) +} + +#[cfg(not(unix))] +pub fn prepare_filesystem(_policy: &SandboxPolicy) -> Result<()> { + Ok(()) +} + +// `effective_gid`/`effective_uid` are intentionally parallel names (same role +// for different identifiers) and the noise from renaming would obscure intent. +#[cfg(unix)] +#[allow(clippy::similar_names)] +pub fn drop_privileges(policy: &SandboxPolicy) -> Result<()> { + drop_privileges_with_identity(policy, ResolvedProcessIdentity::default()) +} + +#[cfg(unix)] +#[allow(clippy::similar_names)] +pub fn drop_privileges_with_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result<()> { + let user_name = match policy.process.run_as_user.as_deref() { + Some(name) if !name.is_empty() => Some(name), + _ => None, + }; + let group_name = match policy.process.run_as_group.as_deref() { + Some(name) if !name.is_empty() => Some(name), + _ => None, + }; + + // If no user/group is configured and we are running as root, fall back to + // "sandbox:sandbox" instead of silently keeping root. This covers the + // local/dev-mode path for drivers that provide no identity metadata. + // For non-root runtimes, the no-op is safe -- we are already unprivileged. + if user_name.is_none() && group_name.is_none() { + if nix::unistd::geteuid().is_root() { + let mut fallback = policy.clone(); + fallback.process.run_as_user = Some("sandbox".into()); + fallback.process.run_as_group = Some("sandbox".into()); + return drop_privileges_with_identity(&fallback, resolved_identity); + } + return Ok(()); + } + + // Resolve UID: numeric values are used directly; names resolve via passwd. + let target_uid = match resolved_identity.uid() { + Some(uid) => Uid::from_raw(uid), + None => match user_name { + Some(name) if name.parse::().is_ok() => { + Uid::from_raw(name.parse().into_diagnostic()?) + } + Some(name) => { + User::from_name(name) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("Sandbox user not found: {name}"))? + .uid + } + None => nix::unistd::geteuid(), + }, + }; + + // Resolve group: if a numeric GID is configured use it directly. + // Otherwise try name resolution, then fall back to current user's primary group. + let target_gid = match resolved_identity.gid() { + Some(gid) => Gid::from_raw(gid), + None => match group_name { + Some(name) if name.parse::().is_ok() => { + Gid::from_raw(name.parse().into_diagnostic()?) + } + Some(name) => { + Group::from_name(name) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("Sandbox group not found: {name}"))? + .gid + } + None => match target_uid.as_raw() { + 0 => nix::unistd::getegid(), + _ => Group::from_gid( + User::from_uid(target_uid) + .into_diagnostic()? + .ok_or_else(|| { + miette::miette!("Failed to resolve user from UID {target_uid}") + })? + .gid, + ) + .into_diagnostic()? + .map_or_else(nix::unistd::getegid, |g| g.gid), + }, + }, + }; + + // Resolve the name for initgroups only for the existing explicit-policy + // path. OCI-derived users carry a numeric UID from the bounded parser and + // must not be looked up again through NSS. + let user_name_is_numeric = user_name.is_some_and(|n| n.parse::().is_ok()); + let initgroups_name = + if user_name.is_some() && !user_name_is_numeric && resolved_identity.uid().is_none() { + Some( + User::from_uid(target_uid) + .into_diagnostic()? + .ok_or_else(|| { + miette::miette!("Failed to resolve user record for UID {target_uid}") + })? + .name, + ) + } else { + None + }; + + if target_uid != nix::unistd::geteuid() { + if resolved_identity.uses_oci_user_fallback() { + // OCI named users use the bounded /etc/group parser shared with + // workspace validation. Numeric OCI users resolve to an empty + // list. Never retain the root supervisor's inherited groups. + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "haiku", + target_os = "redox" + )))] + { + let (_, _, supplementary_gids) = + resolve_filesystem_identity(policy, resolved_identity)?; + nix::unistd::setgroups(&supplementary_gids).into_diagnostic()?; + } + } else if let Some(ref user_name) = initgroups_name { + let user_cstr = CString::new(user_name.as_str()) + .map_err(|_| miette::miette!("Invalid user name"))?; + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "haiku", + target_os = "redox" + ))] + { + let _ = user_cstr; + } + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "haiku", + target_os = "redox" + )))] + { + nix::unistd::initgroups(user_cstr.as_c_str(), target_gid).into_diagnostic()?; + } + } + } + + if target_gid != nix::unistd::getegid() { + nix::unistd::setgid(target_gid).into_diagnostic()?; + } + + // Verify effective GID actually changed (defense-in-depth, CWE-250 / CERT POS37-C) + let effective_gid = nix::unistd::getegid(); + if effective_gid != target_gid { + return Err(miette::miette!( + "Privilege drop verification failed: expected effective GID {}, got {}", + target_gid, + effective_gid + )); + } + + #[cfg(target_os = "linux")] + if nix::unistd::geteuid().is_root() { + drop_capability_bounding_set()?; + } + + if user_name.is_some() { + if target_uid != nix::unistd::geteuid() { + nix::unistd::setuid(target_uid).into_diagnostic()?; + } + + // Verify effective UID actually changed (defense-in-depth, CWE-250 / CERT POS37-C) + let effective_uid = nix::unistd::geteuid(); + if effective_uid != target_uid { + return Err(miette::miette!( + "Privilege drop verification failed: expected effective UID {}, got {}", + target_uid, + effective_uid + )); + } + + // Verify root cannot be re-acquired (CERT POS37-C hardening). + // If we dropped from root, setuid(0) must fail; success means privileges + // were not fully relinquished. + if nix::unistd::setuid(Uid::from_raw(0)).is_ok() && target_uid.as_raw() != 0 { + return Err(miette::miette!( + "Privilege drop verification failed: process can still re-acquire root (UID 0) \ + after switching to UID {}", + target_uid + )); + } + } + + Ok(()) +} + +/// Process exit status. +#[derive(Debug, Clone, Copy)] +pub struct ProcessStatus { + code: Option, + signal: Option, +} + +impl ProcessStatus { + /// Get the conventional exit code when the process exited normally. + #[must_use] + pub const fn exit_code(&self) -> Option { + self.code + } + + /// Get the exit code, or 128 + signal number if killed by signal. + #[must_use] + pub fn code(&self) -> i32 { + self.code + .or_else(|| self.signal.map(|s| 128 + s)) + .unwrap_or(-1) + } + + /// Check if the process exited successfully. + #[must_use] + pub fn success(&self) -> bool { + self.code == Some(0) + } + + /// Get the signal that killed the process, if any. + #[must_use] + pub const fn signal(&self) -> Option { + self.signal + } +} + +impl From for ProcessStatus { + fn from(status: std::process::ExitStatus) -> Self { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + Self { + code: status.code(), + signal: status.signal(), + } + } + + #[cfg(not(unix))] + { + Self { + code: status.code(), + signal: None, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(unix)] + use nix::sys::wait::{WaitStatus, waitpid}; + #[cfg(unix)] + use nix::unistd::{ForkResult, fork}; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + #[cfg(unix)] + use std::mem::size_of; + use std::process::Stdio as StdStdio; + + /// Helper to create a minimal `SandboxPolicy` with the given process policy. + fn policy_with_process(process: ProcessPolicy) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process, + } + } + + #[cfg(unix)] + #[tokio::test] + async fn canonical_tty_environment_replaces_supervisor_identity_defaults() { + let current_user = User::from_uid(nix::unistd::geteuid()) + .expect("look up current user") + .expect("current user entry"); + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(current_user.name.clone()), + run_as_group: None, + }); + let workspace = ResolvedWorkspace::default(); + let mut cmd = Command::new("/usr/bin/env"); + cmd.env_clear() + .env("HOME", "/root") + .env("TERM", "dumb") + .stdout(StdStdio::piped()); + + apply_canonical_process_environment(&mut cmd, &policy, &workspace, true, &HashMap::new()); + + let output = cmd.output().await.expect("run environment probe"); + assert!(output.status.success()); + let environment = String::from_utf8(output.stdout).expect("environment is UTF-8"); + let variables: HashMap<_, _> = environment + .lines() + .filter_map(|line| line.split_once('=')) + .collect(); + + assert_eq!( + variables.get("HOME"), + Some(¤t_user.dir.to_string_lossy().as_ref()) + ); + assert_eq!(variables.get("USER"), Some(¤t_user.name.as_str())); + // SHELL is the shell detected in the current root filesystem, not a + // hardcoded path (bash-less images resolve to /bin/sh). + let expected_shell = openshell_core::shell::detect_login_shell(); + assert_eq!(variables.get("SHELL"), Some(&expected_shell.as_str())); + assert_eq!(variables.get("TERM"), Some(&"xterm-256color")); + } + + /// Unknown names may yield `Ok(None)` (`… not found …`) or `Err` when NSS fails first + /// (e.g. `ENOENT: No such file or directory`). + fn assert_unknown_identity_lookup_failed(msg: &str) { + assert!( + msg.contains("not found") + || msg.contains("ENOENT") + || msg.contains("No such file or directory"), + "expected unknown user/group lookup failure (…not found… or ENOENT): {msg}" + ); + } + + #[test] + #[cfg(unix)] + fn explicit_identity_accepts_non_root_system_ids() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some("101".into()), + run_as_group: Some("102".into()), + }); + + assert!(validate_sandbox_user(&policy).is_ok()); + assert!(validate_sandbox_group(&policy).is_ok()); + } + + #[test] + #[cfg(unix)] + fn resolved_oci_identity_accepts_non_root_system_ids() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some("app".into()), + run_as_group: Some("staff".into()), + }); + let resolved = ResolvedProcessIdentity::new(Some(101), Some(102)); + + assert!(validate_sandbox_user_with_identity(&policy, resolved).is_ok()); + assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); + } + + #[test] + #[cfg(unix)] + fn completed_runtime_identity_rejects_numeric_root() { + let root_user = policy_with_process(ProcessPolicy { + run_as_user: Some("0".into()), + run_as_group: Some("102".into()), + }); + let root_group = policy_with_process(ProcessPolicy { + run_as_user: Some("101".into()), + run_as_group: Some("0".into()), + }); + + assert!(validate_sandbox_user(&root_user).is_err()); + assert!(validate_sandbox_group(&root_group).is_err()); + } + + #[test] + #[cfg(unix)] + fn resolved_oci_components_do_not_repeat_nss_validation() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some("__oci_name_not_in_host_nss__".into()), + run_as_group: Some("__oci_group_not_in_host_nss__".into()), + }); + let resolved = ResolvedProcessIdentity::new(Some(1234), Some(1235)); + + assert!(validate_sandbox_user_with_identity(&policy, resolved).is_ok()); + assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); + } + + #[test] + #[cfg(unix)] + fn explicit_policy_components_keep_existing_validation_path() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some("__explicit_name_not_in_host_nss__".into()), + run_as_group: Some("__oci_group_not_in_host_nss__".into()), + }); + let resolved = ResolvedProcessIdentity::new(None, Some(1235)); + + assert!(validate_sandbox_user_with_identity(&policy, resolved).is_err()); + assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); + } + + #[cfg(target_os = "linux")] + fn capability_bounding_set_clear_available() -> bool { + capctl::caps::CapState::get_current() + .is_ok_and(|state| state.effective.has(capctl::caps::Cap::SETPCAP)) + || capctl::caps::bounding::probe().is_empty() + } + + #[test] + #[cfg(target_os = "linux")] + fn capability_bounding_set_clear_accepts_empty_eperm() { + let remaining = capctl::caps::CapSet::empty(); + + assert!( + validate_capability_bounding_set_clear( + Err(capctl::Error::from_code(libc::EPERM)), + remaining, + || Ok(()), + ) + .is_ok() + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn capability_bounding_set_clear_rejects_nonempty_eperm() { + let mut remaining = capctl::caps::CapSet::empty(); + remaining.add(capctl::caps::Cap::CHOWN); + + let result = validate_capability_bounding_set_clear( + Err(capctl::Error::from_code(libc::EPERM)), + remaining, + || panic!("unknown capabilities should not be checked when known caps remain"), + ); + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Failed to clear child capability bounding set") + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn capability_bounding_set_clear_rejects_nonempty_success() { + let mut remaining = capctl::caps::CapSet::empty(); + remaining.add(capctl::caps::Cap::CHOWN); + + let result = validate_capability_bounding_set_clear(Ok(()), remaining, || { + panic!("unknown capabilities should not be checked when known caps remain") + }); + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("capabilities remain raised") + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn capability_bounding_set_clear_rejects_unknown_eperm() { + let remaining = capctl::caps::CapSet::empty(); + + let result = validate_capability_bounding_set_clear( + Err(capctl::Error::from_code(libc::EPERM)), + remaining, + || Err(capctl::Error::from_code(libc::EPERM)), + ); + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Failed to clear unknown child capability bounding set entries") + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn capability_probe_child() { + if std::env::var_os("OPENSHELL_TEST_PROBE_CHILD_CAPS").is_none() { + return; + } + + assert!( + capctl::caps::bounding::probe().is_empty(), + "child CapBnd should be empty after exec" + ); + } + + #[test] + fn drop_privileges_noop_when_no_user_or_group() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: None, + run_as_group: None, + }); + if nix::unistd::geteuid().is_root() { + // As root, drop_privileges falls back to "sandbox:sandbox". + // If that user exists, it succeeds; if not (e.g. CI), it + // must error rather than silently keep root. + let has_sandbox = User::from_name("sandbox").ok().flatten().is_some(); + assert_eq!(drop_privileges(&policy).is_ok(), has_sandbox); + } else { + assert!(drop_privileges(&policy).is_ok()); + } + } + + #[test] + fn drop_privileges_noop_when_empty_strings() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(String::new()), + run_as_group: Some(String::new()), + }); + if nix::unistd::geteuid().is_root() { + let has_sandbox = User::from_name("sandbox").ok().flatten().is_some(); + assert_eq!(drop_privileges(&policy).is_ok(), has_sandbox); + } else { + assert!(drop_privileges(&policy).is_ok()); + } + } + + #[test] + fn drop_privileges_succeeds_for_current_group() { + // Set only run_as_group (no run_as_user) so that initgroups() is not + // called. initgroups(3) requires CAP_SETGID/root even when the target + // is the current user, so it cannot be exercised without elevated + // privileges. This test covers the setgid() + GID post-condition + // verification path without needing root. + let current_group = Group::from_gid(nix::unistd::getegid()) + .expect("getgrgid") + .expect("current group entry"); + + let policy = policy_with_process(ProcessPolicy { + run_as_user: None, + run_as_group: Some(current_group.name), + }); + + let result = drop_privileges(&policy); + #[cfg(target_os = "linux")] + { + if nix::unistd::geteuid().is_root() && !capability_bounding_set_clear_available() { + let msg = format!("{}", result.unwrap_err()); + assert!( + msg.contains("Failed to clear child capability bounding set"), + "unexpected failure: {msg}" + ); + return; + } + } + assert!(result.is_ok(), "drop_privileges failed: {result:?}"); + } + + #[test] + #[cfg(target_os = "linux")] + #[allow(unsafe_code)] + fn drop_privileges_clears_bounding_set_for_spawned_child_when_permitted() { + use std::os::unix::process::CommandExt; + + if !capability_bounding_set_clear_available() { + eprintln!( + "skipping: CAP_SETPCAP is not effective and the capability bounding set is nonempty" + ); + return; + } + + let current_group = Group::from_gid(nix::unistd::getegid()) + .expect("getgrgid") + .expect("current group entry"); + + let policy = policy_with_process(ProcessPolicy { + run_as_user: None, + run_as_group: Some(current_group.name), + }); + + let mut cmd = std::process::Command::new(std::env::current_exe().expect("current exe")); + cmd.arg("capability_probe_child") + .arg("--nocapture") + .env("OPENSHELL_TEST_PROBE_CHILD_CAPS", "1") + .stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::piped()); + + unsafe { + cmd.pre_exec(move || { + drop_privileges(&policy).map_err(|err| std::io::Error::other(err.to_string())) + }); + } + + let output = cmd.output().expect("spawn child status probe"); + assert!( + output.status.success(), + "status probe failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + #[ignore = "initgroups(3) requires CAP_SETGID; run as root: sudo cargo test -- --ignored"] + fn drop_privileges_succeeds_for_current_user() { + // Exercises the full privilege-drop path including initgroups(), + // setgid(), setuid(), and the root-reacquisition check. Requires + // CAP_SETGID (root) because initgroups(3) calls setgroups(2) + // internally. Fixes: https://github.com/NVIDIA/OpenShell/issues/622 + let current_user = User::from_uid(nix::unistd::geteuid()) + .expect("getpwuid") + .expect("current user entry"); + let current_group = Group::from_gid(nix::unistd::getegid()) + .expect("getgrgid") + .expect("current group entry"); + + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(current_user.name), + run_as_group: Some(current_group.name), + }); + + assert!(drop_privileges(&policy).is_ok()); + } + + #[test] + fn drop_privileges_fails_for_nonexistent_user() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some("__nonexistent_test_user_42__".to_string()), + run_as_group: None, + }); + + let result = drop_privileges(&policy); + assert!(result.is_err()); + let msg = format!("{}", result.unwrap_err()); + assert_unknown_identity_lookup_failed(&msg); + } + + #[test] + fn drop_privileges_fails_for_nonexistent_group() { + let policy = policy_with_process(ProcessPolicy { + run_as_user: None, + run_as_group: Some("__nonexistent_test_group_42__".to_string()), + }); + + let result = drop_privileges(&policy); + assert!(result.is_err()); + let msg = format!("{}", result.unwrap_err()); + assert_unknown_identity_lookup_failed(&msg); + } + + #[cfg(unix)] + #[allow(unsafe_code)] + fn probe_hardened_child(probe: unsafe fn() -> i64) -> i64 { + const HARDEN_FAILED: i64 = -2; + + let mut fds = [0; 2]; + let pipe_rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + assert_eq!( + pipe_rc, + 0, + "pipe failed: {}", + std::io::Error::last_os_error() + ); + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + unsafe { libc::close(fds[0]) }; + let value = match harden_child_process() { + Ok(()) => unsafe { probe() }, + Err(_) => HARDEN_FAILED, + }; + let bytes = value.to_ne_bytes(); + let written = unsafe { libc::write(fds[1], bytes.as_ptr().cast(), bytes.len()) }; + unsafe { + libc::close(fds[1]); + libc::_exit(i32::from(written != bytes.len().cast_signed())); + } + } + ForkResult::Parent { child } => { + unsafe { libc::close(fds[1]) }; + let mut bytes = [0u8; size_of::()]; + let read = unsafe { libc::read(fds[0], bytes.as_mut_ptr().cast(), bytes.len()) }; + unsafe { libc::close(fds[0]) }; + assert_eq!( + read.cast_unsigned(), + bytes.len(), + "expected {} probe bytes, got {}", + bytes.len(), + read + ); + + match waitpid(child, None).expect("waitpid should succeed") { + WaitStatus::Exited(_, 0) => {} + status => panic!("probe child exited unexpectedly: {status:?}"), + } + + i64::from_ne_bytes(bytes) + } + } + } + + #[cfg(unix)] + #[allow(unsafe_code)] + unsafe fn core_dump_limit_is_zero_probe() -> i64 { + let mut limit = std::mem::MaybeUninit::::uninit(); + let rc = unsafe { libc::getrlimit(libc::RLIMIT_CORE, limit.as_mut_ptr()) }; + if rc != 0 { + return -1; + } + let limit = unsafe { limit.assume_init() }; + i64::from(limit.rlim_cur == 0 && limit.rlim_max == 0) + } + + #[test] + #[cfg(unix)] + fn harden_child_process_disables_core_dumps() { + assert_eq!(probe_hardened_child(core_dump_limit_is_zero_probe), 1); + } + + #[cfg(target_os = "linux")] + #[allow(unsafe_code)] + unsafe fn dumpable_flag_probe() -> i64 { + unsafe { i64::from(libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0)) } + } + + #[test] + #[cfg(target_os = "linux")] + fn harden_child_process_marks_process_nondumpable() { + assert_eq!(probe_hardened_child(dumpable_flag_probe), 0); + } + + #[test] + #[cfg(target_os = "linux")] + fn parse_pids_max_detects_limited_runtime() { + assert_eq!( + parse_pids_max("2048\n"), + RuntimePidLimitStatus::Limited(2048) + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn parse_pids_max_detects_unlimited_runtime() { + assert_eq!(parse_pids_max("max\n"), RuntimePidLimitStatus::Unlimited); + } + + #[test] + #[cfg(target_os = "linux")] + fn parse_pids_max_reports_invalid_values() { + let status = parse_pids_max("not-a-number\n"); + assert!(matches!(status, RuntimePidLimitStatus::Unavailable(_))); + } + + #[test] + #[cfg(target_os = "linux")] + fn pid_limit_require_mode_rejects_missing_guardrail_statuses() { + for status in [ + RuntimePidLimitStatus::Unlimited, + RuntimePidLimitStatus::Unavailable("missing".to_string()), + ] { + let result = check_runtime_pid_limit_status(status, RuntimePidLimitMode::Require); + assert!(result.is_err()); + } + } + + #[test] + #[cfg(target_os = "linux")] + fn pid_limit_warn_mode_accepts_missing_guardrail_statuses() { + for status in [ + RuntimePidLimitStatus::Unlimited, + RuntimePidLimitStatus::Unavailable("missing".to_string()), + ] { + let result = check_runtime_pid_limit_status(status, RuntimePidLimitMode::Warn); + assert!(result.is_ok()); + } + } + + #[tokio::test] + async fn inject_provider_env_sets_placeholder_values() { + let mut cmd = Command::new("/usr/bin/env"); + cmd.stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::null()); + + let provider_env = std::iter::once(( + "ANTHROPIC_API_KEY".to_string(), + "openshell:resolve:env:ANTHROPIC_API_KEY".to_string(), + )) + .collect(); + + inject_provider_env(&mut cmd, &provider_env); + + let output = cmd.output().await.expect("spawn env"); + let stdout = String::from_utf8(output.stdout).expect("utf8"); + assert!(stdout.contains("ANTHROPIC_API_KEY=openshell:resolve:env:ANTHROPIC_API_KEY")); + } + + #[cfg(unix)] + fn sandbox_policy_with_read_write( + path: PathBuf, + run_as_user: Option, + run_as_group: Option, + ) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![], + read_write: vec![path], + include_workdir: false, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user, + run_as_group, + }, + } + } + + #[cfg(unix)] + #[test] + fn prepare_read_write_path_creates_missing_directory() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing").join("nested"); + + assert!(prepare_read_write_path(&missing).unwrap()); + assert!(missing.is_dir()); + } + + #[cfg(unix)] + #[test] + fn prepare_read_write_path_preserves_existing_directory() { + let dir = tempfile::tempdir().unwrap(); + let existing = dir.path().join("existing"); + std::fs::create_dir(&existing).unwrap(); + + assert!(!prepare_read_write_path(&existing).unwrap()); + assert!(existing.is_dir()); + } + + #[cfg(unix)] + #[test] + fn prepare_read_write_path_rejects_symlink() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("target"); + let link = dir.path().join("link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let error = prepare_read_write_path(&link).unwrap_err(); + assert!( + error + .to_string() + .contains("is a symlink — refusing to chown"), + "unexpected error: {error}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_filesystem_skips_chown_for_existing_read_write_paths() { + use std::os::unix::fs::MetadataExt; + + if nix::unistd::geteuid().is_root() { + return; + } + + let Ok(Some(current_user)) = User::from_uid(nix::unistd::geteuid()) else { + eprintln!("skipping: current UID has no /etc/passwd entry"); + return; + }; + let restricted_group = Group::from_gid(Gid::from_raw(0)) + .unwrap() + .expect("gid 0 group entry"); + if restricted_group.gid == nix::unistd::getegid() { + return; + } + + let dir = tempfile::tempdir().unwrap(); + let existing = dir.path().join("existing"); + std::fs::create_dir(&existing).unwrap(); + let before = std::fs::metadata(&existing).unwrap(); + + let policy = sandbox_policy_with_read_write( + existing.clone(), + Some(current_user.name), + Some(restricted_group.name), + ); + + prepare_filesystem(&policy).expect("existing path should not be re-owned"); + + let after = std::fs::metadata(&existing).unwrap(); + assert_eq!(after.uid(), before.uid()); + assert_eq!(after.gid(), before.gid()); + } + + #[cfg(unix)] + #[test] + #[allow(clippy::similar_names)] + fn chown_sandbox_home_changes_ownership_recursively() { + use std::os::unix::fs::MetadataExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("file.txt"), "hello").unwrap(); + std::fs::create_dir(root.join("subdir")).unwrap(); + std::fs::write(root.join("subdir").join("nested.txt"), "world").unwrap(); + + let expected_uid = nix::unistd::geteuid(); + let expected_gid = nix::unistd::getegid(); + chown_sandbox_home(&root, Some(expected_uid), Some(expected_gid)).unwrap(); + + for path in &[ + root.clone(), + root.join("file.txt"), + root.join("subdir"), + root.join("subdir").join("nested.txt"), + ] { + let meta = std::fs::metadata(path).unwrap(); + assert_eq!(meta.uid(), expected_uid.as_raw()); + assert_eq!(meta.gid(), expected_gid.as_raw()); + } + } + + #[cfg(unix)] + #[test] + fn chown_sandbox_home_rejects_symlink_root() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("real"); + let link = dir.path().join("link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let err = chown_sandbox_home( + &link, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + ) + .unwrap_err(); + assert!( + err.to_string().contains("symlink"), + "expected symlink rejection: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn chown_sandbox_home_skips_symlink_children() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + let target = dir.path().join("outside"); + std::fs::write(&target, "secret").unwrap(); + symlink(&target, root.join("link")).unwrap(); + + chown_sandbox_home( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + ) + .expect("symlink children should be skipped"); + } + + #[cfg(unix)] + #[test] + fn chown_recursive_skips_erofs_subtree_but_continues_siblings() { + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + + let readonly_dir = root.join("ro-mount"); + std::fs::create_dir(&readonly_dir).unwrap(); + std::fs::write(readonly_dir.join("child-under-ro.txt"), "data").unwrap(); + std::fs::write(root.join("writable-sibling.txt"), "data").unwrap(); + + let chowned = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&chowned); + let readonly_dir_for_chown = readonly_dir.clone(); + let fake_chown = + move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + if path == readonly_dir_for_chown { + return Err(nix::errno::Errno::EROFS); + } + observed.lock().unwrap().push(path.to_path_buf()); + Ok(()) + }; + + chown_children( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &fake_chown, + ) + .expect("read-only subtree should be skipped"); + + let chowned = chowned.lock().unwrap(); + assert!( + !chowned.contains(&readonly_dir.join("child-under-ro.txt")), + "children under EROFS directory must not be traversed" + ); + assert!( + chowned.contains(&root.join("writable-sibling.txt")), + "writable sibling should still be chowned" + ); + } + + #[cfg(unix)] + #[test] + fn chown_recursive_propagates_non_erofs_errors() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + let fake_chown = |_path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + Err(nix::errno::Errno::EPERM) + }; + + let result = chown_recursive( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &fake_chown, + ); + assert!(result.is_err(), "non-EROFS errors should propagate"); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_chowns_only_root() { + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + let child = root.join("image-content.txt"); + std::fs::write(&child, "image-owned").unwrap(); + + let chowned = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&chowned); + let fake_chown = + move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + observed.lock().unwrap().push(path.to_path_buf()); + Ok(()) + }; + + prepare_oci_workspace_with( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + &fake_chown, + ) + .expect("workspace root should be prepared"); + + assert_eq!(*chowned.lock().unwrap(), vec![root]); + assert!(child.exists(), "image-provided child should be untouched"); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_accepts_existing_owner_writable_directory() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + + validate_oci_workspace( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .expect("image owner already has write and traverse authority"); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_accepts_supplementary_group_write_authority() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o070)).unwrap(); + let metadata = std::fs::symlink_metadata(&root).unwrap(); + + validate_oci_workspace( + &root, + Some(Uid::from_raw(metadata.uid().wrapping_add(1))), + Some(Gid::from_raw(metadata.gid().wrapping_add(1))), + &[Gid::from_raw(metadata.gid())], + ) + .expect("supplementary group already has write and traverse authority"); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_unwritable_directory() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let metadata = std::fs::symlink_metadata(&root).unwrap(); + + let error = validate_oci_workspace( + &root, + Some(Uid::from_raw(metadata.uid().wrapping_add(1))), + Some(Gid::from_raw(metadata.gid().wrapping_add(1))), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("not writable and traversable")); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_missing_path() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("missing"); + + let error = validate_oci_workspace( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("does not exist")); + } + + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] + fn effective_identity_validation_honors_named_user_acl() { + const TEST_UID: u32 = 42_234; + const TEST_GID: u32 = 42_235; + const ACL_XATTR_VERSION: u32 = 2; + const ACL_USER_OBJ: u16 = 0x01; + const ACL_USER: u16 = 0x02; + const ACL_GROUP_OBJ: u16 = 0x04; + const ACL_MASK: u16 = 0x10; + const ACL_OTHER: u16 = 0x20; + const ACL_UNDEFINED_ID: u32 = u32::MAX; + + if !nix::unistd::geteuid().is_root() { + return; + } + + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + + let mut acl = ACL_XATTR_VERSION.to_ne_bytes().to_vec(); + for (tag, permissions, id) in [ + (ACL_USER_OBJ, 0o7_u16, ACL_UNDEFINED_ID), + (ACL_USER, 0o7_u16, TEST_UID), + (ACL_GROUP_OBJ, 0o0_u16, ACL_UNDEFINED_ID), + (ACL_MASK, 0o7_u16, ACL_UNDEFINED_ID), + (ACL_OTHER, 0o0_u16, ACL_UNDEFINED_ID), + ] { + acl.extend_from_slice(&tag.to_ne_bytes()); + acl.extend_from_slice(&permissions.to_ne_bytes()); + acl.extend_from_slice(&id.to_ne_bytes()); + } + let path = CString::new(root.as_os_str().as_encoded_bytes()).unwrap(); + let name = c"system.posix_acl_access"; + let result = unsafe { + libc::setxattr( + path.as_ptr(), + name.as_ptr(), + acl.as_ptr().cast(), + acl.len(), + 0, + ) + }; + assert_eq!( + result, + 0, + "setxattr failed: {}", + std::io::Error::last_os_error() + ); + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + let credentials_dropped = unsafe { + libc::setgroups(0, std::ptr::null()) == 0 + && libc::setgid(TEST_GID) == 0 + && libc::setuid(TEST_UID) == 0 + }; + let valid = credentials_dropped + && validate_oci_workspace_as_effective_identity(&root).is_ok(); + unsafe { libc::_exit(i32::from(!valid)) }; + } + ForkResult::Parent { child } => { + assert_eq!( + waitpid(child, None).expect("waitpid should succeed"), + WaitStatus::Exited(child, 0), + "named ACL user should retain workspace authority" + ); + } + } + } + + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] + fn effective_identity_validation_honors_landlock_denial() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.filesystem = FilesystemPolicy { + read_only: vec![root.clone()], + read_write: Vec::new(), + include_workdir: false, + }; + policy.landlock = LandlockPolicy { + compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, + }; + let Ok(prepared) = sandbox::linux::prepare_current_user(&policy, None) else { + return; + }; + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + let denied = sandbox::linux::enforce(prepared).is_ok() + && validate_oci_workspace_as_effective_identity(&root).is_err(); + unsafe { libc::_exit(i32::from(!denied)) }; + } + ForkResult::Parent { child } => { + assert_eq!( + waitpid(child, None).expect("waitpid should succeed"), + WaitStatus::Exited(child, 0), + "kernel-effective validation should honor an enforced LSM denial" + ); + } + } + } + + #[cfg(target_os = "linux")] + #[test] + fn runtime_ca_paths_are_added_to_the_effective_read_only_policy() { + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.filesystem.read_only = vec![PathBuf::from("/usr")]; + let certificate = PathBuf::from("/run/openshell-proxy-ca/ca.crt"); + let bundle = PathBuf::from("/run/openshell-proxy-ca/ca-bundle.crt"); + + let effective = policy_with_runtime_read_only( + &policy, + &[certificate.clone(), bundle.clone(), certificate.clone()], + ); + + assert_eq!(policy.filesystem.read_only, vec![PathBuf::from("/usr")]); + assert_eq!( + effective.filesystem.read_only, + vec![PathBuf::from("/usr"), certificate, bundle] + ); + } + + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] + fn runtime_ca_material_remains_readable_after_landlock_for_non_root_workload() { + let root = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let ca_directory = root.path().join("openshell-proxy-ca"); + std::fs::create_dir(&ca_directory).unwrap(); + std::fs::set_permissions(&ca_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let certificate = ca_directory.join("ca.crt"); + let bundle = ca_directory.join("ca-bundle.crt"); + let denied = root.path().join("not-authorized"); + for path in [&certificate, &bundle, &denied] { + std::fs::write(path, b"public certificate material").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); + } + + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.landlock = LandlockPolicy { + compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, + }; + let runtime_paths = + ca_runtime_read_only_paths(Some(&(certificate.clone(), bundle.clone()))); + let Ok(Some(prepared)) = prepare_child_sandbox(&policy, None, &runtime_paths) else { + return; + }; + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + let dropped = if nix::unistd::geteuid().is_root() { + unsafe { + libc::setgroups(0, std::ptr::null()) == 0 + && libc::setgid(42_235) == 0 + && libc::setuid(42_234) == 0 + } + } else { + true + }; + let valid = dropped + && sandbox::linux::enforce(prepared).is_ok() + && std::fs::read(&certificate).is_ok() + && std::fs::read(&bundle).is_ok() + && std::fs::read(&denied).is_err(); + unsafe { libc::_exit(i32::from(!valid)) }; + } + ForkResult::Parent { child } => assert_eq!( + waitpid(child, None).expect("waitpid should succeed"), + WaitStatus::Exited(child, 0), + "Landlock must preserve non-root access only to admitted public CA material" + ), + } + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_restrictive_parent() { + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().canonicalize().unwrap().join("private"); + let root = parent.join("project"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + let metadata = std::fs::symlink_metadata(&parent).unwrap(); + + let error = validate_oci_workspace( + &root, + Some(Uid::from_raw(metadata.uid().wrapping_add(1))), + Some(Gid::from_raw(metadata.gid().wrapping_add(1))), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("not traversable")); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_symlink_component() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let target = base.join("target"); + let link = base.join("link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let error = validate_oci_workspace( + &link, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("symlink")); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_makes_existing_root_owner_writable() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o555)).unwrap(); + + prepare_oci_workspace_with(&root, None, None, &[], &|_, _, _| Ok(())) + .expect("read-only workspace root should be prepared"); + + let mode = std::fs::symlink_metadata(&root) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o755); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_symlink_root() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let target = base.join("real"); + let link = base.join("link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let err = prepare_oci_workspace( + &link, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!( + err.to_string().contains("symlink"), + "expected symlink rejection: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_symlink_parent() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let target = base.join("real"); + let parent_link = base.join("parent-link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &parent_link).unwrap(); + + let err = prepare_oci_workspace( + &parent_link.join("workspace"), + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!( + err.to_string().contains("symlink"), + "expected parent symlink rejection: {err}" + ); + assert!( + !target.join("workspace").exists(), + "workspace must not be created through a symlink parent" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_parent_traversal() { + let err = prepare_oci_workspace( + Path::new("/tmp/workspace/../escape"), + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!( + err.to_string().contains("must be normalized"), + "expected traversal rejection: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_inaccessible_existing_parent() { + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); + let metadata = std::fs::symlink_metadata(&parent).unwrap(); + let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); + let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); + let root = parent.join("project"); + + let error = prepare_oci_workspace_with( + &root, + Some(different_user), + Some(different_group), + &[], + &|_, _, _| Ok(()), + ) + .unwrap_err(); + + assert!( + error.to_string().contains("is not traversable"), + "unexpected error: {error}" + ); + assert!( + !root.exists(), + "workspace must not be created below an inaccessible parent" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_accepts_supplementary_group_parent() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o710)).unwrap(); + let parent = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o710)).unwrap(); + let metadata = std::fs::symlink_metadata(&parent).unwrap(); + let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); + let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); + let supplementary_group = Gid::from_raw(metadata.gid()); + let root = parent.join("project"); + + prepare_oci_workspace_with( + &root, + Some(different_user), + Some(different_group), + &[supplementary_group], + &|_, _, _| Ok(()), + ) + .expect("supplementary group execute permission should allow traversal"); + + assert!(root.is_dir()); + } + + #[cfg(not(any( + target_os = "aix", + target_os = "haiku", + target_os = "illumos", + target_os = "ios", + target_os = "macos", + target_os = "redox", + target_os = "solaris" + )))] + #[test] + fn named_user_supplementary_groups_include_primary_group() { + let user = User::from_uid(nix::unistd::geteuid()) + .expect("resolve current UID") + .expect("current user exists"); + + let groups = named_user_supplementary_groups(&user.name, user.gid) + .expect("resolve named-user supplementary groups"); + + assert!(groups.contains(&user.gid)); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_non_directory_root() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::write(&root, "not a directory").unwrap(); + + let error = prepare_oci_workspace( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!( + error.to_string().contains("is not a directory"), + "unexpected error: {error}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_propagates_root_chown_error() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + let fake_chown = |_path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + Err(nix::errno::Errno::EROFS) + }; + + let error = prepare_oci_workspace_with( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + &fake_chown, + ) + .unwrap_err(); + + assert!( + error.to_string().contains("Read-only file system"), + "unexpected error: {error}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_creates_missing_root() { + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let missing = dir + .path() + .canonicalize() + .unwrap() + .join("missing") + .join("sandbox"); + let chowned = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&chowned); + let fake_chown = + move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + observed.lock().unwrap().push(path.to_path_buf()); + Ok(()) + }; + + prepare_oci_workspace_with( + &missing, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + &fake_chown, + ) + .expect("missing OCI workspace should be created"); + + assert!(missing.is_dir()); + assert_eq!( + std::fs::symlink_metadata(missing.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o755 + ); + assert_eq!(*chowned.lock().unwrap(), vec![missing]); + } + + #[cfg(unix)] + #[test] + fn rewrite_passwd_modifies_existing_sandbox_entry() { + let dir = tempfile::tempdir().unwrap(); + let passwd = dir.path().join("passwd"); + std::fs::write( + &passwd, + "root:x:0:0:root:/root:/bin/bash\nsandbox:x:1000:1000::/sandbox:/bin/bash\n", + ) + .unwrap(); + + rewrite_passwd_at(&passwd, "5000", "6000").unwrap(); + + let content = std::fs::read_to_string(&passwd).unwrap(); + assert!(content.contains("sandbox:x:5000:6000::/sandbox:/bin/bash")); + assert!(content.contains("root:x:0:0:root:/root:/bin/bash")); + } + + #[cfg(unix)] + #[test] + fn rewrite_passwd_appends_when_no_sandbox_entry() { + let dir = tempfile::tempdir().unwrap(); + let passwd = dir.path().join("passwd"); + std::fs::write(&passwd, "root:x:0:0:root:/root:/bin/bash\n").unwrap(); + + rewrite_passwd_at(&passwd, "5000", "6000").unwrap(); + + let content = std::fs::read_to_string(&passwd).unwrap(); + assert!(content.contains("root:x:0:0:root:/root:/bin/bash")); + assert!(content.contains("sandbox:x:5000:6000::/sandbox:/bin/sh")); + } + + #[cfg(unix)] + #[test] + fn rewrite_group_modifies_existing_sandbox_entry() { + let dir = tempfile::tempdir().unwrap(); + let group = dir.path().join("group"); + std::fs::write(&group, "root:x:0:\nsandbox:x:1000:\n").unwrap(); + + rewrite_group_at(&group, "6000").unwrap(); + + let content = std::fs::read_to_string(&group).unwrap(); + assert!(content.contains("sandbox:x:6000:")); + assert!(content.contains("root:x:0:")); + } + + #[cfg(unix)] + #[test] + fn rewrite_group_appends_when_no_sandbox_entry() { + let dir = tempfile::tempdir().unwrap(); + let group = dir.path().join("group"); + std::fs::write(&group, "root:x:0:\n").unwrap(); + + rewrite_group_at(&group, "6000").unwrap(); + + let content = std::fs::read_to_string(&group).unwrap(); + assert!(content.contains("root:x:0:")); + assert!(content.contains("sandbox:x:6000:")); + } + + #[cfg(unix)] + #[test] + fn rewrite_passwd_leaves_malformed_entry_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let passwd = dir.path().join("passwd"); + // Only 3 fields — slice pattern should fall through instead of panic. + std::fs::write(&passwd, "sandbox:x:1000\n").unwrap(); + rewrite_passwd_at(&passwd, "5000", "6000").unwrap(); + let content = std::fs::read_to_string(&passwd).unwrap(); + assert!(content.contains("sandbox:x:1000")); + } + + #[cfg(unix)] + #[test] + fn rewrite_group_leaves_malformed_entry_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let group = dir.path().join("group"); + // Only 2 fields — slice pattern should fall through instead of panic. + std::fs::write(&group, "sandbox:x\n").unwrap(); + rewrite_group_at(&group, "6000").unwrap(); + let content = std::fs::read_to_string(&group).unwrap(); + assert!(content.contains("sandbox:x")); + } + + #[cfg(unix)] + #[test] + fn rewrite_passwd_preserves_other_entries() { + let dir = tempfile::tempdir().unwrap(); + let passwd = dir.path().join("passwd"); + std::fs::write( + &passwd, + "root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534:nobody:/:/usr/sbin/nologin\nsandbox:x:1000:1000::/sandbox:/bin/bash\n", + ) + .unwrap(); + + rewrite_passwd_at(&passwd, "1234567", "1234567").unwrap(); + + let content = std::fs::read_to_string(&passwd).unwrap(); + assert!(content.contains("root:x:0:0:root:/root:/bin/bash")); + assert!(content.contains("nobody:x:65534:65534:nobody:/:/usr/sbin/nologin")); + assert!(content.contains("sandbox:x:1234567:1234567::/sandbox:/bin/bash")); + assert_eq!(content.lines().count(), 3); + } + + #[tokio::test] + async fn inject_provider_env_skips_supervisor_identity_material() { + let mut cmd = Command::new("/usr/bin/env"); + cmd.env_clear() + .stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::null()); + + let provider_env = HashMap::from([ + ( + "ANTHROPIC_API_KEY".to_string(), + "openshell:resolve:env:ANTHROPIC_API_KEY".to_string(), + ), + ( + openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), + "provider-token".to_string(), + ), + ( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + "/spiffe-workload-api/spire-agent.sock".to_string(), + ), + ]); + + inject_provider_env(&mut cmd, &provider_env); + + let output = cmd.output().await.expect("spawn env"); + assert!(output.status.success()); + let stdout = String::from_utf8(output.stdout).expect("utf8"); + assert!(stdout.contains("ANTHROPIC_API_KEY=openshell:resolve:env:ANTHROPIC_API_KEY")); + assert!(!stdout.contains(openshell_core::sandbox_env::SANDBOX_TOKEN)); + assert!(!stdout.contains(openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET)); + } + + #[tokio::test] + async fn strip_supervisor_only_env_removes_identity_material() { + let mut cmd = Command::new("/usr/bin/env"); + cmd.stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::null()) + .env("OPENSHELL_ENDPOINT", "https://gateway.example.test"); + + for key in SUPERVISOR_ONLY_ENV_VARS { + cmd.env(key, format!("{key}-secret")); + } + + strip_supervisor_only_env(&mut cmd); + + let output = cmd.output().await.expect("spawn env"); + assert!(output.status.success()); + let stdout = String::from_utf8(output.stdout).expect("utf8"); + + for key in SUPERVISOR_ONLY_ENV_VARS { + assert!( + !stdout + .lines() + .any(|line| line.starts_with(&format!("{key}="))), + "{key} must not be inherited by sandbox child processes" + ); + } + assert!(stdout.contains("OPENSHELL_ENDPOINT=https://gateway.example.test")); + } + + #[tokio::test] + async fn transparent_mediation_removes_ambient_proxy_routing() { + let mut cmd = Command::new("/usr/bin/env"); + cmd.env_clear() + .stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::null()) + .env("PATH", "/usr/bin:/bin"); + for key in PROXY_ENV_VARS { + cmd.env(key, "http://ambient-proxy.invalid:3128"); + } + + strip_proxy_env(&mut cmd); + + let output = cmd.output().await.expect("spawn env"); + assert!(output.status.success()); + let stdout = String::from_utf8(output.stdout).expect("utf8"); + for key in PROXY_ENV_VARS { + assert!( + !stdout + .lines() + .any(|line| line.starts_with(&format!("{key}="))), + "{key} must not redirect a transparently mediated process" + ); + } + assert!(stdout.contains("PATH=/usr/bin:/bin")); + } + + // ---- Numeric UID tests (Phase 2) ---- + + #[test] + fn drop_privileges_accepts_numeric_uid() { + // When running as non-root, a numeric UID/GID that matches the + // current process should succeed without any passwd lookup. + if nix::unistd::geteuid().is_root() { + return; + } + + let uid_raw = nix::unistd::geteuid().as_raw(); + let gid_raw = nix::unistd::getegid().as_raw(); + + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(uid_raw.to_string()), + run_as_group: Some(gid_raw.to_string()), + }); + + assert!( + drop_privileges(&policy).is_ok(), + "should accept current process UID/GID as numeric strings" + ); + } + + #[test] + fn drop_privileges_numeric_uid_skips_initgroups() { + // When running as non-root with a numeric user but group matches, + // initgroups should not be called (guard: target_uid != geteuid()). + if nix::unistd::geteuid().is_root() { + return; + } + + let current_uid = nix::unistd::geteuid().as_raw(); + + // Use a different group name that exists (the current one). + let current_group = Group::from_gid(nix::unistd::getegid()) + .expect("should resolve current group") + .expect("current group should exist"); + + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(current_uid.to_string()), // numeric UID, no passwd entry needed + run_as_group: Some(current_group.name), // name-based group + }); + + assert!( + drop_privileges(&policy).is_ok(), + "should accept numeric UID with name-based group (initgroups guarded)" + ); + } + + #[test] + fn numeric_uid_privilege_drop_child() { + if std::env::var_os("OPENSHELL_TEST_NUMERIC_UID_CHILD").is_none() { + return; + } + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some("999999".into()), + run_as_group: Some("999999".into()), + }); + match drop_privileges(&policy) { + Ok(()) => {} + Err(e) => { + assert!( + !e.to_string().contains("Failed to resolve user record"), + "unexpected error for numeric UID without passwd entry: {e}" + ); + } + } + } + + #[test] + fn drop_privileges_numeric_uid_without_passwd_entry_skips_lookup() { + let mut cmd = std::process::Command::new(std::env::current_exe().expect("current exe")); + cmd.arg("numeric_uid_privilege_drop_child") + .arg("--nocapture") + .env("OPENSHELL_TEST_NUMERIC_UID_CHILD", "1") + .stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::piped()); + let output = cmd.output().expect("spawn child"); + assert!( + output.status.success(), + "numeric UID privilege drop child failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } +} diff --git a/crates/openshell-sandbox/src/pty.rs b/crates/openshell-sandbox/src/pty.rs new file mode 100644 index 0000000000..d887342306 --- /dev/null +++ b/crates/openshell-sandbox/src/pty.rs @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workload-side PTY and audited pre-exec setup. + +use std::os::fd::RawFd; +use std::process::Command; + +use nix::pty::Winsize; +use nix::unistd::setsid; +use openshell_core::policy::SandboxPolicy; +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; + +#[allow(unsafe_code)] +pub fn set_winsize(fd: RawFd, winsize: Winsize) -> std::io::Result<()> { + // SAFETY: fd is the owned PTY master and winsize is initialized. + let rc = unsafe { libc::ioctl(fd, libc::TIOCSWINSZ, &winsize) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +/// Install a pre-exec hook that gives the child a dedicated process group. +#[allow(unsafe_code)] +pub fn install_dedicated_process_group(command: &mut Command) { + // SAFETY: the hook invokes only the async-signal-safe setpgid syscall. + unsafe { + command.pre_exec(|| { + if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } +} + +#[allow(unsafe_code, clippy::useless_conversion)] +fn set_controlling_tty(fd: RawFd) -> std::io::Result<()> { + // SAFETY: fd is the slave PTY inherited by this pre-exec child. + let rc = unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[allow( + unsafe_code, + clippy::unnecessary_wraps, + reason = "pre-exec installation remains fallible as the prepared policy evolves" +)] +pub fn install_pre_exec( + command: &mut Command, + policy: SandboxPolicy, + _workdir: Option, + slave_fd: RawFd, + #[cfg(target_os = "linux")] prepared: Option, + #[cfg(target_os = "linux")] + child_hardening: openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + let mut prepared = prepared; + #[cfg(target_os = "linux")] + let mut child_hardening = child_hardening; + // SAFETY: all allocations and policy compilation happened before spawn; + // the hook performs only the audited child transition. + unsafe { + command.pre_exec(move || { + setsid().map_err(|error| std::io::Error::other(error.to_string()))?; + set_controlling_tty(slave_fd)?; + enter_sandbox( + &policy, + #[cfg(target_os = "linux")] + prepared.take(), + #[cfg(target_os = "linux")] + &mut child_hardening, + ) + }); + } + Ok(()) +} + +#[allow( + unsafe_code, + clippy::unnecessary_wraps, + reason = "pre-exec installation remains fallible as the prepared policy evolves" +)] +pub fn install_pre_exec_no_pty( + command: &mut Command, + policy: SandboxPolicy, + _workdir: Option, + #[cfg(target_os = "linux")] prepared: Option, + #[cfg(target_os = "linux")] + child_hardening: openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + let mut prepared = prepared; + #[cfg(target_os = "linux")] + let mut child_hardening = child_hardening; + // SAFETY: all allocations and policy compilation happened before spawn; + // the hook performs only the audited child transition. + unsafe { + command.pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } + enter_sandbox( + &policy, + #[cfg(target_os = "linux")] + prepared.take(), + #[cfg(target_os = "linux")] + &mut child_hardening, + ) + }); + } + Ok(()) +} + +fn enter_sandbox( + policy: &SandboxPolicy, + #[cfg(target_os = "linux")] prepared: Option, + #[cfg(target_os = "linux")] + child_hardening: &mut openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> std::io::Result<()> { + crate::process::harden_child_process() + .map_err(|error| std::io::Error::other(error.to_string()))?; + + #[cfg(target_os = "linux")] + if let Some(prepared) = prepared { + crate::sandbox::linux::enforce_capability_free(prepared, child_hardening) + .map_err(|error| std::io::Error::other(error.to_string()))?; + } + + #[cfg(not(target_os = "linux"))] + crate::sandbox::apply(policy, None) + .map_err(|error| std::io::Error::other(error.to_string()))?; + + #[cfg(target_os = "linux")] + let _ = policy; + + Ok(()) +} diff --git a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs new file mode 100644 index 0000000000..9d4502dc5f --- /dev/null +++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs @@ -0,0 +1,764 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Landlock filesystem sandboxing. + +use landlock::{ + ABI, Access, AccessFs, BitFlags, CompatLevel, Compatible, PathBeneath, PathFd, PathFdError, + Ruleset, RulesetAttr, RulesetCreatedAttr, +}; +use miette::{IntoDiagnostic, Result}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkPolicy, ProcessPolicy, + SandboxPolicy, +}; +use std::os::fd::AsFd; +use std::path::{Path, PathBuf}; +use tracing::debug; + +/// Result of probing the kernel for Landlock support. +#[derive(Debug)] +pub enum LandlockAvailability { + /// Landlock is available with the given ABI version. + Available { abi: i32 }, + /// Kernel does not implement Landlock (ENOSYS). + NotImplemented, + /// Landlock is compiled in but not enabled at boot (EOPNOTSUPP). + NotEnabled, + /// Landlock syscall is blocked, likely by a container seccomp profile (EPERM). + Blocked, + /// Unexpected error from the probe syscall. + Unknown(i32), +} + +impl std::fmt::Display for LandlockAvailability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Available { abi } => write!(f, "available (ABI v{abi})"), + Self::NotImplemented => { + write!(f, "not implemented (kernel lacks CONFIG_SECURITY_LANDLOCK)") + } + Self::NotEnabled => write!( + f, + "not enabled (Landlock built into kernel but not in active LSM list)" + ), + Self::Blocked => write!( + f, + "blocked (container seccomp profile denies Landlock syscalls)" + ), + Self::Unknown(errno) => write!(f, "unexpected probe error (errno {errno})"), + } + } +} + +/// Probe the kernel for Landlock support by issuing the `landlock_create_ruleset` +/// syscall with the version-check flag. +/// +/// This is safe to call from the parent process and does not create any file +/// descriptors or modify process state. +pub fn probe_availability() -> LandlockAvailability { + // landlock_create_ruleset syscall number (same on x86_64 and aarch64). + const SYS_LANDLOCK_CREATE_RULESET: libc::c_long = 444; + // Flag: return the highest supported ABI version instead of creating a ruleset. + const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1 << 0; + + // SAFETY: landlock_create_ruleset(NULL, 0, LANDLOCK_CREATE_RULESET_VERSION) + // is a read-only probe that returns the ABI version or an error code. + // It does not allocate file descriptors or modify process state. + #[allow(unsafe_code)] + let ret = unsafe { + libc::syscall( + SYS_LANDLOCK_CREATE_RULESET, + std::ptr::null::(), + 0_usize, + LANDLOCK_CREATE_RULESET_VERSION, + ) + }; + + if ret >= 0 { + #[allow(clippy::cast_possible_truncation)] + LandlockAvailability::Available { abi: ret as i32 } + } else { + let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + match errno { + libc::ENOSYS => LandlockAvailability::NotImplemented, + libc::EOPNOTSUPP => LandlockAvailability::NotEnabled, + libc::EPERM => LandlockAvailability::Blocked, + other => LandlockAvailability::Unknown(other), + } + } +} + +/// A prepared Landlock ruleset ready to be enforced via `restrict_self()`. +/// +/// Created by [`prepare`] while running as root (so `PathFd::new()` can open +/// any path regardless of DAC permissions). Enforced by [`enforce`] after +/// `drop_privileges()` — `restrict_self()` does not require elevated privileges. +pub struct PreparedRuleset { + ruleset: landlock::RulesetCreated, + compatibility: LandlockCompatibility, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PathOpenMode { + Privileged, + CurrentUser, +} + +/// Phase 1: Open `PathFds` and build the Landlock ruleset **as root**. +/// +/// This must run before `drop_privileges()` so that `PathFd::new()` can open +/// paths that are only accessible to root (e.g. mode 700 directories). +/// +/// Returns `None` if there are no filesystem paths to restrict (no-op). +/// Returns `Some(PreparedRuleset)` on success, or an error. +pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result> { + prepare_with_path_open_mode(policy, workdir, PathOpenMode::Privileged) +} + +/// Phase 1 for already-unprivileged workloads. +/// +/// The sandbox boundary starts as the workload UID, so Landlock path FDs are +/// opened as the same UID that will run the workload. +/// Paths this UID cannot open are already unavailable to the workload; omit +/// them from the allowlist and let the resulting ruleset deny everything else. +pub fn prepare_current_user( + policy: &SandboxPolicy, + workdir: Option<&str>, +) -> Result> { + prepare_with_path_open_mode(policy, workdir, PathOpenMode::CurrentUser) +} + +/// Build the mandatory same-UID self-protection baseline. +/// +/// Landlock is allow-list only. Granting `/` would also grant the protected +/// `/.openshell` subtree, so enumerate the root's children and omit that one +/// hierarchy. Entries the final UID cannot open are already inaccessible and +/// are safely omitted by [`PathOpenMode::CurrentUser`]. +pub fn prepare_capability_free_baseline() -> Result { + let read_write = capability_free_baseline_paths(Path::new("/"))?; + if read_write.is_empty() { + return Err(miette::miette!( + "capability-free Landlock baseline found no usable root entries" + )); + } + + let policy = SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy { + read_only: Vec::new(), + read_write, + include_workdir: false, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy { + compatibility: LandlockCompatibility::HardRequirement, + }, + process: ProcessPolicy::default(), + }; + prepare_with_path_open_mode(&policy, None, PathOpenMode::CurrentUser)?.ok_or_else(|| { + miette::miette!("capability-free Landlock baseline unexpectedly produced no ruleset") + }) +} + +fn capability_free_baseline_paths(root: &Path) -> Result> { + const PRIVATE_ROOT: &str = ".openshell"; + + let mut paths = Vec::new(); + for entry in std::fs::read_dir(root).into_diagnostic()? { + let entry = entry.into_diagnostic()?; + if entry.file_name() != PRIVATE_ROOT { + paths.push(entry.path()); + } + } + paths.sort(); + Ok(paths) +} + +fn prepare_with_path_open_mode( + policy: &SandboxPolicy, + workdir: Option<&str>, + path_open_mode: PathOpenMode, +) -> Result> { + let read_only = policy.filesystem.read_only.clone(); + let mut read_write = policy.filesystem.read_write.clone(); + + if policy.filesystem.include_workdir + && let Some(dir) = workdir + { + let workdir_path = PathBuf::from(dir); + if !read_write.contains(&workdir_path) { + read_write.push(workdir_path); + } + } + + if read_only.is_empty() && read_write.is_empty() { + return Ok(None); + } + + let compatibility = &policy.landlock.compatibility; + + // Probe first: kernels without Landlock (e.g. gVisor's sentry returns + // ENOSYS) would otherwise log misleading "Applying"+"Built" events. + let availability = probe_availability(); + if !matches!(availability, LandlockAvailability::Available { .. }) { + match compatibility { + LandlockCompatibility::BestEffort => { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .severity(openshell_ocsf::SeverityId::High) + .confidence(openshell_ocsf::ConfidenceId::High) + .is_alert(true) + .finding_info( + openshell_ocsf::FindingInfo::new( + "landlock-unavailable", + "Landlock Filesystem Sandbox Unavailable", + ) + .with_desc(&format!( + "Running WITHOUT filesystem restrictions: Landlock is {availability}. \ + Set landlock.compatibility to 'hard_requirement' to make this fatal." + )), + ) + .message(format!( + "Landlock filesystem sandbox unavailable: {availability}" + )) + .build() + ); + return Ok(None); + } + LandlockCompatibility::HardRequirement => { + return Err(miette::miette!( + "Landlock unavailable in hard_requirement mode: {availability}" + )); + } + } + } + + let total_paths = read_only.len() + read_write.len(); + let abi = ABI::V2; + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "applying") + .message(format!( + "Applying Landlock filesystem sandbox [abi:{abi:?} compat:{:?} ro:{} rw:{}]", + policy.landlock.compatibility, + read_only.len(), + read_write.len(), + )) + .build() + ); + + let result: Result = (|| { + let access_all = AccessFs::from_all(abi); + let access_read = AccessFs::from_read(abi); + + let mut ruleset = Ruleset::default(); + ruleset = ruleset + .set_compatibility(compat_level(compatibility)) + .handle_access(access_all) + .into_diagnostic()?; + + let mut ruleset = ruleset.create().into_diagnostic()?; + let mut rules_applied: usize = 0; + + for path in &read_only { + if let Some(path_fd) = try_open_path(path, compatibility, path_open_mode)? { + let allowed_access = access_for_path_fd(&path_fd, access_read, abi)?; + debug!(path = %path.display(), "Landlock allow read-only"); + ruleset = ruleset + .add_rule(PathBeneath::new(path_fd, allowed_access)) + .into_diagnostic()?; + rules_applied += 1; + } + } + + for path in &read_write { + if let Some(path_fd) = try_open_path(path, compatibility, path_open_mode)? { + let allowed_access = access_for_path_fd(&path_fd, access_all, abi)?; + debug!(path = %path.display(), "Landlock allow read-write"); + ruleset = ruleset + .add_rule(PathBeneath::new(path_fd, allowed_access)) + .into_diagnostic()?; + rules_applied += 1; + } + } + + if rules_applied == 0 { + return Err(miette::miette!( + "Landlock ruleset has zero valid paths — all {} path(s) failed to open. \ + Refusing to apply an empty ruleset that would block all filesystem access.", + total_paths, + )); + } + + let skipped = total_paths - rules_applied; + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "built") + .message(format!( + "Landlock ruleset built [rules_applied:{rules_applied} skipped:{skipped}]" + )) + .build() + ); + + Ok(PreparedRuleset { + ruleset, + compatibility: compatibility.clone(), + }) + })(); + + match result { + Ok(prepared) => Ok(Some(prepared)), + Err(err) => { + if matches!(compatibility, LandlockCompatibility::BestEffort) { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .severity(openshell_ocsf::SeverityId::High) + .confidence(openshell_ocsf::ConfidenceId::High) + .is_alert(true) + .finding_info( + openshell_ocsf::FindingInfo::new( + "landlock-unavailable", + "Landlock Filesystem Sandbox Unavailable", + ) + .with_desc(&format!( + "Running WITHOUT filesystem restrictions: {err}. \ + Set landlock.compatibility to 'hard_requirement' to make this fatal." + )), + ) + .message(format!("Landlock filesystem sandbox unavailable: {err}")) + .build() + ); + Ok(None) + } else { + Err(err) + } + } + } +} + +/// Phase 2: Enforce a prepared Landlock ruleset by calling `restrict_self()`. +/// +/// This runs **after** `drop_privileges()`. The `restrict_self()` syscall does +/// not require root — it only restricts the calling thread (and its future +/// children), which is always permitted. +/// +/// Respects the same `best_effort` / `hard_requirement` compatibility as +/// [`prepare`]: if `restrict_self()` fails and the policy is `best_effort`, +/// the error is logged and the sandbox continues without Landlock. +pub fn enforce(prepared: PreparedRuleset) -> Result<()> { + let result = prepared.ruleset.restrict_self().into_diagnostic(); + if let Err(err) = result { + if matches!(prepared.compatibility, LandlockCompatibility::BestEffort) { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .severity(openshell_ocsf::SeverityId::High) + .confidence(openshell_ocsf::ConfidenceId::High) + .is_alert(true) + .finding_info( + openshell_ocsf::FindingInfo::new( + "landlock-enforce-failed", + "Landlock restrict_self Failed", + ) + .with_desc(&format!( + "Ruleset was prepared but restrict_self() failed: {err}. \ + Running WITHOUT filesystem restrictions. \ + Set landlock.compatibility to 'hard_requirement' to make this fatal." + )), + ) + .message(format!( + "Landlock restrict_self failed (best_effort): {err}" + )) + .build() + ); + return Ok(()); + } + return Err(err); + } + Ok(()) +} + +/// Legacy single-phase apply. Kept for non-Linux platforms and tests. +/// On Linux, callers should use [`prepare`] + [`enforce`] for correct +/// privilege ordering. +#[allow(dead_code)] // Retained for backward compat; live callers use prepare+enforce. +pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { + if let Some(prepared) = prepare(policy, workdir)? { + enforce(prepared)?; + } + Ok(()) +} + +/// Tailor a rule's access mask to the inode referenced by its already-open FD. +/// +/// Landlock directory-only rights such as `ReadDir` are invalid for regular +/// files and device nodes in hard-requirement mode. Classifying through the +/// same `PathFd` used by the rule avoids a pathname TOCTOU race. +fn access_for_path_fd( + path_fd: &PathFd, + requested_access: BitFlags, + abi: ABI, +) -> Result> { + let stat = rustix::fs::fstat(path_fd.as_fd()).into_diagnostic()?; + Ok(match rustix::fs::FileType::from_raw_mode(stat.st_mode) { + rustix::fs::FileType::Directory => requested_access, + _ => requested_access & AccessFs::from_file(abi), + }) +} + +/// Attempt to open a path for Landlock rule creation. +/// +/// In `BestEffort` mode, inaccessible paths (missing, permission denied, symlink +/// loops, etc.) are skipped with a warning and `Ok(None)` is returned so the +/// caller can continue building the ruleset from the remaining valid paths. +/// +/// In `HardRequirement` mode, any failure is fatal — the caller propagates the +/// error, which ultimately aborts sandbox startup. +fn try_open_path( + path: &Path, + compatibility: &LandlockCompatibility, + path_open_mode: PathOpenMode, +) -> Result> { + match PathFd::new(path) { + Ok(fd) => Ok(Some(fd)), + Err(err) => { + let reason = classify_path_fd_error(&err); + let is_not_found = matches!( + &err, + PathFdError::OpenCall { source, .. } + if source.kind() == std::io::ErrorKind::NotFound + ); + if matches!(path_open_mode, PathOpenMode::CurrentUser) { + if is_not_found { + debug!( + path = %path.display(), + reason, + "Skipping non-existent Landlock path for current user" + ); + } else { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Other, "already-denied") + .message(format!( + "Skipping inaccessible Landlock path for current user [path:{} error:{err}]", + path.display() + )) + .build() + ); + } + return Ok(None); + } + match compatibility { + LandlockCompatibility::BestEffort => { + // NotFound is expected for stale baseline paths (e.g. + // /app baked into the server-stored policy but absent + // in this container image). Log at debug! to avoid + // polluting SSH exec stdout — the pre_exec hook + // inherits the tracing subscriber whose writer targets + // fd 1 (the pipe/PTY). + // + // Other errors (permission denied, symlink loops, etc.) + // are genuinely unexpected and logged at warn!. + if is_not_found { + debug!( + path = %path.display(), + reason, + "Skipping non-existent Landlock path (best-effort mode)" + ); + } else { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Medium) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Other, "degraded") + .message(format!( + "Skipping inaccessible Landlock path (best-effort) [path:{} error:{err}]", + path.display() + )) + .build() + ); + } + Ok(None) + } + LandlockCompatibility::HardRequirement => Err(miette::miette!( + "Landlock path unavailable in hard_requirement mode: {} ({}): {}", + path.display(), + reason, + err, + )), + } + } + } +} + +/// Classify a [`PathFdError`] into a human-readable reason. +/// +/// `PathFd::new()` wraps `open(path, O_PATH | O_CLOEXEC)` which can fail for +/// several reasons beyond simple non-existence. The `PathFdError::OpenCall` +/// variant wraps the underlying `std::io::Error`. +fn classify_path_fd_error(err: &PathFdError) -> &'static str { + match err { + PathFdError::OpenCall { source, .. } => classify_io_error(source), + // PathFdError is #[non_exhaustive], handle future variants gracefully. + _ => "unexpected error", + } +} + +/// Classify a `std::io::Error` into a human-readable reason string. +fn classify_io_error(err: &std::io::Error) -> &'static str { + match err.kind() { + std::io::ErrorKind::NotFound => "path does not exist", + std::io::ErrorKind::PermissionDenied => "permission denied", + _ => match err.raw_os_error() { + Some(40) => "too many symlink levels", // ELOOP + Some(36) => "path name too long", // ENAMETOOLONG + Some(20) => "path component is not a directory", // ENOTDIR + _ => "unexpected error", + }, + } +} + +fn compat_level(level: &LandlockCompatibility) -> CompatLevel { + match level { + LandlockCompatibility::BestEffort => CompatLevel::BestEffort, + LandlockCompatibility::HardRequirement => CompatLevel::HardRequirement, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy}; + + fn hard_requirement_policy(read_only: Vec, read_write: Vec) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only, + read_write, + include_workdir: false, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy { + compatibility: LandlockCompatibility::HardRequirement, + }, + process: ProcessPolicy::default(), + } + } + + #[test] + fn prepare_hard_requirement_accepts_device_paths() { + if !matches!(probe_availability(), LandlockAvailability::Available { .. }) { + return; + } + + let policy = hard_requirement_policy( + vec![PathBuf::from("/tmp"), PathBuf::from("/dev/urandom")], + vec![PathBuf::from("/dev/null")], + ); + + let result = prepare(&policy, None); + if let Err(err) = result { + panic!("hard_requirement should accept mixed directory and device paths: {err}"); + } + } + + #[test] + fn capability_free_baseline_omits_only_private_root() { + let root = tempfile::tempdir().unwrap(); + for name in ["bin", "etc", "sandbox", ".openshell"] { + std::fs::create_dir(root.path().join(name)).unwrap(); + } + + let paths = capability_free_baseline_paths(root.path()).unwrap(); + assert_eq!( + paths, + ["bin", "etc", "sandbox"] + .map(|name| root.path().join(name)) + .to_vec() + ); + } + fn tailored_access(path: &Path, requested_access: BitFlags) -> BitFlags { + let path_fd = PathFd::new(path).unwrap(); + access_for_path_fd(&path_fd, requested_access, ABI::V2).unwrap() + } + + #[test] + fn access_for_path_fd_preserves_directory_access() { + let dir = tempfile::tempdir().unwrap(); + let requested_access = AccessFs::from_all(ABI::V2); + + assert_eq!( + tailored_access(dir.path(), requested_access), + requested_access + ); + } + + #[test] + fn access_for_path_fd_limits_regular_file_access() { + let file = tempfile::NamedTempFile::new().unwrap(); + let requested_access = AccessFs::from_all(ABI::V2); + + assert_eq!( + tailored_access(file.path(), requested_access), + requested_access & AccessFs::from_file(ABI::V2) + ); + } + + #[test] + fn access_for_path_fd_limits_character_device_access() { + let requested_read = AccessFs::from_read(ABI::V2); + let requested_write = AccessFs::from_all(ABI::V2); + + assert_eq!( + tailored_access(Path::new("/dev/urandom"), requested_read), + requested_read & AccessFs::from_file(ABI::V2) + ); + assert_eq!( + tailored_access(Path::new("/dev/null"), requested_write), + requested_write & AccessFs::from_file(ABI::V2) + ); + } + + #[test] + fn access_for_path_fd_classifies_symlink_target() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("target"); + let link = dir.path().join("link"); + std::fs::File::create(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let requested_access = AccessFs::from_all(ABI::V2); + assert_eq!( + tailored_access(&link, requested_access), + requested_access & AccessFs::from_file(ABI::V2) + ); + } + + #[test] + fn try_open_path_best_effort_returns_none_for_missing_path() { + let result = try_open_path( + &PathBuf::from("/nonexistent/openshell/test/path"), + &LandlockCompatibility::BestEffort, + PathOpenMode::Privileged, + ); + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + } + + #[test] + fn try_open_path_hard_requirement_errors_for_missing_path() { + let result = try_open_path( + &PathBuf::from("/nonexistent/openshell/test/path"), + &LandlockCompatibility::HardRequirement, + PathOpenMode::Privileged, + ); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("hard_requirement"), + "error should mention hard_requirement mode: {err_msg}" + ); + assert!( + err_msg.contains("does not exist"), + "error should include the classified reason: {err_msg}" + ); + } + + #[test] + fn try_open_path_succeeds_for_existing_path() { + let dir = tempfile::tempdir().unwrap(); + let result = try_open_path( + dir.path(), + &LandlockCompatibility::BestEffort, + PathOpenMode::Privileged, + ); + assert!(result.is_ok()); + assert!(result.unwrap().is_some()); + } + + #[test] + fn try_open_path_current_user_skips_missing_path_in_hard_requirement() { + let result = try_open_path( + &PathBuf::from("/nonexistent/openshell/test/path"), + &LandlockCompatibility::HardRequirement, + PathOpenMode::CurrentUser, + ); + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + } + + #[test] + fn classify_not_found() { + let err = std::io::Error::from_raw_os_error(libc::ENOENT); + assert_eq!(classify_io_error(&err), "path does not exist"); + } + + #[test] + fn classify_permission_denied() { + let err = std::io::Error::from_raw_os_error(libc::EACCES); + assert_eq!(classify_io_error(&err), "permission denied"); + } + + #[test] + fn classify_symlink_loop() { + let err = std::io::Error::from_raw_os_error(libc::ELOOP); + assert_eq!(classify_io_error(&err), "too many symlink levels"); + } + + #[test] + fn classify_name_too_long() { + let err = std::io::Error::from_raw_os_error(libc::ENAMETOOLONG); + assert_eq!(classify_io_error(&err), "path name too long"); + } + + #[test] + fn classify_not_a_directory() { + let err = std::io::Error::from_raw_os_error(libc::ENOTDIR); + assert_eq!(classify_io_error(&err), "path component is not a directory"); + } + + #[test] + fn classify_unknown_error() { + let err = std::io::Error::from_raw_os_error(libc::EIO); + assert_eq!(classify_io_error(&err), "unexpected error"); + } + + #[test] + fn classify_path_fd_error_extracts_io_error() { + // Use PathFd::new on a non-existent path to get a real PathFdError + // (the OpenCall variant is #[non_exhaustive] and can't be constructed directly). + let err = PathFd::new("/nonexistent/openshell/classify/test").unwrap_err(); + assert_eq!(classify_path_fd_error(&err), "path does not exist"); + } + + #[test] + fn probe_availability_returns_a_result() { + // The probe should not panic regardless of whether Landlock is available. + // On Linux hosts with Landlock, this returns Available; on Docker Desktop + // linuxkit or older kernels, it returns NotImplemented/NotEnabled/Blocked. + let result = probe_availability(); + let display = format!("{result}"); + assert!( + !display.is_empty(), + "probe_availability Display should produce output" + ); + // Verify the Debug impl works too. + let debug = format!("{result:?}"); + assert!( + !debug.is_empty(), + "probe_availability Debug should produce output" + ); + } +} diff --git a/crates/openshell-sandbox/src/sandbox/linux/mod.rs b/crates/openshell-sandbox/src/sandbox/linux/mod.rs new file mode 100644 index 0000000000..523d33bd0c --- /dev/null +++ b/crates/openshell-sandbox/src/sandbox/linux/mod.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Linux sandbox implementation using Landlock and seccomp. + +mod landlock; +mod seccomp; + +use miette::Result; +use openshell_core::policy::SandboxPolicy; +use std::path::PathBuf; +use std::sync::Once; + +/// Opaque handle to a prepared-but-not-yet-enforced sandbox. +/// Holds the Landlock ruleset with `PathFds` opened before child exec. +pub struct PreparedSandbox { + landlock: Vec, + policy: SandboxPolicy, +} + +/// Phase 1: Prepare sandbox restrictions **as root** (before `drop_privileges`). +/// +/// Opens Landlock `PathFds` while the process still has root privileges, +/// ensuring paths like mode-700 directories are accessible. +pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result { + let landlock = landlock::prepare(policy, workdir)?; + Ok(PreparedSandbox { + landlock: landlock.into_iter().collect(), + policy: policy.clone(), + }) +} + +/// Phase 1 for already-unprivileged workloads. +/// +/// Opens Landlock `PathFds` as the current workload UID. +pub fn prepare_current_user( + policy: &SandboxPolicy, + workdir: Option<&str>, +) -> Result { + let landlock = landlock::prepare_current_user(policy, workdir)?; + Ok(PreparedSandbox { + landlock: landlock.into_iter().collect(), + policy: policy.clone(), + }) +} + +/// Prepare the mandatory capability-free filesystem baseline plus the +/// optional user policy. +/// +/// The baseline is always a hard requirement. It grants access to each +/// top-level filesystem entry independently while deliberately omitting the +/// driver-owned `/.openshell` hierarchy. Applying the user ruleset after the +/// baseline intersects the two policies; it can narrow the baseline but can +/// never make the private hierarchy visible. +pub fn prepare_capability_free( + policy: &SandboxPolicy, + workdir: Option<&str>, +) -> Result { + let baseline = landlock::prepare_capability_free_baseline()?; + let user = landlock::prepare_current_user(policy, workdir)?; + let mut landlock = vec![baseline]; + landlock.extend(user); + Ok(PreparedSandbox { + landlock, + policy: policy.clone(), + }) +} + +/// Phase 2: Enforce prepared sandbox restrictions (after `drop_privileges`). +/// +/// Calls `restrict_self()` for Landlock and applies seccomp filters. +/// Neither operation requires root privileges. +pub fn enforce(prepared: PreparedSandbox) -> Result<()> { + for ruleset in prepared.landlock { + landlock::enforce(ruleset)?; + } + seccomp::apply(&prepared.policy)?; + Ok(()) +} + +/// Enforce the capability-free child filter stack. +/// +/// Landlock precedes sandbox-TGID self-protection. The ordinary workload +/// filter is installed last. The final filter blocks any later seccomp +/// installation, so this order is mandatory for capability-free children. +pub fn enforce_capability_free( + prepared: PreparedSandbox, + child_hardening: &mut openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> Result<()> { + for ruleset in prepared.landlock { + landlock::enforce(ruleset)?; + } + child_hardening + .install() + .map_err(|error| miette::miette!("install child self-protection filter: {error}"))?; + seccomp::apply(&prepared.policy)?; + Ok(()) +} + +/// Apply the supervisor seccomp prelude after privileged bootstrap completes. +pub fn apply_supervisor_prelude() -> Result<()> { + seccomp::apply_supervisor_prelude() +} + +/// Legacy single-phase apply. Kept for backward compatibility. +/// New callers should use [`prepare`] + [`enforce`] for correct privilege ordering. +#[allow(dead_code)] // Retained for backward compat; live callers use prepare+enforce. +pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { + landlock::apply(policy, workdir)?; + seccomp::apply(policy)?; + Ok(()) +} + +/// Probe Landlock availability and emit OCSF logs from the parent process. +/// +/// This must be called **before** `pre_exec` / `fork()` so that the OCSF events +/// are emitted through the parent's tracing subscriber (the child process after +/// fork does not have a working tracing pipeline). +pub fn log_sandbox_readiness(policy: &SandboxPolicy, workdir: Option<&str>) { + static PROBED: Once = Once::new(); + let mut already_probed = true; + PROBED.call_once(|| already_probed = false); + if already_probed { + return; + } + + let mut read_write = policy.filesystem.read_write.clone(); + let read_only = &policy.filesystem.read_only; + + if policy.filesystem.include_workdir + && let Some(dir) = workdir + { + let workdir_path = PathBuf::from(dir); + if !read_write.contains(&workdir_path) { + read_write.push(workdir_path); + } + } + + let total_paths = read_only.len() + read_write.len(); + + if total_paths == 0 { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Other, "skipped") + .message("Landlock filesystem sandbox skipped: no paths configured".to_string()) + .build() + ); + return; + } + + let availability = landlock::probe_availability(); + if let landlock::LandlockAvailability::Available { abi } = &availability { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "probed") + .message(format!( + "Landlock filesystem sandbox available \ + [abi:v{abi} compat:{:?} ro:{} rw:{}]", + policy.landlock.compatibility, + read_only.len(), + read_write.len(), + )) + .build() + ); + } else { + // Landlock is NOT available — this is the critical log that was + // previously invisible because it only fired inside pre_exec. + let is_best_effort = matches!( + policy.landlock.compatibility, + openshell_core::policy::LandlockCompatibility::BestEffort + ); + let (desc, msg) = if is_best_effort { + ( + format!( + "Sandbox will run WITHOUT filesystem restrictions: {availability}. \ + Policy requests {total_paths} path rule(s) \ + (ro:{} rw:{}) but Landlock cannot enforce them. \ + Set landlock.compatibility to 'hard_requirement' to make this fatal.", + read_only.len(), + read_write.len(), + ), + format!( + "Landlock filesystem sandbox unavailable (best_effort, degraded): {availability}" + ), + ) + } else { + ( + format!( + "Landlock is unavailable: {availability}. \ + Policy requires {total_paths} path rule(s) \ + (ro:{} rw:{}) with hard_requirement — sandbox startup will fail.", + read_only.len(), + read_write.len(), + ), + format!( + "Landlock filesystem sandbox unavailable (hard_requirement, will fail): {availability}" + ), + ) + }; + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .severity(openshell_ocsf::SeverityId::High) + .confidence(openshell_ocsf::ConfidenceId::High) + .is_alert(true) + .finding_info( + openshell_ocsf::FindingInfo::new( + "landlock-unavailable", + "Landlock Filesystem Sandbox Unavailable", + ) + .with_desc(&desc), + ) + .message(msg) + .build() + ); + } +} diff --git a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs new file mode 100644 index 0000000000..f3da87aeef --- /dev/null +++ b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs @@ -0,0 +1,892 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Seccomp syscall filtering. +//! +//! The filter uses a default-allow policy with targeted blocks: +//! +//! 1. **Socket domain blocks** -- prevent raw/kernel sockets that bypass the proxy +//! 2. **Unconditional syscall blocks** -- block syscalls that enable sandbox escape +//! (fileless exec, ptrace, BPF, cross-process memory access, `io_uring`, mount) +//! 3. **Conditional syscall blocks** -- block dangerous flag combinations on otherwise +//! needed syscalls (`execveat+AT_EMPTY_PATH`, `unshare+CLONE_NEWUSER`, +//! `seccomp+SET_MODE_FILTER`) +//! +//! ## `AF_NETLINK` policy +//! +//! `AF_NETLINK` sockets are allowed **only** for the `NETLINK_ROUTE` protocol +//! (protocol value 0). All other netlink protocols are blocked with `EPERM`. +//! +//! `NETLINK_ROUTE` is required by `getifaddrs(3)` on Linux (used by Node.js, +//! Python, Go, and many HTTP/gRPC client libraries during startup). Without it +//! those runtimes fail to enumerate network interfaces even when they have no +//! intent to modify them. +//! +//! The risk is contained by existing sandbox layers: +//! - **Privilege drop**: `CAP_NET_ADMIN` is not granted, so all write operations +//! (add/delete routes, addresses, interfaces) fail with `EPERM` regardless. +//! - **Driver outer fence**: direct workload egress is rejected outside this +//! process by Docker network-none, Kubernetes `NetworkPolicy`, or a NIC-less +//! VM. +//! +//! Every other netlink protocol (`NETLINK_SOCK_DIAG`, `NETLINK_NETFILTER`, +//! `NETLINK_AUDIT`, `NETLINK_XFRM`, `NETLINK_GENERIC`, etc.) remains blocked. + +use miette::{IntoDiagnostic, Result}; +use openshell_core::policy::{NetworkMode, SandboxPolicy}; +use seccompiler::{ + SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, SeccompRule, + apply_filter, apply_filter_all_threads, +}; +use std::collections::BTreeMap; +use std::convert::TryInto; +use tracing::debug; + +/// Value of `SECCOMP_SET_MODE_FILTER` (linux/seccomp.h). +const SECCOMP_SET_MODE_FILTER: u64 = 1; + +// libc 0.2.185 omits `SYS_kexec_file_load` from the musl/aarch64 bindings even +// though the kernel exposes syscall 294. Fall back to the literal so the +// supervisor's seccomp filter still blocks fileless kernel-image loads when +// built statically against musl on aarch64. +#[cfg(all(target_arch = "aarch64", target_env = "musl"))] +#[allow(non_upper_case_globals)] +const SYS_kexec_file_load: libc::c_long = 294; +#[cfg(not(all(target_arch = "aarch64", target_env = "musl")))] +use libc::SYS_kexec_file_load; + +/// Apply the supervisor seccomp filter across the running process. +/// +/// This runs after privileged startup helpers complete and synchronizes the +/// filter across all supervisor threads via TSYNC. It intentionally blocks +/// only the privileged escape primitives that the long-lived supervisor no +/// longer needs once bootstrap is complete. +pub fn apply_supervisor_prelude() -> Result<()> { + let filter = build_supervisor_prelude_filter()?; + set_no_new_privs()?; + apply_filter_all_threads(&filter).into_diagnostic()?; + Ok(()) +} + +pub fn apply(policy: &SandboxPolicy) -> Result<()> { + let allow_inet = matches!(policy.network.mode, NetworkMode::Proxy | NetworkMode::Allow); + let main_filter = build_filter(allow_inet)?; + let compatibility_filter = build_compatibility_filter()?; + + set_no_new_privs()?; + apply_runtime_filters(&main_filter, &compatibility_filter)?; + + Ok(()) +} + +fn build_filter(allow_inet: bool) -> Result { + let rules = build_filter_rules(allow_inet)?; + compile_filter(rules, SeccompAction::Errno(libc::EPERM as u32)) +} + +fn build_supervisor_prelude_filter() -> Result { + compile_filter( + build_supervisor_prelude_rules(), + SeccompAction::Errno(libc::EPERM as u32), + ) +} + +fn build_supervisor_prelude_rules() -> BTreeMap> { + let mut rules: BTreeMap> = BTreeMap::new(); + + for syscall in [ + libc::SYS_mount, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_pivot_root, + libc::SYS_umount2, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_kexec_load, + SYS_kexec_file_load, + ] { + rules.entry(syscall).or_default(); + } + + rules +} + +fn set_no_new_privs() -> Result<()> { + // libc/syscall FFI requires unsafe + #[allow(unsafe_code)] + let rc = unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }; + if rc != 0 { + return Err(miette::miette!( + "Failed to set no_new_privs: {}", + std::io::Error::last_os_error() + )); + } + + Ok(()) +} + +fn compile_filter( + rules: BTreeMap>, + blocked_action: SeccompAction, +) -> Result { + let arch = std::env::consts::ARCH + .try_into() + .map_err(|_| miette::miette!("Unsupported architecture for seccomp"))?; + + let filter = + SeccompFilter::new(rules, SeccompAction::Allow, blocked_action, arch).into_diagnostic()?; + + filter.try_into().into_diagnostic() +} + +/// Build a minimal BPF filter for unavailable process APIs. +/// +/// This is a separate filter from the main one because seccomp BPF cannot +/// dereference the `struct clone_args *` pointer that clone3 takes as arg 0, +/// so we cannot selectively block `CLONE_NEWUSER`. We block clone3 +/// unconditionally with ENOSYS so glibc falls back to the older clone +/// syscall (where flags are a direct register argument and CAN be filtered). +/// +/// glibc's clone3 wrapper and process launchers that opportunistically use +/// pidfds check for ENOSYS specifically. EPERM is treated as a hard policy +/// failure instead of triggering their portable fallback paths. +fn build_compatibility_filter() -> Result { + let mut rules: BTreeMap> = BTreeMap::new(); + rules.entry(libc::SYS_clone3).or_default(); + rules.entry(libc::SYS_pidfd_open).or_default(); + compile_filter(rules, SeccompAction::Errno(libc::ENOSYS as u32)) +} + +/// Install the sandbox seccomp filters in the required order. +/// +/// Order matters: +/// 1. Install the compatibility filter first so it can still call +/// `seccomp(SECCOMP_SET_MODE_FILTER)`. +/// 2. Install the main filter second. It blocks further seccomp filter +/// installation with `EPERM`, preserving the original hardening intent. +fn apply_runtime_filters( + main_filter: seccompiler::BpfProgramRef<'_>, + compatibility_filter: seccompiler::BpfProgramRef<'_>, +) -> Result<()> { + apply_filter(compatibility_filter).into_diagnostic()?; + apply_filter(main_filter).into_diagnostic()?; + Ok(()) +} + +fn build_filter_rules(allow_inet: bool) -> Result>> { + let mut rules: BTreeMap> = BTreeMap::new(); + + // --- Socket domain blocks --- + let mut blocked_domains = vec![ + libc::AF_PACKET, + libc::AF_BLUETOOTH, + libc::AF_VSOCK, + // AF_NETLINK is handled separately below: NETLINK_ROUTE (protocol 0) + // is allowed for getifaddrs(3); all other netlink protocols are blocked. + ]; + if !allow_inet { + blocked_domains.push(libc::AF_INET); + blocked_domains.push(libc::AF_INET6); + } + + for domain in blocked_domains { + debug!(domain, "Blocking socket domain via seccomp"); + add_socket_domain_rule(&mut rules, domain)?; + } + + // Allow AF_NETLINK only for NETLINK_ROUTE (protocol 0). + // + // NETLINK_ROUTE is needed by getifaddrs(3) which is called by Node.js, + // Python, Go, and many HTTP/gRPC client libraries during startup to + // enumerate local network interfaces. Blocking it causes runtime errors + // such as "getifaddrs returned an error" in tools like Claude Code. + // + // The rule blocks socket(AF_NETLINK, *, protocol) for any protocol != 0. + // Write operations via NETLINK_ROUTE still require CAP_NET_ADMIN, which + // the sandbox does not grant, so interface/route modification is not possible. + add_netlink_non_route_rule(&mut rules)?; + + // --- Unconditional syscall blocks --- + // These syscalls are blocked entirely (empty rule vec = unconditional EPERM). + + // Fileless binary execution via memfd bypasses Landlock filesystem restrictions. + rules.entry(libc::SYS_memfd_create).or_default(); + // Cross-process memory inspection and code injection. + rules.entry(libc::SYS_ptrace).or_default(); + // Kernel BPF program loading. + rules.entry(libc::SYS_bpf).or_default(); + // Cross-process memory read. + rules.entry(libc::SYS_process_vm_readv).or_default(); + // Cross-process memory write (symmetric with process_vm_readv). + rules.entry(libc::SYS_process_vm_writev).or_default(); + // Process fd theft and signalling via pidfd. pidfd_open is made + // unavailable with ENOSYS by the compatibility filter so runtimes can + // fall back without gaining a handle to the trusted sandbox boundary. + rules.entry(libc::SYS_pidfd_getfd).or_default(); + rules.entry(libc::SYS_pidfd_send_signal).or_default(); + // Async I/O subsystem with extensive CVE history. + rules.entry(libc::SYS_io_uring_setup).or_default(); + // Filesystem mount could subvert Landlock or overlay writable paths. + rules.entry(libc::SYS_mount).or_default(); + // New mount API syscalls (Linux 5.2+) bypass the SYS_mount block entirely. + rules.entry(libc::SYS_fsopen).or_default(); + rules.entry(libc::SYS_fsconfig).or_default(); + rules.entry(libc::SYS_fsmount).or_default(); + rules.entry(libc::SYS_fspick).or_default(); + rules.entry(libc::SYS_move_mount).or_default(); + rules.entry(libc::SYS_open_tree).or_default(); + // Namespace manipulation — setns enters existing namespaces, pivot_root/umount2 + // change the filesystem root. The supervisor calls setns before seccomp is applied, + // so blocking it here is safe. + rules.entry(libc::SYS_setns).or_default(); + rules.entry(libc::SYS_umount2).or_default(); + rules.entry(libc::SYS_pivot_root).or_default(); + // Kernel exploit primitives: userfaultfd enables race-condition exploitation (multiple + // CVEs), perf_event_open enables Spectre-class side channels. Both blocked by Docker's + // default seccomp profile. + rules.entry(libc::SYS_userfaultfd).or_default(); + rules.entry(libc::SYS_perf_event_open).or_default(); + + // --- Conditional syscall blocks --- + + // execveat with AT_EMPTY_PATH enables fileless execution from an anonymous fd. + add_masked_arg_rule( + &mut rules, + libc::SYS_execveat, + 4, // flags argument + libc::AT_EMPTY_PATH as u64, + )?; + + // unshare with CLONE_NEWUSER allows creating user namespaces to escalate privileges. + add_masked_arg_rule( + &mut rules, + libc::SYS_unshare, + 0, // flags argument + libc::CLONE_NEWUSER as u64, + )?; + + // clone with CLONE_NEWUSER achieves the same as unshare via a different syscall. + add_masked_arg_rule( + &mut rules, + libc::SYS_clone, + 0, // flags argument + libc::CLONE_NEWUSER as u64, + )?; + // clone3 is handled by the ENOSYS compatibility filter. + + // seccomp(SECCOMP_SET_MODE_FILTER) would let sandboxed code replace the active filter. + let condition = SeccompCondition::new( + 0, // operation argument + SeccompCmpArgLen::Dword, + SeccompCmpOp::Eq, + SECCOMP_SET_MODE_FILTER, + ) + .into_diagnostic()?; + let rule = SeccompRule::new(vec![condition]).into_diagnostic()?; + rules.entry(libc::SYS_seccomp).or_default().push(rule); + + Ok(rules) +} + +#[allow(clippy::cast_sign_loss)] +fn add_socket_domain_rule(rules: &mut BTreeMap>, domain: i32) -> Result<()> { + let condition = + SeccompCondition::new(0, SeccompCmpArgLen::Dword, SeccompCmpOp::Eq, domain as u64) + .into_diagnostic()?; + + let rule = SeccompRule::new(vec![condition]).into_diagnostic()?; + rules.entry(libc::SYS_socket).or_default().push(rule); + Ok(()) +} + +/// Block `socket(AF_NETLINK, *, protocol)` for every protocol except +/// `NETLINK_ROUTE` (protocol 0). +/// +/// Two AND'd conditions are required: +/// - arg0 == `AF_NETLINK` (domain) +/// - arg2 != 0 (protocol is not `NETLINK_ROUTE`) +/// +/// A seccomp rule fires (and returns EPERM) only when **all** conditions +/// match, so this rule is triggered for any `socket(AF_NETLINK, *, non-zero)` +/// call while leaving `socket(AF_NETLINK, *, 0)` (`NETLINK_ROUTE`) through. +#[allow(clippy::cast_sign_loss)] +fn add_netlink_non_route_rule(rules: &mut BTreeMap>) -> Result<()> { + let domain_condition = SeccompCondition::new( + 0, // domain argument + SeccompCmpArgLen::Dword, + SeccompCmpOp::Eq, + libc::AF_NETLINK as u64, + ) + .into_diagnostic()?; + + let protocol_condition = SeccompCondition::new( + 2, // protocol argument + SeccompCmpArgLen::Dword, + SeccompCmpOp::Ne, + 0, // NETLINK_ROUTE = 0 + ) + .into_diagnostic()?; + + let rule = SeccompRule::new(vec![domain_condition, protocol_condition]).into_diagnostic()?; + rules.entry(libc::SYS_socket).or_default().push(rule); + Ok(()) +} + +/// Block a syscall when a specific bit pattern is set in an argument. +/// +/// Uses `MaskedEq` to check `(arg & flag_bit) == flag_bit`, which triggers +/// EPERM when the flag is present regardless of other bits in the argument. +fn add_masked_arg_rule( + rules: &mut BTreeMap>, + syscall: i64, + arg_index: u8, + flag_bit: u64, +) -> Result<()> { + let condition = SeccompCondition::new( + arg_index, + SeccompCmpArgLen::Dword, + SeccompCmpOp::MaskedEq(flag_bit), + flag_bit, + ) + .into_diagnostic()?; + let rule = SeccompRule::new(vec![condition]).into_diagnostic()?; + rules.entry(syscall).or_default().push(rule); + Ok(()) +} + +#[cfg(test)] +// libc/syscall FFI requires unsafe; these tests fork children and exercise +// blocked syscalls, so unsafe blocks/calls are pervasive. +#[allow( + unsafe_code, + unsafe_op_in_unsafe_fn, + unused_unsafe, + clippy::borrow_as_ptr, + trivial_numeric_casts +)] +mod tests { + use super::*; + + // These tests cover both filter construction (rule map shape and BPF + // compilation) and selected runtime behavior on Linux via forked children. + + #[test] + fn build_filter_proxy_mode_compiles() { + let filter = build_filter(true); + assert!(filter.is_ok(), "build_filter(true) should succeed"); + } + + #[test] + fn build_filter_block_mode_compiles() { + let filter = build_filter(false); + assert!(filter.is_ok(), "build_filter(false) should succeed"); + } + + #[test] + fn build_supervisor_prelude_filter_compiles() { + let filter = build_supervisor_prelude_filter(); + assert!( + filter.is_ok(), + "build_supervisor_prelude_filter() should succeed" + ); + } + + #[test] + fn add_masked_arg_rule_creates_entry() { + let mut rules: BTreeMap> = BTreeMap::new(); + let result = add_masked_arg_rule(&mut rules, libc::SYS_execveat, 4, 0x1000); + assert!(result.is_ok()); + assert!( + rules.contains_key(&libc::SYS_execveat), + "should have an entry for SYS_execveat" + ); + assert_eq!( + rules[&libc::SYS_execveat].len(), + 1, + "should have exactly one rule" + ); + } + + #[test] + fn unconditional_blocks_present_in_filter() { + // Build a real filter and verify all unconditional blocks are present. + let filter_rules = build_filter_rules(true).unwrap(); + + // Unconditional blocks have an empty Vec (no conditions = always match). + let expected = [ + libc::SYS_memfd_create, + libc::SYS_ptrace, + libc::SYS_bpf, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_pidfd_getfd, + libc::SYS_pidfd_send_signal, + libc::SYS_io_uring_setup, + libc::SYS_mount, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_setns, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_userfaultfd, + libc::SYS_perf_event_open, + ]; + + for syscall in expected { + assert!( + filter_rules.contains_key(&syscall), + "syscall {syscall} should be in the rules map" + ); + assert!( + filter_rules[&syscall].is_empty(), + "syscall {syscall} should have empty rules (unconditional block)" + ); + } + } + + #[test] + fn conditional_blocks_have_rules() { + // Build a real filter and verify the conditional syscalls have rule entries + // (non-empty Vec means conditional match). + let filter_rules = build_filter_rules(true).unwrap(); + + for syscall in [ + libc::SYS_execveat, + libc::SYS_unshare, + libc::SYS_clone, + libc::SYS_seccomp, + ] { + assert!( + filter_rules.contains_key(&syscall), + "syscall {syscall} should be in the rules map" + ); + assert!( + !filter_rules[&syscall].is_empty(), + "syscall {syscall} should have conditional rules" + ); + } + } + + #[test] + fn netlink_socket_rules_are_conditional_not_unconditional() { + // SYS_socket must appear in the rules map (for domain blocks and the + // AF_NETLINK+non-ROUTE filter), but it must NOT be an unconditional block + // (empty Vec). An empty Vec would block ALL socket() calls, including + // socket(AF_NETLINK, *, NETLINK_ROUTE=0) which getifaddrs(3) needs. + let filter_rules = build_filter_rules(true).unwrap(); + + assert!( + filter_rules.contains_key(&libc::SYS_socket), + "SYS_socket should be in the rules map (domain blocks present)" + ); + + // The Vec for SYS_socket must be non-empty (rules are + // conditional), which is the opposite of an unconditional block. + assert!( + !filter_rules[&libc::SYS_socket].is_empty(), + "SYS_socket should have conditional rules, not an unconditional block" + ); + } + + #[test] + fn supervisor_prelude_blocks_expected_syscalls() { + let filter_rules = build_supervisor_prelude_rules(); + + for syscall in [ + libc::SYS_mount, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_pivot_root, + libc::SYS_umount2, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_kexec_load, + SYS_kexec_file_load, + ] { + assert!( + filter_rules.contains_key(&syscall), + "syscall {syscall} should be in the supervisor prelude rules" + ); + assert!( + filter_rules[&syscall].is_empty(), + "syscall {syscall} should be unconditionally blocked in the supervisor prelude" + ); + } + } + + #[test] + fn supervisor_prelude_keeps_required_setup_syscalls_available() { + let filter_rules = build_supervisor_prelude_rules(); + + for syscall in [ + libc::SYS_setns, + libc::SYS_clone, + libc::SYS_unshare, + libc::SYS_ptrace, + ] { + assert!( + !filter_rules.contains_key(&syscall), + "syscall {syscall} should remain available during supervisor startup" + ); + } + } + + #[test] + fn compatibility_filter_compiles() { + let bpf = build_compatibility_filter(); + assert!( + bpf.is_ok(), + "process API compatibility filter should compile" + ); + } + + #[test] + fn compatibility_syscalls_are_not_in_main_filter() { + // These APIs must NOT be in the EPERM filter; the compatibility filter + // reports them unavailable so process launchers can fall back. + let filter_rules = build_filter_rules(true).unwrap(); + for syscall in [libc::SYS_clone3, libc::SYS_pidfd_open] { + assert!( + !filter_rules.contains_key(&syscall), + "syscall {syscall} should use the ENOSYS compatibility filter" + ); + } + } + + // --- Behavioral tests --- + // + // These apply seccomp filters in a forked child and verify that blocked + // syscalls actually return the expected errno. They only compile and run + // on Linux (seccomp is a Linux kernel feature). + + /// Fork a child, apply the given filter, invoke `syscall_nr`, and return + /// the errno observed by the child. The child exits 0 if the syscall + /// returned the expected errno, 1 otherwise. + unsafe fn assert_blocked_in_child( + filter: &seccompiler::BpfProgram, + syscall_nr: i64, + expected_errno: i32, + ) { + let pid = libc::fork(); + assert!(pid >= 0, "fork failed"); + if pid == 0 { + // Child: apply filter and try the syscall. + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + apply_filter(filter).expect("apply_filter"); + let ret = libc::syscall(syscall_nr, 0 as libc::c_ulong, 0 as libc::c_ulong); + let errno = *libc::__errno_location(); + if ret == -1 && errno == expected_errno { + libc::_exit(0); + } else { + // Write diagnostic before exiting so test failures are debuggable. + let msg = format!( + "syscall {syscall_nr}: expected errno={expected_errno}, got ret={ret} errno={errno}\n" + ); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + // Parent: wait for child. + let mut status: libc::c_int = 0; + libc::waitpid(pid, &mut status, 0); + assert!( + libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, + "child failed: syscall {syscall_nr} was not blocked with errno {expected_errno}" + ); + } + + unsafe fn install_runtime_filters_in_child( + main_filter: &seccompiler::BpfProgram, + compatibility_filter: &seccompiler::BpfProgram, + ) { + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + if let Err(err) = apply_runtime_filters(main_filter, compatibility_filter) { + let msg = format!("failed to install runtime seccomp filters: {err}\n"); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + + #[test] + fn behavioral_memfd_create_blocked() { + let filter = build_filter(true).unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_memfd_create, libc::EPERM) }; + } + + #[test] + fn behavioral_ptrace_blocked() { + let filter = build_filter(true).unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_ptrace, libc::EPERM) }; + } + + #[test] + fn behavioral_process_vm_writev_blocked() { + let filter = build_filter(true).unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_process_vm_writev, libc::EPERM) }; + } + + #[test] + fn behavioral_userfaultfd_blocked() { + let filter = build_filter(true).unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_userfaultfd, libc::EPERM) }; + } + + #[test] + fn behavioral_perf_event_open_blocked() { + let filter = build_filter(true).unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_perf_event_open, libc::EPERM) }; + } + + #[test] + fn behavioral_setns_blocked() { + let filter = build_filter(true).unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_setns, libc::EPERM) }; + } + + #[test] + fn behavioral_supervisor_prelude_mount_blocked() { + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + if let Err(err) = apply_supervisor_prelude() { + let msg = format!("failed to install supervisor prelude: {err}\n"); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + let ret = libc::syscall( + libc::SYS_mount, + std::ptr::null::(), + std::ptr::null::(), + std::ptr::null::(), + 0 as libc::c_ulong, + std::ptr::null::(), + ); + let errno = *libc::__errno_location(); + if ret == -1 && errno == libc::EPERM { + libc::_exit(0); + } else { + let msg = format!( + "mount: expected EPERM after supervisor prelude, got ret={ret} errno={errno}\n" + ); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + } + + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "mount should be blocked by the supervisor prelude filter" + ); + } + + #[test] + fn behavioral_clone3_returns_enosys() { + // clone3 uses a separate filter that returns ENOSYS (not EPERM) so + // glibc falls back to clone. + let main_filter = build_filter(true).unwrap(); + let compatibility_filter = build_compatibility_filter().unwrap(); + // Apply in the same order as apply(): compatibility filter first, main filter second. + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + install_runtime_filters_in_child(&main_filter, &compatibility_filter); + let ret = libc::syscall(libc::SYS_clone3, 0 as libc::c_ulong, 0 as libc::c_ulong); + let errno = *libc::__errno_location(); + if ret == -1 && errno == libc::ENOSYS { + libc::_exit(0); + } else { + let msg = format!("clone3: expected ENOSYS, got ret={ret} errno={errno}\n"); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + } + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "clone3 should be blocked with ENOSYS, not EPERM" + ); + } + + #[test] + fn behavioral_pidfd_open_returns_enosys() { + let filter = build_compatibility_filter().unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_pidfd_open, libc::ENOSYS) }; + } + + #[test] + fn behavioral_third_filter_install_blocked_after_startup() { + let main_filter = build_filter(true).unwrap(); + let compatibility_filter = build_compatibility_filter().unwrap(); + let third_filter = build_compatibility_filter().unwrap(); + + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + install_runtime_filters_in_child(&main_filter, &compatibility_filter); + match apply_filter(&third_filter) { + Err(seccompiler::Error::Seccomp(e)) + if e.raw_os_error() == Some(libc::EPERM) => + { + libc::_exit(0); + } + Err(err) => { + let msg = + format!("third filter install failed with unexpected error: {err}\n"); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + Ok(()) => { + let msg = "third filter unexpectedly installed\n"; + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + } + } + + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "additional seccomp filter installation should be blocked after startup" + ); + } + + #[test] + fn behavioral_netlink_route_allowed() { + // socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE=0) must succeed (not blocked). + // This is the call getifaddrs(3) makes on Linux to enumerate interfaces. + let filter = build_filter(true).unwrap(); + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + apply_filter(&filter).expect("apply_filter"); + // NETLINK_ROUTE = 0 + let fd = libc::socket(libc::AF_NETLINK, libc::SOCK_RAW, 0); + if fd >= 0 { + libc::close(fd); + libc::_exit(0); + } else { + let errno = *libc::__errno_location(); + let msg = format!( + "socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE): expected success, got errno={errno}\n" + ); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + } + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE) should be allowed for getifaddrs(3)" + ); + } + + #[test] + fn behavioral_netlink_non_route_blocked() { + // socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG=4) must be blocked. + // NETLINK_SOCK_DIAG is representative of non-ROUTE netlink protocols + // that have no legitimate use inside the sandbox. + let filter = build_filter(true).unwrap(); + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + apply_filter(&filter).expect("apply_filter"); + // NETLINK_SOCK_DIAG = 4 + let fd = libc::socket(libc::AF_NETLINK, libc::SOCK_RAW, 4); + let errno = *libc::__errno_location(); + if fd == -1 && errno == libc::EPERM { + libc::_exit(0); + } else { + if fd >= 0 { + libc::close(fd); + } + let msg = format!( + "socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG): expected EPERM, got fd={fd} errno={errno}\n" + ); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + } + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG) should be blocked with EPERM" + ); + } + + #[test] + fn behavioral_block_mode_denies_inet_and_packet_sockets() { + let filter = build_filter(false).unwrap(); + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + apply_filter(&filter).expect("apply block-mode filter"); + for (domain, socket_type, protocol) in [ + (libc::AF_INET, libc::SOCK_STREAM, 0), + (libc::AF_INET6, libc::SOCK_DGRAM, 0), + (libc::AF_PACKET, libc::SOCK_RAW, 0), + ] { + let fd = libc::socket(domain, socket_type, protocol); + let errno = *libc::__errno_location(); + if fd >= 0 || errno != libc::EPERM { + if fd >= 0 { + libc::close(fd); + } + libc::_exit(1); + } + } + let unix_fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if unix_fd < 0 { + libc::_exit(1); + } + libc::close(unix_fd); + libc::_exit(0); + } + } + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "block mode must deny IPv4, IPv6, and packet sockets while retaining Unix IPC" + ); + } +} diff --git a/crates/openshell-sandbox/src/sandbox/mod.rs b/crates/openshell-sandbox/src/sandbox/mod.rs new file mode 100644 index 0000000000..ff44f8ba10 --- /dev/null +++ b/crates/openshell-sandbox/src/sandbox/mod.rs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Platform sandboxing implementation. + +use miette::Result; +use openshell_core::policy::SandboxPolicy; + +#[cfg(target_os = "linux")] +pub mod linux; + +/// Apply sandboxing rules for the current platform. +/// +/// # Errors +/// +/// Returns an error if the sandbox cannot be applied. +// On Linux the spawn path uses `prepare`+`enforce` directly; this single-phase +// apply is only invoked from the non-Linux spawn_impl. +#[cfg_attr(target_os = "linux", allow(dead_code))] +#[cfg_attr(not(target_os = "linux"), allow(clippy::unnecessary_wraps))] +pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { + #[cfg(target_os = "linux")] + { + linux::apply(policy, workdir) + } + + #[cfg(not(target_os = "linux"))] + { + let _ = (policy, workdir); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .severity(openshell_ocsf::SeverityId::Medium) + .finding_info(openshell_ocsf::FindingInfo::new( + "platform-sandbox-unavailable", + "Platform Sandboxing Not Implemented", + ).with_desc("Sandbox policy provided but platform sandboxing is not yet implemented on this OS")) + .message("Platform sandboxing not yet implemented") + .build() + ); + Ok(()) + } +} + +/// Apply seccomp hardening for the long-lived supervisor process itself. +#[cfg_attr(not(target_os = "linux"), allow(clippy::unnecessary_wraps))] +pub fn apply_supervisor_startup_hardening() -> Result<()> { + #[cfg(target_os = "linux")] + { + linux::apply_supervisor_prelude() + } + + #[cfg(not(target_os = "linux"))] + { + Ok(()) + } +} diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs deleted file mode 100644 index 11f3e68e23..0000000000 --- a/crates/openshell-sandbox/src/sidecar_control.rs +++ /dev/null @@ -1,1210 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Local control channel for Kubernetes sidecar topology. -//! -//! The network sidecar owns gateway credentials. The process supervisor in the -//! agent container connects over this Unix socket to receive policy/provider -//! state without mounting gateway credentials into the agent container. - -use miette::{IntoDiagnostic, Result, WrapErr}; -use prost::Message; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; -use std::time::Duration; -use tokio::io::{AsyncBufReadExt, AsyncWrite, AsyncWriteExt, BufReader}; -use tokio::net::UnixListener; -use tokio::net::unix::OwnedWriteHalf; -use tokio::sync::{Mutex, broadcast, mpsc}; -use tracing::{debug, info, warn}; - -#[derive(Debug, Clone)] -pub struct BootstrapData { - pub policy_proto: openshell_core::proto::SandboxPolicy, - pub provider_env_revision: u64, - pub provider_env_generation: u64, - pub provider_child_env: HashMap, - pub agent_proposals_enabled: bool, - pub proxy_ca_cert_path: Option, - pub proxy_ca_bundle_path: Option, -} - -#[derive(Debug, Clone)] -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -pub struct EntrypointStarted { - pub pid: u32, - pub start_session: bool, - pub instance_id: String, - pub exit_code: Option, - pub finalized: bool, -} - -#[derive(Debug, Clone, Copy)] -pub struct ExpectedPeer { - pub uid: u32, - pub gid: u32, -} - -#[derive(Debug, Clone)] -pub enum ControlUpdate { - ProviderEnv { - revision: u64, - generation: u64, - provider_child_env: HashMap, - }, - Policy { - policy_proto: Box, - policy_hash: String, - config_revision: u64, - }, - AgentProposals { - enabled: bool, - config_revision: u64, - }, - MainProcessExitAck { - instance_id: String, - }, -} - -#[derive(Clone)] -pub struct Publisher { - state: Arc>, - updates: broadcast::Sender, -} - -impl Publisher { - pub fn publish_provider_env(&self, revision: u64, provider_child_env: HashMap) { - let mut state = self.state.write().expect("sidecar control state poisoned"); - if revision == state.provider_env_revision { - return; - } - state.provider_env_revision = revision; - state.provider_env_generation = state - .provider_env_generation - .checked_add(1) - .expect("sidecar provider environment generation overflow"); - state.provider_child_env.clone_from(&provider_child_env); - - // Keep generation assignment, bootstrap state, and publication under - // one lock so cloned publishers cannot emit generations out of order. - let _ = self.updates.send(WireServerMessage::ProviderEnvUpdated { - revision, - generation: state.provider_env_generation, - provider_child_env, - }); - } - - pub fn publish_policy( - &self, - policy_proto: openshell_core::proto::SandboxPolicy, - policy_hash: String, - config_revision: u64, - ) { - { - let mut state = self.state.write().expect("sidecar control state poisoned"); - state.policy_proto = policy_proto.clone(); - } - - let _ = self.updates.send(WireServerMessage::PolicyUpdated { - policy_proto: policy_proto.encode_to_vec(), - policy_hash, - config_revision, - }); - } - - pub fn publish_agent_proposals(&self, enabled: bool, config_revision: u64) { - { - let mut state = self.state.write().expect("sidecar control state poisoned"); - if state.agent_proposals_enabled == enabled { - return; - } - state.agent_proposals_enabled = enabled; - } - - let _ = self.updates.send(WireServerMessage::AgentProposalsUpdated { - enabled, - config_revision, - }); - } - - #[cfg(any(target_os = "linux", test))] - pub fn publish_main_process_exit_ack(&self, instance_id: String) { - let _ = self - .updates - .send(WireServerMessage::MainProcessExitAck { instance_id }); - } -} - -pub struct ServerHandle { - publisher: Publisher, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - entrypoint_rx: mpsc::Receiver, - connection_task: tokio::task::JoinHandle<()>, -} - -impl ServerHandle { - pub fn publisher(&self) -> Publisher { - self.publisher.clone() - } - - #[cfg(test)] - pub fn into_entrypoint_receiver(self) -> mpsc::Receiver { - self.entrypoint_rx - } - - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - pub fn into_runtime_parts( - self, - ) -> ( - mpsc::Receiver, - tokio::task::JoinHandle<()>, - ) { - (self.entrypoint_rx, self.connection_task) - } -} - -pub struct ProcessConnection { - pub writer: Arc>, - pub updates: mpsc::UnboundedReceiver, - pub closed: tokio::sync::oneshot::Receiver<()>, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum WireClientMessage { - BootstrapRequest { supervisor_pid: u32 }, - EntrypointStarted { pid: u32, instance_id: String }, - MainProcessExited { instance_id: String, exit_code: i32 }, - MainProcessFinalized { instance_id: String }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum WireServerMessage { - BootstrapResponse { - policy_proto: Vec, - provider_env_revision: u64, - provider_env_generation: u64, - provider_child_env: HashMap, - agent_proposals_enabled: bool, - proxy_ca_cert_path: Option, - proxy_ca_bundle_path: Option, - }, - ProviderEnvUpdated { - revision: u64, - generation: u64, - provider_child_env: HashMap, - }, - PolicyUpdated { - policy_proto: Vec, - policy_hash: String, - config_revision: u64, - }, - AgentProposalsUpdated { - enabled: bool, - config_revision: u64, - }, - MainProcessExitAck { - instance_id: String, - }, -} - -impl BootstrapData { - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - fn to_wire(&self) -> WireServerMessage { - WireServerMessage::BootstrapResponse { - policy_proto: self.policy_proto.encode_to_vec(), - provider_env_revision: self.provider_env_revision, - provider_env_generation: self.provider_env_generation, - provider_child_env: self.provider_child_env.clone(), - agent_proposals_enabled: self.agent_proposals_enabled, - proxy_ca_cert_path: self - .proxy_ca_cert_path - .as_ref() - .map(|path| path.display().to_string()), - proxy_ca_bundle_path: self - .proxy_ca_bundle_path - .as_ref() - .map(|path| path.display().to_string()), - } - } -} - -impl TryFrom for BootstrapData { - type Error = miette::Report; - - fn try_from(message: WireServerMessage) -> Result { - let WireServerMessage::BootstrapResponse { - policy_proto, - provider_env_revision, - provider_env_generation, - provider_child_env, - agent_proposals_enabled, - proxy_ca_cert_path, - proxy_ca_bundle_path, - } = message - else { - return Err(miette::miette!( - "expected sidecar bootstrap response, received update message" - )); - }; - - let policy_proto = openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) - .into_diagnostic() - .wrap_err("failed to decode sidecar bootstrap policy")?; - let policy_proto = canonicalize_sidecar_policy( - policy_proto, - "sidecar bootstrap policy failed validation", - )?; - - Ok(Self { - policy_proto, - provider_env_revision, - provider_env_generation, - provider_child_env, - agent_proposals_enabled, - proxy_ca_cert_path: proxy_ca_cert_path.map(PathBuf::from), - proxy_ca_bundle_path: proxy_ca_bundle_path.map(PathBuf::from), - }) - } -} - -impl TryFrom for ControlUpdate { - type Error = miette::Report; - - fn try_from(message: WireServerMessage) -> Result { - match message { - WireServerMessage::ProviderEnvUpdated { - revision, - generation, - provider_child_env, - } => Ok(Self::ProviderEnv { - revision, - generation, - provider_child_env, - }), - WireServerMessage::PolicyUpdated { - policy_proto, - policy_hash, - config_revision, - } => { - let policy_proto = - openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) - .into_diagnostic() - .wrap_err("failed to decode sidecar policy update")?; - let policy_proto = canonicalize_sidecar_policy( - policy_proto, - "sidecar policy update failed validation", - )?; - Ok(Self::Policy { - policy_proto: Box::new(policy_proto), - policy_hash, - config_revision, - }) - } - WireServerMessage::AgentProposalsUpdated { - enabled, - config_revision, - } => Ok(Self::AgentProposals { - enabled, - config_revision, - }), - WireServerMessage::MainProcessExitAck { instance_id } => { - Ok(Self::MainProcessExitAck { instance_id }) - } - WireServerMessage::BootstrapResponse { .. } => Err(miette::miette!( - "unexpected sidecar bootstrap response after initial handshake" - )), - } - } -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -pub fn spawn_server( - path: &Path, - bootstrap: BootstrapData, - expected_peer: ExpectedPeer, -) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to create sidecar control socket dir {}", - parent.display() - ) - })?; - } - match std::fs::remove_file(path) { - Ok(()) => {} - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => { - return Err(err).into_diagnostic().wrap_err_with(|| { - format!( - "failed to remove stale sidecar control socket {}", - path.display() - ) - }); - } - } - - let listener = UnixListener::bind(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to bind sidecar control socket {}", path.display()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to set permissions on sidecar control socket {}", - path.display() - ) - })?; - } - - let state = Arc::new(RwLock::new(bootstrap)); - let (updates, _) = broadcast::channel(32); - let (entrypoint_tx, entrypoint_rx) = mpsc::channel(8); - let publisher = Publisher { - state: state.clone(), - updates: updates.clone(), - }; - - let connection_task = tokio::spawn(accept_authoritative_connection( - listener, - path.to_path_buf(), - expected_peer, - state, - updates, - entrypoint_tx, - )); - info!(path = %path.display(), "Sidecar control socket listening"); - - Ok(ServerHandle { - publisher, - entrypoint_rx, - connection_task, - }) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -async fn accept_authoritative_connection( - listener: UnixListener, - socket_path: PathBuf, - expected_peer: ExpectedPeer, - state: Arc>, - updates: broadcast::Sender, - entrypoint_tx: mpsc::Sender, -) { - let stream = match listener.accept().await { - Ok((stream, _addr)) => stream, - Err(err) => { - warn!(error = %err, "Failed to accept authoritative sidecar control connection"); - return; - } - }; - - // The process supervisor connects before it launches the workload. Drop - // the listener and unlink its pathname after that first accept so workload - // processes can neither open a second control channel nor impersonate a - // restarted server at the trusted path. - drop(listener); - if let Err(err) = std::fs::remove_file(&socket_path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - path = %socket_path.display(), - error = %err, - "Failed to unlink accepted sidecar control socket" - ); - } - - if let Err(err) = handle_connection(stream, expected_peer, state, updates, entrypoint_tx).await - { - warn!(error = %err, "Authoritative sidecar control connection closed"); - } -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -async fn handle_connection( - stream: tokio::net::UnixStream, - expected_peer: ExpectedPeer, - state: Arc>, - updates: broadcast::Sender, - entrypoint_tx: mpsc::Sender, -) -> Result<()> { - let credentials = stream - .peer_cred() - .into_diagnostic() - .wrap_err("failed to read sidecar control peer credentials")?; - if credentials.uid() != expected_peer.uid || credentials.gid() != expected_peer.gid { - return Err(miette::miette!( - "sidecar control peer identity mismatch: expected uid:gid {}:{}, got {}:{}", - expected_peer.uid, - expected_peer.gid, - credentials.uid(), - credentials.gid(), - )); - } - let peer_pid = credentials - .pid() - .and_then(|pid| u32::try_from(pid).ok()) - .ok_or_else(|| miette::miette!("sidecar control peer PID is unavailable"))?; - - let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); - - let first_line = - lines.next_line().await.into_diagnostic()?.ok_or_else(|| { - miette::miette!("sidecar control client disconnected before bootstrap") - })?; - match decode_client_message(&first_line)? { - WireClientMessage::BootstrapRequest { supervisor_pid } => { - if supervisor_pid == 0 || supervisor_pid != peer_pid { - return Err(miette::miette!( - "sidecar bootstrap PID mismatch: peer PID {peer_pid}, claimed PID {supervisor_pid}" - )); - } - entrypoint_tx - .send(EntrypointStarted { - pid: supervisor_pid, - start_session: false, - instance_id: String::new(), - exit_code: None, - finalized: false, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - WireClientMessage::EntrypointStarted { .. } - | WireClientMessage::MainProcessExited { .. } - | WireClientMessage::MainProcessFinalized { .. } => { - return Err(miette::miette!( - "sidecar control client sent entrypoint event before bootstrap" - )); - } - } - - // Subscribe before taking the bootstrap snapshot so an update can neither - // be missed between the snapshot and the live update stream nor omitted - // from the snapshot itself. - let mut update_rx = updates.subscribe(); - let bootstrap = { - let state = state.read().expect("sidecar control state poisoned"); - state.to_wire() - }; - write_json_line(&mut writer, &bootstrap).await?; - - loop { - tokio::select! { - line = lines.next_line() => { - let Some(line) = line.into_diagnostic()? else { - return Ok(()); - }; - match decode_client_message(&line)? { - WireClientMessage::BootstrapRequest { .. } => { - debug!("Ignoring duplicate sidecar bootstrap request"); - } - WireClientMessage::EntrypointStarted { pid, instance_id } => { - if pid == 0 { - warn!("Ignoring sidecar entrypoint event with pid=0"); - continue; - } - entrypoint_tx - .send(EntrypointStarted { - pid, - start_session: true, - instance_id, - exit_code: None, - finalized: false, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - WireClientMessage::MainProcessExited { - instance_id, - exit_code, - } => { - entrypoint_tx - .send(EntrypointStarted { - pid: 0, - start_session: false, - instance_id, - exit_code: Some(exit_code), - finalized: false, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - WireClientMessage::MainProcessFinalized { instance_id } => { - entrypoint_tx - .send(EntrypointStarted { - pid: 0, - start_session: false, - instance_id, - exit_code: None, - finalized: true, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - } - } - update = update_rx.recv() => { - match update { - Ok(message) => write_json_line(&mut writer, &message).await?, - Err(broadcast::error::RecvError::Lagged(skipped)) => { - warn!(skipped, "Sidecar control client lagged behind updates"); - } - Err(broadcast::error::RecvError::Closed) => return Ok(()), - } - } - } - } -} - -pub async fn connect_process_client( - path: &Path, - timeout: Duration, -) -> Result<(BootstrapData, ProcessConnection)> { - let stream = connect_with_retry(path, timeout).await?; - let (reader, mut writer) = stream.into_split(); - write_json_line( - &mut writer, - &WireClientMessage::BootstrapRequest { - supervisor_pid: std::process::id(), - }, - ) - .await?; - - let mut lines = BufReader::new(reader).lines(); - let first_line = lines - .next_line() - .await - .into_diagnostic()? - .ok_or_else(|| miette::miette!("sidecar control closed before bootstrap response"))?; - let bootstrap = BootstrapData::try_from(decode_server_message(&first_line)?)?; - - let (update_tx, updates) = mpsc::unbounded_channel(); - let (closed_tx, closed) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - while let Ok(Some(line)) = lines.next_line().await { - match decode_server_message(&line).and_then(ControlUpdate::try_from) { - Ok(update) => { - if update_tx.send(update).is_err() { - break; - } - } - Err(err) => { - warn!(error = %err, "Ignoring invalid sidecar control update"); - } - } - } - let _ = closed_tx.send(()); - }); - - Ok(( - bootstrap, - ProcessConnection { - writer: Arc::new(Mutex::new(writer)), - updates, - closed, - }, - )) -} - -async fn connect_with_retry(path: &Path, timeout: Duration) -> Result { - let deadline = tokio::time::Instant::now() + timeout; - loop { - match tokio::net::UnixStream::connect(path).await { - Ok(stream) => return Ok(stream), - Err(err) if tokio::time::Instant::now() < deadline => { - debug!( - path = %path.display(), - error = %err, - "Waiting for sidecar control socket" - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - Err(err) => { - return Err(err).into_diagnostic().wrap_err_with(|| { - format!( - "timed out waiting for sidecar control socket {}", - path.display() - ) - }); - } - } - } -} - -pub async fn send_entrypoint_started( - writer: &Arc>, - pid: u32, - instance_id: String, -) -> Result<()> { - let message = WireClientMessage::EntrypointStarted { pid, instance_id }; - let mut writer = writer.lock().await; - write_json_line(&mut *writer, &message).await -} - -pub async fn send_main_process_exited( - writer: &Arc>, - instance_id: String, - exit_code: i32, -) -> Result<()> { - let message = WireClientMessage::MainProcessExited { - instance_id, - exit_code, - }; - let mut writer = writer.lock().await; - write_json_line(&mut *writer, &message).await -} - -pub async fn send_main_process_finalized( - writer: &Arc>, - instance_id: String, -) -> Result<()> { - let message = WireClientMessage::MainProcessFinalized { instance_id }; - let mut writer = writer.lock().await; - write_json_line(&mut *writer, &message).await -} - -async fn write_json_line(writer: &mut W, value: &T) -> Result<()> -where - W: AsyncWrite + Unpin + Send, - T: Serialize + Sync, -{ - let bytes = serde_json::to_vec(value).into_diagnostic()?; - writer.write_all(&bytes).await.into_diagnostic()?; - writer.write_all(b"\n").await.into_diagnostic()?; - writer.flush().await.into_diagnostic()?; - Ok(()) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn decode_client_message(line: &str) -> Result { - serde_json::from_str(line) - .into_diagnostic() - .wrap_err("failed to decode sidecar client message") -} - -fn decode_server_message(line: &str) -> Result { - serde_json::from_str(line) - .into_diagnostic() - .wrap_err("failed to decode sidecar server message") -} - -fn canonicalize_sidecar_policy( - policy: openshell_core::proto::SandboxPolicy, - error_message: &'static str, -) -> Result { - // Bootstrap and update messages must expose the same canonical typed - // policy to every process-supervisor consumer. Keep validation details - // out of this channel error because they can contain authored values. - openshell_policy::validate_and_canonicalize_sandbox_policy(policy) - .map_err(|_| miette::miette!(error_message)) -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::proto::{McpOptions, NetworkEndpoint, NetworkPolicyRule, SandboxPolicy}; - - fn defaultable_mcp_policy(mcp: Option) -> SandboxPolicy { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "mcp".to_string(), - NetworkPolicyRule { - name: "mcp".to_string(), - endpoints: vec![NetworkEndpoint { - host: "mcp.example.com".to_string(), - port: 443, - protocol: "mcp".to_string(), - mcp, - rules: vec![openshell_core::proto::L7Rule { - allow: Some(openshell_core::proto::L7Allow { - method: "tools/list".to_string(), - ..Default::default() - }), - }], - ..Default::default() - }], - ..Default::default() - }, - ); - policy - } - - fn mcp_versions(policy: &SandboxPolicy) -> &[String] { - policy.network_policies["mcp"].endpoints[0] - .mcp - .as_ref() - .expect("canonical MCP options") - .versions - .as_slice() - } - - fn bootstrap_message(policy: &SandboxPolicy) -> WireServerMessage { - WireServerMessage::BootstrapResponse { - policy_proto: policy.encode_to_vec(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - } - } - - fn policy_update_message(policy: &SandboxPolicy) -> WireServerMessage { - WireServerMessage::PolicyUpdated { - policy_proto: policy.encode_to_vec(), - policy_hash: "hash".to_string(), - config_revision: 1, - } - } - - fn current_peer() -> ExpectedPeer { - ExpectedPeer { - uid: nix::unistd::Uid::current().as_raw(), - gid: nix::unistd::Gid::current().as_raw(), - } - } - - #[test] - fn policy_messages_canonicalize_defaultable_mcp_versions() { - for raw in [ - defaultable_mcp_policy(None), - defaultable_mcp_policy(Some(McpOptions::default())), - ] { - let bootstrap = BootstrapData::try_from(bootstrap_message(&raw)) - .expect("defaultable MCP policy must pass bootstrap ingress"); - assert_eq!(mcp_versions(&bootstrap.policy_proto), ["2025-11-25"]); - - let update = ControlUpdate::try_from(policy_update_message(&raw)) - .expect("defaultable MCP policy must pass update ingress"); - let ControlUpdate::Policy { policy_proto, .. } = update else { - panic!("expected policy update"); - }; - assert_eq!(mcp_versions(&policy_proto), ["2025-11-25"]); - } - } - - #[test] - fn policy_messages_reject_invalid_mcp_versions_without_echoing_values() { - let invalid = defaultable_mcp_policy(Some(McpOptions { - versions: vec!["latest".to_string()], - ..Default::default() - })); - - let bootstrap_error = BootstrapData::try_from(bootstrap_message(&invalid)) - .expect_err("invalid MCP policy must not pass bootstrap ingress") - .to_string(); - assert_eq!( - bootstrap_error, - "sidecar bootstrap policy failed validation" - ); - assert!(!bootstrap_error.contains("latest")); - - let update_error = ControlUpdate::try_from(policy_update_message(&invalid)) - .expect_err("invalid MCP policy must not pass update ingress") - .to_string(); - assert_eq!(update_error, "sidecar policy update failed validation"); - assert!(!update_error.contains("latest")); - } - - #[tokio::test] - async fn bootstrap_round_trips_policy_and_provider_env() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let mut env = HashMap::new(); - env.insert("GITHUB_TOKEN".to_string(), "secret".to_string()); - let bootstrap = BootstrapData { - policy_proto: SandboxPolicy { - version: 7, - ..SandboxPolicy::default() - }, - provider_env_revision: 3, - provider_env_generation: 0, - provider_child_env: env.clone(), - agent_proposals_enabled: true, - proxy_ca_cert_path: Some(PathBuf::from("/tmp/ca.pem")), - proxy_ca_bundle_path: Some(PathBuf::from("/tmp/bundle.pem")), - }; - - let _server = spawn_server(&socket, bootstrap, current_peer()).unwrap(); - let (received, _connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - assert_eq!(received.policy_proto.version, 7); - assert_eq!(received.provider_env_revision, 3); - assert_eq!(received.provider_env_generation, 0); - assert_eq!(received.provider_child_env, env); - assert!(received.agent_proposals_enabled); - assert_eq!( - received.proxy_ca_cert_path, - Some(PathBuf::from("/tmp/ca.pem")) - ); - assert_eq!( - received.proxy_ca_bundle_path, - Some(PathBuf::from("/tmp/bundle.pem")) - ); - } - - #[tokio::test] - async fn provider_env_updates_use_generation_not_fingerprint_order() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: u64::MAX, - provider_env_generation: 7, - provider_child_env: HashMap::from([("TOKEN".to_string(), "first".to_string())]), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let publisher = server.publisher(); - let (_bootstrap, mut connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - publisher.publish_provider_env( - 1, - HashMap::from([("TOKEN".to_string(), "second".to_string())]), - ); - - let update = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - match update { - ControlUpdate::ProviderEnv { - revision, - generation, - provider_child_env, - } => { - assert_eq!(revision, 1); - assert_eq!(generation, 8); - assert_eq!( - provider_child_env.get("TOKEN").map(String::as_str), - Some("second") - ); - } - other => panic!("unexpected sidecar update: {other:?}"), - } - - publisher.publish_provider_env( - 1, - HashMap::from([("TOKEN".to_string(), "duplicate".to_string())]), - ); - assert!( - tokio::time::timeout(Duration::from_millis(50), connection.updates.recv()) - .await - .is_err(), - "an identical fingerprint must remain a no-op" - ); - - publisher.publish_provider_env( - u64::MAX, - HashMap::from([("TOKEN".to_string(), "third".to_string())]), - ); - let update = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - match update { - ControlUpdate::ProviderEnv { - revision, - generation, - provider_child_env, - } => { - assert_eq!(revision, u64::MAX); - assert_eq!(generation, 9); - assert_eq!( - provider_child_env.get("TOKEN").map(String::as_str), - Some("third") - ); - } - other => panic!("unexpected sidecar update: {other:?}"), - } - } - - #[tokio::test] - async fn agent_proposals_update_is_delivered_to_process_client() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let publisher = server.publisher(); - let (_bootstrap, mut connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - publisher.publish_agent_proposals(true, 9); - - let update = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - match update { - ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - assert!(enabled); - assert_eq!(config_revision, 9); - } - other => panic!("unexpected sidecar update: {other:?}"), - } - } - - #[tokio::test] - async fn entrypoint_started_is_delivered_to_server() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let publisher = server.publisher(); - let mut entrypoint_rx = server.into_entrypoint_receiver(); - let (_bootstrap, mut connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - let anchor = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(anchor.pid, std::process::id()); - assert!(!anchor.start_session); - - send_entrypoint_started(&connection.writer, 4242, "instance-1".to_string()) - .await - .unwrap(); - - let started = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(started.pid, 4242); - assert!(started.start_session); - assert_eq!(started.instance_id, "instance-1"); - assert!(started.exit_code.is_none()); - - send_main_process_exited(&connection.writer, "instance-1".to_string(), 0) - .await - .unwrap(); - let terminal = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(terminal.exit_code, Some(0)); - assert!(!terminal.finalized); - - assert!( - tokio::time::timeout(Duration::from_millis(20), connection.updates.recv()) - .await - .is_err(), - "process side must not observe a durable ACK before gateway persistence" - ); - publisher.publish_main_process_exit_ack("instance-1".to_string()); - let ack = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - assert!(matches!( - ack, - ControlUpdate::MainProcessExitAck { instance_id } if instance_id == "instance-1" - )); - - send_main_process_finalized(&connection.writer, "instance-1".to_string()) - .await - .unwrap(); - let delivered = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert!(delivered.exit_code.is_none()); - assert!(delivered.finalized); - } - - #[tokio::test] - async fn second_control_client_is_rejected_after_authoritative_bootstrap() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let _server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - - let (_bootstrap, _connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - let err = tokio::net::UnixStream::connect(&socket) - .await - .expect_err("control listener must be removed after the first bootstrap"); - assert!( - matches!( - err.kind(), - std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused - ), - "unexpected second-client error: {err}" - ); - } - - #[tokio::test] - async fn authoritative_connection_task_ends_when_process_supervisor_disconnects() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let (_entrypoint_rx, connection_task) = server.into_runtime_parts(); - let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - drop(connection); - tokio::time::timeout(Duration::from_secs(1), connection_task) - .await - .expect("server must observe authoritative client disconnect") - .expect("control task must not panic"); - } - - #[tokio::test] - async fn process_client_reports_network_sidecar_restart() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let (_entrypoint_rx, connection_task) = server.into_runtime_parts(); - let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - connection_task.abort(); - let _ = connection_task.await; - tokio::time::timeout(Duration::from_secs(1), connection.closed) - .await - .expect("process supervisor must observe network sidecar disconnect") - .expect("disconnect notifier must remain live"); - } - - #[tokio::test] - async fn bootstrap_rejects_claimed_pid_that_does_not_match_peer_credentials() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let mut entrypoint_rx = server.into_entrypoint_receiver(); - - let mut stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); - write_json_line( - &mut stream, - &WireClientMessage::BootstrapRequest { - supervisor_pid: std::process::id().saturating_add(1), - }, - ) - .await - .unwrap(); - - assert!( - tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .is_none(), - "mismatched bootstrap must not publish a process anchor" - ); - } - - #[test] - fn malformed_client_message_is_rejected() { - let err = decode_client_message("not-json").unwrap_err(); - assert!( - err.to_string() - .contains("failed to decode sidecar client message") - ); - } -} diff --git a/crates/openshell-sandbox/tests/stdout_logging.rs b/crates/openshell-sandbox/tests/stdout_logging.rs index c4f5213de8..ed3a2d5523 100644 --- a/crates/openshell-sandbox/tests/stdout_logging.rs +++ b/crates/openshell-sandbox/tests/stdout_logging.rs @@ -6,21 +6,16 @@ use std::process::Command; #[test] fn startup_logs_go_to_stderr_not_stdout() { let output = Command::new(env!("CARGO_BIN_EXE_openshell-sandbox")) - .arg("--") - .arg("/usr/bin/printf") - .arg("hello") + .arg("--bootstrap") + .arg("/does/not/exist/openshell-boundary.json") .env("OPENSHELL_LOG_LEVEL", "info") .env_remove("RUST_LOG") - .env_remove("OPENSHELL_POLICY_RULES") - .env_remove("OPENSHELL_POLICY_DATA") - .env_remove("OPENSHELL_SANDBOX_ID") - .env_remove("OPENSHELL_ENDPOINT") .output() .expect("spawn openshell-sandbox"); assert!( !output.status.success(), - "expected sandbox startup to fail without a policy source" + "expected sandbox startup to fail without bootstrap material" ); let stdout = String::from_utf8_lossy(&output.stdout); @@ -31,11 +26,9 @@ fn startup_logs_go_to_stderr_not_stdout() { "expected startup logs on stderr only, got stdout: {stdout}" ); assert!( - stderr.contains("Starting sandbox"), - "expected startup log on stderr, got: {stderr}" - ); - assert!( - stderr.contains("Sandbox policy required"), - "expected missing-policy error on stderr, got: {stderr}" + stderr.contains("capability-free sandbox probe") + || stderr.contains("read boundary config") + || stderr.contains("openshell-sandbox requires Linux"), + "expected startup qualification or bootstrap error on stderr, got: {stderr}" ); } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index b3589b8864..d59eedf632 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -173,6 +173,7 @@ mod traced_driver { } const DELETE_PHASE_CAS_RETRY_LIMIT: usize = 3; +const SUPERVISOR_SESSION_CAS_RETRY_LIMIT: usize = 3; #[derive(Clone, Debug, Eq, PartialEq)] pub enum GatewayListenerRequirement { @@ -2900,18 +2901,19 @@ impl ComputeRuntime { SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) }); - if !driver_snapshot_reports_terminal_container_exit(&incoming) - || existing_phase != SandboxPhase::Starting - { + if existing_phase != SandboxPhase::Starting { return self.apply_sandbox_update_locked(incoming, existing).await; } - // A terminal snapshot can already be queued when StartSandbox moves - // the durable phase to Starting. Release the global watch lock, wait - // for that lifecycle operation, and then reread both the driver and - // store before applying the terminal observation. Taking the - // per-sandbox gate only for this ambiguous phase avoids delaying - // unrelated watch events behind slow lifecycle operations. + // Any snapshot can already be queued when StartSandbox moves the + // durable phase to Starting. In particular, an old-generation Ready + // event followed by its terminal event can otherwise promote and then + // stop the new generation before the replacement supervisor connects. + // Release the global watch lock, wait for that lifecycle operation, + // and then reread both the driver and store before applying an + // authoritative observation. Taking the per-sandbox gate only for + // this ambiguous phase avoids delaying unrelated watch events behind + // slow lifecycle operations. let existing_name = existing_sandbox.as_ref().map_or_else( || incoming.name.clone(), |sandbox| sandbox.object_name().to_string(), @@ -2940,7 +2942,7 @@ impl ComputeRuntime { { warn!( sandbox_id = %incoming.id, - "Could not validate terminal driver snapshot; retaining current sandbox state" + "Could not validate driver snapshot during sandbox start; retaining current sandbox state" ); return Ok(()); } @@ -3052,82 +3054,129 @@ impl ComputeRuntime { instance_id: Option<&str>, terminal_delivery_finalized: bool, ) -> Result<(), String> { - let guard = self.sync_lock.lock().await; - - let Some(existing) = self + let _guard = self.sync_lock.lock().await; + let existing = self .store .get_message::(sandbox_id) .await - .map_err(|err| err.to_string())? - else { - return Ok(()); - }; - let current_phase = - SandboxPhase::try_from(existing.phase()).unwrap_or(SandboxPhase::Unknown); - if !connected - && matches!(current_phase, SandboxPhase::Error | SandboxPhase::Completed) - && terminal_delivery_finalized - { - drop(guard); - self.schedule_ephemeral_sandbox_delete(&existing); - return Ok(()); - } - if matches!( - current_phase, - SandboxPhase::Deleting - | SandboxPhase::Error - | SandboxPhase::Stopping - | SandboxPhase::Stopped - | SandboxPhase::Completed - ) { - return Ok(()); - } - if !connected && current_phase != SandboxPhase::Ready { - return Ok(()); - } - let expected_resource_version = sandbox_resource_version(&existing); - - // Use CAS to update sandbox phase based on supervisor session state - let result = self - .store - .update_message_cas::(sandbox_id, expected_resource_version, |sandbox| { - let sandbox_name = sandbox.object_name().to_string(); - if connected { - ensure_supervisor_ready_status(&mut sandbox.status, &sandbox_name); - let status = sandbox.status.get_or_insert_with(Default::default); - status.main_process_instance_id = instance_id.unwrap_or_default().to_string(); - status.exit_code = None; - sandbox.set_phase(SandboxPhase::Ready as i32); - } else { - ensure_supervisor_not_ready_status(&mut sandbox.status, &sandbox_name); - sandbox.set_phase(SandboxPhase::Provisioning as i32); - } - }) - .await; + .map_err(|err| err.to_string())?; + self.set_supervisor_session_state_from_snapshot( + sandbox_id, + connected, + instance_id, + terminal_delivery_finalized, + existing, + ) + .await + } - // Handle not found gracefully (sandbox may have been deleted) - let sandbox = match result { - Ok(s) => s, - Err(crate::persistence::PersistenceError::Database(ref msg)) - if msg.contains("not found") => - { + async fn set_supervisor_session_state_from_snapshot( + &self, + sandbox_id: &str, + connected: bool, + instance_id: Option<&str>, + terminal_delivery_finalized: bool, + mut existing: Option, + ) -> Result<(), String> { + for attempt in 1..=SUPERVISOR_SESSION_CAS_RETRY_LIMIT { + let Some(current) = existing else { return Ok(()); - } - Err(crate::persistence::PersistenceError::Conflict { - current_resource_version, - }) => { + }; + let current_phase = + SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); + if connected + && matches!( + current_phase, + SandboxPhase::Deleting | SandboxPhase::Stopping | SandboxPhase::Stopped + ) + { return Err(format!( - "concurrent modification detected (current resource_version: {})", - current_resource_version - .map_or_else(|| "unknown".to_string(), |v| v.to_string()) + "sandbox is not accepting supervisor sessions while {current_phase:?}" )); } - Err(e) => return Err(e.to_string()), - }; + if !connected + && matches!(current_phase, SandboxPhase::Error | SandboxPhase::Completed) + && terminal_delivery_finalized + { + self.schedule_ephemeral_sandbox_delete(¤t); + return Ok(()); + } + if matches!( + current_phase, + SandboxPhase::Deleting + | SandboxPhase::Error + | SandboxPhase::Stopping + | SandboxPhase::Stopped + | SandboxPhase::Completed + ) { + return Ok(()); + } + if !connected && current_phase != SandboxPhase::Ready { + return Ok(()); + } + let expected_resource_version = sandbox_resource_version(¤t); + let result = self + .store + .update_message_cas::( + sandbox_id, + expected_resource_version, + |sandbox| { + let sandbox_name = sandbox.object_name().to_string(); + if connected { + ensure_supervisor_ready_status(&mut sandbox.status, &sandbox_name); + let status = sandbox.status.get_or_insert_with(Default::default); + status.main_process_instance_id = + instance_id.unwrap_or_default().to_string(); + status.exit_code = None; + sandbox.set_phase(SandboxPhase::Ready as i32); + } else { + ensure_supervisor_not_ready_status(&mut sandbox.status, &sandbox_name); + sandbox.set_phase(SandboxPhase::Provisioning as i32); + } + }, + ) + .await; - self.sandbox_index.update_from_sandbox(&sandbox); - self.sandbox_watch_bus.notify(sandbox_id); - Ok(()) + match result { + Ok(sandbox) => { + self.sandbox_index.update_from_sandbox(&sandbox); + self.sandbox_watch_bus.notify(sandbox_id); + return Ok(()); + } + Err(crate::persistence::PersistenceError::Database(ref message)) + if message.contains("not found") => + { + return Ok(()); + } + Err(crate::persistence::PersistenceError::Conflict { + current_resource_version, + }) if attempt < SUPERVISOR_SESSION_CAS_RETRY_LIMIT => { + debug!( + sandbox_id, + attempt, + ?current_resource_version, + "Retrying supervisor session state after concurrent modification" + ); + existing = self + .store + .get_message::(sandbox_id) + .await + .map_err(|error| error.to_string())?; + } + Err(crate::persistence::PersistenceError::Conflict { + current_resource_version, + }) => { + return Err(format!( + "concurrent modification detected after {attempt} attempts (current resource_version: {})", + current_resource_version + .map_or_else(|| "unknown".to_string(), |version| version.to_string()) + )); + } + Err(error) => return Err(error.to_string()), + } + } + + unreachable!("supervisor session CAS retry loop always returns") } /// Persist a terminal canonical-process result. Successful completion is @@ -4357,6 +4406,12 @@ fn apply_driver_snapshot( SandboxPhase::Stopping } SandboxPhase::Stopping if phase != SandboxPhase::Error => SandboxPhase::Stopping, + // A driver's explicit bootstrap condition is authoritative evidence + // that an accepted StartSandbox operation is provisioning a new + // generation. This observation must be able to recover a stale + // Stopped view so the replacement supervisor can register. A genuine + // stop includes Suspended=True and does not satisfy this predicate. + SandboxPhase::Stopped if driver_snapshot_confirms_starting(incoming) => phase, SandboxPhase::Stopped => SandboxPhase::Stopped, SandboxPhase::Completed => SandboxPhase::Completed, SandboxPhase::Starting if !matches!(phase, SandboxPhase::Ready | SandboxPhase::Error) => { @@ -4429,15 +4484,17 @@ fn driver_snapshot_confirms_stopped(incoming: &DriverSandbox) -> bool { }) } -fn driver_snapshot_reports_terminal_container_exit(incoming: &DriverSandbox) -> bool { +fn driver_snapshot_confirms_starting(incoming: &DriverSandbox) -> bool { incoming.status.as_ref().is_some_and(|status| { - status.conditions.iter().any(|condition| { - condition.status.eq_ignore_ascii_case("false") - && matches!( - condition.reason.to_ascii_lowercase().as_str(), - "containerexited" | "containerstopped" | "containerruntimerestart" - ) - }) + !status.deleting + && status.conditions.iter().any(|condition| { + condition.r#type.eq_ignore_ascii_case("Bootstrapping") + && condition.status.eq_ignore_ascii_case("true") + }) + && !status.conditions.iter().any(|condition| { + condition.r#type.eq_ignore_ascii_case("Suspended") + && condition.status.eq_ignore_ascii_case("true") + }) }) } @@ -7130,6 +7187,9 @@ mod tests { ); register_test_supervisor_session(&runtime, sandbox.object_id()); + driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new( + ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()), + ))); runtime .apply_sandbox_update(ready_driver_sandbox( sandbox.object_id(), @@ -7717,6 +7777,63 @@ mod tests { } } + #[tokio::test] + async fn stale_ready_snapshot_queued_before_start_is_revalidated() { + let driver = ControlledDriver::new(); + driver.block_start(); + let sandbox = sandbox_record( + "sb-start-ready-race", + "sandbox-start-ready-race", + SandboxPhase::Stopped, + ); + let mut current = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + current.status = Some(make_driver_status(make_driver_condition( + "ContainerStarting", + "replacement workload is still starting", + ))); + driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(current))); + let mut runtime = test_runtime(driver.clone()).await; + runtime.driver_info.driver_reports_runtime_readiness = true; + runtime.store.put_message(&sandbox).await.unwrap(); + + let start_runtime = runtime.clone(); + let sandbox_name = sandbox.object_name().to_string(); + let start = + tokio::spawn( + async move { start_runtime.start_sandbox("default", &sandbox_name).await }, + ); + tokio::time::timeout(Duration::from_secs(1), driver.start_started.notified()) + .await + .expect("start did not reach the driver"); + + let stale_ready = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + let update_runtime = runtime.clone(); + let mut update = + tokio::spawn(async move { update_runtime.apply_sandbox_update(stale_ready).await }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut update) + .await + .is_err(), + "queued Ready event must wait for the active start operation" + ); + + driver.release_start(); + start.await.unwrap().unwrap(); + update.await.unwrap().unwrap(); + + let stored = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + let phase = SandboxPhase::try_from(stored.phase()).unwrap_or(SandboxPhase::Unknown); + assert!( + matches!(phase, SandboxPhase::Starting | SandboxPhase::Provisioning), + "the queued Ready event must not promote the sandbox; got {phase:?}" + ); + } + #[tokio::test] async fn live_container_exit_during_start_still_transitions_to_error() { for reason in [ @@ -7879,7 +7996,8 @@ mod tests { // (PodTerminated). Starting from the Starting phase that `start` sets, the // reconciled sandbox must advance to Ready rather than being pinned at // Starting by the stale Suspended condition. - let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; let sandbox = sandbox_record("sb-resumed", "sandbox-resumed", SandboxPhase::Starting); runtime.store.put_message(&sandbox).await.unwrap(); register_test_supervisor_session(&runtime, sandbox.object_id()); @@ -7907,6 +8025,7 @@ mod tests { ..Default::default() }); + driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(resumed.clone()))); runtime.apply_sandbox_update(resumed).await.unwrap(); let current = runtime @@ -7944,6 +8063,73 @@ mod tests { assert_eq!(current.phase(), SandboxPhase::Stopped as i32); } + #[tokio::test] + async fn active_bootstrap_snapshot_recovers_stale_stopped_view() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-restart", "sandbox-restart", SandboxPhase::Stopped); + runtime.store.put_message(&sandbox).await.unwrap(); + let mut bootstrapping = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + let mut status = make_driver_status(make_driver_condition( + "DependenciesNotReady", + "replacement supervisor is starting", + )); + status.conditions.push(DriverCondition { + r#type: "Bootstrapping".to_string(), + status: "True".to_string(), + reason: "GenerationStarting".to_string(), + message: "Replacement generation is starting".to_string(), + last_transition_time: String::new(), + }); + bootstrapping.status = Some(status); + + runtime.apply_sandbox_update(bootstrapping).await.unwrap(); + + let current = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(current.phase(), SandboxPhase::Provisioning as i32); + } + + #[tokio::test] + async fn suspended_bootstrap_snapshot_does_not_revive_stopped_sandbox() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-stopped", "sandbox-stopped", SandboxPhase::Stopped); + runtime.store.put_message(&sandbox).await.unwrap(); + let mut status = make_driver_status(make_driver_condition( + "DependenciesNotReady", + "dependencies are unavailable", + )); + status.conditions.push(DriverCondition { + r#type: "Bootstrapping".to_string(), + status: "True".to_string(), + reason: "GenerationStarting".to_string(), + message: "Replacement generation is starting".to_string(), + last_transition_time: String::new(), + }); + status.conditions.push(DriverCondition { + r#type: "Suspended".to_string(), + status: "True".to_string(), + reason: "PodTerminated".to_string(), + message: "Sandbox is suspended".to_string(), + last_transition_time: String::new(), + }); + let mut suspended = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + suspended.status = Some(status); + + runtime.apply_sandbox_update(suspended).await.unwrap(); + + let current = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(current.phase(), SandboxPhase::Stopped as i32); + } + #[tokio::test] async fn begin_sandbox_delete_retries_after_stale_snapshot_conflict() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; @@ -9577,6 +9763,66 @@ mod tests { ); } + #[tokio::test] + async fn supervisor_session_connected_rejects_stopped_sandbox() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Stopped); + runtime.store.put_message(&sandbox).await.unwrap(); + + let error = runtime + .supervisor_session_connected("sb-1", "stale-generation") + .await + .unwrap_err(); + + assert!(error.contains("Stopped")); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Stopped as i32); + } + + #[tokio::test] + async fn supervisor_session_connected_retries_a_stale_store_snapshot() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); + runtime.store.put_message(&sandbox).await.unwrap(); + let stale = runtime.store.get_message::("sb-1").await.unwrap(); + + runtime + .store + .update_message_cas::("sb-1", 0, |sandbox| { + sandbox.set_current_policy_version(7); + }) + .await + .unwrap(); + + runtime + .set_supervisor_session_state_from_snapshot( + "sb-1", + true, + Some("test-generation"), + false, + stale, + ) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Ready + ); + assert_eq!(stored.current_policy_version(), 7); + } + #[tokio::test] async fn supervisor_session_disconnected_demotes_ready_sandbox() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index e0ff130ebc..54ecdea832 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -368,7 +368,7 @@ async fn handle_create_sandbox_inner( }; // Leave an omitted command empty rather than persisting a concrete shell: - // the supervisor resolves the default login shell against the sandbox image + // the sandbox boundary resolves the default login shell against the agent image // (bash when present, otherwise /bin/sh on minimal images like Alpine), // which the gateway cannot do since it does not see the sandbox filesystem. // The default is an interactive login shell, so request a TTY. @@ -2287,10 +2287,7 @@ fn sandbox_relay_reachable(state: &ServerState, sandbox: &Sandbox) -> bool { let phase = SandboxPhase::try_from(sandbox.phase()).ok(); matches!(phase, Some(SandboxPhase::Ready)) || (matches!(phase, Some(SandboxPhase::Completed | SandboxPhase::Error)) - && state.supervisor_sessions.has_session(sandbox.object_id()) - && !state - .supervisor_sessions - .terminal_delivery_finalized(sandbox.object_id())) + && state.supervisor_sessions.has_session(sandbox.object_id())) } pub(super) async fn handle_create_ssh_session( @@ -5894,6 +5891,9 @@ mod tests { .supervisor_sessions .finalize_main_process_exit("sandbox-work") ); + assert!(sandbox_relay_reachable(&state, &sandbox)); + + assert!(state.supervisor_sessions.disconnect("sandbox-work")); assert!(!sandbox_relay_reachable(&state, &sandbox)); } @@ -6657,7 +6657,16 @@ mod tests { let mut sandbox = test_sandbox("cross-ws", Vec::new()); sandbox.metadata.as_mut().unwrap().workspace = "other-workspace".to_string(); + sandbox.set_phase(SandboxPhase::Completed as i32); state.store.put_message(&sandbox).await.unwrap(); + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + let _ = state.supervisor_sessions.register( + sandbox.object_id().to_string(), + "retained-terminal-session".to_string(), + tx, + shutdown_tx, + ); // --- handle_watch_sandbox --- let err = handle_watch_sandbox( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8c8afdf08..5230d7df3a 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -698,15 +698,6 @@ pub(crate) async fn run_server( ) .await?; - if let Err(err) = state.compute.start_persisted_sandboxes().await { - warn!(error = %err, "Failed to start persisted sandboxes during startup"); - } - - state.compute.spawn_watchers(shutdown_rx.clone()); - ssh_sessions::spawn_session_reaper(store.clone(), Duration::from_secs(3600)); - supervisor_session::spawn_relay_reaper(state.clone(), Duration::from_secs(30)); - provider_refresh::spawn_refresh_worker(state.clone(), Duration::from_secs(60)); - // Create the multiplexed service let service = MultiplexService::new(state.clone()); @@ -791,6 +782,19 @@ pub(crate) async fn run_server( ))); } + // Restored supervisors need the callback listeners while the compute + // driver reconciles persisted sandboxes. Serve them before starting that + // reconciliation so policy fetch and supervisor-session registration + // cannot deadlock gateway startup. + if let Err(err) = state.compute.start_persisted_sandboxes().await { + warn!(error = %err, "Failed to start persisted sandboxes during startup"); + } + + state.compute.spawn_watchers(shutdown_rx.clone()); + ssh_sessions::spawn_session_reaper(store.clone(), Duration::from_secs(3600)); + supervisor_session::spawn_relay_reaper(state.clone(), Duration::from_secs(30)); + provider_refresh::spawn_refresh_worker(state.clone(), Duration::from_secs(60)); + shutdown_signal().await; info!("Shutdown signal received; stopping gateway"); state.gateway_shutting_down.store(true, Ordering::Release); diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index c8491dc1eb..5e69d7215f 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -785,26 +785,36 @@ pub async fn handle_connect_supervisor( return Err(Status::internal("failed to send session accepted")); } - if superseded { - state - .supervisor_sessions - .replay_pending_relays(&sandbox_id, &tx) - .await; - } - if let Err(err) = state .compute .supervisor_session_connected(&sandbox_id, &hello.instance_id) .await { + // Do not expose SessionAccepted to the supervisor when the gateway + // could not durably record the connection. Dropping the buffered + // response forces a reconnect, which gives the state transition a + // fresh chance instead of leaving a healthy-looking supervisor tied + // to a sandbox that never reaches Ready. + state + .supervisor_sessions + .remove_if_current(&sandbox_id, &session_id); warn!( sandbox_id = %sandbox_id, session_id = %session_id, error = %err, "supervisor session: failed to mark sandbox ready" ); - } else { - state.telemetry.sandbox_session_connected(&sandbox_id); + return Err(Status::aborted( + "failed to persist supervisor session state; reconnect", + )); + } + state.telemetry.sandbox_session_connected(&sandbox_id); + + if superseded { + state + .supervisor_sessions + .replay_pending_relays(&sandbox_id, &tx) + .await; } // Step 4: Spawn the session loop that reads inbound messages. diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 34d9c32a47..b3b048e394 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -11,12 +11,16 @@ repository.workspace = true rust-version.workspace = true [dependencies] +openshell-binary-identity = { path = "../openshell-binary-identity" } openshell-core = { path = "../openshell-core", features = ["oauth"] } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-router = { path = "../openshell-router" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +async-trait = "0.1" + apollo-parser = { workspace = true } aws-sigv4 = { version = "1", features = ["sign-http", "http1"] } aws-credential-types = { version = "1", features = ["hardcoded-credentials"] } @@ -25,6 +29,7 @@ http = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } flate2 = "1" +futures = { workspace = true } glob = { workspace = true } hex = "0.4" hickory-proto = "0.26.1" @@ -62,7 +67,6 @@ tonic = { workspace = true } temp-env = "0.3" tokio = { workspace = true, features = ["test-util"] } tokio-tungstenite = { workspace = true } -futures = { workspace = true } tracing-subscriber = { workspace = true } tokio-stream = { workspace = true, features = ["net"] } diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index e3fa6d36b4..bb6001f023 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -900,8 +900,8 @@ _matching_endpoint_configs := [cfg | # Full matched endpoint records are kept separate from the legacy # endpoint-config list, which intentionally contains only connection/L7 # metadata. The policy name and array index identify the endpoint within this -# policy generation while the complete endpoint preserves explicit protocol -# markers needed by later policy-DNS correlation. +# policy generation while the complete endpoint preserves protocol markers +# needed by later policy-DNS correlation. _policy_endpoint_records(policy_name, policy) := [record | some endpoint_index, ep in policy.endpoints @@ -922,12 +922,15 @@ _matching_endpoint_records := [record | # Endpoints eligible for policy DNS are a policy-data snapshot, not an # authorization decision. In particular, they do not depend on input.exec or -# grant access to any process. Only endpoints that explicitly opt into raw TCP -# and provide a resolvable host plus concrete ports are materialized. +# grant access to any process. Every supported endpoint protocol is carried by +# TCP, and an omitted protocol is the default L4 TCP form. Endpoints with a +# resolvable host plus concrete ports are therefore materialized regardless of +# whether later stream handling is L4, HTTP, WebSocket, or another L7 adapter. policy_dns_eligible_endpoint_records := [record | some policy_name, policy in data.network_policies some endpoint_index, ep in policy.endpoints - lower(object.get(ep, "protocol", "")) == "tcp" + protocol := lower(object.get(ep, "protocol", "tcp")) + protocol in {"tcp", "rest", "websocket", "graphql", "sql", "json-rpc", "mcp"} object.get(ep, "host", "") != "" ports := object.get(ep, "ports", []) count(ports) > 0 diff --git a/crates/openshell-supervisor-network/src/identity_source.rs b/crates/openshell-supervisor-network/src/identity_source.rs new file mode 100644 index 0000000000..f5446ddbda --- /dev/null +++ b/crates/openshell-supervisor-network/src/identity_source.rs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The in-pod binary-identity resolver (RFC 0012 runtime contract). +//! +//! RFC 0012 delivers executable identity on every +//! [`MediatedConnection`](openshell_isolation_interface::contract::MediatedConnection): +//! the backend resolves identity for the accepted connection before mediation. +//! An unresolved identity denies that connection. This is the in-pod +//! resolution mechanism — procfs, keyed by the workload-side TCP peer port — +//! kept in this crate on purpose: the proxy that consumes identity is here, and +//! so are procfs and the binary identity cache. Stronger backends may use a +//! different resolution mechanism without changing the contract. The result +//! type lives in the lower `openshell-isolation-interface` crate (network -> +//! interface -> core, acyclic). +//! +//! The legacy listener still resolves identity in the proxy hot path. The RFC +//! 0012 co-located source invokes this resolver before returning each accepted +//! connection, so mediation consumes the bound identity result. + +use std::sync::Arc; +use std::sync::atomic::AtomicU32; + +#[cfg(target_os = "linux")] +use openshell_binary_identity::ProcfsIdentityResolver as SharedProcfsIdentityResolver; +use openshell_isolation_interface::contract::{BinaryIdentity, ResolveError}; + +/// In-pod binary-identity resolver: reads and hashes the executable resolved +/// for an accepted connection from procfs. Resolution fails closed; it never +/// fabricates identity fields. +#[derive(Clone)] +pub struct ProcfsIdentityResolver { + /// The workload entrypoint PID, whose network namespace owns the peer + /// sockets the proxy resolves. Published once the agent starts. + pub entrypoint_pid: Arc, +} + +impl ProcfsIdentityResolver { + /// Resolve the executable identity behind an accepted workload connection. + pub fn resolve_connection( + &self, + workload_addr: std::net::SocketAddr, + proxy_addr: std::net::SocketAddr, + ) -> Result { + // procfs resolution is Linux-only; on other targets the supervisor has + // no procfs to read, so resolution fails closed. + #[cfg(target_os = "linux")] + { + self.resolve_via_procfs(workload_addr, proxy_addr) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (workload_addr, proxy_addr); + Err(ResolveError::Failed( + "no procfs on this platform; identity resolution unavailable".to_string(), + )) + } + } +} + +#[cfg(target_os = "linux")] +impl ProcfsIdentityResolver { + fn resolve_via_procfs( + &self, + workload_addr: std::net::SocketAddr, + proxy_addr: std::net::SocketAddr, + ) -> Result { + use std::sync::atomic::Ordering; + + let entrypoint_pid = self.entrypoint_pid.load(Ordering::Acquire); + if entrypoint_pid == 0 { + // No workload yet: nothing to attribute the connection to. Fail + // closed so a binary-scoped rule cannot match an unattributed peer. + return Err(ResolveError::NotFound); + } + + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let owners = crate::procfs::resolve_tcp_peer_socket_owners(entrypoint_pid, connection) + .map_err(|_| ResolveError::NotFound)?; + let resolver = SharedProcfsIdentityResolver::for_process_tree(entrypoint_pid); + let mut identities = Vec::with_capacity(owners.owners.len()); + for owner in owners.owners { + identities.push(resolver.resolve(owner.pid)?); + } + let Some(identity) = identities.first().cloned() else { + return Err(ResolveError::NotFound); + }; + if identities.iter().skip(1).any(|candidate| { + candidate.binary_path != identity.binary_path + || candidate.binary_digest != identity.binary_digest + || candidate.ancestors != identity.ancestors + || candidate.cmdline_paths != identity.cmdline_paths + }) { + return Err(ResolveError::Failed( + "shared socket owners have different policy identities".to_string(), + )); + } + Ok(identity) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Stands in for the mediation service: a binary-scoped rule can only be + /// authorized by a resolved identity carrying the fields it requires. + fn admits_binary_rule(result: Result) -> bool { + matches!(result, Ok(identity) if identity.binary_digest.is_some()) + } + + #[test] + fn fails_closed_before_the_workload_starts() { + // entrypoint_pid == 0 means no agent yet; identity must fail closed so a + // binary-scoped rule cannot be satisfied by an unattributed connection. + let resolver = ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(0)), + }; + assert!(!admits_binary_rule(resolver.resolve_connection( + "127.0.0.1:12345".parse().unwrap(), + "127.0.0.1:3128".parse().unwrap(), + ))); + } + + #[test] + fn unknown_peer_fails_closed() { + // A peer port no live workload connection owns must resolve to an error, + // never a fabricated identity. + let resolver = ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(u32::MAX - 1)), + }; + assert!(!admits_binary_rule(resolver.resolve_connection( + "127.0.0.1:1".parse().unwrap(), + "127.0.0.1:3128".parse().unwrap(), + ))); + } +} diff --git a/crates/openshell-supervisor-network/src/inference_routes.rs b/crates/openshell-supervisor-network/src/inference_routes.rs index 22b406b8dd..90a24aa0e3 100644 --- a/crates/openshell-supervisor-network/src/inference_routes.rs +++ b/crates/openshell-supervisor-network/src/inference_routes.rs @@ -106,6 +106,22 @@ pub async fn build_inference_context( sandbox_id: Option<&str>, openshell_endpoint: Option<&str>, inference_routes: Option<&str>, +) -> Result>> { + build_inference_context_with_host_gateway( + sandbox_id, + openshell_endpoint, + inference_routes, + None, + ) + .await +} + +#[allow(clippy::similar_names)] +pub async fn build_inference_context_with_host_gateway( + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + inference_routes: Option<&str>, + host_gateway_ip: Option, ) -> Result>> { use openshell_router::Router; use openshell_router::config::RouterConfig; @@ -250,13 +266,18 @@ pub async fn build_inference_context( // Partition routes by name into user-facing and system caches. let (user_routes, system_routes) = partition_routes(routes); - let router = - Router::new().map_err(|e| miette::miette!("failed to initialize inference router: {e}"))?; + let inference_router = Router::with_dns_overrides(host_gateway_ip.into_iter().flat_map(|ip| { + crate::proxy::HOST_GATEWAY_ALIASES + .iter() + .copied() + .map(move |host| (host, ip)) + })) + .map_err(|e| miette::miette!("failed to initialize inference router: {e}"))?; let patterns = crate::l7::inference::default_patterns(); let ctx = Arc::new(crate::proxy::InferenceContext::new( patterns, - router, + inference_router, user_routes, system_routes, )); diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 93315a671a..fb5db6b94a 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -3301,16 +3301,30 @@ fn parse_status_code(headers: &str) -> Option { code_str.parse().ok() } -/// Check if the response headers contain `Connection: close`. +/// Check whether the response is delimited by closing the connection. +/// +/// HTTP/1.0 closes by default unless the server explicitly negotiates +/// keep-alive. HTTP/1.1 keeps connections alive by default unless the server +/// sends `Connection: close`. fn parse_connection_close(headers: &str) -> bool { + let http_1_0 = headers + .lines() + .next() + .is_some_and(|line| line.starts_with("HTTP/1.0 ")); for line in headers.lines().skip(1) { let lower = line.to_ascii_lowercase(); if lower.starts_with("connection:") { let val = lower.split_once(':').map_or("", |(_, v)| v.trim()); - return val.contains("close"); + return if http_1_0 { + !val.split(',') + .any(|token| token.trim().eq_ignore_ascii_case("keep-alive")) + } else { + val.split(',') + .any(|token| token.trim().eq_ignore_ascii_case("close")) + }; } } - false + http_1_0 } fn response_is_event_stream(headers: &str) -> bool { @@ -5477,6 +5491,10 @@ mod tests { assert!(!parse_connection_close( "HTTP/1.1 200 OK\r\nHost: x\r\n\r\n" )); + assert!(parse_connection_close("HTTP/1.0 200 OK\r\nHost: x\r\n\r\n")); + assert!(!parse_connection_close( + "HTTP/1.0 200 OK\r\nConnection: keep-alive\r\n\r\n" + )); } #[test] @@ -5549,6 +5567,41 @@ mod tests { ); } + #[tokio::test] + async fn relay_response_http_1_0_defaults_to_connection_close() { + let response = b"HTTP/1.0 200 OK\r\nServer: test\r\n\r\nhello world"; + let (mut upstream_read, mut upstream_write) = tokio::io::duplex(4096); + let (mut client_read, mut client_write) = tokio::io::duplex(4096); + + tokio::spawn(async move { + upstream_write.write_all(response).await.unwrap(); + upstream_write.shutdown().await.unwrap(); + }); + + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(2), + relay_response( + "POST", + &mut upstream_read, + &mut client_write, + RelayResponseOptions::default(), + ), + ) + .await + .expect("HTTP/1.0 close-delimited response should not deadlock") + .expect("HTTP/1.0 response should relay"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + + client_write.shutdown().await.unwrap(); + let mut received = Vec::new(); + client_read.read_to_end(&mut received).await.unwrap(); + assert!( + received + .windows(b"hello world".len()) + .any(|value| value == b"hello world") + ); + } + #[tokio::test] async fn relay_response_no_framing_event_stream_reads_until_eof() { let response = diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index 2275a60d34..4b8be4736e 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -8,7 +8,7 @@ //! store, terminates TLS from the client (presenting dynamic certs per hostname), //! inspects the plaintext HTTP, then re-encrypts to upstream using real root CAs. -use miette::{IntoDiagnostic, Result, miette}; +use miette::{IntoDiagnostic, Result, WrapErr, miette}; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName}; use rustls::{ClientConfig, ServerConfig}; @@ -17,7 +17,6 @@ use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::TcpStream; use tokio_rustls::{TlsAcceptor, TlsConnector}; const MAX_CACHED_CERTS: usize = 256; @@ -67,6 +66,76 @@ impl SandboxCa { pub fn cert_pem(&self) -> &str { &self.ca_cert_pem } + + /// Returns the CA private key in PKCS#8 PEM format. + pub fn private_key_pem(&self) -> String { + self.ca_key.serialize_pem() + } + + /// Load a durable CA certificate and matching private key from absolute paths. + pub fn load_from_paths(certificate_path: &Path, private_key_path: &Path) -> Result { + if !certificate_path.is_absolute() || !private_key_path.is_absolute() { + return Err(miette!( + "proxy CA certificate and key paths must be absolute" + )); + } + if certificate_path == private_key_path { + return Err(miette!( + "proxy CA certificate and private key must use different paths" + )); + } + let certificate_pem = std::fs::read_to_string(certificate_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA certificate {}", certificate_path.display()) + })?; + let private_key_pem = std::fs::read_to_string(private_key_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA private key {}", private_key_path.display()) + })?; + Self::from_pem(&certificate_pem, &private_key_pem) + } + + /// Load a durable CA while preserving the exact certificate bytes supplied + /// by the provisioner for boundary launch replay. + pub fn from_pem(certificate_pem: &str, private_key_pem: &str) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let ca_key = KeyPair::from_pem(private_key_pem) + .into_diagnostic() + .wrap_err("parse proxy CA private key")?; + let certificates = rustls_pemfile::certs(&mut certificate_pem.as_bytes()) + .collect::, _>>() + .into_diagnostic() + .wrap_err("parse proxy CA certificate")?; + if certificates.len() != 1 { + return Err(miette!( + "proxy CA certificate file must contain exactly one certificate" + )); + } + let private_key = rustls_pemfile::private_key(&mut private_key_pem.as_bytes()) + .into_diagnostic() + .wrap_err("parse proxy CA private key")? + .ok_or_else(|| miette!("proxy CA private key file contains no private key"))?; + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .into_diagnostic() + .wrap_err("proxy CA certificate and private key do not match")?; + + let params = CertificateParams::from_ca_cert_pem(certificate_pem) + .into_diagnostic() + .wrap_err("parse proxy CA signing certificate")?; + let ca_cert = params + .self_signed(&ca_key) + .into_diagnostic() + .wrap_err("initialize proxy CA signer")?; + Ok(Self { + ca_cert, + ca_key, + ca_cert_pem: certificate_pem.to_string(), + }) + } } /// A leaf certificate chain and private key for a specific hostname. @@ -170,11 +239,14 @@ impl ProxyTlsState { /// Accept TLS from a sandbox client, presenting a dynamic cert for the hostname. /// /// Returns a TLS stream that can be used for plaintext HTTP inspection. -pub async fn tls_terminate_client( - client: TcpStream, +pub async fn tls_terminate_client( + client: S, tls_state: &ProxyTlsState, hostname: &str, -) -> Result { +) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ let acceptor = tls_state.acceptor_for(hostname)?; let tls_stream = acceptor.accept(client).await.into_diagnostic()?; Ok(tls_stream) @@ -559,4 +631,23 @@ mod tests { "bundle should contain at least one cert", ); } + + #[test] + fn durable_ca_round_trip_preserves_certificate_bytes() { + let generated = SandboxCa::generate().unwrap(); + let certificate = generated.cert_pem().to_string(); + let private_key = generated.private_key_pem(); + let loaded = SandboxCa::from_pem(&certificate, &private_key).unwrap(); + + assert_eq!(loaded.cert_pem(), certificate); + assert_eq!(loaded.private_key_pem(), private_key); + } + + #[test] + fn durable_ca_rejects_mismatched_key_and_relative_paths() { + let certificate = SandboxCa::generate().unwrap(); + let other_key = SandboxCa::generate().unwrap(); + assert!(SandboxCa::from_pem(certificate.cert_pem(), &other_key.private_key_pem()).is_err()); + assert!(SandboxCa::load_from_paths(Path::new("ca.pem"), Path::new("ca.key")).is_err()); + } } diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index 4fec48b300..f5537d28f3 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -9,6 +9,7 @@ //! aggregate them. pub mod identity; +pub mod identity_source; pub mod inference_routes; pub mod l7; pub mod opa; @@ -18,6 +19,7 @@ pub mod procfs; pub mod proxy; pub mod run; pub mod sigv4; +mod spiffe_endpoint; mod token_grant; pub mod upstream_proxy; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 98632686e9..3696470228 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -92,15 +92,6 @@ pub struct NetworkInput { pub cmdline_paths: Vec, } -pub(crate) fn network_binary_identity_required() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY).map_or(true, |value| { - !matches!( - value.as_str(), - "relaxed" | "disabled" | "endpoint-only" | "false" | "0" - ) - }) -} - fn inject_runtime_policy_data(data: &mut serde_json::Value, require_binary_identity: bool) { let Some(obj) = data.as_object_mut() else { return; @@ -318,7 +309,7 @@ impl OpaEngine { engine .add_policy_from_file(policy_path) .map_err(|e| miette::miette!("{e}"))?; - let require_binary_identity = network_binary_identity_required(); + let require_binary_identity = true; emit_binary_identity_mode(require_binary_identity, "files"); let data_json = preprocess_yaml_data( &yaml_str, @@ -335,7 +326,7 @@ impl OpaEngine { /// /// Preprocesses the YAML data to expand access presets and validate L7 config. pub fn from_strings(policy: &str, data_yaml: &str) -> Result { - Self::from_strings_with_options(policy, data_yaml, network_binary_identity_required(), None) + Self::from_strings_with_options(policy, data_yaml, true, None) } pub fn from_strings_with_middleware_config( @@ -343,12 +334,7 @@ impl OpaEngine { data_yaml: &str, validate_middleware_config: Option<&MiddlewareConfigValidator>, ) -> Result { - Self::from_strings_with_options( - policy, - data_yaml, - network_binary_identity_required(), - validate_middleware_config, - ) + Self::from_strings_with_options(policy, data_yaml, true, validate_middleware_config) } #[cfg(test)] @@ -401,11 +387,7 @@ impl OpaEngine { /// gap between user-specified symlink paths (e.g., `/usr/bin/python3`) and /// kernel-resolved canonical paths (e.g., `/usr/bin/python3.11`). pub fn from_proto_with_pid(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> Result { - Self::from_proto_with_pid_and_binary_identity_required( - proto, - entrypoint_pid, - network_binary_identity_required(), - ) + Self::from_proto_with_pid_and_binary_identity_required(proto, entrypoint_pid, true) } fn from_proto_with_pid_and_binary_identity_required( @@ -815,6 +797,7 @@ impl OpaEngine { /// generation comparison and callback linearizes state derived from an OPA /// snapshot with every policy reload and fail-closed transition. Callers /// must not perform I/O or other long-running work in `operation`. + #[allow(dead_code)] pub(crate) fn with_current_generation( &self, expected_generation: u64, @@ -2452,13 +2435,13 @@ process: "#; #[test] - fn policy_dns_snapshot_is_tcp_only_stable_and_generation_consistent() { + fn policy_dns_snapshot_includes_every_tcp_carried_endpoint() { let engine = OpaEngine::from_strings(TEST_POLICY, POLICY_DNS_SNAPSHOT_DATA).unwrap(); let snapshot = engine.policy_dns_eligibility_snapshot().unwrap(); assert_eq!(snapshot.generation, engine.current_generation()); - assert_eq!(snapshot.endpoints.len(), 2); + assert_eq!(snapshot.endpoints.len(), 4); assert_eq!(snapshot.endpoints[0].policy_name, "dns_transport"); assert_eq!(snapshot.endpoints[0].endpoint_index, 0); assert_eq!( @@ -2471,15 +2454,17 @@ process: panic!("eligible endpoint must retain concrete ports"); }; assert_eq!(ports.as_ref(), &[53.into(), 853.into()]); - assert_eq!(snapshot.endpoints[1].endpoint_index, 4); + assert_eq!(snapshot.endpoints[1].endpoint_index, 1); + assert_eq!(snapshot.endpoints[2].endpoint_index, 2); + assert_eq!(snapshot.endpoints[3].endpoint_index, 4); engine .reload(TEST_POLICY, POLICY_DNS_SNAPSHOT_DATA) .unwrap(); let reloaded = engine.policy_dns_eligibility_snapshot().unwrap(); assert_eq!(reloaded.generation, snapshot.generation + 1); - assert_eq!(reloaded.endpoints.len(), 2); - assert_eq!(reloaded.endpoints[1].endpoint_index, 4); + assert_eq!(reloaded.endpoints.len(), 4); + assert_eq!(reloaded.endpoints[3].endpoint_index, 4); } #[test] @@ -2497,7 +2482,13 @@ process: fn policy_dns_snapshot_accepts_the_default_multi_policy_shape() { let engine = OpaEngine::from_strings(TEST_POLICY, TEST_DATA_YAML).unwrap(); let snapshot = engine.policy_dns_eligibility_snapshot().unwrap(); - assert!(snapshot.endpoints.is_empty()); + assert_eq!(snapshot.endpoints.len(), 15); + assert!( + snapshot + .endpoints + .iter() + .any(|endpoint| endpoint.policy_name == "claude_code") + ); } #[test] diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index b7dd13ca9a..8b260294c9 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -34,7 +34,7 @@ pub(crate) use store::{ use crate::opa::OpaEngine; use crate::proxy::destination::{build_validation_plan, filter_resolved_addresses}; -use crate::proxy::is_host_gateway_alias; +use crate::proxy::{INFERENCE_LOCAL_HOST, INFERENCE_LOCAL_PORT, is_host_gateway_alias}; use openshell_core::host_pattern::HostSelector; use openshell_ocsf::{ ActionId, ActivityId, ConfigStateChangeBuilder, DispositionId, Endpoint, @@ -122,11 +122,16 @@ impl PolicyDnsService { .policy .policy_dns_eligibility_snapshot() .map_err(|error| PolicyDnsError::Policy(error.to_string()))?; - let eligible = eligible_endpoints( - &snapshot.endpoints, - &normalized_name, - self.trusted_host_gateway, - )?; + let system_inference = normalized_name.as_str() == INFERENCE_LOCAL_HOST; + let eligible = if system_inference { + vec![system_inference_endpoint(family)?] + } else { + eligible_endpoints( + &snapshot.endpoints, + &normalized_name, + self.trusted_host_gateway, + )? + }; if eligible.is_empty() { emit_dns_denial( &normalized_name, @@ -139,18 +144,34 @@ impl PolicyDnsService { // The trusted resolver is invoked only after the immutable snapshot // proved policy eligibility. It never consults sandbox resolver state. let endpoint_context = eligible_endpoint_context(&eligible); - let trusted_answer = match self.resolver.resolve(&normalized_name, family).await { - Ok(answer) => answer, - Err(error) => { - emit_dns_failure( - &normalized_name, - family, - &endpoint_context, - snapshot.generation, - resolver_failure_detail(&error), - "Policy DNS trusted resolver query failed", - ); - return Err(PolicyDnsError::Resolver(error)); + let trusted_answer = if system_inference { + TrustedAnswer { + addresses: vec![family_loopback(family)], + ttl: MAX_MAPPING_TTL, + } + } else if is_host_gateway_alias(normalized_name.as_str()) { + let address = self + .trusted_host_gateway + .filter(|address| family.accepts(*address)) + .ok_or(PolicyDnsError::NoValidAddress)?; + TrustedAnswer { + addresses: vec![address], + ttl: MAX_MAPPING_TTL, + } + } else { + match self.resolver.resolve(&normalized_name, family).await { + Ok(answer) => answer, + Err(error) => { + emit_dns_failure( + &normalized_name, + family, + &endpoint_context, + snapshot.generation, + resolver_failure_detail(&error), + "Policy DNS trusted resolver query failed", + ); + return Err(PolicyDnsError::Resolver(error)); + } } }; let ttl = clamp_mapping_ttl(trusted_answer.ttl); @@ -253,6 +274,28 @@ impl PolicyDnsService { } } +fn family_loopback(family: AddressFamily) -> std::net::IpAddr { + match family { + AddressFamily::Ipv4 => std::net::Ipv4Addr::LOCALHOST.into(), + AddressFamily::Ipv6 => std::net::Ipv6Addr::LOCALHOST.into(), + } +} + +fn system_inference_endpoint(family: AddressFamily) -> Result { + let address = family_loopback(family); + let destination_plan = crate::proxy::destination::build_pinned_validation_plan(vec![address]) + .map_err(|error| PolicyDnsError::Policy(error.reason))?; + Ok(EligibleEndpoint { + endpoint_id: PolicyEndpointId { + policy_name: "openshell-system-inference".to_string(), + endpoint_index: 0, + }, + ports: vec![INFERENCE_LOCAL_PORT], + destination_plan, + contract_fingerprint: "openshell-system-inference-local".to_string(), + }) +} + struct EligibleEndpoint { endpoint_id: PolicyEndpointId, ports: Vec, @@ -286,6 +329,7 @@ fn eligible_endpoints( name.as_str(), name.as_str(), trusted_host_gateway, + None, &raw_allowed_ips, exact_declared_host, ) @@ -612,6 +656,36 @@ process: { run_as_user: sandbox, run_as_group: sandbox } assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } + #[tokio::test] + async fn publishes_system_inference_without_upstream_resolution_or_user_policy() { + let service = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); + let now = Instant::now(); + + let answer = service + .answer_query(INFERENCE_LOCAL_HOST, AddressFamily::Ipv4, now) + .await + .unwrap(); + + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); + let mapping = service + .store + .lookup( + answer.address, + INFERENCE_LOCAL_PORT, + answer.policy_generation, + now, + ) + .unwrap(); + assert_eq!( + mapping.record.normalized_name.as_str(), + INFERENCE_LOCAL_HOST + ); + assert_eq!( + mapping.pinned_addresses(), + [IpAddr::V4(Ipv4Addr::LOCALHOST)] + ); + } + #[tokio::test] async fn eligible_nxdomain_fails_without_publishing_a_mapping() { let policy = Arc::new( @@ -805,26 +879,42 @@ process: { run_as_user: sandbox, run_as_group: sandbox } .unwrap(); assert_eq!(mapping.record.contracts[0].pinned_addresses, [trusted]); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn reserved_gateway_alias_accepts_an_exact_private_backend_gateway() { + let trusted: IpAddr = "172.23.0.1".parse().unwrap(); + let service = gateway_service(Vec::new(), Some(trusted)); + let now = Instant::now(); + + let answer = service + .answer_query("host.openshell.internal", AddressFamily::Ipv4, now) + .await + .unwrap(); + let mapping = service + .store + .lookup(answer.address, 8080, answer.policy_generation, now) + .unwrap(); + + assert_eq!(mapping.record.contracts[0].pinned_addresses, [trusted]); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } #[tokio::test] - async fn reserved_gateway_alias_rejects_mismatch_metadata_private_and_wrong_family_answers() { + async fn reserved_gateway_alias_rejects_wrong_address_family_without_resolver_fallback() { let trusted: IpAddr = "169.254.1.2".parse().unwrap(); - for (family, address) in [ - (AddressFamily::Ipv4, "169.254.1.3"), - (AddressFamily::Ipv4, "169.254.169.254"), - (AddressFamily::Ipv4, "10.2.3.4"), - (AddressFamily::Ipv6, "fe80::2"), - ] { - let service = gateway_service(vec![address.parse().unwrap()], Some(trusted)); - let result = service - .answer_query("host.openshell.internal", family, Instant::now()) - .await; - assert!( - matches!(result, Err(PolicyDnsError::NoValidAddress)), - "{address} must not satisfy the trusted gateway contract" - ); - } + let service = gateway_service(vec!["fe80::2".parse().unwrap()], Some(trusted)); + let result = service + .answer_query( + "host.openshell.internal", + AddressFamily::Ipv6, + Instant::now(), + ) + .await; + + assert!(matches!(result, Err(PolicyDnsError::NoValidAddress))); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } struct BlockingResolver { diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index ad6095efa9..9e866ebb03 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -7,8 +7,10 @@ use super::resolver::MAX_DNS_MESSAGE_BYTES; use super::store::{ResolvedEndpointStore, StoreConfig, SyntheticPools}; use super::{PolicyDnsService, SocketTrustedResolver, wire}; use crate::opa::OpaEngine; +use futures::{FutureExt as _, StreamExt as _, stream::FuturesUnordered}; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_isolation_interface::contract::{DnsMediationSource, DnsTransport}; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -20,6 +22,7 @@ const IPV6_POOL_PREFIX: u8 = 119; const IPV4_EPOCH_WINDOWS: u64 = 1 << (IPV4_POOL_PREFIX - 15); const MAX_MAPPINGS: usize = 1024; const MAX_CONCURRENT_UDP_QUERIES: usize = 64; +const MEDIATION_ACCEPT_WINDOW: usize = 32; #[derive(Debug, Clone)] pub(crate) struct PolicyDnsRuntimeConfig { @@ -63,6 +66,97 @@ pub(crate) struct PolicyDnsRuntime { } impl PolicyDnsRuntime { + /// Start policy DNS over an isolation-backend exchange source. No UDP or + /// TCP listener is bound in the supervisor namespace. + pub(crate) fn start_mediated( + policy: Arc, + source: Arc, + trusted_host_gateway: Option, + config: PolicyDnsRuntimeConfig, + mut engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream = trusted_resolver_from_resolv_conf()?; + let store = Arc::new(ResolvedEndpointStore::new( + StoreConfig::new(config.pools, MAX_MAPPINGS) + .map_err(|error| miette::miette!(error.to_string()))?, + )); + let service = Arc::new(PolicyDnsService::new( + policy, + SocketTrustedResolver::new(upstream), + store.clone(), + trusted_host_gateway, + )); + let task = tokio::spawn(async move { + if engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + let mut accepts = FuturesUnordered::new(); + for _ in 0..MEDIATION_ACCEPT_WINDOW { + let source = source.clone(); + accepts.push(async move { source.accept().await }.boxed()); + } + loop { + let Some(Ok(query)) = accepts.next().await else { + return; + }; + let source = source.clone(); + accepts.push(async move { source.accept().await }.boxed()); + let service = service.clone(); + tokio::spawn(async move { + let timing = query.timing.clone(); + let response = match query.transport { + DnsTransport::Udp => { + wire::handle_udp_query_with_ipv6(&service, &query.request, false).await + } + DnsTransport::Tcp => { + wire::handle_tcp_query_with_ipv6(&service, &query.request, false).await + } + } + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process(format!( + "policy DNS response failed: {error}" + )) + }); + if query.response.send(response).is_err() { + tracing::warn!("sandbox DNS response channel closed before delivery"); + } + tracing::debug!( + target: "openshell::dns_timing", + notification_to_queue_us = timing + .sandbox_notification_to_queue + .as_micros(), + queue_wait_us = timing.sandbox_queue_wait.as_micros(), + supervisor_processing_us = timing + .supervisor_received_at + .elapsed() + .as_micros(), + "mediated DNS query timing" + ); + }); + } + }); + let expiry_store = store.clone(); + let expiry_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let _ = expiry_store.expire(std::time::Instant::now()); + } + }); + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "ready") + .message("Policy DNS connected to isolation boundary") + .build() + ); + Ok(Self { + store, + tasks: vec![task, expiry_task], + }) + } + pub(crate) fn start( policy: Arc, udp: tokio::net::UdpSocket, diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index ea2cd7c0f8..439cdcfa36 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -73,6 +73,17 @@ pub(crate) struct MappingLookup { } impl MappingLookup { + pub(crate) fn pinned_addresses(&self) -> Vec { + let mut seen = HashSet::new(); + self.record + .contracts + .iter() + .filter(|contract| contract.port == self.port) + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .filter(|address| seen.insert(*address)) + .collect() + } + pub(crate) fn endpoint_ids(&self) -> impl Iterator { self.record .contracts diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 84df3801ce..400e2f722f 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -11,9 +11,11 @@ use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; #[cfg(target_os = "linux")] -use crate::policy_dns::{MappingLookupError, PolicyEndpointId, ResolvedEndpointStore}; +use crate::policy_dns::PolicyEndpointId; +use crate::policy_dns::{MappingLookupError, ResolvedEndpointStore}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; +use futures::{FutureExt as _, StreamExt as _, stream::FuturesUnordered}; use miette::{IntoDiagnostic, Result}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::denial::DenialEvent; @@ -24,6 +26,10 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation_interface::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, MediationTiming, + NetworkMediationSource, NetworkOpenResult, PendingNetworkOpen, ResolveError, +}; use openshell_ocsf::{ ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, @@ -36,16 +42,37 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; -use tokio::net::{TcpListener, TcpStream}; +use tokio::net::TcpListener; +#[cfg(any(target_os = "linux", test))] +use tokio::net::TcpStream; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; +type ProxyClient = tokio::io::BufReader; +type AcceptedProxyConnection = ( + BoundaryDuplexStream, + Option>, + Option<(SocketAddr, SocketAddr)>, + Option, +); + +struct TransparentOpen { + destination: SocketAddr, + authorization: Option<(EgressDecision, destination::UpstreamConnector)>, +} + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation_interface::contract::BackendError), +} + use self::destination::{ - DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, - validate_destination, + DestinationDenial, DestinationDenialKind, DestinationRequest, build_pinned_validation_plan, + build_validation_plan, validate_destination, }; use self::egress::{ EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, @@ -53,6 +80,25 @@ use self::egress::{ }; const MAX_HEADER_BYTES: usize = 8192; +const MEDIATION_ACCEPT_WINDOW: usize = 32; + +struct NetworkOpenTimingGuard { + timing: MediationTiming, + operation: &'static str, +} + +impl Drop for NetworkOpenTimingGuard { + fn drop(&mut self) { + tracing::debug!( + target: "openshell::network_open_timing", + operation = self.operation, + notification_to_queue_us = self.timing.sandbox_notification_to_queue.as_micros(), + queue_wait_us = self.timing.sandbox_queue_wait.as_micros(), + supervisor_processing_us = self.timing.supervisor_received_at.elapsed().as_micros(), + "mediated network open timing" + ); + } +} const TUNNEL_PROTOCOL_PEEK_BYTES: usize = crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE.len(); #[cfg(not(test))] const TUNNEL_PROTOCOL_PEEK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(100); @@ -62,12 +108,10 @@ const TUNNEL_PROTOCOL_PEEK_TIMEOUT: std::time::Duration = std::time::Duration::f const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(5); #[cfg(test)] const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(1); -const INFERENCE_LOCAL_HOST: &str = "inference.local"; -const INFERENCE_LOCAL_PORT: u16 = 443; +pub(crate) const INFERENCE_LOCAL_HOST: &str = "inference.local"; +pub(crate) const INFERENCE_LOCAL_PORT: u16 = 443; const FORWARD_ENCODED_SLASH_REJECTION_DETAIL: &str = "request-target contains an encoded '/' (%2F) which is not allowed on this endpoint"; -#[cfg(target_os = "linux")] -const SIDECAR_SUPERVISOR_TOPOLOGY: &str = "sidecar"; fn emit_credential_endpoint_mismatch(host: &str, port: u16, policy_name: &str) { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -97,7 +141,7 @@ fn emit_credential_endpoint_mismatch(host: &str, port: u16, policy_name: &str) { /// machine. Traffic to these names is eligible for the trusted-gateway SSRF /// exemption when the resolved IP matches the driver-injected value read from /// `/etc/hosts` at proxy startup. -const HOST_GATEWAY_ALIASES: &[&str] = &[ +pub(crate) const HOST_GATEWAY_ALIASES: &[&str] = &[ "host.openshell.internal", "host.containers.internal", "host.docker.internal", @@ -255,6 +299,9 @@ impl ProxyHandle { activity_tx: Option, engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, + backend_host_gateway: Option, + network_mediation_source: Option>, + policy_dns_store: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -270,15 +317,27 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let source_backed = network_mediation_source.is_some(); + let listener = if source_backed { + None + } else { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) .severity(SeverityId::Informational) .status(StatusId::Success) .dst_endpoint(Endpoint::from_ip(local_addr.ip(), local_addr.port())) - .message(format!("Proxy listening on {local_addr}")) + .message(if source_backed { + "Proxy consuming isolation-boundary streams".to_string() + } else { + format!("Proxy listening on {local_addr}") + }) .build(); ocsf_emit!(event); } @@ -287,6 +346,7 @@ impl ProxyHandle { // runs. This is read once at startup so later /etc/hosts modifications // by sandbox workloads cannot influence the stored value. let trusted_host_gateway: Arc> = Arc::new(detect_trusted_host_gateway()); + let backend_host_gateway = Arc::new(backend_host_gateway); if let Some(ref ip) = *trusted_host_gateway { tracing::info!( %ip, @@ -368,14 +428,81 @@ impl ProxyHandle { } } + let mut network_accepts = network_mediation_source.as_ref().map(|source| { + let accepts = FuturesUnordered::new(); + for _ in 0..MEDIATION_ACCEPT_WINDOW { + let source = source.clone(); + accepts.push(async move { source.accept().await }.boxed()); + } + accepts + }); + // Transparent opens require policy evaluation and destination + // validation before the sandbox may complete connect(2). Keep + // those potentially expensive operations out of the accept loop: + // serial preauthorization turns bursts of DNS-driven TCP opens + // into head-of-line blocking even though the source itself can + // accept a window of requests concurrently. + let (preauthorized_tx, mut preauthorized_rx) = + mpsc::channel(MEDIATION_ACCEPT_WINDOW * 2); let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + let accepts = network_accepts + .as_mut() + .expect("mediation source has an accept window"); + tokio::select! { + pending = accepts.next() => { + let pending = pending.expect("accept window is never empty"); + let source = source.clone(); + accepts.push(async move { source.accept().await }.boxed()); + match pending { + Ok(connection) => { + let tx = preauthorized_tx.clone(); + let dns_store = policy_dns_store.clone(); + let opa = opa_engine.clone(); + let backend_gateway = *backend_host_gateway; + let trusted_gateway = *trusted_host_gateway; + tokio::spawn(async move { + if let Some(connection) = preauthorize_transparent_open( + connection, + dns_store.as_ref(), + &opa, + backend_gateway, + trusted_gateway, + ) + .await + { + let _ = tx.send(connection).await; + } + }); + continue; + } + Err(error) => Err(ProxyAcceptError::Source(error)), + } + } + Some(connection) = preauthorized_rx.recv() => Ok(connection), + } + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + set_tcp_nodelay_best_effort(&stream); + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr), None) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs, transparent_destination)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -384,8 +511,10 @@ impl ProxyHandle { let policy_local = policy_local_ctx.clone(); let proposals = agent_proposals.clone(); let gw = trusted_host_gateway.clone(); + let backend_gw = backend_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); let credentials = provider_credentials.clone(); + let dns_store = policy_dns_store.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); @@ -398,8 +527,12 @@ impl ProxyHandle { let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, + transparent_destination, + dns_store, opa, cache, spid, @@ -407,6 +540,7 @@ impl ProxyHandle { inf, policy_local, proposals, + backend_gw, gw, up_proxy, credentials, @@ -427,7 +561,19 @@ impl ProxyHandle { } }); } - Err(err) => { + Err(ProxyAcceptError::Source(err)) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!( + "Network-mediation source failed; proxy accept loop exiting: {err}" + )) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(err)) => { match classify_accept_error( &err, &mut consecutive_resource_errors, @@ -465,7 +611,7 @@ impl ProxyHandle { }); Ok(Self { - http_addr: Some(local_addr), + http_addr: (!source_backed).then_some(local_addr), join, exited_rx: Some(exited_rx), }) @@ -481,6 +627,218 @@ impl ProxyHandle { } } +async fn preauthorize_transparent_open( + connection: PendingNetworkOpen, + policy_dns_store: Option<&Arc>, + opa_engine: &OpaEngine, + backend_host_gateway: Option, + trusted_host_gateway: Option, +) -> Option { + let PendingNetworkOpen { + stream, + binary_identity, + destination, + socket: _, + policy_generation: _, + timing, + result, + } = connection; + let _timing = NetworkOpenTimingGuard { + timing, + operation: "tcp", + }; + let host = match transparent_destination_host(destination, policy_dns_store, opa_engine) { + Ok(host) => host, + Err(error) => { + warn!(%destination, %error, "Denied staged transparent connection"); + emit_staged_transparent_denial( + destination, + &binary_identity, + &error.to_string(), + "transparent_tcp_mapping_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + }; + if host != INFERENCE_LOCAL_HOST || destination.port() != INFERENCE_LOCAL_PORT { + let mut decision = authorize_supplied_identity( + opa_engine, + EgressIntent::connect(host.clone(), destination.port()), + &binary_identity, + ); + if let NetworkAction::Deny { reason } = &decision.action { + warn!(%destination, %reason, "Denied staged transparent connection"); + emit_staged_transparent_denial( + destination, + &binary_identity, + reason, + "transparent_tcp_policy_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + if let Err(denial) = + hydrate_destination_plan(&mut decision, backend_host_gateway, trusted_host_gateway) + { + warn!(%destination, reason = %denial.reason, "Denied staged transparent destination"); + emit_staged_transparent_denial( + destination, + &binary_identity, + &denial.reason, + "transparent_tcp_destination_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + if let Some(mapping) = policy_dns_store.and_then(|store| { + store + .lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) + .ok() + }) { + let Ok(plan) = build_pinned_validation_plan(mapping.pinned_addresses()) else { + emit_staged_transparent_denial( + destination, + &binary_identity, + "policy DNS produced an invalid pinned destination", + "transparent_tcp_destination_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + }; + decision.endpoint.destination = Some(plan); + } + let plan = decision + .endpoint + .destination + .as_ref() + .expect("destination plan hydrated"); + let connector = match validate_destination(DestinationRequest { + host: &host, + port: destination.port(), + sandbox_entrypoint_pid: 0, + plan, + }) + .await + { + Ok(connector) => connector, + Err(denial) => { + warn!(%destination, reason = %denial.reason, "Denied staged transparent destination"); + emit_staged_transparent_denial( + destination, + &binary_identity, + &denial.reason, + "transparent_tcp_destination_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + }; + if result.send(NetworkOpenResult::RelayReady).is_err() { + return None; + } + return Some(( + stream, + Some(binary_identity), + None, + Some(TransparentOpen { + destination, + authorization: Some((decision, connector)), + }), + )); + } + if result.send(NetworkOpenResult::RelayReady).is_err() { + return None; + } + Some(( + stream, + Some(binary_identity), + None, + Some(TransparentOpen { + destination, + authorization: None, + }), + )) +} + +fn emit_staged_transparent_denial( + destination: SocketAddr, + identity: &Result, + reason: &str, + status_detail: &'static str, +) { + let (binary, ancestors, cmdline) = identity.as_ref().map_or_else( + |_| ("-".to_string(), "-".to_string(), "-".to_string()), + |identity| { + ( + identity.binary_path.display().to_string(), + identity + .ancestors + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(" -> "), + identity + .cmdline_paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "), + ) + }, + ); + ocsf_emit!( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_ip(destination.ip(), destination.port())) + .actor_process(Process::from_bypass(&binary, "-", &ancestors).with_cmd_line(&cmdline)) + .message(format!("Transparent TCP denied before relay: {reason}")) + .status_detail(status_detail) + .build() + ); +} + +fn transparent_destination_host( + destination: SocketAddr, + policy_dns_store: Option<&Arc>, + opa_engine: &OpaEngine, +) -> Result { + let Some(store) = policy_dns_store else { + return Ok(destination.ip().to_string()); + }; + match store.lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) { + Ok(mapping) => Ok(mapping.record.normalized_name.as_str().to_string()), + Err(MappingLookupError::Missing) => Ok(destination.ip().to_string()), + Err(error) => Err(miette::miette!( + "transparent destination mapping is unavailable: {error}" + )), + } +} + impl Drop for ProxyHandle { fn drop(&mut self) { self.join.abort(); @@ -1192,21 +1550,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1584,8 +1945,8 @@ fn build_forward_destination_deny_ocsf_event( } #[allow(clippy::too_many_arguments)] -async fn deny_connect_destination( - client: &mut TcpStream, +async fn deny_connect_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1597,7 +1958,10 @@ async fn deny_connect_destination( decision: &EgressDecision, denial_tx: &Option>, activity_tx: &Option, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_connect_destination_deny_ocsf_event( denial, peer_addr, host, port, binary, pid, ancestors, cmdline, @@ -1630,8 +1994,8 @@ async fn deny_connect_destination( } #[allow(clippy::too_many_arguments)] -async fn deny_forward_destination( - client: &mut TcpStream, +async fn deny_forward_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1646,7 +2010,10 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_forward_destination_deny_ocsf_event( denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, @@ -1681,9 +2048,105 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + provider_credentials: Option, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + None, + None, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + backend_host_gateway, + trusted_host_gateway, + upstream_proxy, + provider_credentials, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +/// Adapt a transparent application stream to the existing CONNECT pipeline. +/// The synthetic CONNECT request is supervisor-owned and its successful 200 +/// response is consumed before bytes are returned to the workload. +fn virtual_connect_stream( + workload: BoundaryDuplexStream, + authority: String, +) -> BoundaryDuplexStream { + let (handler, bridge) = tokio::io::duplex(64 * 1024); + let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); + let (mut workload_read, mut workload_write) = tokio::io::split(workload); + tokio::spawn(async move { + let request = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); + if bridge_write.write_all(request.as_bytes()).await.is_ok() { + let _ = tokio::io::copy(&mut workload_read, &mut bridge_write).await; + } + let _ = bridge_write.shutdown().await; + }); + tokio::spawn(async move { + let mut header = Vec::with_capacity(256); + let mut byte = [0_u8; 1]; + while header.len() < MAX_HEADER_BYTES { + match bridge_read.read(&mut byte).await { + Ok(0) | Err(_) => return, + Ok(_) => header.push(byte[0]), + } + if header.ends_with(b"\r\n\r\n") { + break; + } + } + if !header.starts_with(b"HTTP/1.1 200 ") && !header.starts_with(b"HTTP/1.0 200 ") { + let _ = workload_write.shutdown().await; + return; + } + let _ = tokio::io::copy(&mut bridge_read, &mut workload_write).await; + let _ = workload_write.shutdown().await; + }); + Box::new(handler) +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, + transparent_open: Option, + policy_dns_store: Option>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1691,6 +2154,7 @@ async fn handle_tcp_connection( inference_ctx: Option>, policy_local_ctx: Option>, agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, trusted_host_gateway: Arc>, upstream_proxy: Arc>, provider_credentials: Option, @@ -1705,6 +2169,23 @@ async fn handle_tcp_connection( denial_tx: Option>, activity_tx: Option, ) -> Result<()> { + let (mut preauthorized_decision, prevalidated_connector) = if let Some(transparent) = + transparent_open + { + let destination = transparent.destination; + let host = + transparent_destination_host(destination, policy_dns_store.as_ref(), &opa_engine)?; + let (decision, connector) = transparent + .authorization + .map_or((None, None), |(decision, connector)| { + (Some(decision), Some(connector)) + }); + let authority = format!("{host}:{}", destination.port()); + client = tokio::io::BufReader::new(virtual_connect_stream(client.into_inner(), authority)); + (decision, connector) + } else { + (None, None) + }; let mut buf = vec![0u8; MAX_HEADER_BYTES]; let mut used = 0usize; @@ -1757,11 +2238,14 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, policy_local_ctx, agent_proposals, + backend_host_gateway, trusted_host_gateway, provider_credentials, secret_resolver, @@ -1803,22 +2287,33 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); // Evaluate OPA policy with process-identity binding. // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(decision) = preauthorized_decision.take() { + decision + } else if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -1935,25 +2430,28 @@ async fn handle_tcp_connection( let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - match hydrate_destination_plan(&mut decision, *trusted_host_gateway) { - Ok(()) => {} - Err(denial) => { - deny_connect_destination( - &mut client, - &denial, - workload_addr, - &host_lc, - port, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - &decision, - &denial_tx, - &activity_tx, - ) - .await?; - return Ok(()); + if prevalidated_connector.is_none() { + match hydrate_destination_plan(&mut decision, *backend_host_gateway, *trusted_host_gateway) + { + Ok(()) => {} + Err(denial) => { + deny_connect_destination( + &mut client, + &denial, + workload_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + &decision, + &denial_tx, + &activity_tx, + ) + .await?; + return Ok(()); + } } } let destination_plan = decision @@ -1964,32 +2462,36 @@ async fn handle_tcp_connection( // Defense-in-depth: resolve DNS and reject connections to internal IPs. let dns_connect_start = std::time::Instant::now(); - let connector = match validate_destination(DestinationRequest { - host: &raw_host, - port, - sandbox_entrypoint_pid, - plan: destination_plan, - }) - .await - { - Ok(connector) => connector, - Err(denial) => { - deny_connect_destination( - &mut client, - &denial, - workload_addr, - &host_lc, - port, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - &decision, - &denial_tx, - &activity_tx, - ) - .await?; - return Ok(()); + let connector = if let Some(connector) = prevalidated_connector { + connector + } else { + match validate_destination(DestinationRequest { + host: &raw_host, + port, + sandbox_entrypoint_pid, + plan: destination_plan, + }) + .await + { + Ok(connector) => connector, + Err(denial) => { + deny_connect_destination( + &mut client, + &denial, + workload_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + &decision, + &denial_tx, + &activity_tx, + ) + .await?; + return Ok(()); + } } }; @@ -2205,7 +2707,7 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; @@ -2637,18 +3139,6 @@ fn authorize_egress_intent( } }; - if !crate::opa::network_binary_identity_required() { - let result = evaluate_endpoint_only_opa(engine, intent); - debug!( - "authorize_egress_intent endpoint-only: host={} port={} transport={:?} action={:?}", - result.intent.destination.host, - result.intent.destination.port, - result.intent.transport, - result.action - ); - return result; - } - let entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); let Some(proc_net_anchor_pid) = proc_net_anchor_pid(entrypoint_pid) else { return deny( @@ -2726,18 +3216,10 @@ fn authorize_egress_intent( #[cfg(target_os = "linux")] fn proc_net_anchor_pid(entrypoint_pid: u32) -> Option { - if entrypoint_pid != 0 { - return Some(entrypoint_pid); - } - sidecar_topology_enabled().then(std::process::id) -} - -#[cfg(target_os = "linux")] -fn sidecar_topology_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .is_ok_and(|value| value == SIDECAR_SUPERVISOR_TOPOLOGY) + (entrypoint_pid != 0).then_some(entrypoint_pid) } +#[cfg(test)] fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> EgressDecision { let input = crate::opa::NetworkInput { host: intent.destination.host.clone(), @@ -2780,6 +3262,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.authorize_egress(&input) { + Ok(authorization) => EgressDecision { + intent, + action: authorization.action.clone(), + policy_generation: authorization.generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::from_authorization(&authorization), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2789,10 +3342,6 @@ fn authorize_egress_intent( _entrypoint_pid: &AtomicU32, intent: EgressIntent, ) -> EgressDecision { - if !crate::opa::network_binary_identity_required() { - return evaluate_endpoint_only_opa(engine, intent); - } - EgressDecision { intent, action: NetworkAction::Deny { @@ -2824,13 +3373,16 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. -async fn handle_inference_interception( - client: TcpStream, +async fn handle_inference_interception( + client: S, host: &str, port: u16, tls_state: Option<&Arc>, inference_ctx: Option<&Arc>, -) -> Result { +) -> Result +where + S: TokioAsyncRead + TokioAsyncWrite + Unpin + Send, +{ let Some(ctx) = inference_ctx else { return Ok(InferenceOutcome::Denied { reason: "cluster inference context not configured".to_string(), @@ -3419,13 +3971,16 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } -async fn reject_stale_connect_policy( - client: &mut TcpStream, +async fn reject_stale_connect_policy( + client: &mut C, host: &str, port: u16, activity_tx: Option<&ActivitySender>, error: miette::Report, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ warn!( host, port, @@ -3464,6 +4019,7 @@ fn hydrate_tls_mode(decision: &mut EgressDecision) { fn hydrate_destination_plan( decision: &mut EgressDecision, + backend_host_gateway: Option, trusted_host_gateway: Option, ) -> std::result::Result<(), DestinationDenial> { let host = decision.intent.destination.host.clone(); @@ -3472,6 +4028,7 @@ fn hydrate_destination_plan( let plan = build_validation_plan( &host, &host.to_ascii_lowercase(), + backend_host_gateway, trusted_host_gateway, &raw_allowed_ips, exact_declared_host, @@ -4818,12 +5375,15 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, policy_local_ctx: Option>, agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, trusted_host_gateway: Arc>, provider_credentials: Option, secret_resolver: Option>, @@ -4922,19 +5482,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -5580,7 +6148,7 @@ async fn handle_forward_proxy( // - Otherwise: reject internal IPs, allow public IPs through. // When the policy host is already a literal IP address, treat it as // implicitly allowed — the user explicitly declared the destination. - match hydrate_destination_plan(&mut decision, *trusted_host_gateway) { + match hydrate_destination_plan(&mut decision, *backend_host_gateway, *trusted_host_gateway) { Ok(()) => {} Err(denial) => { deny_forward_destination( @@ -5884,6 +6452,17 @@ async fn handle_forward_proxy( ), ) .await?; + client.shutdown().await.into_diagnostic()?; + let mut discard = [0_u8; 1024]; + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match client.read(&mut discard).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; } else { respond( client, @@ -6160,8 +6739,9 @@ fn normalize_host(raw_host: &str) -> &str { raw_host.strip_suffix('.').unwrap_or(raw_host) } -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; + client.flush().await.into_diagnostic()?; Ok(()) } @@ -6300,11 +6880,14 @@ const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (C /// HTTP status (the flaw this replaces). Returns `true` when the connection was /// refused (the caller must stop) and `false` when the caller should proceed to /// establish the tunnel. -async fn refuse_connect_when_tls_unavailable( - client: &mut TcpStream, +async fn refuse_connect_when_tls_unavailable( + client: &mut C, tls_state_present: bool, effective_tls_skip: bool, -) -> Result { +) -> Result +where + C: TokioAsyncWrite + Unpin, +{ if tls_state_present || effective_tls_skip { return Ok(false); } @@ -6355,6 +6938,189 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn supplied_identity_preserves_authorized_endpoint_metadata() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + inspected: + name: inspected + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + request_body_credential_rewrite: true + allowed_ips: ["192.0.2.0/24"] + rules: + - allow: { method: GET, path: /allowed } + binaries: + - path: /usr/bin/python3 +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = Ok(ContractBinaryIdentity { + binary_path: PathBuf::from("/usr/bin/python3"), + binary_digest: Some("00".repeat(32).parse().expect("digest")), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }); + + let mut decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &identity, + ); + + assert_eq!(query_allowed_ips(&decision), ["192.0.2.0/24"]); + hydrate_l7_route(&mut decision); + let route = decision + .endpoint + .l7_route + .expect("supplied identity must retain L7 metadata"); + assert_eq!(route.configs.len(), 1); + assert!(route.configs[0].config.request_body_credential_rewrite); + } + + #[tokio::test] + async fn staged_transparent_open_waits_for_l4_policy() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: 203.0.113.7 + port: 443 + - host: 169.254.169.254 + port: 80 + binaries: + - path: /usr/bin/curl +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .unwrap(); + let identity = || { + Ok(ContractBinaryIdentity { + binary_path: PathBuf::from("/usr/bin/curl"), + binary_digest: Some("00".repeat(32).parse().unwrap()), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }) + }; + let pending = |destination: &str| { + let (stream, _peer) = tokio::io::duplex(64); + let (result, completion) = tokio::sync::oneshot::channel(); + ( + PendingNetworkOpen { + stream: Box::new(stream), + binary_identity: identity(), + destination: destination.parse().unwrap(), + socket: openshell_isolation_interface::contract::NetworkSocketMetadata { + socket_cookie: 7, + nonblocking: false, + process_generation: 1, + }, + policy_generation: engine.current_generation(), + timing: MediationTiming::default(), + result, + }, + completion, + ) + }; + + let (allowed, allowed_result) = pending("203.0.113.7:443"); + assert!( + preauthorize_transparent_open(allowed, None, &engine, None, None) + .await + .is_some() + ); + assert_eq!(allowed_result.await.unwrap(), NetworkOpenResult::RelayReady); + + let (unsafe_destination, unsafe_result) = pending("169.254.169.254:80"); + assert!( + preauthorize_transparent_open(unsafe_destination, None, &engine, None, None) + .await + .is_none() + ); + assert_eq!( + unsafe_result.await.unwrap(), + NetworkOpenResult::Denied { + errno: libc::EACCES + } + ); + + let (denied, denied_result) = pending("203.0.113.8:443"); + assert!( + preauthorize_transparent_open(denied, None, &engine, None, None) + .await + .is_none() + ); + assert_eq!( + denied_result.await.unwrap(), + NetworkOpenResult::Denied { + errno: libc::EACCES + } + ); + } + + struct FailedMediationSource; + + #[tokio::test] + async fn virtual_connect_is_portless_and_hides_the_synthetic_handshake() { + let (workload, mut workload_peer) = tokio::io::duplex(1024); + let mut handler = virtual_connect_stream(Box::new(workload), "api.example.com:443".into()); + + workload_peer.write_all(b"client-tls").await.unwrap(); + let mut request = vec![0_u8; 128]; + let length = handler.read(&mut request).await.unwrap(); + let request = &request[..length]; + assert!(request.starts_with(b"CONNECT api.example.com:443 HTTP/1.1\r\n")); + + handler + .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\nserver-tls") + .await + .unwrap(); + let mut response = [0_u8; 10]; + workload_peer.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"server-tls"); + } + + #[async_trait::async_trait] + impl NetworkMediationSource for FailedMediationSource { + async fn accept( + &self, + ) -> std::result::Result< + PendingNetworkOpen, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unavailable( + "test source unavailable".to_string(), + ), + ) + } + } + struct DenyWebSocketPreflight; #[tonic::async_trait] @@ -6530,6 +7296,7 @@ network_policies: {} AgentProposals::default(), Arc::new(None), Arc::new(None), + Arc::new(None), None, None, None, @@ -6541,6 +7308,49 @@ network_policies: {} client.await.unwrap() } + #[tokio::test] + async fn terminal_mediation_source_failure_stops_proxy() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, + ) + .expect("engine"), + ); + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let mut handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + None, + Some(Arc::new(FailedMediationSource)), + None, + ) + .await + .expect("proxy starts before source accept"); + let exited = handle + .take_exit_receiver() + .expect("proxy exposes its exit receiver"); + + tokio::time::timeout(std::time::Duration::from_secs(1), exited) + .await + .expect("source failure must stop the proxy") + .expect_err("proxy task drops the exit sender"); + assert!(handle.join.is_finished()); + } + #[tokio::test] async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { for host in ["api.example.com", "unmatched.example.com"] { @@ -6626,7 +7436,13 @@ network_policies: .expect("read proxy response"); response }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); tokio::time::timeout( std::time::Duration::from_secs(30), @@ -6636,12 +7452,15 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), None, AgentProposals::default(), Arc::new(None), + Arc::new(None), None, None, None, @@ -6760,7 +7579,13 @@ network_policies: .await .unwrap(); }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); let handler = tokio::spawn(async move { handle_forward_proxy( @@ -6769,12 +7594,15 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), None, AgentProposals::default(), Arc::new(None), + Arc::new(None), None, None, None, @@ -7354,14 +8182,14 @@ network_policies: let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); - let (server, _) = listener.accept().await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut tokio::io::BufReader::new(&mut server)) .await .expect("peek should succeed") .expect("client sent bytes"); @@ -12101,6 +12929,7 @@ network_policies: None, // inference_ctx None, // policy_local_ctx AgentProposals::default(), // agent_proposals + Arc::new(None), // backend_host_gateway Arc::new(None), // trusted_host_gateway Arc::new(None), // upstream_proxy None, // provider_credentials @@ -12170,6 +12999,7 @@ network_policies: AgentProposals::default(), Arc::new(None), Arc::new(None), + Arc::new(None), None, None, None, diff --git a/crates/openshell-supervisor-network/src/proxy/destination.rs b/crates/openshell-supervisor-network/src/proxy/destination.rs index 1ce514133a..16e0e03999 100644 --- a/crates/openshell-supervisor-network/src/proxy/destination.rs +++ b/crates/openshell-supervisor-network/src/proxy/destination.rs @@ -29,6 +29,10 @@ pub(crate) enum AddressAuthorization { TrustedGatewayAlias { expected_ip: IpAddr, }, + /// A backend-provided host-side dial target. The backend is the trusted + /// authority for this mapping, so the supervisor does not consult its own + /// resolver before dialing it. + BackendPinnedGateway(IpAddr), /// Addresses already resolved and authorized by policy DNS. This mode must /// never resolve `DestinationRequest::host` again before constructing the /// unopened connector. @@ -79,11 +83,16 @@ impl DestinationDenial { pub(crate) fn build_validation_plan( host: &str, normalized_host: &str, + backend_host_gateway: Option, trusted_host_gateway: Option, raw_allowed_ips: &[String], exact_declared_endpoint_host: bool, ) -> Result { let address_authorization = if is_host_gateway_alias(normalized_host) + && let Some(expected_ip) = backend_host_gateway + { + AddressAuthorization::BackendPinnedGateway(expected_ip) + } else if is_host_gateway_alias(normalized_host) && let Some(expected_ip) = trusted_host_gateway { AddressAuthorization::TrustedGatewayAlias { expected_ip } @@ -140,7 +149,8 @@ pub(crate) fn filter_resolved_addresses( resolved_ips: &[IpAddr], ) -> Result, DestinationDenial> { let (kind, control_plane_blocked) = match &plan.address_authorization { - AddressAuthorization::TrustedGatewayAlias { .. } => { + AddressAuthorization::TrustedGatewayAlias { .. } + | AddressAuthorization::BackendPinnedGateway(_) => { (DestinationDenialKind::TrustedGateway, true) } AddressAuthorization::ExplicitAllowedIps(_) @@ -211,6 +221,20 @@ pub(crate) fn filter_resolved_addresses( None } } + AddressAuthorization::BackendPinnedGateway(expected_ip) => { + if is_cloud_metadata_ip(ip) { + Some(format!( + "{host} resolves to cloud metadata address {ip}, connection rejected" + )) + } else if ip != *expected_ip { + Some(format!( + "{host} resolves to {ip} which does not match backend host gateway \ + {expected_ip}, connection rejected" + )) + } else { + None + } + } AddressAuthorization::PinnedResolved(pinned) if !pinned.contains(&ip) => Some(format!( "{host} resolves to unpinned address {ip}, connection rejected" )), @@ -296,6 +320,23 @@ pub(crate) async fn validate_destination( DestinationDenial::new(DestinationDenialKind::TrustedGateway, reason) })? } + AddressAuthorization::BackendPinnedGateway(ip) => { + if BLOCKED_CONTROL_PLANE_PORTS.contains(&port) { + return Err(DestinationDenial::new( + DestinationDenialKind::TrustedGateway, + format!("port {port} is a blocked control-plane port, connection rejected"), + )); + } + if is_cloud_metadata_ip(*ip) { + return Err(DestinationDenial::new( + DestinationDenialKind::TrustedGateway, + format!( + "backend host gateway resolves to cloud metadata address {ip}, connection rejected" + ), + )); + } + vec![SocketAddr::new(*ip, port)] + } AddressAuthorization::ExplicitAllowedIps(networks) => { resolve_and_check_allowed_ips(host, port, networks, sandbox_entrypoint_pid) .await @@ -381,6 +422,7 @@ mod tests { "api.example.test", "api.example.test", None, + None, &["not-an-ip".to_string()], false, ) @@ -516,10 +558,26 @@ mod tests { #[test] fn validation_mode_precedence_is_explicit_and_stable() { + let backend_ip = IpAddr::V4(Ipv4Addr::LOCALHOST); let trusted_ip = IpAddr::V4(Ipv4Addr::new(169, 254, 1, 2)); + let backend = build_validation_plan( + "host.openshell.internal", + "host.openshell.internal", + Some(backend_ip), + Some(trusted_ip), + &["10.0.0.0/8".to_string()], + true, + ) + .unwrap(); + assert_eq!( + backend.address_authorization, + AddressAuthorization::BackendPinnedGateway(backend_ip) + ); + let trusted = build_validation_plan( "host.openshell.internal", "host.openshell.internal", + None, Some(trusted_ip), &["10.0.0.0/8".to_string()], true, @@ -536,6 +594,7 @@ mod tests { "10.2.3.4", "10.2.3.4", None, + None, &["10.0.0.0/8".to_string()], true, ) @@ -545,21 +604,24 @@ mod tests { AddressAuthorization::ExplicitAllowedIps(vec!["10.0.0.0/8".parse().unwrap()]) ); - let implicit = build_validation_plan("10.2.3.4", "10.2.3.4", None, &[], true).unwrap(); + let implicit = + build_validation_plan("10.2.3.4", "10.2.3.4", None, None, &[], true).unwrap(); assert_eq!( implicit.address_authorization, AddressAuthorization::ImplicitIpLiteral("10.2.3.4".parse().unwrap()) ); let declared = - build_validation_plan("private.example", "private.example", None, &[], true).unwrap(); + build_validation_plan("private.example", "private.example", None, None, &[], true) + .unwrap(); assert_eq!( declared.address_authorization, AddressAuthorization::ExactDeclaredHost ); let default = - build_validation_plan("*.example.com", "*.example.com", None, &[], false).unwrap(); + build_validation_plan("*.example.com", "*.example.com", None, None, &[], false) + .unwrap(); assert_eq!( default.address_authorization, AddressAuthorization::DefaultPublicOnly diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 314596b048..55c4dd9099 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] + //! Transport-neutral egress inputs and authorization results. //! //! Explicit proxy adapters normalize their protocol-specific request into an diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 186d156086..5ed4fd1c68 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -490,26 +490,20 @@ async fn exercise_benchmark_request(proxy_addr: SocketAddr, target: SocketAddr, #[test] #[ignore = "manual proxy allocation/query/latency baseline"] fn proxy_performance_baseline() { - temp_env::with_vars( - [( - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, - Some("endpoint-only"), - )], - || { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .unwrap() - .block_on(async { - // Benchmark the full fail-closed path using a declared loopback - // destination. This is deterministic and never opens a listener - // outside the local process, so it does not trigger host firewall - // prompts during manual baseline collection. - let target: SocketAddr = "127.0.0.1:18080".parse().unwrap(); - - let policy = format!( - r#" + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() + .block_on(async { + // Benchmark the full fail-closed path using a declared loopback + // destination. This is deterministic and never opens a listener + // outside the local process, so it does not trigger host firewall + // prompts during manual baseline collection. + let target: SocketAddr = "127.0.0.1:18080".parse().unwrap(); + + let policy = format!( + r#" network_policies: proxy_compatibility: name: proxy_compatibility @@ -518,93 +512,92 @@ network_policies: port: {port} tls: skip binaries: - - path: "/**" + - path: "/no-such-benchmark-binary" "#, - host = target.ip(), - port = target.port(), - ); - let engine = Arc::new( - OpaEngine::from_strings_with_binary_identity_required( - include_str!("../../../data/sandbox-policy.rego"), - &policy, - false, - ) - .unwrap(), - ); - let proxy_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let proxy_addr = proxy_listener.local_addr().unwrap(); - let proxy_engine = engine.clone(); - let proxy_task = tokio::spawn(async move { - while let Ok((stream, _)) = proxy_listener.accept().await { - let engine = proxy_engine.clone(); - tokio::spawn(async move { - Box::pin(handle_tcp_connection( - stream, - engine, - Arc::new(BinaryIdentityCache::new()), - Arc::new(AtomicU32::new(0)), - None, - None, - None, - AgentProposals::default(), - Arc::new(None), - Arc::new(None), - None, - None, - None, - None, - None, - )) - .await - .unwrap(); - }); - } + host = target.ip(), + port = target.port(), + ); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + include_str!("../../../data/sandbox-policy.rego"), + &policy, + true, + ) + .unwrap(), + ); + let proxy_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_addr = proxy_listener.local_addr().unwrap(); + let proxy_engine = engine.clone(); + let proxy_task = tokio::spawn(async move { + while let Ok((stream, _)) = proxy_listener.accept().await { + let engine = proxy_engine.clone(); + tokio::spawn(async move { + Box::pin(handle_tcp_connection( + stream, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(std::process::id())), + None, + None, + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + Arc::new(None), + None, + None, + None, + None, + None, + )) + .await + .unwrap(); }); - - for connect in [true, false] { - exercise_benchmark_request(proxy_addr, target, connect).await; - } - - let iterations = std::env::var("OPENSHELL_PROXY_BASELINE_ITERATIONS") - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or(25); - let mut results = serde_json::Map::new(); - for (name, connect) in [("connect", true), ("forward", false)] { - crate::test_alloc::reset(); - crate::opa::reset_test_opa_query_count(); - let started = std::time::Instant::now(); - for _ in 0..iterations { - exercise_benchmark_request(proxy_addr, target, connect).await; - } - let elapsed = started.elapsed(); - let queries = crate::opa::test_opa_query_count(); - let (allocations, allocated_bytes) = crate::test_alloc::snapshot(); - let expected_queries = 4; - assert_eq!(queries, expected_queries * iterations); - results.insert( - name.to_string(), - serde_json::json!({ - "allocated_bytes_per_request": allocated_bytes / iterations, - "allocations_per_request": allocations / iterations, - "latency_ns_per_request": elapsed.as_nanos() / u128::from(iterations), - "opa_queries_per_request": queries / iterations, - }), - ); - } - println!( - "{}", - serde_json::json!({ - "iterations": iterations, - "proxy_performance_baseline": results, - "scenario": "declared_loopback_destination_denied", - "schema_version": 1, - }) - ); - - proxy_task.abort(); - }); - }, - ); + } + }); + + for connect in [true, false] { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + + let iterations = std::env::var("OPENSHELL_PROXY_BASELINE_ITERATIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(25); + let mut results = serde_json::Map::new(); + for (name, connect) in [("connect", true), ("forward", false)] { + crate::test_alloc::reset(); + crate::opa::reset_test_opa_query_count(); + let started = std::time::Instant::now(); + for _ in 0..iterations { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + let elapsed = started.elapsed(); + let queries = crate::opa::test_opa_query_count(); + let (allocations, allocated_bytes) = crate::test_alloc::snapshot(); + let expected_queries = 4; + assert_eq!(queries, expected_queries * iterations); + results.insert( + name.to_string(), + serde_json::json!({ + "allocated_bytes_per_request": allocated_bytes / iterations, + "allocations_per_request": allocations / iterations, + "latency_ns_per_request": elapsed.as_nanos() / u128::from(iterations), + "opa_queries_per_request": queries / iterations, + }), + ); + } + println!( + "{}", + serde_json::json!({ + "iterations": iterations, + "proxy_performance_baseline": results, + "scenario": "declared_loopback_destination_denied", + "schema_version": 1, + }) + ); + + proxy_task.abort(); + }); } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 2a71702b4b..7f136492cb 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,6 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +use openshell_isolation_interface::contract::{DnsMediationSource, NetworkMediationSource}; #[cfg(target_os = "linux")] pub struct TransparentRuntimeSetup { @@ -155,6 +156,7 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, + _mediated_policy_dns: Option, #[cfg(target_os = "linux")] _policy_dns: Option, #[cfg(target_os = "linux")] @@ -196,7 +198,10 @@ pub async fn run_networking( agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, + host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, + network_mediation_source: Option>, + dns_mediation_source: Option>, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -313,10 +318,29 @@ pub async fn run_networking( // the proxy, so it's owned here. let identity_cache = opa_engine.map(|_| Arc::new(BinaryIdentityCache::new())); - // Generate ephemeral CA and TLS state for HTTPS L7 inspection. + // Load a provisioned CA when the boundary lifetime outlives this control + // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - match SandboxCa::generate() { + let configured_ca = match ( + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), + ) { + (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( + std::path::Path::new(&certificate), + std::path::Path::new(&private_key), + )?), + (None, None) => None, + _ => { + return Err(miette::miette!( + "{} and {} must be configured together", + openshell_core::sandbox_env::PROXY_CA_CERT, + openshell_core::sandbox_env::PROXY_CA_KEY, + )); + } + }; + let durable_ca = configured_ca.is_some(); + match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); @@ -356,7 +380,11 @@ pub async fn run_networking( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .message("TLS termination enabled: ephemeral CA generated") + .message(if durable_ca { + "TLS termination enabled: provisioned CA loaded" + } else { + "TLS termination enabled: ephemeral CA generated" + }) .build() ); (Some(state), Some(paths)) @@ -402,6 +430,21 @@ pub async fn run_networking( (None, None) }; + let mediated_policy_dns = if let Some(source) = dns_mediation_source { + let engine = opa_engine + .cloned() + .ok_or_else(|| miette::miette!("Mediated DNS requires an OPA engine"))?; + Some(crate::policy_dns::PolicyDnsRuntime::start_mediated( + engine, + source, + host_gateway_ip, + crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(0)?, + engine_ready_rx.clone(), + )?) + } else { + None + }; + let proxy_handle = if matches!(policy.network.mode, NetworkMode::Proxy) { let proxy_policy = policy.network.proxy.as_ref().ok_or_else(|| { miette::miette!("Network mode is set to proxy but no proxy configuration was provided") @@ -426,10 +469,11 @@ pub async fn run_networking( }); // Build inference context for local routing of intercepted inference calls. - let inference_ctx = crate::inference_routes::build_inference_context( + let inference_ctx = crate::inference_routes::build_inference_context_with_host_gateway( sandbox_id, openshell_endpoint, inference_routes, + host_gateway_ip, ) .await?; @@ -447,6 +491,11 @@ pub async fn run_networking( activity_tx.clone(), engine_ready_rx, upstream_proxy_args, + host_gateway_ip, + network_mediation_source, + mediated_policy_dns + .as_ref() + .map(|runtime| runtime.store.clone()), ) .await?; Some(proxy_handle) @@ -492,6 +541,7 @@ pub async fn run_networking( proxy: proxy_handle, ca_file_paths, policy_local_ctx, + _mediated_policy_dns: mediated_policy_dns, #[cfg(target_os = "linux")] _policy_dns: policy_dns, #[cfg(target_os = "linux")] diff --git a/crates/openshell-supervisor-network/src/spiffe_endpoint.rs b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs new file mode 100644 index 0000000000..b3b6816f1e --- /dev/null +++ b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +/// Convert a path to a SPIFFE Workload API endpoint URL. +/// +/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. +/// Otherwise, assume it is a Unix socket path and prepend `unix:`. +#[allow(dead_code)] +pub fn workload_api_endpoint(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.starts_with("unix:") || path.starts_with("tcp:") { + path.into_owned() + } else { + format!("unix:{path}") + } +} diff --git a/crates/openshell-supervisor-network/src/upstream_proxy.rs b/crates/openshell-supervisor-network/src/upstream_proxy.rs index f95c5a3e81..490602c4d8 100644 --- a/crates/openshell-supervisor-network/src/upstream_proxy.rs +++ b/crates/openshell-supervisor-network/src/upstream_proxy.rs @@ -83,6 +83,9 @@ const CONNECT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); pub struct ProxyEndpoint { host: String, port: u16, + /// Optional driver-pinned address used only for the TCP dial. The + /// configured host remains authoritative for TLS identity and logging. + dial_ip: Option, /// Pre-computed `Basic ` header value from the proxy auth file. /// Never logged. proxy_authorization: Option, @@ -106,6 +109,7 @@ impl std::fmt::Debug for ProxyEndpoint { f.debug_struct("ProxyEndpoint") .field("host", &self.host) .field("port", &self.port) + .field("dial_ip", &self.dial_ip) .field("proxy_authorization", &self.proxy_authorization.is_some()) .field("tls", &self.tls.is_some()) .finish() @@ -334,6 +338,9 @@ pub struct UpstreamProxyArgs { /// `http://host:port` or `https://host:port` corporate proxy URL, or /// `None` for direct egress. pub https_proxy: Option, + /// Optional compute-driver-selected IP for reaching the proxy from the + /// supervisor's network namespace without changing its TLS identity. + pub proxy_dial_ip: Option, /// Comma-separated `NO_PROXY` list. pub no_proxy: Option, /// Path to the root-only credential mount (`user:pass`). @@ -354,6 +361,7 @@ pub struct UpstreamProxyArgs { // Supervisor CLI flag names for the corporate-proxy settings, used as the // dispatch keys in `from_lookup` and in operator-facing error messages. const ARG_HTTPS_PROXY: &str = "--upstream-proxy"; +const ARG_PROXY_DIAL_IP: &str = "--upstream-proxy-dial-ip"; const ARG_NO_PROXY: &str = "--upstream-no-proxy"; const ARG_PROXY_AUTH_FILE: &str = "--upstream-proxy-auth-file"; const ARG_PROXY_AUTH_ALLOW_INSECURE: &str = "--upstream-proxy-auth-allow-insecure"; @@ -391,6 +399,8 @@ impl UpstreamProxyConfig { Self::from_lookup(|name| { if name == ARG_HTTPS_PROXY { args.https_proxy.clone() + } else if name == ARG_PROXY_DIAL_IP { + args.proxy_dial_ip.map(|ip| ip.to_string()) } else if name == ARG_NO_PROXY { args.no_proxy.clone() } else if name == ARG_PROXY_AUTH_FILE { @@ -424,6 +434,12 @@ impl UpstreamProxyConfig { let https = var(ARG_HTTPS_PROXY)? .map(|url| parse_proxy_url(&url, ARG_HTTPS_PROXY)) .transpose()?; + let proxy_dial_ip = var(ARG_PROXY_DIAL_IP)? + .map(|raw| { + raw.parse::() + .map_err(|error| format!("{ARG_PROXY_DIAL_IP} is invalid: {error}")) + }) + .transpose()?; let auth_file = var(ARG_PROXY_AUTH_FILE)?; let auth_allow_insecure = var(ARG_PROXY_AUTH_ALLOW_INSECURE)?; let connect_by_hostname_raw = var(ARG_PROXY_CONNECT_BY_HOSTNAME)?; @@ -435,6 +451,7 @@ impl UpstreamProxyConfig { // silently running with direct egress. for (name, value) in [ (ARG_PROXY_AUTH_FILE, &auth_file), + (ARG_PROXY_DIAL_IP, &proxy_dial_ip.map(|ip| ip.to_string())), (ARG_PROXY_AUTH_ALLOW_INSECURE, &auth_allow_insecure), (ARG_PROXY_CONNECT_BY_HOSTNAME, &connect_by_hostname_raw), (ARG_NO_PROXY, &no_proxy_list), @@ -446,6 +463,7 @@ impl UpstreamProxyConfig { } return Ok(None); }; + https.dial_ip = proxy_dial_ip; // CONNECT-target mode. The default binds the tunnel to a validated // address; hostname CONNECT re-opens proxy-side DNS resolution and @@ -592,6 +610,7 @@ fn parse_proxy_url(raw: &str, var_name: &str) -> Result<(ProxyEndpoint, bool), S ProxyEndpoint { host: addr.host, port: addr.port, + dial_ip: None, proxy_authorization: None, tls: None, }, @@ -1000,7 +1019,10 @@ async fn connect_via_inner( port: u16, target: ConnectTarget, ) -> std::io::Result { - let tcp = TcpStream::connect((endpoint.host.as_str(), endpoint.port)).await?; + let tcp = match endpoint.dial_ip { + Some(ip) => TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await?, + None => TcpStream::connect((endpoint.host.as_str(), endpoint.port)).await?, + }; set_tcp_nodelay_best_effort(&tcp); // For an `https://` proxy, wrap the connection in TLS (verifying the proxy // certificate against the configured roots) before the CONNECT handshake. @@ -1113,6 +1135,7 @@ mod tests { ARG_PROXY_AUTH_ALLOW_INSECURE as PROXY_AUTH_ALLOW_INSECURE, ARG_PROXY_AUTH_FILE as PROXY_AUTH_FILE, ARG_PROXY_CA_BUNDLE as PROXY_CA_BUNDLE, ARG_PROXY_CONNECT_BY_HOSTNAME as PROXY_CONNECT_BY_HOSTNAME, + ARG_PROXY_DIAL_IP as PROXY_DIAL_IP, }; fn config_from(pairs: &[(&str, &str)]) -> Result, String> { @@ -1834,6 +1857,7 @@ mod tests { ProxyEndpoint { host: addr.ip().to_string(), port: addr.port(), + dial_ip: None, proxy_authorization: auth.map(str::to_string), tls: None, } @@ -2241,14 +2265,16 @@ mod tests { // -- TLS (https://) proxies -- - /// A fake `https://` proxy: a TLS server with a self-signed cert for - /// 127.0.0.1 that answers CONNECT with 200. Returns the listen address, + /// A fake `https://` proxy: a TLS server with a self-signed cert for the + /// requested identity that answers CONNECT with 200. Returns the listen address, /// the server task (yielding the received CONNECT request), and the /// server certificate PEM to use as the corporate CA bundle. - async fn fake_tls_proxy() -> (SocketAddr, tokio::task::JoinHandle, String) { + async fn fake_tls_proxy( + tls_identity: &str, + ) -> (SocketAddr, tokio::task::JoinHandle, String) { install_crypto_provider(); let key = rcgen::KeyPair::generate().unwrap(); - let cert = rcgen::CertificateParams::new(vec!["127.0.0.1".to_string()]) + let cert = rcgen::CertificateParams::new(vec![tls_identity.to_string()]) .unwrap() .self_signed(&key) .unwrap(); @@ -2288,18 +2314,23 @@ mod tests { #[tokio::test] async fn connect_via_https_proxy_with_corporate_ca_bundle() { - let (addr, handle, cert_pem) = fake_tls_proxy().await; + const PROXY_IDENTITY: &str = "proxy.corp.test"; + let (addr, handle, cert_pem) = fake_tls_proxy(PROXY_IDENTITY).await; let ca_file = tempfile::NamedTempFile::new().unwrap(); std::fs::write(ca_file.path(), cert_pem).unwrap(); - let proxy_url = format!("https://{addr}"); + let proxy_url = format!("https://{PROXY_IDENTITY}:{}", addr.port()); let ca_path = ca_file.path().to_string_lossy().into_owned(); + let dial_ip = addr.ip().to_string(); let cfg = config_ok(&[ (HTTPS_PROXY, proxy_url.as_str()), + (PROXY_DIAL_IP, dial_ip.as_str()), (PROXY_CA_BUNDLE, ca_path.as_str()), ]); let endpoint = &cfg.https; assert!(endpoint.tls.is_some()); + assert_eq!(endpoint.host, PROXY_IDENTITY); + assert_eq!(endpoint.dial_ip, Some(addr.ip())); let stream = connect_via(endpoint, "api.example.com", 443, ConnectTarget::Hostname) .await @@ -2314,7 +2345,7 @@ mod tests { async fn connect_via_https_proxy_rejects_untrusted_cert() { // No corporate CA bundle: the self-signed proxy cert must not verify // against the built-in / system roots, so the handshake fails closed. - let (addr, _handle, _cert_pem) = fake_tls_proxy().await; + let (addr, _handle, _cert_pem) = fake_tls_proxy("127.0.0.1").await; let proxy_url = format!("https://{addr}"); let cfg = config_ok(&[(HTTPS_PROXY, proxy_url.as_str())]); let endpoint = &cfg.https; diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 2e2120f1d0..cfcea38555 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openshell-supervisor-process" -description = "Process component of the OpenShell supervisor: entrypoint spawn, SSH server, supervisor session, netns, bypass monitor" +description = "Process access and gateway session runtime for the OpenShell supervisor" version.workspace = true edition.workspace = true license.workspace = true @@ -12,14 +12,14 @@ rust-version.workspace = true [dependencies] openshell-core = { path = "../openshell-core" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } -openshell-policy = { path = "../openshell-policy" } anyhow = { workspace = true } +async-trait = "0.1" base64 = { workspace = true } bytes = { workspace = true } hex = "0.4" -ipnet = "2" miette = { workspace = true } nix = { workspace = true } rand = "0.10" @@ -35,14 +35,6 @@ uuid = { workspace = true } [target.'cfg(unix)'.dependencies] libc = "0.2" -rustix = { workspace = true } - -[target.'cfg(target_os = "linux")'.dependencies] -capctl = "0.2.4" -landlock = "0.4" -seccompiler = "0.5" -socket2 = { workspace = true } -tempfile = "3" [dev-dependencies] tempfile = "3" diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs deleted file mode 100644 index 44847b0d13..0000000000 --- a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs +++ /dev/null @@ -1,651 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Bypass detection monitor — reads kernel log messages from `/dev/kmsg` to -//! detect and report direct connection attempts that bypass the HTTP CONNECT -//! proxy. -//! -//! When the sandbox network namespace has nftables log rules installed (see -//! `NetworkNamespace::install_bypass_rules`), the kernel writes a log line for -//! each dropped packet. This module reads those messages, parses the nftables -//! LOG format, and emits structured tracing events + denial aggregator entries. -//! -//! ## Graceful degradation -//! -//! If `/dev/kmsg` cannot be opened (e.g., restricted container environment), -//! the monitor logs a one-time warning and returns. The nftables reject rules -//! still provide fast-fail UX — the monitor only adds diagnostic visibility. - -mod procfs; - -use openshell_core::activity::{ActivitySender, try_record_activity}; -use openshell_core::denial::DenialEvent; -use openshell_ocsf::{ - ActionId, ActivityId, ConfidenceId, DetectionFindingBuilder, DispositionId, Endpoint, - FindingInfo, NetworkActivityBuilder, Process, SeverityId, ocsf_emit, -}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; -use tokio::sync::mpsc; -use tracing::debug; - -/// A parsed nftables log entry from `/dev/kmsg`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BypassEvent { - /// Destination IP address. - pub dst_addr: String, - /// Destination port. - pub dst_port: u16, - /// Source port (used for process identity resolution). - pub src_port: u16, - /// Protocol (TCP or UDP). - pub proto: String, - /// UID of the process that initiated the connection. - pub uid: Option, -} - -/// Parse a nftables log line from `/dev/kmsg`. -/// -/// Expected format (from the kernel LOG target): -/// ```text -/// ...,;openshell:bypass::IN= OUT=veth-s-... SRC=10.200.0.2 DST=93.184.216.34 -/// LEN=60 ... PROTO=TCP SPT=48012 DPT=443 ... UID=1000 -/// ``` -/// -/// Returns `None` if the line doesn't match the expected prefix or is malformed. -pub fn parse_kmsg_line(line: &str, namespace_prefix: &str) -> Option { - // Check that this line contains our namespace prefix. - let prefix_pos = line.find(namespace_prefix)?; - let relevant = &line[prefix_pos + namespace_prefix.len()..]; - - let dst_addr = extract_field(relevant, "DST=")?; - let dst_port = extract_field(relevant, "DPT=")?.parse::().ok()?; - let src_port = extract_field(relevant, "SPT=") - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - let proto = extract_field(relevant, "PROTO=") - .unwrap_or_else(|| "unknown".to_string()) - .to_lowercase(); - let uid = extract_field(relevant, "UID=").and_then(|s| s.parse::().ok()); - - Some(BypassEvent { - dst_addr, - dst_port, - src_port, - proto, - uid, - }) -} - -fn build_bypass_ocsf_events( - event: &BypassEvent, - binary: &str, - binary_pid: &str, - ancestors: &str, -) -> (openshell_ocsf::OcsfEvent, openshell_ocsf::OcsfEvent) { - let hint = hint_for_event(event); - let reason = "direct connection bypassed HTTP CONNECT proxy"; - let dst_port = event.dst_port.to_string(); - let dst_ep = event.dst_addr.parse::().map_or_else( - |_| Endpoint::from_domain(&event.dst_addr, event.dst_port), - |ip| Endpoint::from_ip(ip, event.dst_port), - ); - - let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Refuse) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .dst_endpoint(dst_ep) - .actor_process(Process::from_bypass(binary, binary_pid, ancestors)) - .firewall_rule("bypass-detect", "nftables") - .observation_point(3) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} action=reject reason={reason}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - - let finding_event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .is_alert(true) - .confidence(ConfidenceId::High) - .finding_info(FindingInfo::new("bypass-detect", "Proxy Bypass Detected").with_desc(reason)) - .remediation(hint) - .evidence_pairs(&[ - ("dst_addr", event.dst_addr.as_str()), - ("dst_port", dst_port.as_str()), - ("proto", event.proto.as_str()), - ("binary", binary), - ("binary_pid", binary_pid), - ("ancestors", ancestors), - ]) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} hint={hint}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - - (net_event, finding_event) -} - -/// Extract a single space-delimited field value from a nftables log line. -/// -/// Given `"DST="` and a string like `"...DST=93.184.216.34 LEN=60..."`, -/// returns `Some("93.184.216.34")`. -fn extract_field(s: &str, key: &str) -> Option { - let start = s.find(key)? + key.len(); - let rest = &s[start..]; - let end = rest.find(' ').unwrap_or(rest.len()); - let value = &rest[..end]; - if value.is_empty() { - None - } else { - Some(value.to_string()) - } -} - -/// Generate a protocol-appropriate hint for the bypass event. -fn hint_for_event(event: &BypassEvent) -> &'static str { - if event.proto == "udp" && event.dst_port == 53 { - "DNS queries should route through the sandbox proxy; check resolver configuration" - } else if event.proto == "udp" { - "UDP traffic must route through the sandbox proxy" - } else { - "ensure process honors HTTP_PROXY/HTTPS_PROXY; for Node.js set NODE_USE_ENV_PROXY=1" - } -} - -/// Spawn the bypass monitor as a background tokio task. -/// -/// Uses `dmesg --follow` to tail the kernel ring buffer for nftables log -/// entries matching the given namespace. Falls back gracefully if `dmesg` -/// is not available. -/// -/// We use `dmesg` rather than reading `/dev/kmsg` directly because the -/// container runtime's device cgroup policy blocks direct `/dev/kmsg` access -/// even with `CAP_SYSLOG`. The `dmesg` command reads via the `syslog(2)` -/// syscall which is permitted with `CAP_SYSLOG`. -/// -/// Returns a `JoinHandle` if the monitor was started, or `None` if `dmesg` -/// is not available. -pub fn spawn( - namespace_name: String, - entrypoint_pid: Arc, - denial_tx: Option>, - activity_tx: Option, -) -> Option> { - use std::io::BufRead; - use std::process::{Command, Stdio}; - - // Verify dmesg is available before spawning the monitor. - let dmesg_check = Command::new("dmesg") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - - if !dmesg_check.is_ok_and(|s| s.success()) { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .severity(SeverityId::Low) - .message( - "dmesg not available; bypass detection monitor will not run. \ - Bypass REJECT rules still provide fast-fail behavior.", - ) - .build(); - ocsf_emit!(event); - return None; - } - - let namespace_prefix = format!("openshell:bypass:{namespace_name}:"); - debug!( - namespace = %namespace_name, - "Starting bypass detection monitor via dmesg --follow" - ); - - let handle = tokio::task::spawn_blocking(move || { - // Start dmesg in follow mode to tail new kernel messages. - let mut child = match Command::new("dmesg") - .args(["--follow", "--notime"]) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn() - { - Ok(c) => c, - Err(e) => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .severity(SeverityId::Low) - .message(format!( - "Failed to start dmesg --follow; bypass monitor will not run: {e}" - )) - .build(); - ocsf_emit!(event); - return; - } - }; - - let Some(stdout) = child.stdout.take() else { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .severity(SeverityId::Low) - .message("dmesg --follow produced no stdout; bypass monitor will not run") - .build(); - ocsf_emit!(event); - return; - }; - - let reader = std::io::BufReader::new(stdout); - for line in reader.lines() { - let line = match line { - Ok(l) => l, - Err(e) => { - debug!(error = %e, "Error reading dmesg line, continuing"); - continue; - } - }; - - let Some(event) = parse_kmsg_line(&line, &namespace_prefix) else { - continue; - }; - - // Attempt process identity resolution (best-effort, TCP only). - let pid = entrypoint_pid.load(Ordering::Acquire); - let (binary, binary_pid, ancestors) = - if event.proto == "tcp" && event.src_port > 0 && pid > 0 { - resolve_process_identity(pid, event.src_port) - } else { - ("-".to_string(), "-".to_string(), "-".to_string()) - }; - - // Dual-emit: Network Activity [4001] + Detection Finding [2004] - let (net_event, finding_event) = - build_bypass_ocsf_events(&event, &binary, &binary_pid, &ancestors); - ocsf_emit!(net_event); - ocsf_emit!(finding_event); - - // Send to denial aggregator if available. - if let Some(ref tx) = denial_tx { - let ancestors_vec: Vec = if ancestors == "-" { - vec![] - } else { - ancestors.split(" -> ").map(String::from).collect() - }; - - let _ = tx.send(DenialEvent { - host: event.dst_addr.clone(), - port: event.dst_port, - binary: binary.clone(), - ancestors: ancestors_vec, - deny_reason: "direct connection bypassed HTTP CONNECT proxy".to_string(), - denial_stage: "bypass".to_string(), - l7_method: None, - l7_path: None, - }); - } - if let Some(ref tx) = activity_tx { - let _ = try_record_activity(tx, true, "bypass"); - } - } - - // Clean up the dmesg child process. - let _ = child.kill(); - let _ = child.wait(); - debug!("Bypass monitor: dmesg reader exited"); - }); - - Some(handle) -} - -/// Resolve process identity from a TCP source port. -/// -/// Returns `(binary_path, pid, ancestors)` as display strings. -/// Falls back to `("-", "-", "-")` on any failure (race condition, etc.). -fn resolve_process_identity(entrypoint_pid: u32, src_port: u16) -> (String, String, String) { - match procfs::resolve_tcp_peer_socket_owners(entrypoint_pid, src_port) { - Ok(socket_owners) => { - let mut identities = Vec::new(); - for owner in &socket_owners.owners { - let Ok(binary_path) = procfs::binary_path(owner.pid.cast_signed()) else { - continue; - }; - let ancestors = procfs::collect_ancestor_binaries(owner.pid, entrypoint_pid); - identities.push((owner.pid, binary_path, ancestors)); - } - - if identities.is_empty() { - return ("-".to_string(), "-".to_string(), "-".to_string()); - } - - identities.sort_by_key(|(pid, _, _)| *pid); - let first_identity = (identities[0].1.clone(), identities[0].2.clone()); - let ambiguous = identities - .iter() - .skip(1) - .any(|(_, binary_path, ancestors)| { - binary_path != &first_identity.0 || ancestors != &first_identity.1 - }); - - if ambiguous { - let pids = identities - .iter() - .map(|(pid, _, _)| pid.to_string()) - .collect::>() - .join(", "); - let owner_summary = identities - .iter() - .map(|(pid, binary_path, ancestors)| { - let ancestors_str = if ancestors.is_empty() { - "-".to_string() - } else { - ancestors - .iter() - .map(|p| p.display().to_string()) - .collect::>() - .join(" -> ") - }; - format!( - "pid={pid} binary={} ancestors=[{ancestors_str}]", - binary_path.display() - ) - }) - .collect::>() - .join("; "); - return ("ambiguous".to_string(), pids, owner_summary); - } - - let (pid, binary_path, ancestors) = identities.remove(0); - let ancestors_str = if ancestors.is_empty() { - "-".to_string() - } else { - ancestors - .iter() - .map(|p| p.display().to_string()) - .collect::>() - .join(" -> ") - }; - ( - binary_path.display().to_string(), - pid.to_string(), - ancestors_str, - ) - } - Err(_) => ("-".to_string(), "-".to_string(), "-".to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_kmsg_line_tcp_bypass() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=10.200.0.2 DST=93.184.216.34 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=12345 \ - DF PROTO=TCP SPT=48012 DPT=443 WINDOW=65535 RES=0x00 SYN URGP=0 UID=1000"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "93.184.216.34"); - assert_eq!(event.dst_port, 443); - assert_eq!(event.src_port, 48012); - assert_eq!(event.proto, "tcp"); - assert_eq!(event.uid, Some(1000)); - } - - #[test] - fn parse_kmsg_line_udp_dns_bypass() { - let line = "6,5678,9012,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=10.200.0.2 DST=8.8.8.8 LEN=40 TOS=0x00 PREC=0x00 TTL=64 ID=0 \ - DF PROTO=UDP SPT=53421 DPT=53 LEN=32 UID=1000"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "8.8.8.8"); - assert_eq!(event.dst_port, 53); - assert_eq!(event.src_port, 53421); - assert_eq!(event.proto, "udp"); - assert_eq!(event.uid, Some(1000)); - } - - #[test] - fn parse_kmsg_line_no_uid() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=10.200.0.2 DST=10.0.0.5 LEN=60 PROTO=TCP SPT=12345 DPT=6379"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "10.0.0.5"); - assert_eq!(event.dst_port, 6379); - assert_eq!(event.proto, "tcp"); - assert_eq!(event.uid, None); - } - - #[test] - fn parse_kmsg_line_wrong_namespace_returns_none() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-other:IN= OUT=veth \ - SRC=10.200.0.2 DST=1.2.3.4 PROTO=TCP SPT=1111 DPT=80"; - - let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); - assert!(result.is_none()); - } - - #[test] - fn parse_kmsg_line_unrelated_message_returns_none() { - let line = "6,1234,5678,-;audit: type=1400 audit(1234567890.123:1): something else"; - let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); - assert!(result.is_none()); - } - - #[test] - fn parse_kmsg_line_missing_dst_returns_none() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth \ - SRC=10.200.0.2 PROTO=TCP SPT=1111 DPT=80"; - // Missing DST= field - let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); - assert!(result.is_none()); - } - - #[test] - fn parse_kmsg_line_ipv6_address() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=fd00::2 DST=2001:4860:4860::8888 LEN=60 PROTO=TCP SPT=55555 DPT=443 UID=1000"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "2001:4860:4860::8888"); - assert_eq!(event.dst_port, 443); - assert_eq!(event.proto, "tcp"); - } - - #[test] - fn hint_for_tcp_event() { - let event = BypassEvent { - dst_addr: "1.2.3.4".to_string(), - dst_port: 443, - src_port: 12345, - proto: "tcp".to_string(), - uid: None, - }; - assert!(hint_for_event(&event).contains("HTTP_PROXY")); - } - - #[test] - fn hint_for_dns_bypass() { - let event = BypassEvent { - dst_addr: "8.8.8.8".to_string(), - dst_port: 53, - src_port: 12345, - proto: "udp".to_string(), - uid: None, - }; - assert!(hint_for_event(&event).contains("DNS")); - } - - #[test] - fn hint_for_non_dns_udp() { - let event = BypassEvent { - dst_addr: "1.2.3.4".to_string(), - dst_port: 5060, - src_port: 12345, - proto: "udp".to_string(), - uid: None, - }; - assert!(hint_for_event(&event).contains("UDP")); - } - - #[test] - fn bypass_ocsf_contract_is_stable() { - let event = BypassEvent { - dst_addr: "93.184.216.34".to_string(), - dst_port: 443, - src_port: 48012, - proto: "tcp".to_string(), - uid: Some(1000), - }; - let (network, finding) = - build_bypass_ocsf_events(&event, "/usr/bin/curl", "42", "/usr/bin/sh"); - let network = serde_json::to_value(network).unwrap(); - assert_eq!(network["class_name"], "Network Activity"); - assert_eq!(network["activity_name"], "Refuse"); - assert_eq!(network["action"], "Denied"); - assert_eq!(network["disposition"], "Blocked"); - assert_eq!(network["severity"], "Medium"); - assert!(network.get("status").is_none()); - assert_eq!(network["dst_endpoint"]["ip"], "93.184.216.34"); - assert_eq!(network["dst_endpoint"]["port"], 443); - assert_eq!(network["actor"]["process"]["name"], "/usr/bin/curl"); - assert_eq!(network["firewall_rule"]["name"], "bypass-detect"); - assert_eq!(network["firewall_rule"]["type"], "nftables"); - assert_eq!(network["observation_point_id"], 3); - assert!( - network["message"] - .as_str() - .unwrap() - .contains("action=reject") - ); - - let finding = serde_json::to_value(finding).unwrap(); - assert_eq!(finding["class_name"], "Detection Finding"); - assert_eq!(finding["action"], "Denied"); - assert_eq!(finding["disposition"], "Blocked"); - assert_eq!(finding["severity"], "Medium"); - assert_eq!(finding["confidence"], "High"); - assert_eq!(finding["is_alert"], true); - assert_eq!(finding["finding_info"]["uid"], "bypass-detect"); - assert_eq!(finding["finding_info"]["title"], "Proxy Bypass Detected"); - assert_eq!(finding["evidences"][0]["data"]["dst_port"], "443"); - } - - #[test] - fn resolve_process_identity_surfaces_ambiguous_shared_socket() { - use std::ffi::CString; - use std::net::{TcpListener, TcpStream}; - use std::os::fd::AsRawFd; - use std::time::{Duration, Instant}; - - if !std::path::Path::new("/bin/sleep").exists() { - eprintln!("skipping: /bin/sleep not available"); - return; - } - - let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener"); - let listener_port = listener.local_addr().unwrap().port(); - let stream = TcpStream::connect(("127.0.0.1", listener_port)).expect("connect"); - let peer_port = stream.local_addr().unwrap().port(); - let (_accepted, _) = listener.accept().expect("accept"); - - let fd = stream.as_raw_fd(); - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - unsafe { - let flags = libc::fcntl(fd, libc::F_GETFD); - assert!(flags >= 0, "F_GETFD failed"); - assert_eq!( - libc::fcntl(fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC), - 0, - "F_SETFD failed" - ); - } - - let sleep_path = CString::new("/bin/sleep").unwrap(); - let arg0 = CString::new("sleep").unwrap(); - let arg1 = CString::new("30").unwrap(); - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - let child_pid = unsafe { libc::fork() }; - assert!(child_pid >= 0, "fork failed"); - if child_pid == 0 { - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - unsafe { - libc::execl( - sleep_path.as_ptr(), - arg0.as_ptr(), - arg1.as_ptr(), - std::ptr::null::(), - ); - libc::_exit(127); - } - } - - if std::fs::read_link(format!("/proc/{child_pid}/exe")).is_err() - || std::fs::read_dir(format!("/proc/{child_pid}/fd")).is_err() - { - #[allow(unsafe_code)] - unsafe { - libc::kill(child_pid, libc::SIGKILL); - libc::waitpid(child_pid, std::ptr::null_mut(), 0); - } - eprintln!("skipping: cannot read /proc/{child_pid} (restricted /proc)"); - return; - } - - let deadline = Instant::now() + Duration::from_secs(2); - loop { - if let Ok(link) = std::fs::read_link(format!("/proc/{child_pid}/exe")) - && link.to_string_lossy().contains("sleep") - { - break; - } - assert!( - Instant::now() < deadline, - "child pid {child_pid} did not exec into sleep within 2s" - ); - std::thread::sleep(Duration::from_millis(20)); - } - - let (binary, pid, ancestors) = resolve_process_identity(std::process::id(), peer_port); - - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - unsafe { - libc::kill(child_pid, libc::SIGKILL); - libc::waitpid(child_pid, std::ptr::null_mut(), 0); - } - - assert_eq!(binary, "ambiguous"); - assert!(pid.contains(&std::process::id().to_string())); - assert!(pid.contains(&child_pid.to_string())); - assert!(ancestors.contains("binary=")); - } - - #[test] - fn extract_field_basic() { - let s = "DST=1.2.3.4 LEN=60"; - assert_eq!(extract_field(s, "DST="), Some("1.2.3.4".to_string())); - assert_eq!(extract_field(s, "LEN="), Some("60".to_string())); - } - - #[test] - fn extract_field_missing() { - let s = "DST=1.2.3.4 LEN=60"; - assert_eq!(extract_field(s, "PROTO="), None); - } - - #[test] - fn extract_field_at_end_of_string() { - let s = "DST=1.2.3.4"; - assert_eq!(extract_field(s, "DST="), Some("1.2.3.4".to_string())); - } -} diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs b/crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs deleted file mode 100644 index 98bf9634a6..0000000000 --- a/crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs +++ /dev/null @@ -1,318 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Linux `/proc` filesystem reading for bypass-monitor process identity. -//! -//! Trimmed copy of `openshell-supervisor-network`'s `procfs` module: only -//! the helpers the bypass monitor calls when resolving the originating PID -//! and binary for an nftables LOG entry. The networking leaf keeps its own -//! richer copy because it also needs sha256 hashing, cmdline scraping, and -//! ambiguity-failure helpers for its proxy identity cache. - -use miette::Result; -use std::collections::HashSet; -use std::path::PathBuf; - -/// Where a socket owner was discovered while scanning `/proc`. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum SocketOwnerSource { - /// Owner was found in the entrypoint process tree at the given BFS depth. - Descendant { depth: usize }, - /// Owner was found by scanning all of `/proc` after the descendant scan. - ProcFallback, -} - -/// A process with an fd pointing at a target socket inode. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct SocketOwner { - pub pid: u32, - pub source: SocketOwnerSource, -} - -/// All process owners for a TCP peer socket. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct TcpPeerSocketOwners { - pub inode: u64, - pub owners: Vec, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct DescendantPid { - pid: u32, - depth: usize, -} - -/// Read the binary path of a process via `/proc/{pid}/exe` symlink. -/// -/// Strips the kernel-added `" (deleted)"` suffix when the raw readlink -/// target cannot be stat'd, so callers see a clean path. See the networking -/// crate's procfs documentation for the full rationale. -pub fn binary_path(pid: i32) -> Result { - use std::ffi::OsString; - use std::io::ErrorKind; - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - const DELETED_SUFFIX: &[u8] = b" (deleted)"; - - let link = format!("/proc/{pid}/exe"); - let target = std::fs::read_link(&link).map_err(|e| { - miette::miette!( - "Failed to read /proc/{pid}/exe: {e}. \ - Cannot determine binary identity — denying request. \ - Hint: the proxy may need CAP_SYS_PTRACE or to run as the same user." - ) - })?; - - let raw_target_missing = - matches!(std::fs::metadata(&target), Err(err) if err.kind() == ErrorKind::NotFound); - - let bytes = target.as_os_str().as_bytes(); - if raw_target_missing && bytes.ends_with(DELETED_SUFFIX) { - let stripped = bytes[..bytes.len() - DELETED_SUFFIX.len()].to_vec(); - return Ok(PathBuf::from(OsString::from_vec(stripped))); - } - - Ok(target) -} - -/// Resolve all process owners for the TCP peer inside a sandbox network namespace. -pub fn resolve_tcp_peer_socket_owners( - entrypoint_pid: u32, - peer_port: u16, -) -> Result { - let inode = parse_proc_net_tcp(entrypoint_pid, peer_port)?; - let owners = find_socket_inode_owners(inode, entrypoint_pid)?; - Ok(TcpPeerSocketOwners { inode, owners }) -} - -/// Read the `PPid` (parent PID) from `/proc//status`. -fn read_ppid(pid: u32) -> Option { - let status = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?; - for line in status.lines() { - if let Some(rest) = line.strip_prefix("PPid:") { - return rest.trim().parse().ok(); - } - } - None -} - -/// Walk the process tree upward from `pid`, collecting binary paths. -/// -/// Stops at PID 1 (init), `stop_pid` (the entrypoint process), or after -/// 64 ancestors. The returned vec excludes `pid` itself. -#[allow(clippy::similar_names)] -pub fn collect_ancestor_binaries(pid: u32, stop_pid: u32) -> Vec { - const MAX_DEPTH: usize = 64; - let mut ancestors = Vec::new(); - let mut current = pid; - - for _ in 0..MAX_DEPTH { - let ppid = match read_ppid(current) { - Some(p) if p > 0 && p != current => p, - _ => break, - }; - - if let Ok(path) = binary_path(ppid.cast_signed()) { - ancestors.push(path); - } - - if ppid == stop_pid || ppid == 1 { - break; - } - current = ppid; - } - - ancestors -} - -fn parse_proc_net_tcp(pid: u32, peer_port: u16) -> Result { - for suffix in &["tcp", "tcp6"] { - let path = format!("/proc/{pid}/net/{suffix}"); - let Ok(content) = std::fs::read_to_string(&path) else { - continue; - }; - - for line in content.lines().skip(1) { - let fields: Vec<&str> = line.split_whitespace().collect(); - if fields.len() < 10 { - continue; - } - - let local_addr = fields[1]; - let local_port = match local_addr.rsplit_once(':') { - Some((_, port_hex)) => u16::from_str_radix(port_hex, 16).unwrap_or(0), - None => continue, - }; - - let state = fields[3]; - if state != "01" { - continue; - } - - if local_port == peer_port { - let inode: u64 = fields[9] - .parse() - .map_err(|_| miette::miette!("Failed to parse inode from {}", fields[9]))?; - if inode == 0 { - continue; - } - return Ok(inode); - } - } - } - - Err(miette::miette!( - "No ESTABLISHED TCP connection found for port {} in /proc/{}/net/tcp{{,6}}", - peer_port, - pid - )) -} - -fn find_socket_inode_owners(inode: u64, entrypoint_pid: u32) -> Result> { - let target = format!("socket:[{inode}]"); - let mut owners = Vec::new(); - let mut checked = HashSet::new(); - - let descendants = collect_descendant_pids_with_depth(entrypoint_pid); - - for descendant in &descendants { - checked.insert(descendant.pid); - if check_pid_fds(descendant.pid, &target) { - owners.push(SocketOwner { - pid: descendant.pid, - source: SocketOwnerSource::Descendant { - depth: descendant.depth, - }, - }); - } - } - - if let Ok(proc_dir) = std::fs::read_dir("/proc") { - let mut proc_pids = Vec::new(); - for entry in proc_dir.flatten() { - let name = entry.file_name(); - if let Ok(pid) = name.to_string_lossy().parse::() { - proc_pids.push(pid); - } - } - proc_pids.sort_unstable(); - - for pid in proc_pids { - if checked.contains(&pid) { - continue; - } - checked.insert(pid); - if check_pid_fds(pid, &target) { - owners.push(SocketOwner { - pid, - source: SocketOwnerSource::ProcFallback, - }); - } - } - } - - if !owners.is_empty() { - return Ok(owners); - } - - Err(miette::miette!( - "No process found owning socket inode {} \ - (scanned {} descendants of entrypoint PID {}). \ - Hint: the container may need --cap-add=SYS_PTRACE to read /proc//fd/ \ - for processes running as a different user.", - inode, - descendants.len(), - entrypoint_pid - )) -} - -fn check_pid_fds(pid: u32, target: &str) -> bool { - let fd_dir = format!("/proc/{pid}/fd"); - let Some(fds) = std::fs::read_dir(&fd_dir).ok() else { - return false; - }; - for fd_entry in fds.flatten() { - if let Ok(link) = std::fs::read_link(fd_entry.path()) - && link.to_string_lossy() == target - { - return true; - } - } - false -} - -fn collect_descendant_pids_with_depth(root_pid: u32) -> Vec { - let mut pids = vec![DescendantPid { - pid: root_pid, - depth: 0, - }]; - let mut seen = HashSet::from([root_pid]); - let mut i = 0; - while i < pids.len() { - let pid = pids[i].pid; - let child_depth = pids[i].depth + 1; - let task_dir = format!("/proc/{pid}/task"); - if let Ok(tasks) = std::fs::read_dir(&task_dir) { - for task_entry in tasks.flatten() { - let children_path = task_entry.path().join("children"); - if let Ok(children_str) = std::fs::read_to_string(&children_path) { - for child in children_str.split_whitespace() { - if let Ok(child_pid) = child.parse::() - && seen.insert(child_pid) - { - pids.push(DescendantPid { - pid: child_pid, - depth: child_depth, - }); - } - } - } - } - } - i += 1; - } - pids -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn binary_path_reads_current_process() { - let pid = std::process::id().cast_signed(); - let path = binary_path(pid).unwrap(); - assert!(path.exists()); - } - - #[test] - #[allow(clippy::similar_names)] - fn read_ppid_returns_parent() { - let pid = std::process::id(); - let ppid = read_ppid(pid); - assert!(ppid.is_some(), "Should be able to read PPid of self"); - assert!(ppid.unwrap() > 0, "PPid should be > 0"); - } - - #[test] - fn read_ppid_nonexistent_pid() { - let result = read_ppid(999_999_999); - assert!(result.is_none()); - } - - #[test] - fn collect_ancestor_binaries_returns_parents() { - let pid = std::process::id(); - let ancestors = collect_ancestor_binaries(pid, 1); - assert!( - !ancestors.is_empty(), - "Should have at least one ancestor binary" - ); - for path in &ancestors { - assert!( - !path.as_os_str().is_empty(), - "Ancestor path should not be empty" - ); - } - } -} diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs new file mode 100644 index 0000000000..8f51bf2397 --- /dev/null +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-owned access-plane assembly for a remote sandbox. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use miette::Result; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward, BoundaryProcess}; +use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit}; + +fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { + openshell_ocsf::ctx::ctx() +} + +/// Supervisor-owned SSH and gateway-session tasks for a running sandbox. +pub struct BoundaryAccess { + instance_id: String, + terminating: Arc, + ssh_task: Option>, + session_task: Option>, + session_readiness: Option>, + main_session: Option>, +} + +impl BoundaryAccess { + /// Stable supervisor instance ID used for lifecycle reporting. + #[must_use] + pub fn instance_id(&self) -> &str { + &self.instance_id + } + + /// Observe whether the gateway has accepted the current supervisor + /// session. The value returns to false while the session reconnects. + #[must_use] + pub fn session_readiness(&self) -> Option> { + self.session_readiness.clone() + } + + /// Publish the canonical process's terminal status to attached clients. + pub async fn publish_main_exit(&self, exit_code: i32, attachment_expected: bool) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + let _ = main_session + .finish_remote(exit_code, attachment_expected) + .await; + } + + /// Release terminal delivery after the gateway acknowledges the exit, then + /// wait for attached clients to consume the terminal status. + pub async fn drain_main_terminal_delivery(&self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + main_session.mark_terminal_reported(); + main_session.wait_for_terminal_attachments().await; + } +} + +impl Drop for BoundaryAccess { + fn drop(&mut self) { + self.terminating.store(true, Ordering::Release); + if let Some(task) = self.ssh_task.take() { + task.abort(); + } + if let Some(task) = self.session_task.take() { + task.abort(); + } + } +} + +/// Start the supervisor access plane using sandbox-supplied exec and +/// loopback-forwarding capabilities. +#[allow(clippy::too_many_arguments)] +pub async fn start_boundary_access( + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option<&str>, + shared_ssh_socket: bool, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + boundary_exec: Arc, + port_forward: Arc, + agent: Arc, +) -> Result { + let instance_id = uuid::Uuid::new_v4().to_string(); + let terminating = Arc::new(AtomicBool::new(false)); + let Some(ssh_socket_path) = ssh_socket_path.map(std::path::PathBuf::from) else { + return Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: None, + session_task: None, + session_readiness: None, + main_session: None, + }); + }; + + let attachment = agent + .attach() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let main_session = crate::main_session::MainSession::from_boundary(attachment, agent); + + let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); + let listen_path = ssh_socket_path.clone(); + let ssh_port_forward = port_forward.clone(); + let ssh_main_session = main_session.clone(); + let ssh_task = tokio::spawn(async move { + if let Err(error) = crate::ssh::run_ssh_server( + listen_path, + ssh_ready_tx, + ca_file_paths, + shared_ssh_socket, + ssh_port_forward, + boundary_exec, + Some(ssh_main_session), + ) + .await + { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message(format!("SSH server failed: {error}")) + .build() + ); + } + }); + + match tokio::time::timeout(Duration::from_secs(10), ssh_ready_rx).await { + Ok(Ok(Ok(()))) => {} + Ok(Ok(Err(error))) => { + ssh_task.abort(); + return Err(error.context("SSH server failed during startup")); + } + Ok(Err(_)) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server task ended before signaling readiness" + )); + } + Err(_) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server did not start within 10 seconds" + )); + } + } + + let (session_task, session_readiness) = match (openshell_endpoint, sandbox_id) { + (Some(endpoint), Some(id)) => { + let (task, mut accepted) = crate::supervisor_session::spawn_with_readiness( + endpoint.to_string(), + id.to_string(), + ssh_socket_path, + port_forward, + None, + terminating.clone(), + instance_id.clone(), + ); + let accepted_result = + tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready)) + .await + .map(|result| result.map(|_| ())); + match accepted_result { + Ok(Ok(())) => (Some(task), Some(accepted)), + Ok(Err(_)) => { + task.abort(); + return Err(miette::miette!( + "supervisor session ended before gateway acceptance" + )); + } + Err(_) => { + task.abort(); + return Err(miette::miette!( + "gateway did not accept supervisor session within 10 seconds" + )); + } + } + } + _ => (None, None), + }; + + Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: Some(ssh_task), + session_task, + session_readiness, + main_session: Some(main_session), + }) +} + +/// Report the canonical process exit until the gateway acknowledges it. +pub async fn report_main_process_exit( + endpoint: &str, + sandbox_id: &str, + instance_id: &str, + exit_code: i32, +) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::report_main_process_exit( + endpoint, + sandbox_id, + instance_id, + exit_code, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process exit report failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} + +/// Finalize canonical process terminal delivery until acknowledged. +pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::finalize_main_process_exit( + endpoint, + sandbox_id, + instance_id, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process finalization failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn expected_post_exit_attachment_is_preserved_for_remote_main() { + let main_session = crate::main_session::MainSession::inert(); + let access = BoundaryAccess { + instance_id: "instance".to_string(), + terminating: Arc::new(AtomicBool::new(false)), + ssh_task: None, + session_task: None, + session_readiness: None, + main_session: Some(main_session.clone()), + }; + + access.publish_main_exit(7, true).await; + + main_session + .begin_terminal_attachment() + .expect("declared CLI attachment must remain valid after a fast remote main exits"); + main_session.end_terminal_attachment(); + } +} diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index 743942faa4..023a6c8e73 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -1,30 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process component of the `OpenShell` supervisor. +//! Access-plane component of the `OpenShell` supervisor. //! -//! Owns the entrypoint process spawn, SSH server, supervisor session, network -//! namespace, bypass monitor, child environment construction, skills install, -//! and log push. Populated by follow-up commits as modules migrate out of -//! `openshell-sandbox`. +//! Owns SSH access, retained canonical-process I/O, gateway supervisor +//! sessions, skills, and log forwarding. Workload spawning and in-sandbox +//! enforcement live exclusively in `openshell-sandbox`. -pub mod child_env; pub mod debug_rpc; -#[cfg(unix)] -pub mod identity; +pub mod delegated; pub mod log_push; pub mod main_session; -pub mod managed_children; -pub mod process; -pub mod run; -pub mod sandbox; pub mod skills; pub mod ssh; pub mod supervisor_session; mod unix_socket; - -#[cfg(target_os = "linux")] -pub mod bypass_monitor; -#[cfg(target_os = "linux")] -pub mod netns; diff --git a/crates/openshell-supervisor-process/src/main_session.rs b/crates/openshell-supervisor-process/src/main_session.rs index 00dd2ea53c..3ffa24d1d0 100644 --- a/crates/openshell-supervisor-process/src/main_session.rs +++ b/crates/openshell-supervisor-process/src/main_session.rs @@ -17,10 +17,22 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Notify; use tokio::sync::watch; -use crate::process::ProcessIo; +use openshell_isolation_interface::contract::{ + BoundaryProcess, BoundarySignal, BoundaryTerminal, ProcessAttachment, +}; const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; +/// Canonical-process I/O retained by the supervisor session multiplexer. +pub enum ProcessIo { + Pty(std::fs::File), + Pipes { + stdin: tokio::process::ChildStdin, + stdout: tokio::process::ChildStdout, + stderr: tokio::process::ChildStderr, + }, +} + #[derive(Clone, Debug)] pub enum MainOutput { Stdout(Bytes), @@ -186,6 +198,8 @@ pub struct MainSession { input_owner: Mutex>, next_owner: AtomicU64, pty_master: Option>, + boundary_process: Option>, + boundary_terminal: Option>, readers_remaining: AtomicUsize, readers_done: Notify, finished: std::sync::atomic::AtomicBool, @@ -194,6 +208,7 @@ pub struct MainSession { } impl MainSession { + const REMOTE_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); #[cfg(test)] pub fn inert() -> Arc { let (input, _input_rx) = tokio::sync::mpsc::channel(64); @@ -205,6 +220,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master: None, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(0), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -256,6 +273,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(if terminal { 1 } else { 2 }), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -270,6 +289,81 @@ impl MainSession { session } + /// Build the control-side multiplexer around a boundary-owned admitted + /// process. Process lifecycle and PTY operations remain delegated to the + /// boundary process handle. + #[must_use] + pub fn from_boundary( + attachment: ProcessAttachment, + process: Arc, + ) -> Arc { + let ProcessAttachment { + stdin, + stdout, + stderr, + terminal, + } = attachment; + let terminal_mode = terminal.is_some(); + let (input, mut input_rx) = tokio::sync::mpsc::channel::>(64); + let session = Arc::new(Self { + pid: 0, + terminal: terminal_mode, + input, + output: OutputLog::new(), + input_owner: Mutex::new(None), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: Some(process), + boundary_terminal: terminal, + readers_remaining: AtomicUsize::new(if terminal_mode { 1 } else { 2 }), + readers_done: Notify::new(), + finished: std::sync::atomic::AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + let stdout_session = Arc::clone(&session); + tokio::spawn(async move { + let mut stdout = stdout; + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stdout_session.reader_finished(); + }); + if let Some(mut stderr) = stderr { + let stderr_session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stderr_session + .publish(MainOutput::Stderr(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stderr_session.reader_finished(); + }); + } + tokio::spawn(async move { + let mut stdin = stdin; + while let Some(data) = input_rx.recv().await { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + }); + session + } + fn start_io( this: &Arc, io: ProcessIo, @@ -380,10 +474,39 @@ impl MainSession { /// /// Returns whether terminal delivery must complete before shutdown. pub async fn finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.wait_for_output_readers().await; + self.complete_finish(exit_code, attachment_expected) + } + + /// Finish a remotely owned process without allowing descendants that keep + /// inherited output descriptors open to block terminal publication forever. + pub async fn finish_remote(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.finish_remote_with_timeout( + exit_code, + attachment_expected, + Self::REMOTE_OUTPUT_DRAIN_TIMEOUT, + ) + .await + } + + async fn finish_remote_with_timeout( + &self, + exit_code: i32, + attachment_expected: bool, + timeout: std::time::Duration, + ) -> bool { + let _ = tokio::time::timeout(timeout, self.wait_for_output_readers()).await; + self.complete_finish(exit_code, attachment_expected) + } + + async fn wait_for_output_readers(&self) { let notified = self.readers_done.notified(); if self.readers_remaining.load(Ordering::Acquire) != 0 { notified.await; } + } + + fn complete_finish(&self, exit_code: i32, attachment_expected: bool) -> bool { let delivery_pending = { let mut state = self .terminal_attachments @@ -410,6 +533,23 @@ impl MainSession { self.output.subscribe() } + /// Return the bounded output sequence range currently retained for a + /// replacement supervisor. A nonzero first sequence is an explicit + /// truncation watermark rather than silent data loss. + #[must_use] + pub fn output_window(&self) -> (u64, u64, bool) { + let state = self + .output + .state + .lock() + .expect("main output log lock poisoned"); + let first_sequence = state + .events + .front() + .map_or(state.next_sequence, |event| event.sequence); + (first_sequence, state.next_sequence, first_sequence != 0) + } + /// Wait until the gateway durably acknowledges the main-process result. pub async fn wait_for_terminal_reported(&self) { let notified = self.output.terminal_reported_notify.notified(); @@ -499,7 +639,16 @@ impl MainSession { } } - pub fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + pub async fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + if let Some(terminal) = self.boundary_terminal.as_ref() { + let _ = terminal + .resize( + u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ) + .await; + return; + } let Some(master) = self.pty_master.as_ref() else { return; }; @@ -515,9 +664,23 @@ impl MainSession { } } - pub fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), nix::errno::Errno> { + pub async fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), String> { + if let Some(process) = self.boundary_process.as_ref() { + let signal = match signal { + nix::sys::signal::Signal::SIGHUP => BoundarySignal::Hup, + nix::sys::signal::Signal::SIGINT => BoundarySignal::Int, + nix::sys::signal::Signal::SIGKILL => BoundarySignal::Kill, + nix::sys::signal::Signal::SIGTERM => BoundarySignal::Term, + other => return Err(format!("boundary signal {other:?} is unsupported")), + }; + return process + .signal(signal) + .await + .map_err(|error| error.to_string()); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); nix::sys::signal::kill(nix::unistd::Pid::from_raw(-pid), signal) + .map_err(|error| error.to_string()) } #[must_use] @@ -544,6 +707,83 @@ fn set_nonblocking(file: &std::fs::File) -> Result<(), nix::errno::Errno> { #[cfg(test)] mod tests { use super::*; + use openshell_isolation_interface::contract::{ + BackendError, BoundaryExitStatus, BoundaryInput, BoundaryOutput, + }; + + struct TestBoundaryProcess { + signals: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryProcess for TestBoundaryProcess { + async fn wait(&self) -> Result { + Ok(BoundaryExitStatus::Exited(0)) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.signals.lock().unwrap().push(signal); + Ok(()) + } + + async fn terminate(&self) -> Result<(), BackendError> { + Ok(()) + } + } + + struct TestBoundaryTerminal { + size: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryTerminal for TestBoundaryTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } + } + + #[tokio::test] + async fn boundary_attachment_drives_main_io_signal_and_terminal() { + let (stdin, mut stdin_peer) = tokio::io::duplex(1024); + let (stdout, mut stdout_peer) = tokio::io::duplex(1024); + let process = Arc::new(TestBoundaryProcess { + signals: Mutex::new(Vec::new()), + }); + let terminal = Arc::new(TestBoundaryTerminal { + size: Mutex::new(None), + }); + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let attachment = ProcessAttachment { + stdin, + stdout, + stderr: None, + terminal: Some(terminal.clone()), + }; + let session = MainSession::from_boundary(attachment, process.clone()); + let mut output = session.subscribe(); + + stdout_peer.write_all(b"ready\n").await.unwrap(); + assert!(matches!( + output.recv().await.unwrap(), + MainOutput::Stdout(data) if data == b"ready\n"[..] + )); + + let (_owner, input) = session.acquire_input().unwrap(); + input.send(b"hello\n".to_vec()).await.unwrap(); + let mut received = [0_u8; 6]; + stdin_peer.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"hello\n"); + + session.resize(120, 40, 0, 0).await; + assert_eq!(*terminal.size.lock().unwrap(), Some((120, 40))); + session + .signal_group(nix::sys::signal::Signal::SIGINT) + .await + .unwrap(); + assert_eq!(*process.signals.lock().unwrap(), vec![BoundarySignal::Int]); + } #[test] fn input_lease_has_one_owner_and_can_be_reacquired() { @@ -632,6 +872,27 @@ mod tests { .expect("closing the attachment should wake the waiter"); } + #[tokio::test] + async fn remote_finish_bounds_output_drain_before_publishing_exit() { + let mut session = MainSession::inert(); + Arc::get_mut(&mut session) + .expect("sole test session reference") + .readers_remaining = AtomicUsize::new(1); + let mut output = session.subscribe(); + + session + .finish_remote_with_timeout(19, false, std::time::Duration::from_millis(10)) + .await; + + assert!(matches!( + output + .recv() + .await + .expect("terminal status after bounded drain"), + MainOutput::Exit(19) + )); + } + #[tokio::test] async fn declared_attachment_waits_for_connection_then_natural_close() { let session = MainSession::inert(); diff --git a/crates/openshell-supervisor-process/src/managed_children.rs b/crates/openshell-supervisor-process/src/managed_children.rs deleted file mode 100644 index 311c80693f..0000000000 --- a/crates/openshell-supervisor-process/src/managed_children.rs +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Process-wide tracker for sandbox-managed child PIDs. -//! -//! The supervisor spawns several long-lived children (the entrypoint, SSH -//! sessions). Each registers its PID here on spawn and removes it on exit so -//! the orchestrator's `SIGCHLD` reaper can distinguish supervised processes -//! from incidental zombies. - -#![cfg(target_os = "linux")] - -use std::collections::HashSet; -use std::sync::{LazyLock, Mutex}; - -static MANAGED_CHILDREN: LazyLock>> = - LazyLock::new(|| Mutex::new(HashSet::new())); - -/// Add `pid` to the supervised-child set. Non-positive or out-of-range values -/// are silently ignored. -pub fn register(pid: u32) { - let Ok(pid) = i32::try_from(pid) else { - return; - }; - if pid <= 0 { - return; - } - if let Ok(mut children) = MANAGED_CHILDREN.lock() { - children.insert(pid); - } -} - -/// Remove `pid` from the supervised-child set. Non-positive or out-of-range -/// values are silently ignored. -pub fn unregister(pid: u32) { - let Ok(pid) = i32::try_from(pid) else { - return; - }; - if pid <= 0 { - return; - } - if let Ok(mut children) = MANAGED_CHILDREN.lock() { - children.remove(&pid); - } -} - -/// Return `true` if `pid` is currently in the supervised-child set. -#[must_use] -pub fn is_managed(pid: i32) -> bool { - MANAGED_CHILDREN - .lock() - .is_ok_and(|children| children.contains(&pid)) -} diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs deleted file mode 100644 index 2b4ea554ed..0000000000 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ /dev/null @@ -1,1239 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Network namespace isolation for sandboxed processes. -//! -//! Creates an isolated network namespace with a veth pair connecting -//! the sandbox to the host. This ensures the sandboxed process can only -//! communicate through the proxy running on the host side of the veth. - -mod nft_ruleset; - -use miette::{IntoDiagnostic, Result}; -use std::net::IpAddr; -use std::os::unix::io::RawFd; -use std::path::Path; -use std::process::Command; -use tracing::{debug, warn}; -use uuid::Uuid; - -/// Default subnet for sandbox networking. -const SUBNET_PREFIX: &str = "10.200.0"; -const HOST_IP_SUFFIX: u8 = 1; -const SANDBOX_IP_SUFFIX: u8 = 2; -/// Unprivileged port owned by the supervisor's policy DNS service. Workload -/// queries still target the standard DNS port and nftables redirects them to -/// this listener before the bypass fence runs. -pub const POLICY_DNS_PORT: u16 = 15_053; -pub const TRANSPARENT_TCP_PORT: u16 = 15_001; -const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; -const NSENTER_SEARCH_PATHS: &[&str] = &[ - "/usr/bin/nsenter", - "/bin/nsenter", - "/usr/sbin/nsenter", - "/sbin/nsenter", -]; - -/// Handle to a network namespace with veth pair. -/// -/// The namespace and veth interfaces are automatically cleaned up on drop. -#[derive(Debug)] -pub struct NetworkNamespace { - /// Namespace name (e.g., "sandbox-{uuid}") - name: String, - /// Host-side veth interface name - veth_host: String, - /// Sandbox-side veth interface name (inside namespace, used only during setup) - _veth_sandbox: String, - /// Host-side IP address (proxy binds here) - host_ip: IpAddr, - /// Sandbox-side IP address - sandbox_ip: IpAddr, - /// File descriptor for the namespace (for setns) - ns_fd: Option, -} - -impl NetworkNamespace { - /// Create a new isolated network namespace with veth pair. - /// - /// Sets up: - /// - A new network namespace named `sandbox-{uuid}` - /// - A veth pair connecting host and sandbox - /// - IP addresses on both ends (10.200.0.1/24 and 10.200.0.2/24) - /// - Default route in sandbox pointing to host - /// - /// # Errors - /// - /// Returns an error if namespace creation or network setup fails. - pub fn create() -> Result { - let id = Uuid::new_v4(); - let short_id = &id.to_string()[..8]; - let name = format!("sandbox-{short_id}"); - let veth_host = format!("veth-h-{short_id}"); - let veth_sandbox = format!("veth-s-{short_id}"); - - let host_ip: IpAddr = format!("{SUBNET_PREFIX}.{HOST_IP_SUFFIX}").parse().unwrap(); - let sandbox_ip: IpAddr = format!("{SUBNET_PREFIX}.{SANDBOX_IP_SUFFIX}") - .parse() - .unwrap(); - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "creating") - .message(format!( - "Creating network namespace [ns:{name} host_veth:{veth_host} sandbox_veth:{veth_sandbox}]" - )) - .build() - ); - - // Create the namespace - run_ip(&["netns", "add", &name])?; - - // Create veth pair - if let Err(e) = run_ip(&[ - "link", - "add", - &veth_host, - "type", - "veth", - "peer", - "name", - &veth_sandbox, - ]) { - // Cleanup namespace on failure - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Move sandbox veth into namespace - if let Err(e) = run_ip(&["link", "set", &veth_sandbox, "netns", &name]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Configure host side - let host_cidr = format!("{host_ip}/24"); - if let Err(e) = run_ip(&["addr", "add", &host_cidr, "dev", &veth_host]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - if let Err(e) = run_ip(&["link", "set", &veth_host, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Configure sandbox side (inside namespace) - let sandbox_cidr = format!("{sandbox_ip}/24"); - if let Err(e) = run_ip_netns(&name, &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - if let Err(e) = run_ip_netns(&name, &["link", "set", &veth_sandbox, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Bring up loopback in namespace - if let Err(e) = run_ip_netns(&name, &["link", "set", "lo", "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Add default route via host - let host_ip_str = host_ip.to_string(); - if let Err(e) = run_ip_netns(&name, &["route", "add", "default", "via", &host_ip_str]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Open the namespace file descriptor for later use with setns - let ns_path = openshell_core::container_paths::netns_path(&name); - let ns_fd = match nix::fcntl::open( - ns_path.as_path(), - nix::fcntl::OFlag::O_RDONLY, - nix::sys::stat::Mode::empty(), - ) { - Ok(fd) => Some(fd), - Err(e) => { - warn!(error = %e, "Failed to open namespace fd, will use nsenter fallback"); - None - } - }; - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "created") - .message(format!( - "Network namespace created [ns:{name} host_ip:{host_ip} sandbox_ip:{sandbox_ip}]" - )) - .build() - ); - - Ok(Self { - name, - veth_host, - _veth_sandbox: veth_sandbox, - host_ip, - sandbox_ip, - ns_fd, - }) - } - - /// Get the host-side IP address (proxy should bind to this). - #[must_use] - pub const fn host_ip(&self) -> IpAddr { - self.host_ip - } - - /// Get the sandbox-side IP address. - #[must_use] - pub const fn sandbox_ip(&self) -> IpAddr { - self.sandbox_ip - } - - /// Get the namespace name. - #[must_use] - pub fn name(&self) -> &str { - &self.name - } - - /// Enter this network namespace. - /// - /// Must be called from the child process after fork, before exec. - /// Uses `setns()` to switch the calling process into the namespace. - /// - /// # Errors - /// - /// Returns an error if setns fails. - /// - /// # Safety - /// - /// This function should only be called in a `pre_exec` context after fork. - pub fn enter(&self) -> Result<()> { - if let Some(fd) = self.ns_fd { - debug!(namespace = %self.name, "Entering network namespace via setns"); - // SAFETY: setns is safe to call after fork, before exec - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - let result = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if result != 0 { - return Err(miette::miette!( - "setns failed: {}", - std::io::Error::last_os_error() - )); - } - Ok(()) - } else { - Err(miette::miette!( - "No namespace file descriptor available for setns" - )) - } - } - - /// Get the namespace file descriptor for use with clone/unshare. - #[must_use] - pub const fn ns_fd(&self) -> Option { - self.ns_fd - } - - /// Install nftables rules for bypass detection inside the namespace. - /// - /// Sets up OUTPUT chain rules that: - /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) - /// 2. ACCEPT loopback traffic - /// 3. ACCEPT established/related connections (response packets) - /// 4. LOG + REJECT all other TCP/UDP traffic (bypass attempts) - /// - /// This provides two benefits: - /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of - /// a 30-second timeout when they bypass the proxy - /// - **Diagnostics**: nftables LOG entries are picked up by the bypass - /// monitor to emit structured tracing events - /// - /// Degrades gracefully if `nft` is not available — the namespace - /// still provides isolation via routing, just without fast-fail and - /// diagnostic logging. - pub fn install_bypass_rules(&self, proxy_port: u16) -> Result<()> { - let Some(nft_path) = find_nft() else { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Disabled, "degraded") - .message(format!( - "nft not found; bypass detection rules will not be installed [ns:{}]", - self.name - )) - .build() - ); - return Ok(()); - }; - - let host_ip_str = self.host_ip.to_string(); - let log_prefix = format!("openshell:bypass:{}:", &self.name); - - // The kernel's nf_log_syslog module suppresses log output from - // non-init network namespaces by default. Enable it so the bypass - // monitor can see log entries from the sandbox namespace. - enable_nf_log_all_netns(); - - let commands = - nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Disabled, "failed") - .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", - self.name - )) - .build() - ); - return Err(e); - } - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "installed") - .message(format!( - "Bypass detection rules installed [ns:{}]", - self.name - )) - .build() - ); - - Ok(()) - } - - /// Replace the ordinary bypass fence with the policy-DNS and transparent - /// TCP ruleset. This is fail-closed: callers must not release workload - /// execution unless every required rule was installed. - pub fn install_transparent_tcp_rules( - &self, - proxy_port: u16, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - ) -> Result<()> { - self.validate_synthetic_pool_routes(synthetic_ipv4_cidr, synthetic_ipv6_cidr)?; - // The inner namespace has an IPv4 default route, but not an IPv6 - // default route. Install only the active synthetic IPv6 epoch so the - // kernel reaches the nft OUTPUT hook; REDIRECT then reroutes it to - // the local transparent listener. - run_ip_netns( - &self.name, - &["-6", "route", "replace", synthetic_ipv6_cidr, "dev", "lo"], - )?; - let nft_path = find_nft().ok_or_else(|| { - miette::miette!( - "trusted nft helper not found; policy DNS and transparent TCP require nftables" - ) - })?; - let host_ip = self.host_ip.to_string(); - let log_prefix = format!("openshell:bypass:{}:", self.name); - let commands = nft_ruleset::generate_transparent_tcp_commands( - &host_ip, - proxy_port, - POLICY_DNS_PORT, - TRANSPARENT_TCP_PORT, - synthetic_ipv4_cidr, - synthetic_ipv6_cidr, - Some(&log_prefix), - ); - run_nft_commands_netns(&self.name, &nft_path, &commands)?; - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "installed") - .message(format!( - "Policy DNS and transparent TCP capture installed [ns:{}]", - self.name - )) - .build() - ); - Ok(()) - } - - fn validate_synthetic_pool_routes( - &self, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - ) -> Result<()> { - let reserved = [ - synthetic_ipv4_cidr - .parse::() - .into_diagnostic()?, - synthetic_ipv6_cidr - .parse::() - .into_diagnostic()?, - ]; - for family in ["-4", "-6"] { - let routes = - run_ip_netns_output(&self.name, &[family, "route", "show", "table", "all"])?; - if let Some((route, pool)) = first_route_overlap(&routes, &reserved) { - return Err(miette::miette!( - "synthetic address pool {pool} overlaps workload route {route}; refusing to enable policy DNS" - )); - } - } - Ok(()) - } - - /// Bind IPv4 and IPv6 transparent listeners inside the workload network - /// namespace without moving an async runtime worker into that namespace. - pub async fn bind_transparent_tcp_listeners( - &self, - ) -> std::io::Result> { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result> { - #[allow(unsafe_code)] - if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { - return Err(std::io::Error::last_os_error()); - } - let mut listeners = Vec::with_capacity(2); - for (domain, address) in [ - ( - socket2::Domain::IPV4, - format!("0.0.0.0:{TRANSPARENT_TCP_PORT}"), - ), - ( - socket2::Domain::IPV6, - format!("[::]:{TRANSPARENT_TCP_PORT}"), - ), - ] { - let socket = socket2::Socket::new( - domain, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - socket.set_reuse_address(true)?; - if domain == socket2::Domain::IPV6 { - socket.set_only_v6(true)?; - } - let address: std::net::SocketAddr = address.parse().map_err(|error| { - std::io::Error::other(format!("invalid listener address: {error}")) - })?; - socket.bind(&address.into())?; - socket.listen(128)?; - let listener: std::net::TcpListener = socket.into(); - listener.set_nonblocking(true)?; - listeners.push(listener); - } - Ok(listeners) - })(); - let _ = tx.send(result); - }); - rx.await - .map_err(|_| std::io::Error::other("netns bind thread panicked"))?? - .into_iter() - .map(tokio::net::TcpListener::from_std) - .collect() - } - - /// Bind UDP and TCP DNS listeners inside the workload network namespace. - /// The workload keeps its image-provided resolver configuration; nftables - /// redirects port 53 to these sockets before the bypass fence runs. - pub async fn bind_policy_dns_sockets( - &self, - ) -> std::io::Result<(tokio::net::UdpSocket, tokio::net::TcpListener)> { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result<(std::net::UdpSocket, std::net::TcpListener)> { - #[allow(unsafe_code)] - if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { - return Err(std::io::Error::last_os_error()); - } - // Bind the exact REDIRECT destination instead of INADDR_ANY. - // For UDP this keeps replies sourced from loopback so - // conntrack can reverse the port/address translation before - // delivering them to libc in nested rootless namespaces. - let address: std::net::SocketAddr = format!("127.0.0.1:{POLICY_DNS_PORT}") - .parse() - .map_err(|error| { - std::io::Error::other(format!("invalid DNS listener address: {error}")) - })?; - - let udp = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::DGRAM, - Some(socket2::Protocol::UDP), - )?; - udp.set_reuse_address(true)?; - udp.bind(&address.into())?; - udp.set_nonblocking(true)?; - - let tcp = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - tcp.set_reuse_address(true)?; - tcp.bind(&address.into())?; - tcp.listen(128)?; - tcp.set_nonblocking(true)?; - - Ok((udp.into(), tcp.into())) - })(); - let _ = tx.send(result); - }); - let (udp, tcp) = rx - .await - .map_err(|_| std::io::Error::other("netns DNS bind thread panicked"))??; - Ok(( - tokio::net::UdpSocket::from_std(udp)?, - tokio::net::TcpListener::from_std(tcp)?, - )) - } - - /// Bind a TCP listener inside this network namespace on a dedicated thread. - /// - /// Spawns a short-lived OS thread that enters the namespace via `setns`, - /// binds a `std::net::TcpListener`, then exits. The listener fd is handed - /// back as a non-blocking `tokio::net::TcpListener`. Using a dedicated - /// thread (not `spawn_blocking`) avoids contaminating the tokio thread - /// pool's namespace state. - /// - /// Returns `Err` if the namespace has no fd, `setns` fails, or bind fails. - pub async fn bind_tcp_in_netns(&self, addr: &str) -> std::io::Result { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let addr = addr.to_string(); - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - // SAFETY: setns is safe to call; this is a dedicated thread - // that exits after binding. The thread's namespace state does - // not contaminate any thread pool. - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpListener::bind(&addr) - })(); - let _ = tx.send(result); - }); - - let std_listener = rx - .await - .map_err(|_| std::io::Error::other("netns bind thread panicked"))??; - std_listener.set_nonblocking(true)?; - tokio::net::TcpListener::from_std(std_listener) - } -} - -impl Drop for NetworkNamespace { - fn drop(&mut self) { - debug!(namespace = %self.name, "Cleaning up network namespace"); - - // Close the fd if we have one - if let Some(fd) = self.ns_fd.take() { - let _ = nix::unistd::close(fd); - } - - // Delete the host-side veth (this also removes the peer) - if let Err(e) = run_ip(&["link", "delete", &self.veth_host]) { - warn!( - error = %e, - veth = %self.veth_host, - "Failed to delete veth interface" - ); - } - - // Delete the namespace - if let Err(e) = run_ip(&["netns", "delete", &self.name]) { - warn!( - error = %e, - namespace = %self.name, - "Failed to delete network namespace" - ); - } - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Disabled, "cleaned_up") - .message(format!("Network namespace cleaned up [ns:{}]", self.name)) - .build() - ); - } -} - -/// Create the workload's network namespace and install bypass detection -/// rules. Returns `None` when the policy is not in proxy mode. -/// -/// The namespace is shared infrastructure: the proxy binds to its host-side -/// veth IP and reads /dev/kmsg from inside it for bypass detection, while -/// the workload child and SSH sessions enter it via `setns()`. -/// -/// # Errors -/// -/// Returns an error if proxy mode is requested but the namespace cannot be -/// created (e.g., missing `CAP_NET_ADMIN` / `CAP_SYS_ADMIN` or `iproute2`). -/// Failure to install nftables bypass-detection rules is non-fatal and is -/// reported via OCSF instead. -pub fn create_netns_for_proxy( - policy: &openshell_core::policy::SandboxPolicy, -) -> Result> { - use openshell_core::policy::NetworkMode; - use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; - - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Ok(None); - } - match NetworkNamespace::create() { - Ok(ns) => { - let proxy_port = policy - .network - .proxy - .as_ref() - .and_then(|p| p.http_addr) - .map_or(3128, |addr| addr.port()); - if let Err(e) = ns.install_bypass_rules(proxy_port) { - ocsf_emit!( - ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "degraded") - .message(format!( - "Failed to install bypass detection rules (non-fatal): {e}" - )) - .build() - ); - } - Ok(Some(ns)) - } - Err(e) => Err(miette::miette!( - "Network namespace creation failed and proxy mode requires isolation. \ - Ensure CAP_NET_ADMIN and CAP_SYS_ADMIN are available and iproute2 is installed. \ - Error: {e}" - )), - } -} - -/// Install pod-network bypass enforcement for Kubernetes sidecar topology. -/// -/// This runs in the current network namespace, not in a per-workload netns. -/// The rules allow loopback and the sidecar proxy UID, then reject direct -/// TCP/UDP egress from other UIDs so traffic must use the sidecar's local -/// proxy. -/// -/// # Errors -/// -/// Returns an error when `nft` is unavailable or the ruleset cannot be loaded. -pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { - match install_sidecar_nft_bypass_rules(proxy_uid) { - Ok(()) => Ok(()), - Err(nft_error) => { - warn!( - error = %nft_error, - "Failed to install nftables sidecar rules; trying iptables-legacy fallback" - ); - install_sidecar_iptables_legacy_bypass_rules(proxy_uid).map_err(|iptables_error| { - miette::miette!( - "sidecar nft ruleset load failed: {nft_error}; sidecar iptables-legacy fallback failed: {iptables_error}" - ) - }) - } - } -} - -fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { - let nft_cmd = find_nft().ok_or_else(|| { - miette::miette!( - "trusted nft helper not found; sidecar network enforcement requires nftables" - ) - })?; - let log_prefix = Some("openshell:sidecar-bypass:"); - let commands = nft_ruleset::generate_sidecar_bypass_commands(proxy_uid, log_prefix); - run_nft_commands_current_namespace(&nft_cmd, &commands) -} - -const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; -const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; - -fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { - let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { - miette::miette!( - "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" - ) - })?; - - let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { - Some(find_ip6tables_legacy().ok_or_else(|| { - miette::miette!( - "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" - ) - })?) - } else { - warn!( - "Skipping IPv6 sidecar iptables-legacy fallback because the current namespace has no non-loopback IPv6 interface" - ); - None - }; - - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); - - if let Err(e) = install_sidecar_iptables_legacy_family_rules( - &ipv4_filter_tool, - proxy_uid, - "icmp-port-unreachable", - ) { - cleanup_sidecar_iptables_legacy_rule_families( - &ipv4_filter_tool, - ipv6_fence_tool.as_deref(), - ); - return Err(e); - } - - if let Some(ipv6_fence_tool) = ipv6_fence_tool - && let Err(e) = install_sidecar_iptables_legacy_family_rules( - &ipv6_fence_tool, - proxy_uid, - "icmp6-port-unreachable", - ) - { - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, Some(&ipv6_fence_tool)); - return Err(e); - } - - Ok(()) -} - -fn current_namespace_has_non_loopback_ipv6() -> Result { - match std::fs::read_to_string(PROC_NET_IF_INET6_PATH) { - Ok(content) => Ok(has_non_loopback_ipv6_interface(&content)), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(e) => Err(miette::miette!( - "failed to inspect {PROC_NET_IF_INET6_PATH} before installing sidecar IPv6 fence: {e}" - )), - } -} - -fn has_non_loopback_ipv6_interface(content: &str) -> bool { - content.lines().any(|line| { - line.split_whitespace() - .nth(5) - .is_some_and(|iface| iface != "lo") - }) -} - -fn install_sidecar_iptables_legacy_family_rules( - cmd: &str, - proxy_uid: u32, - udp_reject_with: &str, -) -> Result<()> { - let proxy_uid_arg = proxy_uid.to_string(); - let commands: Vec> = vec![ - vec!["-N", SIDECAR_IPTABLES_CHAIN], - vec!["-A", SIDECAR_IPTABLES_CHAIN, "-o", "lo", "-j", "ACCEPT"], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-m", - "conntrack", - "--ctstate", - "ESTABLISHED,RELATED", - "-j", - "ACCEPT", - ], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-m", - "owner", - "--uid-owner", - &proxy_uid_arg, - "-j", - "ACCEPT", - ], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-p", - "tcp", - "-j", - "REJECT", - "--reject-with", - "tcp-reset", - ], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-p", - "udp", - "-j", - "REJECT", - "--reject-with", - udp_reject_with, - ], - vec!["-A", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], - ]; - - for args in commands { - if let Err(e) = run_iptables_legacy_current_namespace(cmd, &args) { - cleanup_sidecar_iptables_legacy_rules(cmd); - return Err(e); - } - } - - Ok(()) -} - -fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { - while run_iptables_legacy_current_namespace( - iptables_cmd, - &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], - ) - .is_ok() - {} - let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-F", SIDECAR_IPTABLES_CHAIN]); - let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); -} - -fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { - cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); - if let Some(ipv6_cmd) = ipv6_cmd { - cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); - } -} - -/// Run an `ip` command on the host. -fn run_ip(args: &[&str]) -> Result<()> { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - - debug!(command = %format!("{ip_path} {}", args.join(" ")), "Running ip command"); - - let output = Command::new(ip_path) - .args(args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "{ip_path} {} failed: {}", - args.join(" "), - stderr.trim() - )); - } - - Ok(()) -} - -fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { - debug!( - command = %format!("{iptables_cmd} {}", args.join(" ")), - "Running iptables-legacy sidecar command" - ); - - let output = Command::new(iptables_cmd) - .args(args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "{iptables_cmd} {} failed: {}", - args.join(" "), - stderr.trim() - )); - } - - Ok(()) -} - -/// Run a sequence of nft commands in the current network namespace. -/// -/// Each command is executed as a separate `nft` invocation to avoid atomic -/// batch rollback (where one unsupported expression like `ct state` or `log` -/// causes the entire transaction, including table creation, to fail). -/// -/// Commands marked as non-required are allowed to fail with a warning. -/// Required commands that fail abort the sequence immediately. -fn run_nft_commands_current_namespace( - nft_cmd: &str, - commands: &[nft_ruleset::NftCommand], -) -> Result<()> { - for cmd in commands { - let args_str = cmd.args.join(" "); - debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); - - let output = Command::new(nft_cmd) - .args(&cmd.args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - if cmd.required { - return Err(miette::miette!( - "{nft_cmd} {args_str} failed: {}", - stderr.trim() - )); - } - warn!( - command = %args_str, - error = %stderr.trim(), - "non-required nft command failed (continuing)" - ); - } - } - Ok(()) -} - -/// Run an `ip` command inside a network namespace via `nsenter --net=`. -/// -/// We use `nsenter` instead of `ip netns exec` because `ip netns exec` -/// remounts `/sys` to reflect the target namespace's sysfs entries. That -/// sysfs remount requires real `CAP_SYS_ADMIN` in the host user namespace, -/// which is unavailable in rootless container runtimes (e.g. rootless -/// Podman). `nsenter --net=` enters only the network namespace without -/// changing the mount namespace, avoiding the sysfs remount entirely. -/// The supervisor's operations (addr add, link set, route add) are all -/// netlink-based and do not need sysfs access. -fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { - run_ip_netns_output(netns, args).map(|_| ()) -} - -fn run_ip_netns_output(netns: &str, args: &[&str]) -> Result { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - let mut full_args = vec![net_flag.as_str(), "--", ip_path]; - full_args.extend(args); - - debug!( - command = %format!("{nsenter_path} {}", full_args.join(" ")), - "Running ip in namespace via nsenter" - ); - - let output = Command::new(nsenter_path) - .args(&full_args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "{nsenter_path} --net={} {ip_path} {} failed: {}", - ns_path.display(), - args.join(" "), - stderr.trim() - )); - } - - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) -} - -fn first_route_overlap( - routes: &str, - reserved: &[ipnet::IpNet], -) -> Option<(ipnet::IpNet, ipnet::IpNet)> { - routes.lines().find_map(|line| { - line.split_whitespace().find_map(|token| { - let route = token - .parse::() - .ok() - .or_else(|| token.parse::().ok().map(ipnet::IpNet::from))?; - reserved - .iter() - .copied() - .find(|pool| { - let same_family = route.addr().is_ipv4() == pool.addr().is_ipv4(); - let overlaps = - route.contains(&pool.network()) || pool.contains(&route.network()); - same_family && overlaps - }) - .map(|pool| (route, pool)) - }) - }) -} - -/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. -/// -/// Each command is executed as a separate invocation to avoid atomic batch -/// rollback. See [`run_nft_commands_current_namespace`] for rationale. -fn run_nft_commands_netns( - netns: &str, - nft_cmd: &str, - commands: &[nft_ruleset::NftCommand], -) -> Result<()> { - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - for cmd in commands { - let args_str = cmd.args.join(" "); - debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), - "Running nft command in namespace" - ); - - let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; - let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); - full_args.extend(&arg_refs); - - let output = Command::new(nsenter_path) - .args(&full_args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - if cmd.required { - return Err(miette::miette!( - "nft {args_str} failed in netns {netns}: {}", - stderr.trim() - )); - } - warn!( - command = %args_str, - error = %stderr.trim(), - netns = %netns, - "non-required nft command failed in namespace (continuing)" - ); - } - } - Ok(()) -} - -const NF_LOG_ALL_NETNS_PATH: &str = "/proc/sys/net/netfilter/nf_log_all_netns"; - -/// Enable nftables logging from non-init network namespaces. -/// -/// The kernel's `nf_log_syslog` module silently suppresses log output from -/// non-init network namespaces unless `net.netfilter.nf_log_all_netns` is -/// set to 1. Since sandbox bypass rules live in a per-sandbox network -/// namespace, the bypass monitor can't see log entries without this. -fn enable_nf_log_all_netns() { - use std::path::Path; - if !Path::new(NF_LOG_ALL_NETNS_PATH).exists() { - debug!("nf_log_all_netns sysctl not available (may already be set by init)"); - return; - } - match std::fs::write(NF_LOG_ALL_NETNS_PATH, "1") { - Ok(()) => { - debug!("Enabled nf_log_all_netns for non-init namespace logging"); - } - Err(e) => { - debug!( - error = %e, - "Could not enable nf_log_all_netns; bypass log rules may not produce output" - ); - } - } -} - -/// Well-known paths where nft may be installed. -const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; -const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/iptables-legacy", - "/sbin/iptables-legacy", - "/usr/bin/iptables-legacy", -]; -const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/ip6tables-legacy", - "/sbin/ip6tables-legacy", - "/usr/bin/ip6tables-legacy", -]; - -fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { - paths - .iter() - .copied() - .find(|path| { - let path = Path::new(path); - path.is_absolute() && path.is_file() - }) - .ok_or_else(|| { - miette::miette!( - "trusted {name} helper not found; checked {}", - paths.join(", ") - ) - }) -} - -/// Find the nft binary path, checking well-known locations. -fn find_nft() -> Option { - find_trusted_binary("nft", NFT_SEARCH_PATHS) - .ok() - .map(String::from) -} - -fn find_iptables_legacy() -> Option { - find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) -} - -fn find_ip6tables_legacy() -> Option { - find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - // These tests require root and network namespace support - // Run with: sudo cargo test -- --ignored - - #[test] - fn find_trusted_binary_uses_absolute_existing_file() { - let tempdir = tempfile::tempdir().unwrap(); - let helper = tempdir.path().join("ip"); - fs::write(&helper, b"test helper").unwrap(); - let helper = helper.to_str().unwrap(); - - assert_eq!( - find_trusted_binary("ip", &["relative-ip", "/missing/ip", helper]).unwrap(), - helper - ); - } - - #[test] - fn find_trusted_binary_rejects_missing_helpers() { - let err = - find_trusted_binary("nsenter", &["relative-nsenter", "/missing/nsenter"]).unwrap_err(); - - assert!(err.to_string().contains("trusted nsenter helper not found")); - } - - #[test] - fn nft_search_paths_are_absolute() { - for path in NFT_SEARCH_PATHS { - assert!( - path.starts_with('/'), - "NFT_SEARCH_PATHS entry must be absolute: {path}" - ); - } - } - - #[test] - fn iptables_legacy_search_paths_are_absolute() { - for path in IPTABLES_LEGACY_SEARCH_PATHS { - assert!( - path.starts_with('/'), - "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" - ); - } - } - - #[test] - fn ip6tables_legacy_search_paths_are_absolute() { - for path in IP6TABLES_LEGACY_SEARCH_PATHS { - assert!( - path.starts_with('/'), - "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" - ); - } - } - - #[test] - fn non_loopback_ipv6_detector_ignores_empty_input() { - assert!(!has_non_loopback_ipv6_interface("")); - assert!(!has_non_loopback_ipv6_interface("\n\n")); - } - - #[test] - fn non_loopback_ipv6_detector_ignores_loopback() { - let content = "00000000000000000000000000000001 01 80 10 80 lo\n"; - - assert!(!has_non_loopback_ipv6_interface(content)); - } - - #[test] - fn non_loopback_ipv6_detector_detects_pod_interface() { - let content = "\ -00000000000000000000000000000001 01 80 10 80 lo -fe800000000000000000000000000001 02 40 20 80 eth0 -"; - - assert!(has_non_loopback_ipv6_interface(content)); - } - - #[test] - fn route_overlap_detects_reserved_pool_collision() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n198.18.0.0/15 dev eth1\n"; - let (route, pool) = first_route_overlap(routes, &reserved).expect("collision"); - assert_eq!(route.to_string(), "198.18.0.0/15"); - assert_eq!(pool.to_string(), "198.18.1.0/25"); - } - - #[test] - fn route_overlap_ignores_default_and_unrelated_routes() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n10.200.0.0/24 dev veth\n"; - assert_eq!(first_route_overlap(routes, &reserved), None); - } - - #[test] - #[ignore = "requires root privileges"] - fn test_create_and_drop_namespace() { - let ns = NetworkNamespace::create().expect("Failed to create namespace"); - let name = ns.name().to_string(); - - // Verify namespace exists - let ns_path = openshell_core::container_paths::netns_path(&name); - assert!(ns_path.exists(), "Namespace file should exist"); - - // Verify IPs are set correctly - assert_eq!( - ns.host_ip().to_string(), - format!("{SUBNET_PREFIX}.{HOST_IP_SUFFIX}") - ); - assert_eq!( - ns.sandbox_ip().to_string(), - format!("{SUBNET_PREFIX}.{SANDBOX_IP_SUFFIX}") - ); - - // Drop should clean up - drop(ns); - - // Verify namespace is gone - assert!( - !Path::new(&ns_path).exists(), - "Namespace should be cleaned up" - ); - } -} diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs deleted file mode 100644 index aef95b6068..0000000000 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ /dev/null @@ -1,825 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! nftables ruleset generation for sandbox network bypass enforcement. -//! -//! This module provides pure functions to generate nftables rulesets that enforce -//! the sandbox network policy: all traffic must go through the proxy, with bypass -//! attempts logged and rejected. -//! -//! Rulesets are returned as a sequence of individual nft commands rather than a -//! monolithic file. Running each command as a separate `nft` invocation avoids -//! `nft -f` atomic batch semantics, where a single unsupported expression (e.g. -//! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the -//! entire transaction including table/chain creation. - -const DNS_DESTINATION_PORT: &str = "53"; - -/// A single nft command with metadata about whether it is required. -pub struct NftCommand { - /// The nft command arguments (e.g. `["add", "table", "inet", "openshell_bypass"]`). - pub args: Vec, - /// When false, failure of this command is non-fatal; the caller should - /// log a warning and continue with the remaining commands. - pub required: bool, -} - -/// Generate nft commands for sandbox network bypass enforcement. -/// -/// Creates an `inet` family table (handles both IPv4 and IPv6) with rules that: -/// 1. Accept traffic to the proxy (IPv4 only) -/// 2. Accept loopback traffic -/// 3. Accept established/related connections (optional; requires `nf_conntrack`) -/// 4. Reject TCP and UDP bypass attempts (both IPv4 and IPv6) -/// -/// If `log_prefix` is provided, log rules are inserted before each reject rule -/// so that bypass attempts are recorded in the kernel ring buffer before being -/// rejected. Log rules are always non-required since they need `nf_log` support. -pub fn generate_bypass_commands( - host_ip: &str, - proxy_port: u16, - log_prefix: Option<&str>, -) -> Vec { - let table = "openshell_bypass"; - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", table]), - nft_cmd(true, &["flush", "table", "inet", table]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - table, - "output", - "{ type filter hook output priority 0; policy accept; }", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "ip", - "daddr", - host_ip, - "tcp", - "dport", - &proxy_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", "rule", "inet", table, "output", "oifname", "lo", "accept", - ], - ), - nft_cmd( - false, - &[ - "add", - "rule", - "inet", - table, - "output", - "ct", - "state", - "established,related", - "accept", - ], - ), - ]; - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - cmds -} - -/// Generate the combined policy-DNS, transparent-TCP, and bypass fence. -/// -/// DNS may reach only the supervisor's trusted listener. TCP addressed to the -/// reserved synthetic pools is redirected before the terminal bypass reject; -/// all other direct TCP/UDP retains the existing fast-fail behavior. -pub fn generate_transparent_tcp_commands( - host_ip: &str, - proxy_port: u16, - dns_port: u16, - transparent_port: u16, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - log_prefix: Option<&str>, -) -> Vec { - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", "openshell_transparent"]), - nft_cmd(true, &["flush", "table", "inet", "openshell_transparent"]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - "openshell_transparent", - "output", - "{ type nat hook output priority dstnat; policy accept; }", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "meta", - "nfproto", - "ipv4", - "udp", - "dport", - DNS_DESTINATION_PORT, - "redirect", - "to", - &format!(":{dns_port}"), - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "ip", - "daddr", - synthetic_ipv4_cidr, - "tcp", - "dport", - "1-65535", - "redirect", - "to", - &format!(":{transparent_port}"), - ], - ), - // Synthetic destinations must take precedence over the generic TCP - // DNS capture. A policy endpoint may legitimately use TCP port 53; - // that connection belongs to transparent TCP, not the DNS listener. - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "meta", - "nfproto", - "ipv4", - "tcp", - "dport", - DNS_DESTINATION_PORT, - "redirect", - "to", - &format!(":{dns_port}"), - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "ip6", - "daddr", - synthetic_ipv6_cidr, - "tcp", - "dport", - "1-65535", - "redirect", - "to", - &format!(":{transparent_port}"), - ], - ), - ]; - let mut bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); - // NAT REDIRECT rewrites both DNS and synthetic TCP to loopback before the - // filter hook. Some kernels retain the packet's pre-REDIRECT output - // interface for filter matching, so `oifname lo accept` alone is not - // portable. Admit only packets that the kernel records as DNATed to the - // supervisor listeners. A direct dial to either port has no DNAT status - // and still reaches the terminal bypass reject. Transparent TCP - // authorization after accept remains bound by SO_ORIGINAL_DST plus the - // synthetic-address mapping. - let insertion = bypass - .iter() - .position(|command| { - command.args.iter().any(|arg| arg == "log") - || command.args.iter().any(|arg| arg == "reject") - }) - .unwrap_or(bypass.len()); - bypass.splice( - insertion..insertion, - [ - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "udp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &transparent_port.to_string(), - "accept", - ], - ), - ], - ); - cmds.extend(bypass); - cmds -} - -/// Generate nft commands for Kubernetes sidecar enforcement. -/// -/// The network sidecar and the process supervisor share a pod network -/// namespace. The sidecar runs as `proxy_uid` and owns external egress; -/// sandbox traffic must use loopback services hosted by that sidecar -/// (gateway forward and HTTP CONNECT proxy). The generated fence rejects -/// TCP/UDP bypass attempts from non-proxy UIDs; other L4 protocols are outside -/// the sidecar policy fence. -pub fn generate_sidecar_bypass_commands( - proxy_uid: u32, - log_prefix: Option<&str>, -) -> Vec { - let table = "openshell_sidecar_bypass"; - let uid_str = proxy_uid.to_string(); - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", table]), - nft_cmd(true, &["flush", "table", "inet", table]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - table, - "output", - "{ type filter hook output priority 0; policy accept; }", - ], - ), - nft_cmd( - true, - &[ - "add", "rule", "inet", table, "output", "oifname", "lo", "accept", - ], - ), - nft_cmd( - false, - &[ - "add", - "rule", - "inet", - table, - "output", - "ct", - "state", - "established,related", - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", "rule", "inet", table, "output", "meta", "skuid", &uid_str, "accept", - ], - ), - ]; - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - cmds -} - -fn nft_cmd(required: bool, args: &[&str]) -> NftCommand { - NftCommand { - args: args.iter().map(|s| (*s).to_string()).collect(), - required, - } -} - -fn nft_quote(s: &str) -> String { - // nft quoted strings don't support escape sequences; strip any embedded - // double-quotes that would terminate the string early. - format!("\"{}\"", s.replace('"', "")) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn cmd_str(cmd: &NftCommand) -> String { - cmd.args.join(" ") - } - - fn all_strs(cmds: &[NftCommand]) -> String { - cmds.iter().map(cmd_str).collect::>().join("\n") - } - - #[test] - fn generates_bypass_commands_with_proxy_rule() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - assert!(text.contains("add table inet openshell_bypass")); - assert!(text.contains("add chain inet openshell_bypass output")); - assert!(text.contains("ip daddr 10.0.2.2 tcp dport 8080 accept")); - } - - #[test] - fn bypass_commands_have_table_and_chain() { - let cmds = generate_bypass_commands("192.168.1.1", 3128, None); - let text = all_strs(&cmds); - assert!(text.contains("add table inet openshell_bypass")); - assert!(text.contains("type filter hook output priority 0; policy accept;")); - } - - #[test] - fn proxy_accept_rule_uses_provided_ip_and_port() { - let cmds = generate_bypass_commands("172.16.0.1", 9999, None); - let text = all_strs(&cmds); - assert!(text.contains("ip daddr 172.16.0.1 tcp dport 9999 accept")); - } - - #[test] - fn rules_are_ordered_accept_then_reject() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - let proxy_pos = text.find("ip daddr").unwrap(); - let lo_pos = text.find("oifname lo").unwrap(); - let ct_pos = text.find("ct state established,related").unwrap(); - let reject_pos = text.find("reject with icmp type").unwrap(); - - assert!(proxy_pos < lo_pos); - assert!(lo_pos < ct_pos); - assert!(ct_pos < reject_pos); - } - - #[test] - fn transparent_rules_precede_bypass_rejects_and_scope_dns() { - let commands = generate_transparent_tcp_commands( - "10.200.0.1", - 3128, - 15053, - 15001, - "198.18.0.0/24", - "fd23:6f70:656e::/48", - None, - ); - let text = all_strs(&commands); - assert!(text.contains("meta nfproto ipv4 udp dport 53 redirect to :15053")); - assert!(text.contains("meta nfproto ipv4 tcp dport 53 redirect to :15053")); - assert!(!text.contains("udp dport 53 accept")); - assert!(text.contains("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001")); - assert!( - text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") - ); - assert!(!text.contains("meta mark")); - assert!(text.contains("ct status dnat udp dport 15053 accept")); - assert!(text.contains("ct status dnat tcp dport 15053 accept")); - assert!(text.contains("ct status dnat tcp dport 15001 accept")); - for (protocol, port) in [("udp", "15053"), ("tcp", "15053"), ("tcp", "15001")] { - assert!(!commands.iter().any(|command| { - command.args.ends_with(&[ - protocol.to_string(), - "dport".to_string(), - port.to_string(), - "accept".to_string(), - ]) && !command.args.windows(3).any(|window| { - window == ["ct".to_string(), "status".to_string(), "dnat".to_string()] - }) - })); - } - assert!(text.contains("oifname lo accept")); - assert!( - text.find("ct status dnat tcp dport 15053 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!( - text.find("ct status dnat udp dport 15053 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto udp reject") - .unwrap() - ); - assert!( - text.find("ct status dnat tcp dport 15001 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") - .unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!(!text.contains("meta nfproto ipv6 udp dport 53 redirect")); - assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") - .unwrap() - < text - .find("meta nfproto ipv4 tcp dport 53 redirect to :15053") - .unwrap(), - "synthetic TCP:53 must reach transparent TCP before generic DNS capture" - ); - } - - #[test] - fn both_ipv4_and_ipv6_reject_types_are_present() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - let icmp_count = text - .matches("reject with icmp type port-unreachable") - .count(); - let icmpv6_count = text - .matches("reject with icmpv6 type port-unreachable") - .count(); - assert_eq!(icmp_count, 2, "need IPv4 ICMP rejects for TCP + UDP"); - assert_eq!(icmpv6_count, 2, "need IPv6 ICMPv6 rejects for TCP + UDP"); - } - - #[test] - fn no_log_commands_omit_log_rules() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - assert!( - !text.contains("log prefix"), - "no-log commands must not contain log rules" - ); - } - - #[test] - fn log_commands_contain_prefix_for_tcp_and_udp() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let text = all_strs(&cmds); - let count = text - .matches("log prefix \"openshell:bypass:test:\"") - .count(); - assert_eq!(count, 2, "need log rules for both TCP and UDP"); - assert!(text.contains("tcp flags syn limit rate 5/second burst 10 packets")); - assert!(text.contains("meta l4proto udp limit rate 5/second burst 10 packets")); - } - - #[test] - fn log_rules_appear_before_reject_rules() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let text = all_strs(&cmds); - let tcp_log_pos = text.find("tcp flags syn").unwrap(); - let tcp_reject_pos = text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap(); - let udp_log_pos = text.find("meta l4proto udp limit rate").unwrap(); - let udp_reject_pos = text - .find("meta nfproto ipv4 meta l4proto udp reject") - .unwrap(); - - assert!( - tcp_log_pos < tcp_reject_pos, - "TCP log rule must come before TCP reject rule" - ); - assert!( - udp_log_pos < udp_reject_pos, - "UDP log rule must come before UDP reject rule" - ); - } - - #[test] - fn ct_state_rule_is_not_required() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let ct_cmd = cmds - .iter() - .find(|c| cmd_str(c).contains("ct state")) - .unwrap(); - assert!( - !ct_cmd.required, - "ct state rule should be non-required (needs nf_conntrack)" - ); - } - - #[test] - fn log_rules_are_not_required() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - for cmd in &cmds { - if cmd_str(cmd).contains("log prefix") { - assert!( - !cmd.required, - "log rules should be non-required (needs nf_log)" - ); - } - } - } - - #[test] - fn sidecar_commands_allow_supervisor_uid_and_loopback() { - let cmds = generate_sidecar_bypass_commands(1337, None); - let text = all_strs(&cmds); - assert!(text.contains("add table inet openshell_sidecar_bypass")); - assert!(text.contains("oifname lo accept")); - assert!(text.contains("meta skuid 1337 accept")); - } - - #[test] - fn sidecar_commands_reject_tcp_and_udp_egress() { - let cmds = generate_sidecar_bypass_commands(0, Some("openshell:sidecar:test:")); - let text = all_strs(&cmds); - assert!(text.contains("meta nfproto ipv4 meta l4proto tcp reject")); - assert!(text.contains("meta nfproto ipv6 meta l4proto tcp reject")); - assert!(text.contains("meta nfproto ipv4 meta l4proto udp reject")); - assert!(text.contains("meta nfproto ipv6 meta l4proto udp reject")); - assert_eq!( - text.matches("log prefix \"openshell:sidecar:test:\"") - .count(), - 2 - ); - } - - #[test] - fn log_prefix_is_quoted_as_nft_string_literal() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - for cmd in &cmds { - let s = cmd_str(cmd); - if let Some(idx) = s.find("log prefix ") { - let after_prefix = &s[idx + "log prefix ".len()..]; - assert!( - after_prefix.starts_with('"'), - "log prefix value must be an nft-quoted string, got: {after_prefix}" - ); - } - } - } - - #[test] - fn nft_quote_wraps_in_double_quotes() { - assert_eq!(nft_quote("simple"), "\"simple\""); - assert_eq!(nft_quote("has:colons:"), "\"has:colons:\""); - assert_eq!(nft_quote("has\"quote"), "\"hasquote\""); - assert_eq!(nft_quote("has\\backslash"), "\"has\\backslash\""); - } -} diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs deleted file mode 100644 index 8c47e789ba..0000000000 --- a/crates/openshell-supervisor-process/src/run.rs +++ /dev/null @@ -1,830 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Workload supervision entry point. -//! -//! Spawns the SSH server, optional supervisor session, the entrypoint child -//! process, and waits for it to exit (with optional timeout). Long-running -//! background tasks that aren't strictly tied to the workload's lifetime -//! (policy poll loop, denial aggregator, symlink resolver) live in the -//! orchestrator, not here. - -use miette::{IntoDiagnostic, Result}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use std::time::Duration; -use tokio::time::timeout; -use tracing::info; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, DispositionId, LaunchTypeId, Process as OcsfProcess, - ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, -}; - -#[cfg(target_os = "linux")] -use crate::netns::NetworkNamespace; -use openshell_core::policy::{NetworkMode, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; - -#[cfg(target_os = "linux")] -use openshell_core::activity::ActivitySender; -#[cfg(target_os = "linux")] -use openshell_core::denial::DenialEvent; - -#[cfg(target_os = "linux")] -use crate::managed_children; -use crate::process::{ - ProcessEnforcementMode, ProcessHandle, ProcessStatus, ResolvedProcessIdentity, - ResolvedWorkspace, -}; - -pub enum SidecarExitReport { - Exited { - instance_id: String, - exit_code: i32, - ack: tokio::sync::oneshot::Sender>, - }, - Finalized { - instance_id: String, - ack: tokio::sync::oneshot::Sender>, - }, -} - -fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { - openshell_ocsf::ctx::ctx() -} - -/// Spawn the workload entrypoint, wire up SSH and supervisor session, and -/// wait for the entrypoint child to exit. -/// -/// # Errors -/// -/// Returns an error if SSH server startup fails, if the entrypoint child -/// fails to spawn, or if waiting for the child returns an OS error. -#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] -pub async fn run_process( - program: &str, - args: &[String], - workspace: ResolvedWorkspace, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, - ssh_socket_path: Option, - shared_ssh_socket: bool, - ssh_exit_tx: Option>, - policy: &SandboxPolicy, - resolved_process_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - entrypoint_pid: Arc, - entrypoint_started_tx: Option>, - sidecar_exit_tx: Option>, - provider_credentials: ProviderCredentialState, - provider_env: std::collections::HashMap, - ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, - agent_proposals: AgentProposals, - #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, - #[cfg(target_os = "linux")] bypass_denial_tx: Option< - tokio::sync::mpsc::UnboundedSender, - >, - #[cfg(target_os = "linux")] bypass_activity_tx: Option, -) -> Result { - // Platform drivers with a resolved numeric UID/GID retain the legacy - // account-file update. OCI-image identity leaves those environment values - // empty, so the image's account files remain unchanged. - #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() { - crate::process::update_sandbox_passwd_entries()?; - } - - // Validate the completed process identity before exposing a child. - #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() { - crate::process::validate_sandbox_user_with_identity(policy, resolved_process_identity)?; - crate::process::validate_sandbox_group_with_identity(policy, resolved_process_identity)?; - } - - // Create read_write directories and chown newly-created ones to the - // sandbox user/group. Runs as the supervisor (root) before the child - // is forked so the workload sees writable paths it owns. - #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() { - crate::process::prepare_filesystem_with_identity( - policy, - resolved_process_identity, - workspace.root(), - workspace.home().is_some(), - )?; - } - - // Eagerly fetch initial settings and install the agent skill if the - // proposals flag is on at startup, rather than waiting for the policy - // poll loop's first tick. In offline/file-mode there is no gateway, so - // the flag stays at its default (false) and no skill is installed. - install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; - - // Provider token grants may mount supervisor-only identity sockets such as - // the SPIFFE Workload API. Prepare the child mount namespace that hides - // those mounts before supervisor seccomp hardening removes the needed - // namespace syscalls. - #[cfg(target_os = "linux")] - crate::process::prepare_supervisor_identity_mount_namespace_from_env()?; - - // Install the supervisor seccomp prelude before spawning any workload-side - // tasks. By this point the orchestrator has finished privileged startup - // helpers (network namespace setup, identity mount namespace setup, - // nftables probes via run_networking), and the SSH listener and entrypoint - // child have not been exposed yet. - crate::sandbox::apply_supervisor_startup_hardening()?; - - // Spawn the bypass detection monitor. It tails dmesg for nftables LOG - // entries fired by rules installed on the workload's network namespace - // and reports direct connection attempts that would have bypassed the - // proxy. Spawn it before the entrypoint child so the first packets are - // not missed. Best-effort: returns None when dmesg is unavailable. - #[cfg(target_os = "linux")] - let _bypass_handle = netns.and_then(|ns| { - crate::bypass_monitor::spawn( - ns.name().to_string(), - entrypoint_pid.clone(), - bypass_denial_tx, - bypass_activity_tx, - ) - }); - - // Verify the runtime PID limit can accommodate the policy's pid_max. - #[cfg(target_os = "linux")] - { - let pid_limit_mode = if std::env::var_os("OPENSHELL_REQUIRE_RUNTIME_PID_LIMIT").is_some() { - crate::process::RuntimePidLimitMode::Require - } else { - crate::process::RuntimePidLimitMode::Warn - }; - crate::process::check_runtime_pid_limit(pid_limit_mode)?; - } - - // Zombie reaper — openshell-sandbox may run as PID 1 in containers and - // must reap orphaned grandchildren (e.g. background daemons started by - // coding agents) to prevent zombie accumulation. - // - // Use waitid(..., WNOWAIT) so we can inspect exited children before - // actually reaping them. This avoids racing explicit `child.wait()` calls - // for managed children (entrypoint and SSH session processes). - #[cfg(target_os = "linux")] - tokio::spawn(async { - use nix::sys::wait::{Id, WaitPidFlag, WaitStatus, waitid, waitpid}; - use tokio::signal::unix::{SignalKind, signal}; - use tokio::time::MissedTickBehavior; - - let mut sigchld = match signal(SignalKind::child()) { - Ok(s) => s, - Err(e) => { - tracing::warn!(error = %e, "Failed to register SIGCHLD handler for zombie reaping"); - return; - } - }; - let mut retry = tokio::time::interval(Duration::from_secs(5)); - retry.set_missed_tick_behavior(MissedTickBehavior::Skip); - - loop { - tokio::select! { - _ = sigchld.recv() => {} - _ = retry.tick() => {} - } - - loop { - let status = match waitid( - Id::All, - WaitPidFlag::WEXITED | WaitPidFlag::WNOHANG | WaitPidFlag::WNOWAIT, - ) { - Ok(WaitStatus::StillAlive) | Err(nix::errno::Errno::ECHILD) => break, - Ok(status) => status, - Err(nix::errno::Errno::EINTR) => continue, - Err(e) => { - tracing::debug!(error = %e, "waitid error during zombie reaping"); - break; - } - }; - - let Some(pid) = status.pid() else { - break; - }; - - if managed_children::is_managed(pid.as_raw()) { - // Let the explicit waiter own this child status. - break; - } - - match waitpid(pid, Some(WaitPidFlag::WNOHANG)) { - Ok(WaitStatus::StillAlive) - | Err(nix::errno::Errno::ECHILD | nix::errno::Errno::EINTR) => {} - Ok(reaped) => { - tracing::debug!(?reaped, "Reaped orphaned child process"); - } - Err(e) => { - tracing::debug!(error = %e, "waitpid error during orphan reap"); - break; - } - } - } - } - }); - - // Hard network policy enforcement for SSH sessions and the persistent - // supervisor session: each session's pre-exec hook calls setns(fd, - // CLONE_NEWNET) so it lands inside the workload's network namespace. - // Without this, SSH-spawned shells run in the host namespace and bypass - // the proxy entirely. - #[cfg(target_os = "linux")] - let ssh_netns_fd = netns.and_then(NetworkNamespace::ns_fd); - #[cfg(not(target_os = "linux"))] - let ssh_netns_fd: Option = None; - - #[cfg(target_os = "linux")] - let mut handle = ProcessHandle::spawn( - program, - args, - &workspace, - interactive, - policy, - resolved_process_identity, - enforcement_mode, - netns, - ca_file_paths.as_ref(), - &provider_env, - )?; - - #[cfg(not(target_os = "linux"))] - let mut handle = ProcessHandle::spawn( - program, - args, - &workspace, - interactive, - policy, - resolved_process_identity, - enforcement_mode, - ca_file_paths.as_ref(), - &provider_env, - )?; - - let main_pid = handle.pid(); - let main_session = crate::main_session::MainSession::new(handle.take_io(), main_pid); - let main_instance_id = uuid::Uuid::new_v4().to_string(); - - // SSH-spawned shells get http_proxy=http://: exported into - // their env so cooperative tools (curl, npm, Node) route through the - // CONNECT proxy. Linux uses the netns host_ip; on other targets fall back - // to the policy-declared http_addr directly. - #[cfg(target_os = "linux")] - let ssh_proxy_url = ssh_proxy_url_for_policy(policy, netns.map(NetworkNamespace::host_ip)); - #[cfg(not(target_os = "linux"))] - let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); - - let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); - if let Some(listen_path) = ssh_socket_path.clone() { - let policy_clone = policy.clone(); - let workspace_clone = workspace.clone(); - let proxy_url = ssh_proxy_url; - let netns_fd = ssh_netns_fd; - let ca_paths = ca_file_paths.clone(); - let provider_credentials_clone = provider_credentials.clone(); - let main_session_clone = Arc::clone(&main_session); - let user_env_clone: std::collections::HashMap = - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default(); - - let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); - - tokio::spawn(async move { - let _ssh_exit_guard = ssh_exit_tx; - if let Err(err) = crate::ssh::run_ssh_server( - listen_path, - ssh_ready_tx, - policy_clone, - workspace_clone, - netns_fd, - proxy_url, - ca_paths, - provider_credentials_clone, - user_env_clone, - resolved_process_identity, - enforcement_mode, - shared_ssh_socket, - main_session_clone, - ) - .await - { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message(format!("SSH server failed: {err}")) - .build() - ); - } - }); - - // Wait for the SSH server to bind before advertising its relay. The - // main process is already supervised; MainSession retains any output - // produced while this endpoint is being prepared. - match timeout(Duration::from_secs(10), ssh_ready_rx).await { - Ok(Ok(Ok(()))) => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .message("SSH server is ready to accept connections") - .build() - ); - } - Ok(Ok(Err(err))) => { - return Err(err.context("SSH server failed during startup")); - } - Ok(Err(_)) => { - return Err(miette::miette!( - "SSH server task panicked before signaling ready" - )); - } - Err(_) => { - return Err(miette::miette!( - "SSH server did not start within 10 seconds" - )); - } - } - } - - let supervisor_terminating = Arc::new(AtomicBool::new(false)); - // A canonical process may have completed while the SSH socket was being - // prepared. Detect that exit before entering the main wait path. - let early_exit = handle.try_wait().into_diagnostic()?; - - // Spawn the persistent supervisor session if we have a gateway endpoint - // and sandbox identity. The session provides relay channels for SSH - // connect and ExecSandbox through the gateway. - let supervisor_session_task = if let (Some(endpoint), Some(id), Some(socket)) = - (openshell_endpoint, sandbox_id, ssh_socket_path.as_ref()) - { - let task = crate::supervisor_session::spawn( - endpoint.to_string(), - id.to_string(), - socket.clone(), - ssh_netns_fd, - None, - Arc::clone(&supervisor_terminating), - main_instance_id.clone(), - ); - info!("supervisor session task spawned"); - Some(task) - } else { - None - }; - - // Store the entrypoint PID so the proxy can resolve TCP peer identity - entrypoint_pid.store(handle.pid(), Ordering::Release); - if let Some(tx) = entrypoint_started_tx { - let _ = tx.send((handle.pid(), main_instance_id.clone())); - } - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .launch_type(LaunchTypeId::Spawn) - .process(OcsfProcess::new(program, i64::from(handle.pid()))) - .message(format!("Process started: pid={}", handle.pid())) - .build() - ); - - let outcome = if let Some(status) = early_exit { - ProcessWaitOutcome::Exited(status) - } else { - wait_for_process_exit_or_shutdown(&mut handle, timeout_secs, &supervisor_terminating) - .await? - }; - - let (rendered_code, drain_terminal) = match outcome { - ProcessWaitOutcome::Exited(status) => (status.code(), true), - ProcessWaitOutcome::TimedOut => { - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message("Process timed out, killing") - .build() - ); - (124, false) - } - ProcessWaitOutcome::ShutdownSignal { signal, status } => { - info!( - signal, - exit_code = status.code(), - "Entrypoint exited after supervisor shutdown signal" - ); - (status.code(), false) - } - }; - let terminal_delivery_pending = main_session - .finish( - rendered_code, - drain_terminal && await_main_process_attachment, - ) - .await; - - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .exit_code(rendered_code) - .message(format!("Process exited with code {rendered_code}")) - .build() - ); - - if outcome.should_report_main_process_exit() { - if let Some(tx) = sidecar_exit_tx.as_ref() { - report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code) - .await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); - } - } else { - info!( - instance_id = %main_instance_id, - "skipping main-process exit report during supervisor shutdown" - ); - } - main_session.mark_terminal_reported(); - if outcome.should_report_main_process_exit() && drain_terminal && terminal_delivery_pending { - // The peer's SSH channel-close confirms that the terminal frames sent - // above traversed russh and the relay. Detached commands have no active - // attachment and never enter this wait. - main_session.wait_for_terminal_attachments().await; - } - if outcome.should_report_main_process_exit() { - if let Some(tx) = sidecar_exit_tx.as_ref() { - finalize_sidecar_main_process_exit(tx, &main_instance_id).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await; - info!(instance_id = %main_instance_id, "main-process terminal delivery finalized"); - } - } - - supervisor_terminating.store(true, Ordering::Release); - if let Some(task) = supervisor_session_task { - task.abort(); - } - - Ok(rendered_code) -} - -async fn report_main_process_exit_until_ack( - endpoint: &str, - sandbox_id: &str, - instance_id: &str, - exit_code: i32, -) { - let mut retry_delay = Duration::from_millis(250); - loop { - match crate::supervisor_session::report_main_process_exit( - endpoint, - sandbox_id, - instance_id, - exit_code, - ) - .await - { - Ok(()) => return, - Err(error) => { - tracing::warn!(%error, "main-process exit report failed; retrying"); - tokio::time::sleep(retry_delay).await; - retry_delay = (retry_delay * 2).min(Duration::from_secs(2)); - } - } - } -} - -async fn finalize_main_process_exit_until_ack(endpoint: &str, sandbox_id: &str, instance_id: &str) { - let mut retry_delay = Duration::from_millis(250); - loop { - match crate::supervisor_session::finalize_main_process_exit( - endpoint, - sandbox_id, - instance_id, - ) - .await - { - Ok(()) => return, - Err(error) => { - tracing::warn!(%error, "main-process terminal finalization failed; retrying"); - tokio::time::sleep(retry_delay).await; - retry_delay = (retry_delay * 2).min(Duration::from_secs(2)); - } - } - } -} - -async fn report_sidecar_main_process_exit( - tx: &tokio::sync::mpsc::Sender, - instance_id: &str, - exit_code: i32, -) -> Result<()> { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send(SidecarExitReport::Exited { - instance_id: instance_id.to_string(), - exit_code, - ack: ack_tx, - }) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error)) -} - -async fn finalize_sidecar_main_process_exit( - tx: &tokio::sync::mpsc::Sender, - instance_id: &str, -) -> Result<()> { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send(SidecarExitReport::Finalized { - instance_id: instance_id.to_string(), - ack: ack_tx, - }) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error)) -} - -enum ProcessWaitOutcome { - Exited(ProcessStatus), - TimedOut, - ShutdownSignal { - signal: &'static str, - status: ProcessStatus, - }, -} - -impl ProcessWaitOutcome { - /// A gateway acknowledgement is required for ordinary canonical-process - /// completion, but cannot be awaited after the supervisor itself has been - /// asked to terminate. At that point the gateway may already be shutting - /// down and no longer able to acknowledge the report. - fn should_report_main_process_exit(&self) -> bool { - !matches!(self, Self::ShutdownSignal { .. }) - } -} - -async fn wait_for_process_exit_or_shutdown( - handle: &mut ProcessHandle, - timeout_secs: u64, - terminating: &AtomicBool, -) -> Result { - let pid = handle.pid(); - let wait = handle.wait(); - tokio::pin!(wait); - - if timeout_secs > 0 { - let deadline = tokio::time::sleep(Duration::from_secs(timeout_secs)); - tokio::pin!(deadline); - tokio::select! { - result = &mut wait => { - Ok(ProcessWaitOutcome::Exited(result.into_diagnostic()?)) - } - () = &mut deadline => { - terminating.store(true, Ordering::Release); - terminate_then_kill_pid(pid).await; - Ok(ProcessWaitOutcome::TimedOut) - } - signal = wait_for_supervisor_shutdown_signal() => { - terminating.store(true, Ordering::Release); - signal_entrypoint_for_shutdown(pid, signal); - let status = (&mut wait).await.into_diagnostic()?; - Ok(ProcessWaitOutcome::ShutdownSignal { signal, status }) - } - } - } else { - tokio::select! { - result = &mut wait => { - Ok(ProcessWaitOutcome::Exited(result.into_diagnostic()?)) - } - signal = wait_for_supervisor_shutdown_signal() => { - terminating.store(true, Ordering::Release); - signal_entrypoint_for_shutdown(pid, signal); - let status = (&mut wait).await.into_diagnostic()?; - Ok(ProcessWaitOutcome::ShutdownSignal { signal, status }) - } - } - } -} - -#[cfg(unix)] -async fn terminate_then_kill_pid(pid: u32) { - signal_pid(pid, nix::sys::signal::Signal::SIGTERM, "process timeout"); - tokio::time::sleep(Duration::from_millis(100)).await; - signal_pid(pid, nix::sys::signal::Signal::SIGKILL, "process timeout"); -} - -#[cfg(not(unix))] -async fn terminate_then_kill_pid(_pid: u32) {} - -#[cfg(unix)] -fn signal_entrypoint_for_shutdown(pid: u32, signal: &'static str) { - signal_pid(pid, nix::sys::signal::Signal::SIGTERM, signal); -} - -#[cfg(not(unix))] -fn signal_entrypoint_for_shutdown(_pid: u32, _signal: &'static str) {} - -#[cfg(unix)] -fn signal_pid(pid: u32, signal: nix::sys::signal::Signal, reason: &'static str) { - let raw_pid = i32::try_from(pid).unwrap_or(i32::MAX); - if let Err(error) = nix::sys::signal::kill(nix::unistd::Pid::from_raw(-raw_pid), signal) { - tracing::warn!( - pid, - signal = ?signal, - reason, - error = %error, - "failed to signal entrypoint process group" - ); - } -} - -#[cfg(unix)] -async fn wait_for_supervisor_shutdown_signal() -> &'static str { - use tokio::signal::unix::{SignalKind, signal}; - - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(signal) => signal, - Err(error) => { - tracing::warn!( - error = %error, - "Failed to install SIGTERM handler; supervisor shutdown detection disabled" - ); - return std::future::pending::<&'static str>().await; - } - }; - - let _ = sigterm.recv().await; - info!("Received SIGTERM, shutting down supervisor process"); - "SIGTERM" -} - -#[cfg(not(unix))] -async fn wait_for_supervisor_shutdown_signal() -> &'static str { - std::future::pending::<&'static str>().await -} - -fn ssh_proxy_url_for_policy( - policy: &SandboxPolicy, - netns_proxy_host: Option, -) -> Option { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return None; - } - - let proxy = policy.network.proxy.as_ref()?; - if let Some(host) = netns_proxy_host { - let port = proxy.http_addr.map_or(3128, |addr| addr.port()); - return Some(format!("http://{host}:{port}")); - } - - proxy.http_addr.map(|addr| format!("http://{addr}")) -} - -/// Eagerly fetch initial settings and install the agent-driven policy -/// proposal skill if the flag is on at startup. -/// -/// Without this, the skill would only get installed on the policy poll -/// loop's first false→true transition, which can be ~10 s after launch — -/// long enough for an agent to start running without seeing it. -/// -/// Best-effort: any failure (no gateway, RPC error, install failure) is -/// logged but does not fail sandbox startup. -async fn install_initial_agent_skill( - sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, - agent_proposals: &AgentProposals, -) { - use openshell_core::proto::setting_value; - - if let (Some(id), Some(endpoint)) = (sandbox_id, openshell_endpoint) - && let Ok(client) = - openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await - && let Ok(result) = client.poll_settings(id).await - { - let initial = result - .settings - .get(openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) - .unwrap_or(false); - agent_proposals.set_enabled(initial); - } - - if agent_proposals.enabled() { - match crate::skills::install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill" - ), - Err(error) => tracing::warn!( - error = %error, - "Failed to install sandbox agent skill" - ), - } - } else { - tracing::debug!( - "agent_policy_proposals_enabled is false at startup; skipping skill install" - ); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - - fn policy(mode: NetworkMode, http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode, - proxy: http_addr.map(|http_addr| ProxyPolicy { - http_addr: Some(http_addr), - }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - #[test] - fn ssh_proxy_url_uses_policy_addr_without_netns() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); - - assert_eq!( - ssh_proxy_url_for_policy(&policy, None).as_deref(), - Some("http://127.0.0.1:3128") - ); - } - - #[test] - fn ssh_proxy_url_prefers_netns_host_with_policy_port() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); - - assert_eq!( - ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), - Some("http://10.200.0.1:8080") - ); - } - - #[test] - fn ssh_proxy_url_skips_non_proxy_mode() { - let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); - - assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); - } - - #[cfg(unix)] - #[test] - fn supervisor_shutdown_exit_skips_gateway_acknowledgement() { - use std::os::unix::process::ExitStatusExt; - - let status = ProcessStatus::from(std::process::ExitStatus::from_raw(libc::SIGTERM)); - - assert!(ProcessWaitOutcome::Exited(status).should_report_main_process_exit()); - assert!(ProcessWaitOutcome::TimedOut.should_report_main_process_exit()); - assert!( - !ProcessWaitOutcome::ShutdownSignal { - signal: "SIGTERM", - status, - } - .should_report_main_process_exit() - ); - } -} diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index fbd6d9275b..9f9aac35db 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -3,24 +3,11 @@ //! Embedded SSH server for sandbox access. -use crate::child_env; use crate::main_session::{MainOutput, MainSession}; -#[cfg(target_os = "linux")] -use crate::managed_children; -use crate::process::{ - ProcessEnforcementMode, ResolvedProcessIdentity, ResolvedWorkspace, - drop_privileges_with_identity, is_supervisor_only_env_var, session_user_and_home, -}; -use crate::sandbox; #[cfg(unix)] use libc; use miette::{IntoDiagnostic, Result}; -use nix::pty::{Winsize, openpty}; -use nix::unistd::setsid; use openshell_core::VERSION; -use openshell_core::net::set_tcp_nodelay_best_effort; -use openshell_core::policy::SandboxPolicy; -use openshell_core::provider_credentials::ProviderCredentialState; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, }; @@ -29,16 +16,35 @@ use russh::server::{Auth, ChannelOpenHandle, Handle, Session}; use russh::{ChannelId, ChannelOpenFailure, Sig}; use std::borrow::Cow; use std::collections::HashMap; -use std::io::{Read, Write}; -use std::os::fd::{AsRawFd, RawFd}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use std::sync::{Arc, mpsc}; use std::time::Duration; use tokio::net::UnixListener; use tracing::warn; const NO_LOGIN_SHELL_ENV: (&str, &str) = ("OPENSHELL_NO_LOGIN_SHELL", "1"); +const MAIN_DETACH_PREFIX: u8 = 0x10; +const MAIN_DETACH_KEY: u8 = 0x11; + +fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { + let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); + for &byte in data { + if *prefix_pending { + if byte == MAIN_DETACH_KEY { + *prefix_pending = false; + return (forward, true); + } + forward.push(MAIN_DETACH_PREFIX); + *prefix_pending = false; + } + if byte == MAIN_DETACH_PREFIX { + *prefix_pending = true; + } else { + forward.push(byte); + } + } + (forward, false) +} /// Perform SSH server initialization: generate a host key, build the config, /// and bind the Unix socket listener. Extracted so that startup errors can be @@ -52,13 +58,11 @@ type SshServerInit = ( fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, - enforcement_mode: ProcessEnforcementMode, shared_socket: bool, ) -> Result { let mut rng = rand::rng(); let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; - // TODO: while building the SSH config, refactor the server_id to be "SSH-2.0-OpenShell_" from `openshell_core::VERSION` let mut config = russh::server::Config { server_id: russh::SshId::Standard(Cow::Owned(format!("SSH-2.0-OpenShell_{VERSION}"))), auth_rejection_time: Duration::from_secs(1), @@ -69,17 +73,14 @@ fn ssh_server_init( let config = Arc::new(config); let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone())); - // In full enforcement mode the supervisor normally starts as root and can - // isolate the SSH socket in a root-only directory before spawning - // unprivileged children. Sidecar topology is different: the gateway relay - // runs in the network sidecar as a different UID, so the shared sidecar - // state directory must stay group-accessible. Sidecar mode uses a Linux - // abstract socket instead, so the workload cannot unlink the relay target. + // A driver may place the supervisor in another container, so an explicitly + // shared socket retains group access. Linux abstract sockets avoid a + // workload-replaceable filesystem inode. let abstract_socket = crate::unix_socket::is_abstract(listen_path); if !abstract_socket && let Some(parent) = listen_path.parent() { std::fs::create_dir_all(parent).into_diagnostic()?; #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() && !shared_socket { + if !shared_socket { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o700); std::fs::set_permissions(parent, perms).into_diagnostic()?; @@ -94,7 +95,7 @@ fn ssh_server_init( let listener = UnixListener::bind(runtime_path.as_ref()).into_diagnostic()?; // Tighten filesystem-socket permissions. Abstract sockets have no inode; - // sidecar relay connections authenticate the listener with SO_PEERCRED. + // local relay connections authenticate the listener with SO_PEERCRED. #[cfg(unix)] if !abstract_socket { use std::os::unix::fs::PermissionsExt; @@ -119,71 +120,44 @@ fn ssh_server_init( pub async fn run_ssh_server( listen_path: PathBuf, ready_tx: tokio::sync::oneshot::Sender>, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, shared_socket: bool, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init( - &listen_path, - &ca_file_paths, - enforcement_mode, - shared_socket, - ) { - Ok(v) => { - // Signal that the SSH server has bound the socket and is ready to - // accept connections. The parent task awaits this before spawning - // the entrypoint process, ensuring exec requests won't race - // against server startup. - let _ = ready_tx.send(Ok(())); - v - } - Err(err) => { - let _ = ready_tx.send(Err(err)); - return Ok(()); - } - }; - - let mut consecutive_resource_errors: u32 = 0; - let mut consecutive_unknown_errors: u32 = 0; + let (listener, config, _ca_paths) = + match ssh_server_init(&listen_path, &ca_file_paths, shared_socket) { + Ok(v) => { + // Signal that the SSH server has bound the socket and is ready to + // accept connections. The parent task awaits this before spawning + // the entrypoint process, ensuring exec requests won't race + // against server startup. + let _ = ready_tx.send(Ok(())); + v + } + Err(err) => { + let _ = ready_tx.send(Err(err)); + return Ok(()); + } + }; + let mut consecutive_resource_errors = 0; + let mut consecutive_unknown_errors = 0; loop { match listener.accept().await { Ok((stream, _peer)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; let config = config.clone(); - let policy = policy.clone(); - let workspace = workspace.clone(); - let proxy_url = proxy_url.clone(); - let ca_paths = ca_paths.clone(); - let provider_credentials = provider_credentials.clone(); - let user_environment = user_environment.clone(); - let main_session = Arc::clone(&main_session); + let port_forward = port_forward.clone(); + let boundary_exec = boundary_exec.clone(); + let main_session = main_session.clone(); tokio::spawn(async move { - if let Err(err) = handle_connection( - stream, - config, - policy, - workspace, - netns_fd, - proxy_url, - ca_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ) - .await + if let Err(err) = + handle_connection(stream, config, port_forward, boundary_exec, main_session) + .await { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -196,45 +170,31 @@ pub async fn run_ssh_server( } }); } - Err(err) => { - match classify_ssh_accept_error( - &err, - &mut consecutive_resource_errors, - &mut consecutive_unknown_errors, - ) { - SshAcceptAction::Terminal => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message(format!( - "SSH accept loop exiting on terminal error: {err}" - )) - .build() - ); - break; - } - SshAcceptAction::Retry { backoff, severity } => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(severity) - .status(StatusId::Failure) - .message(format!( - "SSH accept error (retrying in {}ms): {err}", - backoff.as_millis(), - )) - .build() - ); - tokio::time::sleep(backoff).await; - } + Err(error) => match classify_ssh_accept_error( + &error, + &mut consecutive_resource_errors, + &mut consecutive_unknown_errors, + ) { + SshAcceptAction::Terminal => { + return Err(error).into_diagnostic(); } - } + SshAcceptAction::Retry { backoff, severity } => { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(severity) + .status(StatusId::Failure) + .message(format!( + "SSH accept error (retrying in {}ms): {error}", + backoff.as_millis() + )) + .build() + ); + tokio::time::sleep(backoff).await; + } + }, } } - - Ok(()) } const MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS: u32 = 10; @@ -249,13 +209,13 @@ enum SshAcceptAction { } fn classify_ssh_accept_error( - err: &std::io::Error, + error: &std::io::Error, consecutive_resource_errors: &mut u32, consecutive_unknown_errors: &mut u32, ) -> SshAcceptAction { #[cfg(unix)] if matches!( - err.raw_os_error(), + error.raw_os_error(), Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) ) { return SshAcceptAction::Terminal; @@ -263,7 +223,7 @@ fn classify_ssh_accept_error( #[cfg(unix)] if matches!( - err.raw_os_error(), + error.raw_os_error(), Some( libc::EMFILE | libc::ENFILE @@ -286,26 +246,20 @@ fn classify_ssh_accept_error( ) ) { *consecutive_unknown_errors = 0; - - #[cfg(unix)] - let is_resource_pressure = matches!( - err.raw_os_error(), + let resource_pressure = matches!( + error.raw_os_error(), Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) ); - #[cfg(not(unix))] - let is_resource_pressure = false; - - if is_resource_pressure { + if resource_pressure { *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); - let backoff_ms = 100u64 - .saturating_mul(1u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) + let backoff_ms = 100_u64 + .saturating_mul(1_u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) .min(5_000); return SshAcceptAction::Retry { backoff: Duration::from_millis(backoff_ms), severity: SeverityId::Medium, }; } - *consecutive_resource_errors = 0; return SshAcceptAction::Retry { backoff: Duration::from_millis(100), @@ -313,24 +267,25 @@ fn classify_ssh_accept_error( }; } - #[cfg(unix)] #[cfg(target_os = "linux")] - if matches!(err.raw_os_error(), Some(libc::ENONET)) { - *consecutive_unknown_errors = 0; + if error.raw_os_error() == Some(libc::ENONET) { *consecutive_resource_errors = 0; + *consecutive_unknown_errors = 0; return SshAcceptAction::Retry { backoff: Duration::from_millis(100), severity: SeverityId::Low, }; } + *consecutive_resource_errors = 0; *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS { - return SshAcceptAction::Terminal; - } - SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, + SshAcceptAction::Terminal + } else { + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } } } @@ -338,16 +293,9 @@ fn classify_ssh_accept_error( async fn handle_connection( stream: tokio::net::UnixStream, config: Arc, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -363,18 +311,7 @@ async fn handle_connection( .build() ); - let handler = SshHandler::new( - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ); + let handler = SshHandler::new(port_forward, boundary_exec, main_session); russh::server::run_stream(config, stream, handler) .await .map_err(|err| miette::miette!("ssh stream error: {err}"))?; @@ -387,13 +324,12 @@ async fn handle_connection( /// sender. This allows `window_change_request` to resize the correct PTY when /// multiple channels are open simultaneously (e.g. parallel shells, shell + /// sftp, etc.). -// Several independent per-channel boolean flags (login-shell opt-out and the -// main-attachment state bits) legitimately live side by side here. #[allow(clippy::struct_excessive_bools)] #[derive(Default)] struct ChannelState { input_sender: Option, - pty_master: Option, + process: Option>, + terminal: Option>, pty_request: Option, no_login_shell: bool, main_input_owner: Option, @@ -403,37 +339,6 @@ struct ChannelState { main_output_task: Option, } -const MAIN_DETACH_PREFIX: u8 = 0x10; // Ctrl-P -const MAIN_DETACH_KEY: u8 = 0x11; // Ctrl-Q - -/// Remove the `OpenShell` detach sequence from canonical-main input. -/// -/// A trailing Ctrl-P remains pending across SSH data frames. If the following -/// byte is not Ctrl-Q, both bytes are forwarded unchanged. Bytes after a -/// completed detach sequence are discarded because the attachment is closing. -fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { - let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); - - for &byte in data { - if *prefix_pending { - if byte == MAIN_DETACH_KEY { - *prefix_pending = false; - return (forward, true); - } - forward.push(MAIN_DETACH_PREFIX); - *prefix_pending = false; - } - - if byte == MAIN_DETACH_PREFIX { - *prefix_pending = true; - } else { - forward.push(byte); - } - } - - (forward, false) -} - enum InputSender { Process(mpsc::Sender>), Main(tokio::sync::mpsc::Sender>), @@ -454,28 +359,26 @@ impl InputSender { } struct SshHandler { - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + /// Loopback port-forward, injected by the orchestrator (RFC 0012). In-pod + /// this connects from inside the workload netns; a delegated backend + /// tunnels into its guest. The handler does not know which. + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, channels: HashMap, } impl Drop for SshHandler { fn drop(&mut self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; for state in self.channels.values_mut() { if state.main_attached { - self.main_session.end_terminal_attachment(); - state.main_attached = false; + main_session.end_terminal_attachment(); } if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + main_session.release_input(owner); } if let Some(task) = state.main_output_task.take() { task.abort(); @@ -485,29 +388,14 @@ impl Drop for SshHandler { } impl SshHandler { - #[allow(clippy::too_many_arguments)] fn new( - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Self { Self { - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, + port_forward, + boundary_exec, main_session, channels: HashMap::new(), } @@ -550,15 +438,23 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, _session: &mut Session, ) -> Result<(), Self::Error> { - if let Some(state) = self.channels.remove(&channel) { - if state.main_attached { - self.main_session.end_terminal_attachment(); - } - if let Some(owner) = state.main_input_owner { - self.main_session.release_input(owner); + if let Some(mut state) = self.channels.remove(&channel) { + if state.main_attached + && let Some(main_session) = self.main_session.as_ref() + { + main_session.end_terminal_attachment(); + if let Some(owner) = state.main_input_owner.take() { + main_session.release_input(owner); + } + if let Some(task) = state.main_output_task.take() { + task.abort(); + } + return Ok(()); } - if let Some(task) = state.main_output_task { - task.abort(); + if let Some(process) = state.process { + // Channel ownership defines the exec lifetime. Closing an SSH + // channel must not strand an in-boundary process. + let _ = process.terminate().await; } } Ok(()) @@ -574,12 +470,6 @@ impl russh::server::Handler for SshHandler { reply: ChannelOpenHandle, _session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - reply - .reject(ChannelOpenFailure::AdministrativelyProhibited) - .await; - return Ok(()); - } // Validate port range before truncating u32 -> u16. The SSH protocol // uses u32 for ports, but valid TCP ports are 0-65535. Without this // check, port 65537 truncates to port 1 (privileged). @@ -599,9 +489,8 @@ impl russh::server::Handler for SshHandler { return Ok(()); } - // Only allow forwarding to loopback destinations to prevent the - // sandbox SSH server from being used as a generic proxy. - if !is_loopback_host(host_to_connect) { + let target = direct_tcpip_target(host_to_connect, port_to_connect); + if target.is_none() { ocsf_emit!(SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Refuse) .action(ActionId::Denied) @@ -620,16 +509,13 @@ impl russh::server::Handler for SshHandler { let host = host_to_connect.to_string(); // SSH protocol port is bounded by u32 but only u16 is meaningful; // saturate as a guard for malformed clients. - let port = u16::try_from(port_to_connect).unwrap_or(u16::MAX); - let netns_fd = self.netns_fd; - - // Confirm the channel before spawning: the task below writes to it, and - // the peer must see the open-confirmation first. + let port = u16::try_from(port_to_connect).expect("port range checked above"); + let target = target.expect("loopback target checked above"); + let port_forward = self.port_forward.clone(); reply.accept().await; tokio::spawn(async move { - let addr = format!("{host}:{port}"); - let tcp = match connect_in_netns(&addr, netns_fd).await { + let mut tcp_stream = match port_forward.connect(target).await { Ok(stream) => stream, Err(err) => { ocsf_emit!( @@ -637,7 +523,9 @@ impl russh::server::Handler for SshHandler { .activity(ActivityId::Fail) .severity(SeverityId::Low) .status(StatusId::Failure) - .message(format!("direct-tcpip: failed to connect to {addr}: {err}")) + .message(format!( + "direct-tcpip: failed to connect to {host}:{port}: {err}" + )) .build() ); let _ = channel.close().await; @@ -646,7 +534,6 @@ impl russh::server::Handler for SshHandler { }; let mut channel_stream = channel.into_stream(); - let mut tcp_stream = tcp; let _ = tokio::io::copy_bidirectional(&mut channel_stream, &mut tcp_stream).await; }); @@ -694,18 +581,17 @@ impl russh::server::Handler for SshHandler { return Ok(()); }; if state.main_attached { - self.main_session - .resize(col_width, row_height, pixel_width, pixel_height); - } else if let Some(master) = state.pty_master.as_ref() { - let winsize = Winsize { - ws_row: to_u16(row_height.max(1)), - ws_col: to_u16(col_width.max(1)), - ws_xpixel: to_u16(pixel_width), - ws_ypixel: to_u16(pixel_height), - }; - if let Err(e) = unsafe_pty::set_winsize(master.as_raw_fd(), winsize) { - warn!("failed to resize PTY for channel {channel:?}: {e}"); + if let Some(main_session) = self.main_session.as_ref() { + main_session + .resize(col_width, row_height, pixel_width, pixel_height) + .await; } + } else if let Some(terminal) = state.terminal.as_ref() + && let Err(e) = terminal + .resize(to_u16(col_width.max(1)), to_u16(row_height.max(1))) + .await + { + warn!("failed to resize PTY for channel {channel:?}: {e}"); } Ok(()) } @@ -715,10 +601,6 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; // Only allocate a PTY when the client explicitly requested one via // pty_request. VS Code Remote-SSH sends shell_request *without* a @@ -726,7 +608,7 @@ impl russh::server::Handler for SshHandler { // endings. Forcing a PTY here caused CRLF translation which made // VS Code misdetect the platform as Windows (and then try to run // `powershell`). - self.start_shell(channel, session.handle(), None)?; + self.start_shell(channel, session.handle(), None).await?; Ok(()) } @@ -736,16 +618,13 @@ impl russh::server::Handler for SshHandler { data: &[u8], session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; let command = String::from_utf8_lossy(data).trim().to_string(); if command.is_empty() { return Ok(()); } - self.start_shell(channel, session.handle(), Some(command))?; + self.start_shell(channel, session.handle(), Some(command)) + .await?; Ok(()) } @@ -756,12 +635,11 @@ impl russh::server::Handler for SshHandler { session: &mut Session, ) -> Result<(), Self::Error> { if name == "openshell-main" { - if !self.channels.contains_key(&channel) { - return Err(anyhow::anyhow!( - "subsystem_request on unknown channel {channel:?}" - )); - } - if self.main_session.begin_terminal_attachment().is_err() { + let Some(main_session) = self.main_session.clone() else { + session.channel_failure(channel)?; + return Ok(()); + }; + if !begin_main_attachment(&main_session, self.channels.contains_key(&channel)) { session.channel_failure(channel)?; return Ok(()); } @@ -771,34 +649,33 @@ impl russh::server::Handler for SshHandler { .expect("main channel existence checked above"); state.main_attached = true; if let Some(pty) = state.pty_request.take() { - self.main_session.resize( - pty.col_width, - pty.row_height, - pty.pixel_width, - pty.pixel_height, - ); + main_session + .resize( + pty.col_width, + pty.row_height, + pty.pixel_width, + pty.pixel_height, + ) + .await; } - let (input, input_warning) = if state.main_read_only { + let (input, warning) = if state.main_read_only { (None, None) } else { - match self.main_session.acquire_input() { + match main_session.acquire_input() { Ok((owner, input)) => { state.main_input_owner = Some(owner); (Some(InputSender::Main(input)), None) } - Err(error) => { - warn!(%error, "main process input lease unavailable; attaching read-only"); - (None, Some(error)) - } + Err(error) => (None, Some(error)), } }; - state.main_detach_prefix_pending = false; state.input_sender = input; - let mut output = self.main_session.subscribe(); - let terminal_delivery = Arc::clone(&self.main_session); + state.main_detach_prefix_pending = false; + let mut output = main_session.subscribe(); + let terminal_delivery = main_session.clone(); let handle = session.handle(); session.channel_success(channel)?; - if let Some(error) = input_warning { + if let Some(error) = warning { let _ = handle .extended_data( channel, @@ -810,13 +687,13 @@ impl russh::server::Handler for SshHandler { let output_task = tokio::spawn(async move { loop { match output.recv().await { + Ok(MainOutput::Exit(code)) => { + terminal_delivery.wait_for_terminal_reported().await; + let _ = + send_main_output(&handle, channel, MainOutput::Exit(code)).await; + break; + } Ok(event) => { - if let MainOutput::Exit(code) = event { - terminal_delivery.wait_for_terminal_reported().await; - let _ = send_main_output(&handle, channel, MainOutput::Exit(code)) - .await; - break; - } let _ = send_main_output(&handle, channel, event).await; } Err(error) => { @@ -840,31 +717,24 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { state.main_output_task = Some(output_task.abort_handle()); } - } else if name == "sftp" && !self.main_session.finished() { + } else if name == "sftp" { session.channel_success(channel)?; // sftp-server speaks the SFTP binary protocol over stdin/stdout, - // which is exactly what spawn_pipe_exec wires up. This enables + // which the boundary executor preserves as separate pipes. This enables // modern scp (SFTP-based, OpenSSH 9.0+) and SFTP clients to // transfer files into and out of the sandbox. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - Some("/usr/lib/openssh/sftp-server".to_string()), - false, - session.handle(), + self.start_exec_spec( channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &self.provider_credentials.child_env_with_gcp_resolved(), - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - let state = self.channels.get_mut(&channel).ok_or_else(|| { - anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") - })?; - state.input_sender = Some(InputSender::Process(input_sender)); + session.handle(), + openshell_isolation_interface::contract::ExecSpec { + program: "/usr/lib/openssh/sftp-server".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }, + ) + .await?; } else { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -889,11 +759,9 @@ impl russh::server::Handler for SshHandler { ) -> Result<(), Self::Error> { // Accept the env request so the client knows we handled it, but we // don't actually propagate arbitrary variables — the sandbox - // environment is controlled via policy. We must reply so VSCode - // doesn't stall. Two exceptions carry supervisor signals the SSH - // protocol has no native field for: - // - OPENSHELL_NO_LOGIN_SHELL: gateway login-shell opt-out. - // - OPENSHELL_MAIN_READ_ONLY: read-only main attachment. + // environment is controlled via policy. The login-shell opt-out is a + // supervisor signal carried over SSH because the protocol has no + // native field for it. if variable_name == NO_LOGIN_SHELL_ENV.0 && let Some(state) = self.channels.get_mut(&channel) { @@ -919,38 +787,17 @@ impl russh::server::Handler for SshHandler { warn!("data on unknown channel {channel:?}"); return Ok(()); }; - - let main_attached = state.main_attached; - let (forward, detach) = if main_attached { + let (forward, detach) = if state.main_attached { filter_main_detach_sequence(&mut state.main_detach_prefix_pending, data) } else { (data.to_vec(), false) }; - let send_error = (!forward.is_empty()) + let error = (!forward.is_empty()) .then(|| state.input_sender.as_ref()?.send(forward).err()) .flatten(); - - if let Some(error) = send_error { - let handle = session.handle(); - if main_attached { - self.close_main_attachment(channel, handle, Some(error)) - .await; - } else { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; closing attachment\n").into_bytes(), - ) - .await; - let _ = handle.close(channel).await; - } - return Ok(()); - } - if detach { - self.close_main_attachment(channel, session.handle(), None) + if state.main_attached && (detach || error.is_some()) { + self.close_main_attachment(channel, session.handle(), error) .await; - return Ok(()); } Ok(()) } @@ -967,8 +814,12 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached && let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() { - self.main_session.release_input(owner); + // A canonical process outlives one SSH attachment. Release + // this channel's lease without closing process stdin so a + // replacement attachment can become the input owner. + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -984,47 +835,192 @@ impl russh::server::Handler for SshHandler { signal: Sig, _session: &mut Session, ) -> Result<(), Self::Error> { - if !self + if self .channels .get(&channel) .is_some_and(|state| state.main_attached) { + let signal = match signal { + Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), + Sig::INT => Some(nix::sys::signal::Signal::SIGINT), + Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), + Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), + Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + _ => None, + }; + if let (Some(signal), Some(main_session)) = (signal, self.main_session.as_ref()) + && let Err(error) = main_session.signal_group(signal).await + { + warn!(%error, ?signal, "failed to signal canonical main process group"); + } return Ok(()); } + let Some(process) = self + .channels + .get(&channel) + .and_then(|state| state.process.clone()) + else { + return Ok(()); + }; let signal = match signal { - Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), - Sig::INT => Some(nix::sys::signal::Signal::SIGINT), - Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), - Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), - Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + Sig::HUP => Some(openshell_isolation_interface::contract::BoundarySignal::Hup), + Sig::INT => Some(openshell_isolation_interface::contract::BoundarySignal::Int), + Sig::KILL => Some(openshell_isolation_interface::contract::BoundarySignal::Kill), + Sig::TERM => Some(openshell_isolation_interface::contract::BoundarySignal::Term), _ => None, }; if let Some(signal) = signal - && let Err(error) = self.main_session.signal_group(signal) + && let Err(error) = process.signal(signal).await { - warn!(%error, ?signal, "failed to signal canonical main process group"); + warn!(%error, ?signal, "failed to signal boundary exec process"); } Ok(()) } } -async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { - match event { - MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), - MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), - MainOutput::Exit(code) => { - let eof_sent = handle.eof(channel).await.is_ok(); - let status_sent = handle - .exit_status_request(channel, code.max(0).unsigned_abs()) +impl SshHandler { + async fn start_shell( + &mut self, + channel: ChannelId, + handle: Handle, + command: Option, + ) -> anyhow::Result<()> { + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; + let no_login_shell = state.no_login_shell; + let pty = state.pty_request.take(); + let pty_requested = pty.is_some(); + let (program, args) = command.map_or_else( + || { + if pty_requested { + ("/bin/bash".to_string(), vec!["-i".to_string()]) + } else { + ("/bin/bash".to_string(), vec![]) + } + }, + |command| { + ( + "/bin/bash".to_string(), + vec![login_shell_flag(no_login_shell).to_string(), command], + ) + }, + ); + let env = pty + .as_ref() + .map(|request| vec![("TERM".to_string(), request.term.clone())]) + .unwrap_or_default(); + self.start_exec_spec( + channel, + handle, + openshell_isolation_interface::contract::ExecSpec { + program, + args, + env, + workdir: None, + pty: pty_requested, + }, + ) + .await?; + if let (Some(pty), Some(terminal)) = ( + pty, + self.channels + .get(&channel) + .and_then(|state| state.terminal.as_ref()), + ) { + terminal + .resize(to_u16(pty.col_width.max(1)), to_u16(pty.row_height.max(1))) .await - .is_ok(); - let close_sent = handle.close(channel).await.is_ok(); - eof_sent && status_sent && close_sent + .map_err(|error| anyhow::anyhow!(error.to_string()))?; } + Ok(()) + } + + async fn start_exec_spec( + &mut self, + channel: ChannelId, + handle: Handle, + spec: openshell_isolation_interface::contract::ExecSpec, + ) -> anyhow::Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut exec = self + .boundary_exec + .exec(spec) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("exec on unknown channel {channel:?}"))?; + state.process = Some(exec.process.clone()); + state.terminal = exec.terminal.take(); + + if let Some(mut stdin) = exec.stdin.take() { + let (sender, receiver) = mpsc::channel::>(); + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + while let Ok(bytes) = receiver.recv() { + if runtime.block_on(stdin.write_all(&bytes)).is_err() { + break; + } + } + }); + state.input_sender = Some(InputSender::Process(sender)); + } + + let mut stdout = exec.stdout; + let stdout_handle = handle.clone(); + let stdout_task = tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stdout_handle.data(channel, buffer[..size].to_vec()).await; + } + } + } + }); + let stderr_task = exec.stderr.map(|mut stderr| { + let stderr_handle = handle.clone(); + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stderr_handle + .extended_data(channel, 1, buffer[..size].to_vec()) + .await; + } + } + } + }) + }); + tokio::spawn(async move { + let status = exec.process.wait().await; + let _ = stdout_task.await; + if let Some(task) = stderr_task { + let _ = task.await; + } + let code = match status { + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code)) => { + code.max(0).cast_unsigned() + } + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + )) => (128_i32.saturating_add(signal)).max(0).cast_unsigned(), + Err(_) => 1, + }; + let _ = handle.eof(channel).await; + let _ = handle.exit_status_request(channel, code).await; + let _ = handle.close(channel).await; + }); + Ok(()) } -} -impl SshHandler { async fn close_main_attachment( &mut self, channel: ChannelId, @@ -1033,11 +1029,15 @@ impl SshHandler { ) { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached { - self.main_session.end_terminal_attachment(); + if let Some(main_session) = self.main_session.as_ref() { + main_session.end_terminal_attachment(); + } state.main_attached = false; } - if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + if let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() + { + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -1058,118 +1058,33 @@ impl SshHandler { let _ = handle.exit_status_request(channel, 0).await; let _ = handle.close(channel).await; } +} - fn start_shell( - &mut self, - channel: ChannelId, - handle: Handle, - command: Option, - ) -> anyhow::Result<()> { - let provider_env = self.provider_credentials.child_env_with_gcp_resolved(); - let state = self - .channels - .get_mut(&channel) - .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; - let no_login_shell = state.no_login_shell; - if let Some(pty) = state.pty_request.take() { - // PTY was requested — allocate a real PTY (interactive shell or - // exec that explicitly asked for a terminal). - let (pty_master, input_sender) = spawn_pty_shell( - &self.policy, - &self.workspace, - command, - no_login_shell, - &pty, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.pty_master = Some(pty_master); - state.input_sender = Some(InputSender::Process(input_sender)); - } else { - // No PTY requested — use plain pipes so stdout/stderr are - // separate and output has clean LF line endings. This is the - // path VSCode Remote-SSH exec commands take. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - command, - no_login_shell, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.input_sender = Some(InputSender::Process(input_sender)); +fn begin_main_attachment(main_session: &MainSession, channel_exists: bool) -> bool { + channel_exists && main_session.begin_terminal_attachment().is_ok() +} + +async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { + match event { + MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), + MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), + MainOutput::Exit(code) => { + let eof = handle.eof(channel).await.is_ok(); + let status = handle + .exit_status_request(channel, code.max(0).unsigned_abs()) + .await + .is_ok(); + let close = handle.close(channel).await.is_ok(); + eof && status && close } - Ok(()) } } -/// Connect a TCP stream to `addr` inside the sandbox network namespace. -/// -/// The SSH supervisor runs in the host network namespace while sandbox child -/// processes run in an isolated network namespace (with their own loopback). -/// A plain `TcpStream::connect("127.0.0.1:port")` from the supervisor would -/// hit the host loopback, not the sandbox loopback where services are listening. -/// -/// On Linux, we spawn a dedicated OS thread, call `setns` to enter the sandbox -/// namespace, create the socket there, then convert it to a tokio `TcpStream`. -/// We use `std::thread::spawn` (not `spawn_blocking`) because `setns` changes -/// the calling thread's network namespace permanently — a tokio blocking-pool -/// thread could be reused for unrelated tasks and must not be contaminated. -/// On non-Linux platforms (no network namespace support), we connect directly. -pub async fn connect_in_netns( - addr: &str, - netns_fd: Option, -) -> std::io::Result { - #[cfg(target_os = "linux")] - if let Some(fd) = netns_fd { - let addr = addr.to_string(); - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - // Enter the sandbox network namespace on this dedicated thread. - // SAFETY: setns is safe to call; this is a dedicated thread that - // will exit after the connection is established. - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect(&addr) - })(); - let _ = tx.send(result); - }); - - let std_stream = rx - .await - .map_err(|_| std::io::Error::other("netns connect thread panicked"))??; - std_stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(std_stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - #[cfg(not(target_os = "linux"))] - let _ = netns_fd; - - let stream = tokio::net::TcpStream::connect(addr).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) +const fn login_shell_flag(no_login_shell: bool) -> &'static str { + if no_login_shell { "-c" } else { "-lc" } } +#[allow(dead_code)] #[derive(Clone)] struct PtyRequest { term: String, @@ -1191,575 +1106,6 @@ impl Default for PtyRequest { } } -#[allow(clippy::too_many_arguments)] -fn apply_child_env( - cmd: &mut Command, - session_home: &str, - session_user: &str, - term: &str, - proxy_url: Option<&str>, - ca_file_paths: Option<&(PathBuf, PathBuf)>, - provider_env: &HashMap, - user_environment: &HashMap, -) { - let path = std::env::var("PATH").unwrap_or_else(|_| "/usr/local/bin:/usr/bin:/bin".into()); - - cmd.env_clear() - .env(openshell_core::sandbox_env::SANDBOX, "1") - .env("HOME", session_home) - .env("USER", session_user) - .env("SHELL", openshell_core::shell::detect_login_shell()) - .env("PATH", &path) - .env("TERM", term); - - for (key, value) in user_environment { - if !key.starts_with("OPENSHELL_") { - cmd.env(key, value); - } - } - - if let Some(url) = proxy_url { - for (key, value) in child_env::proxy_env_vars(url) { - cmd.env(key, value); - } - } - - if let Some((ca_cert_path, combined_bundle_path)) = ca_file_paths { - for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - } - - for (key, value) in provider_env { - if is_supervisor_only_env_var(key) { - continue; - } - cmd.env(key, value); - } -} - -const fn login_shell_flag(no_login_shell: bool) -> &'static str { - if no_login_shell { "-c" } else { "-lc" } -} - -/// Build the shell command for an SSH session using a shell that exists in the -/// sandbox image (minimal images such as Alpine ship only `/bin/sh`, not bash). -/// -/// `no_command_arg` is appended only when no explicit command is given: `-i` -/// for an interactive PTY session, or `None` for the non-PTY stdin path (a -/// bare shell already reads piped stdin line-by-line). With an explicit -/// command the login-shell flag is used per `no_login_shell`. -fn build_ssh_shell_command( - shell: &str, - command: Option, - no_login_shell: bool, - no_command_arg: Option<&str>, -) -> Command { - let mut cmd = Command::new(shell); - match command { - None => { - if let Some(arg) = no_command_arg { - cmd.arg(arg); - } - } - Some(command) => { - cmd.arg(login_shell_flag(no_login_shell)).arg(command); - } - } - cmd -} - -#[allow(clippy::too_many_arguments)] -fn spawn_pty_shell( - policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, - command: Option, - no_login_shell: bool, - pty: &PtyRequest, - handle: Handle, - channel: ChannelId, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_env: &HashMap, - user_environment: &HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, -) -> anyhow::Result<(std::fs::File, mpsc::Sender>)> { - let winsize = Winsize { - ws_row: to_u16(pty.row_height.max(1)), - ws_col: to_u16(pty.col_width.max(1)), - ws_xpixel: to_u16(pty.pixel_width), - ws_ypixel: to_u16(pty.pixel_height), - }; - let openpty = openpty(Some(&winsize), None)?; - let master = std::fs::File::from(openpty.master); - let slave = std::fs::File::from(openpty.slave); - let slave_fd = slave.as_raw_fd(); - - let stdin = slave.try_clone()?; - let stdout = slave.try_clone()?; - let stderr = slave; - let mut reader = master.try_clone()?; - let mut writer = master.try_clone()?; - - // Resolve a shell present in the sandbox image; interactive PTY sessions - // pass `-i` when no command is given. Runs in the supervisor, so it - // inspects the sandbox filesystem. - let shell = openshell_core::shell::detect_login_shell(); - let mut cmd = build_ssh_shell_command(&shell, command, no_login_shell, Some("-i")); - - let term = if pty.term.is_empty() { - "xterm-256color" - } else { - pty.term.as_str() - }; - - // Derive USER and HOME from the policy's run_as_user when available, - // falling back to "sandbox" / "/sandbox" for backward compatibility. - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); - apply_child_env( - &mut cmd, - &session_home, - &session_user, - term, - proxy_url.as_deref(), - ca_file_paths.as_deref(), - provider_env, - user_environment, - ); - cmd.stdin(stdin).stdout(stdout).stderr(stderr); - - if let Some(dir) = workspace.root() { - cmd.current_dir(dir); - } - - // Probe Landlock availability from the parent process where tracing works. - #[cfg(target_os = "linux")] - if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - } - - // Phase 1: Prepare Landlock ruleset before the child applies it. - #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; - - #[cfg(unix)] - { - unsafe_pty::install_pre_exec( - &mut cmd, - policy.clone(), - workspace.owned_root(), - slave_fd, - netns_fd, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared_sandbox, - ); - } - - #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; - #[cfg(not(target_os = "linux"))] - let mut child = cmd.spawn()?; - #[cfg(target_os = "linux")] - let child_pid = child.id(); - #[cfg(target_os = "linux")] - managed_children::register(child_pid); - let master_file = master; - - let (sender, receiver) = mpsc::channel::>(); - std::thread::spawn(move || { - while let Ok(bytes) = receiver.recv() { - if writer.write_all(&bytes).is_err() { - break; - } - let _ = writer.flush(); - } - }); - - let runtime = tokio::runtime::Handle::current(); - let runtime_reader = runtime.clone(); - let handle_clone = handle.clone(); - // Signal from the reader thread to the exit thread that all output has - // been forwarded. The exit thread waits for this before sending the - // exit-status and closing the channel, ensuring the correct SSH protocol - // ordering: data → EOF → exit-status → close. - let (reader_done_tx, reader_done_rx) = mpsc::channel::<()>(); - std::thread::spawn(move || { - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let data = buf[..n].to_vec(); - let handle_clone = handle_clone.clone(); - let _ = runtime_reader - .block_on(async move { handle_clone.data(channel, data).await }); - } - } - } - // Send EOF to indicate no more data will be sent on this channel. - let eof_handle = handle_clone.clone(); - let _ = runtime_reader.block_on(async move { eof_handle.eof(channel).await }); - // Notify the exit thread that all output has been forwarded. - let _ = reader_done_tx.send(()); - }); - - let handle_exit = handle; - let runtime_exit = runtime; - std::thread::spawn(move || { - let status = child.wait().ok(); - #[cfg(target_os = "linux")] - managed_children::unregister(child_pid); - let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); - // Wait for the reader thread to finish forwarding all output before - // sending exit-status and closing the channel. This prevents the - // race where close() was called before exit_status_request(). - // - // Use a timeout because a backgrounded grandchild process (e.g. - // `nohup daemon &`) may hold the PTY slave open indefinitely, - // preventing the reader from reaching EOF. Two seconds is enough - // for any remaining buffered data to drain. - let _ = reader_done_rx.recv_timeout(Duration::from_secs(2)); - drop(runtime_exit.spawn(async move { - let _ = handle_exit.exit_status_request(channel, code).await; - let _ = handle_exit.close(channel).await; - })); - }); - - Ok((master_file, sender)) -} - -/// Spawn a command using plain pipes (no PTY). -/// -/// stdout is forwarded as SSH channel data and stderr as SSH extended data -/// (type 1), preserving the separation that clients like `VSCode` Remote-SSH -/// expect. Output retains clean LF line endings (no CRLF translation). -#[allow(clippy::too_many_arguments)] -fn spawn_pipe_exec( - policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, - command: Option, - no_login_shell: bool, - handle: Handle, - channel: ChannelId, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_env: &HashMap, - user_environment: &HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, -) -> anyhow::Result>> { - // Resolve a shell present in the sandbox image; minimal images (e.g. Alpine) - // don't ship bash, only `/bin/sh`. Runs in the supervisor, so it inspects - // the sandbox filesystem. No command → read from stdin with no `-i`: - // interactive mode reads .bashrc, writes prompts to stderr, and can add - // just enough latency for VS Code Remote-SSH's platform detection to time - // out and fall back to "windows". A plain shell with piped stdin already - // reads commands line-by-line (script mode), which is what VS Code expects. - let shell = openshell_core::shell::detect_login_shell(); - let mut cmd = build_ssh_shell_command(&shell, command, no_login_shell, None); - - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); - apply_child_env( - &mut cmd, - &session_home, - &session_user, - "dumb", - proxy_url.as_deref(), - ca_file_paths.as_deref(), - provider_env, - user_environment, - ); - cmd.stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - if let Some(dir) = workspace.root() { - cmd.current_dir(dir); - } - - // Probe Landlock availability from the parent process where tracing works. - #[cfg(target_os = "linux")] - if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - } - - // Phase 1: Prepare Landlock ruleset before the child applies it. - #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; - - #[cfg(unix)] - { - unsafe_pty::install_pre_exec_no_pty( - &mut cmd, - policy.clone(), - workspace.owned_root(), - netns_fd, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared_sandbox, - ); - } - - #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; - #[cfg(not(target_os = "linux"))] - let mut child = cmd.spawn()?; - #[cfg(target_os = "linux")] - let child_pid = child.id(); - #[cfg(target_os = "linux")] - managed_children::register(child_pid); - - let child_stdin = child.stdin.take(); - let child_stdout = child.stdout.take().expect("stdout must be piped"); - let child_stderr = child.stderr.take().expect("stderr must be piped"); - - // stdin writer thread - let (sender, receiver) = mpsc::channel::>(); - std::thread::spawn(move || { - let Some(mut stdin) = child_stdin else { - return; - }; - while let Ok(bytes) = receiver.recv() { - if stdin.write_all(&bytes).is_err() { - break; - } - let _ = stdin.flush(); - } - }); - - let runtime = tokio::runtime::Handle::current(); - - // Signal from the reader threads to the exit thread that all output has - // been forwarded. - let (reader_done_tx, reader_done_rx) = mpsc::channel::<()>(); - - // stdout reader - let stdout_handle = handle.clone(); - let stdout_runtime = runtime.clone(); - let reader_done_stdout = reader_done_tx.clone(); - std::thread::spawn(move || { - let mut reader = child_stdout; - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let data = buf[..n].to_vec(); - let h = stdout_handle.clone(); - let _ = stdout_runtime.block_on(async move { h.data(channel, data).await }); - } - } - } - let _ = reader_done_stdout.send(()); - }); - - // stderr reader — sends as extended data (type 1) - let stderr_handle = handle.clone(); - let stderr_runtime = runtime.clone(); - std::thread::spawn(move || { - let mut reader = child_stderr; - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let data = buf[..n].to_vec(); - let h = stderr_handle.clone(); - let _ = stderr_runtime - .block_on(async move { h.extended_data(channel, 1, data).await }); - } - } - } - let _ = reader_done_tx.send(()); - }); - - // Exit waiter thread - let handle_exit = handle; - let runtime_exit = runtime; - std::thread::spawn(move || { - let status = child.wait().ok(); - #[cfg(target_os = "linux")] - managed_children::unregister(child_pid); - let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); - // Wait for both reader threads. - let _ = reader_done_rx.recv_timeout(Duration::from_secs(2)); - let _ = reader_done_rx.recv_timeout(Duration::from_secs(1)); - drop(runtime_exit.spawn(async move { - let _ = handle_exit.eof(channel).await; - let _ = handle_exit.exit_status_request(channel, code).await; - let _ = handle_exit.close(channel).await; - })); - }); - - Ok(sender) -} - -mod unsafe_pty { - #[cfg(not(target_os = "linux"))] - use super::sandbox; - use super::{ - Command, ProcessEnforcementMode, RawFd, ResolvedProcessIdentity, SandboxPolicy, Winsize, - drop_privileges_with_identity, setsid, - }; - #[cfg(unix)] - use std::os::unix::process::CommandExt; - - #[allow(unsafe_code)] - pub fn set_winsize(fd: RawFd, winsize: Winsize) -> std::io::Result<()> { - let rc = unsafe { libc::ioctl(fd, libc::TIOCSWINSZ, &winsize) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - } - - #[allow(unsafe_code)] - // `libc::TIOCSCTTY` is `u32` on macOS/BSD and `u64` on Linux; allow the - // cross-platform conversion so the same expression compiles everywhere. - #[allow(clippy::useless_conversion)] - fn set_controlling_tty(fd: RawFd) -> std::io::Result<()> { - let rc = unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - } - - #[allow(unsafe_code)] - #[allow(clippy::too_many_arguments)] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) - )] - pub fn install_pre_exec( - cmd: &mut Command, - policy: SandboxPolicy, - _workdir: Option, - slave_fd: RawFd, - netns_fd: Option, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - #[cfg(target_os = "linux")] prepared: Option, - ) { - // Wrap in Option so we can .take() it out of the FnMut closure. - // pre_exec is only called once (after fork, before exec). - #[cfg(target_os = "linux")] - let mut prepared = prepared; - unsafe { - cmd.pre_exec(move || { - setsid().map_err(|err| std::io::Error::other(err.to_string()))?; - set_controlling_tty(slave_fd)?; - - enter_netns_and_sandbox( - netns_fd, - &policy, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared.take(), - ) - }); - } - } - - /// Pre-exec hook for pipe-based (non-PTY) exec. - /// - /// Skips `setsid` and `TIOCSCTTY` since there is no controlling terminal. - #[allow(unsafe_code)] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) - )] - pub fn install_pre_exec_no_pty( - cmd: &mut Command, - policy: SandboxPolicy, - _workdir: Option, - netns_fd: Option, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - #[cfg(target_os = "linux")] prepared: Option, - ) { - #[cfg(target_os = "linux")] - let mut prepared = prepared; - unsafe { - cmd.pre_exec(move || { - enter_netns_and_sandbox( - netns_fd, - &policy, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared.take(), - ) - }); - } - } - - fn enter_netns_and_sandbox( - netns_fd: Option, - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - #[cfg(target_os = "linux")] prepared: Option, - ) -> std::io::Result<()> { - // Enter network namespace before dropping privileges. - // This ensures SSH shell processes are isolated to the same - // network namespace as the entrypoint, forcing all traffic - // through the veth pair and CONNECT proxy. - #[cfg(target_os = "linux")] - if let Some(fd) = netns_fd { - #[allow(unsafe_code)] - let result = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if result != 0 { - return Err(std::io::Error::last_os_error()); - } - } - - #[cfg(not(target_os = "linux"))] - let _ = netns_fd; - - // Drop privileges. initgroups/setgid/setuid need /etc/group and - // /etc/passwd which would be blocked if Landlock were already enforced. - if enforcement_mode.uses_privileged_process_setup() { - drop_privileges_with_identity(policy, resolved_identity) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - crate::process::harden_child_process() - .map_err(|err| std::io::Error::other(err.to_string()))?; - - // Phase 2: Enforce the prepared Landlock ruleset + seccomp. - // restrict_self() does not require root. - #[cfg(target_os = "linux")] - if let Some(prepared) = prepared { - crate::sandbox::linux::enforce(prepared) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - - #[cfg(not(target_os = "linux"))] - if enforcement_mode.enforces_child_sandbox() { - sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; - } - - Ok(()) - } -} - fn to_u16(value: u32) -> u16 { u16::try_from(value.min(u32::from(u16::MAX))).unwrap_or(u16::MAX) } @@ -1799,60 +1145,259 @@ fn is_loopback_host(host: &str) -> bool { } } +/// Resolve a (loopback-validated) destination host string to an `IpAddr`, +/// mapping `localhost` to `127.0.0.1`. +/// +/// Returns `None` for anything that does not parse to an IP, so +/// [`LoopbackTarget::new`] never sees a hostname. +fn loopback_ip(host: &str) -> Option { + let host = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + if host.eq_ignore_ascii_case("localhost") { + return Some(std::net::Ipv4Addr::LOCALHOST.into()); + } + host.parse().ok() +} + +fn direct_tcpip_target( + host: &str, + port: u32, +) -> Option { + if !is_loopback_host(host) { + return None; + } + let port = u16::try_from(port).ok()?; + let ip = loopback_ip(host)?; + openshell_isolation_interface::contract::LoopbackTarget::new(ip, port).ok() +} + #[cfg(test)] #[allow( clippy::doc_markdown, - unsafe_code, - reason = "Test code: doc text references identifiers and uses libc::winsize zero-init." + reason = "Test documentation references protocol and API identifiers." )] mod tests { use super::*; - use std::ffi::OsStr; - use std::process::Stdio; + use std::io::Write as _; + use std::process::{Command, Stdio}; + + struct AcceptAnyServerKey; + + impl russh::client::Handler for AcceptAnyServerKey { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + _server_public_key: &russh::keys::PublicKey, + ) -> Result { + Ok(true) + } + } + + struct TestPortForward; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryPortForward for TestPortForward { + async fn connect( + &self, + target: openshell_isolation_interface::contract::LoopbackTarget, + ) -> std::result::Result< + openshell_isolation_interface::contract::BoundaryDuplexStream, + openshell_isolation_interface::contract::BackendError, + > { + let stream = tokio::net::TcpStream::connect((target.host(), target.port())) + .await + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process( + error.to_string(), + ) + })?; + Ok(Box::new(stream)) + } + } + + struct RejectingExec; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryExec for RejectingExec { + async fn exec( + &self, + _spec: openshell_isolation_interface::contract::ExecSpec, + ) -> std::result::Result< + openshell_isolation_interface::contract::ExecSession, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unsupported( + "exec is not used by direct-tcpip tests".into(), + ), + ) + } + } + + async fn authenticated_test_client() -> russh::client::Handle { + let host_key = { + let mut rng = rand::rng(); + PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") + }; + let mut server_config = russh::server::Config { + auth_rejection_time: Duration::from_millis(1), + ..Default::default() + }; + server_config.keys.push(host_key); + + let handler = SshHandler::new( + Arc::new(TestPortForward), + Arc::new(RejectingExec), + Some(MainSession::inert()), + ); + let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + if let Ok(session) = + russh::server::run_stream(Arc::new(server_config), server_stream, handler).await + { + let _ = session.await; + } + }); + + let mut client = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + client_stream, + AcceptAnyServerKey, + ) + .await + .expect("SSH handshake should complete over the duplex"); + let auth = client + .authenticate_none("sandbox") + .await + .expect("auth_none should not error"); + assert!(matches!(auth, russh::client::AuthResult::Success)); + client + } + + #[cfg(unix)] + #[test] + fn transient_accept_errors_retry_with_bounded_backoff() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let aborted = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + assert_eq!( + classify_ssh_accept_error(&aborted, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } + ); + + let exhausted = std::io::Error::from_raw_os_error(libc::EMFILE); + let first = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + let second = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + assert_eq!( + first, + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Medium, + } + ); + assert_eq!( + second, + SshAcceptAction::Retry { + backoff: Duration::from_millis(200), + severity: SeverityId::Medium, + } + ); + } - /// Regression test: SSH sessions run the shell they are given, never a - /// hardcoded bash, so sh-only images (e.g. Alpine) work. Covers both the - /// interactive PTY path (`-i` when no command) and the non-PTY path. + #[cfg(unix)] #[test] - fn build_ssh_shell_command_uses_given_shell() { - // PTY, no command → given shell + interactive flag. - let cmd = build_ssh_shell_command("/bin/sh", None, false, Some("-i")); - assert_eq!(cmd.get_program(), OsStr::new("/bin/sh")); - assert_eq!(cmd.get_args().collect::>(), vec![OsStr::new("-i")]); - - // Non-PTY, no command → bare shell, no args (reads piped stdin). - let cmd = build_ssh_shell_command("/bin/sh", None, false, None); - assert_eq!(cmd.get_program(), OsStr::new("/bin/sh")); - assert_eq!(cmd.get_args().count(), 0); - - // Explicit command → login-shell flag + command, still on the given shell. - let cmd = build_ssh_shell_command("/bin/sh", Some("echo hi".into()), false, Some("-i")); - assert_eq!(cmd.get_program(), OsStr::new("/bin/sh")); + fn invalid_listener_accept_error_is_terminal() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let error = std::io::Error::from_raw_os_error(libc::EBADF); assert_eq!( - cmd.get_args().collect::>(), - vec![OsStr::new("-lc"), OsStr::new("echo hi")] + classify_ssh_accept_error(&error, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Terminal ); + } + + #[test] + fn direct_tcpip_target_rejects_non_loopback_and_out_of_range_ports() { + assert!(direct_tcpip_target("10.0.0.1", 80).is_none()); + assert!(direct_tcpip_target("127.0.0.1", 65_537).is_none()); + } - // OPENSHELL_NO_LOGIN_SHELL → plain -c. - let cmd = build_ssh_shell_command("/bin/sh", Some("echo hi".into()), true, None); + #[test] + fn direct_tcpip_target_accepts_loopback_destinations() { + let target = direct_tcpip_target("localhost", 8_080).expect("loopback target"); assert_eq!( - cmd.get_args().collect::>(), - vec![OsStr::new("-c"), OsStr::new("echo hi")] + target.host(), + std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) ); + assert_eq!(target.port(), 8_080); + } + + #[tokio::test] + async fn direct_tcpip_handler_rejects_invalid_destinations() { + for (host, port) in [("10.0.0.1", 80), ("127.0.0.1", 65_537)] { + let client = authenticated_test_client().await; + let error = client + .channel_open_direct_tcpip(host, port, "127.0.0.1", 0) + .await + .expect_err("invalid forwarding destination must be refused"); + assert!(matches!( + error, + russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) + )); + } } - /// Regression test: the direct-tcpip connect path sets `TCP_NODELAY`. #[tokio::test] - async fn connect_in_netns_sets_tcp_nodelay() { + async fn direct_tcpip_handler_relays_loopback_bytes() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); + .expect("bind loopback echo listener"); + let port = listener.local_addr().expect("listener address").port(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept forwarded stream"); + let mut payload = [0_u8; 4]; + socket.read_exact(&mut payload).await.expect("read payload"); + socket.write_all(&payload).await.expect("echo payload"); + }); - let stream = connect_in_netns(&addr.to_string(), None) + let client = authenticated_test_client().await; + let channel = client + .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); + .expect("loopback forwarding must be allowed"); + let mut stream = channel.into_stream(); + stream.write_all(b"ping").await.expect("write channel"); + let mut echoed = [0_u8; 4]; + tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut echoed)) + .await + .expect("forwarded response timeout") + .expect("read channel"); + assert_eq!(&echoed, b"ping"); + } + + #[tokio::test] + async fn main_attachment_accepts_declared_session_after_process_exit() { + let main_session = MainSession::inert(); + assert!(main_session.finish(23, true).await); + assert!(main_session.finished()); + + assert!(begin_main_attachment(&main_session, true)); + let mut output = main_session.subscribe(); + assert!(matches!( + output.recv().await.expect("retained terminal status"), + MainOutput::Exit(23) + )); + main_session.end_terminal_attachment(); } #[cfg(unix)] @@ -1869,15 +1414,14 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn ssh_server_init_full_enforcement_keeps_private_socket() { + async fn ssh_server_init_keeps_private_socket() { let temp = tempfile::tempdir().unwrap(); let parent = temp.path().join("ssh"); std::fs::create_dir_all(&parent).unwrap(); set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = - ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, false).unwrap(); + let (listener, _, _) = ssh_server_init(&socket, &None, false).unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o700); @@ -1893,8 +1437,7 @@ mod tests { set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = - ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, true).unwrap(); + let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o775); @@ -1905,8 +1448,7 @@ mod tests { #[tokio::test] async fn ssh_server_abstract_socket_cannot_be_replaced_while_bound() { let socket = PathBuf::from(format!("@openshell-ssh-test-{}", uuid::Uuid::new_v4())); - let (listener, _, _) = - ssh_server_init(&socket, &None, ProcessEnforcementMode::NetworkOnly, true).unwrap(); + let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); assert!( !socket.exists(), @@ -1971,41 +1513,9 @@ mod tests { assert_eq!(output.stdout, b"hello"); } - /// Command execution selects a login shell by default and a non-login shell - /// under `--no-login-shell`, so user startup files are sourced only in the - /// default case. - #[cfg(unix)] - #[test] - fn login_shell_flag_controls_profile_sourcing() { - let home = tempfile::tempdir().unwrap(); - std::fs::write(home.path().join(".bash_profile"), "echo LOGIN_MARKER\n").unwrap(); - - let run = |flag: &str| -> String { - let out = Command::new("bash") - .arg(flag) - .arg("true") - .env("HOME", home.path()) - .env_remove("BASH_ENV") // isolate: -c still reads BASH_ENV if set - .output() - .expect("spawn bash"); - String::from_utf8_lossy(&out.stdout).into_owned() - }; - - assert_eq!(login_shell_flag(true), "-c"); - assert_eq!(login_shell_flag(false), "-lc"); - assert!( - run("-lc").contains("LOGIN_MARKER"), - "login shell must source .bash_profile" - ); - assert!( - !run("-c").contains("LOGIN_MARKER"), - "non-login shell must not source it" - ); - } - - /// Verify that the stdin writer delivers all buffered data before exiting - /// when the sender is dropped. This ensures channel_eof doesn't cause - /// data loss — only signals "no more data after this". + /// Verify that the stdin writer delivers all buffered data before exiting + /// when the sender is dropped. This ensures channel_eof doesn't cause + /// data loss — only signals "no more data after this". #[test] fn stdin_writer_delivers_buffered_data_before_eof() { let (sender, receiver) = mpsc::channel::>(); @@ -2105,62 +1615,6 @@ mod tests { assert!(!is_loopback_host("[]")); } - // ----------------------------------------------------------------------- - // Per-channel PTY state tests (#543) - // ----------------------------------------------------------------------- - - #[test] - fn set_winsize_applies_to_correct_pty() { - // Verify that set_winsize applies to a specific PTY master FD, - // which is the mechanism that per-channel tracking relies on. - // With the old single-pty_master design, a window_change_request - // for channel N would resize whatever PTY was stored last — - // potentially belonging to a different channel. - let pty_a = openpty(None, None).expect("openpty a"); - let pty_b = openpty(None, None).expect("openpty b"); - let master_a = std::fs::File::from(pty_a.master); - let master_b = std::fs::File::from(pty_b.master); - let fd_a = master_a.as_raw_fd(); - let fd_b = master_b.as_raw_fd(); - assert_ne!(fd_a, fd_b, "two PTYs must have distinct FDs"); - - // Close the slave ends to avoid leaking FDs in the test. - drop(std::fs::File::from(pty_a.slave)); - drop(std::fs::File::from(pty_b.slave)); - - // Resize only PTY B. - let winsize_b = Winsize { - ws_row: 50, - ws_col: 120, - ws_xpixel: 0, - ws_ypixel: 0, - }; - unsafe_pty::set_winsize(fd_b, winsize_b).expect("set_winsize on PTY B"); - - // Resize PTY A to a different size. - let winsize_a = Winsize { - ws_row: 24, - ws_col: 80, - ws_xpixel: 0, - ws_ypixel: 0, - }; - unsafe_pty::set_winsize(fd_a, winsize_a).expect("set_winsize on PTY A"); - - // Read back sizes via ioctl to verify independence. - let mut actual_a: libc::winsize = unsafe { std::mem::zeroed() }; - let mut actual_b: libc::winsize = unsafe { std::mem::zeroed() }; - #[allow(unsafe_code)] - unsafe { - libc::ioctl(fd_a, libc::TIOCGWINSZ, &mut actual_a); - libc::ioctl(fd_b, libc::TIOCGWINSZ, &mut actual_b); - } - - assert_eq!(actual_a.ws_row, 24, "PTY A should be 24 rows"); - assert_eq!(actual_a.ws_col, 80, "PTY A should be 80 cols"); - assert_eq!(actual_b.ws_row, 50, "PTY B should be 50 rows"); - assert_eq!(actual_b.ws_col, 120, "PTY B should be 120 cols"); - } - #[test] fn channel_state_independent_input_senders() { // Verify that each channel gets its own input sender so that @@ -2211,604 +1665,4 @@ mod tests { .unwrap(); assert_eq!(rx_b.recv().unwrap(), b"still-alive"); } - - #[test] - fn main_detach_filter_forwards_ctrl_c_unchanged() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x03after"); - - assert_eq!(forward, b"before\x03after"); - assert!(!detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_removes_sequence_and_trailing_input() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x10\x11after"); - - assert_eq!(forward, b"before"); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_recognizes_sequence_across_frames() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"before\x10"); - assert_eq!(forward, b"before"); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x11"); - assert!(forward.is_empty()); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_forwards_unmatched_prefix() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x10"); - assert!(forward.is_empty()); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"x"); - assert_eq!(forward, b"\x10x"); - assert!(!detach); - assert!(!prefix_pending); - } - - // ----------------------------------------------------------------------- - // session_user_and_home tests (Phase 2: numeric UID support) - // ----------------------------------------------------------------------- - - #[test] - fn session_user_and_home_returns_numeric_uid_as_user() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1000".into()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "1000"); - // Numeric UID has no passwd entry — defaults to /sandbox. - assert_eq!(home, "/sandbox"); - } - - #[test] - fn session_user_and_home_uses_driver_workspace_when_supplied() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1234".into()), - run_as_group: Some("1235".into()), - }, - }; - - let (user, home) = session_user_and_home(&policy, Some("/workspace/project")); - assert_eq!(user, "1234"); - assert_eq!(home, "/workspace/project"); - } - - #[test] - fn session_user_and_home_returns_name_from_passwd() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("sandbox".into()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "sandbox"); - // Name-based — should resolve via passwd (or /home/{user}). - assert!(!home.is_empty()); - } - - #[test] - fn session_user_and_home_defaults_to_sandbox_when_empty() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some(String::new()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "sandbox"); - assert_eq!(home, "/sandbox"); - } - - #[test] - fn session_user_and_home_defaults_to_sandbox_when_none() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "sandbox"); - assert_eq!(home, "/sandbox"); - } - - #[test] - fn session_user_and_home_handles_large_numeric_uid() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1000660000".into()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "1000660000"); - assert_eq!(home, "/sandbox"); - } - - /// `install_pre_exec_no_pty` runs drop_privileges and succeeds when the - /// current user/group is already the configured one (no actual uid change). - /// - /// This exercises the pre_exec hook end-to-end without needing root: a policy - /// with no run_as_user/group is a no-op when the process is already unprivileged. - #[cfg(unix)] - #[test] - fn pre_exec_always_calls_drop_privileges() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, - }; - - // No user/group configured and not running as root → drop_privileges is - // a no-op, so spawn succeeds regardless of the effective UID. - let policy = SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - }; - - // Skip if running as root: drop_privileges would try to switch to - // "sandbox" which may not exist in the test environment. - if rustix::process::geteuid().is_root() { - return; - } - - let mut cmd = Command::new("echo"); - cmd.arg("drop-privileges-ok"); - cmd.stdout(Stdio::piped()); - - unsafe_pty::install_pre_exec_no_pty( - &mut cmd, - policy, - None, - None, // no netns fd - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::Full, - #[cfg(target_os = "linux")] - Some( - sandbox::linux::prepare( - &SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - }, - None, - ) - .expect("prepare should succeed in test environment"), - ), - ); - - let output = cmd - .spawn() - .expect("spawn must succeed") - .wait_with_output() - .expect("wait_with_output"); - assert!(output.status.success(), "echo should exit 0"); - assert!( - String::from_utf8_lossy(&output.stdout).contains("drop-privileges-ok"), - "echo output should contain 'drop-privileges-ok'" - ); - } - - /// SSH pre-exec uses the numeric identity resolved from OCI metadata rather - /// than looking the preserved declaration up through host NSS. - #[cfg(unix)] - #[test] - fn pre_exec_uses_resolved_oci_identity() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, - }; - - if rustix::process::geteuid().is_root() { - return; - } - - let policy = SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("__oci_user_not_in_host_nss__".into()), - run_as_group: Some("__oci_group_not_in_host_nss__".into()), - }, - }; - let resolved = ResolvedProcessIdentity::new( - Some(rustix::process::geteuid().as_raw()), - Some(rustix::process::getegid().as_raw()), - ); - - let mut cmd = Command::new("echo"); - cmd.arg("resolved-identity-ok"); - cmd.stdout(Stdio::piped()); - - unsafe_pty::install_pre_exec_no_pty( - &mut cmd, - policy, - None, - None, - resolved, - ProcessEnforcementMode::Full, - #[cfg(target_os = "linux")] - None, - ); - - let output = cmd - .spawn() - .expect("spawn should use resolved numeric identity") - .wait_with_output() - .expect("wait should succeed"); - assert!(output.status.success()); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - "resolved-identity-ok" - ); - } - - // ----------------------------------------------------------------------- - // direct-tcpip authorization wiring (SEC-007) - // - // The `loopback_host_*` tests above cover the predicate in isolation. - // These drive the real `russh::server::Handler` over an in-memory duplex - // so the deny path itself is covered: channel-open authorization travels - // through a reply handle rather than the handler's return value, so a - // handler that never rejects anything still type-checks and still passes - // every predicate test. - // ----------------------------------------------------------------------- - - struct AcceptAnyServerKey; - - impl russh::client::Handler for AcceptAnyServerKey { - type Error = russh::Error; - - async fn check_server_key( - &mut self, - _server_public_key: &russh::keys::PublicKey, - ) -> Result { - Ok(true) - } - } - - fn forwarding_test_policy() -> SandboxPolicy { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - - SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - } - } - - /// Serve `SshHandler` on one end of an in-memory duplex and return an - /// authenticated client handle for the other end. - /// - /// The handler gets `netns_fd: None` so `connect_in_netns` performs a plain - /// TCP connect, making the forwarding path reachable without a network - /// namespace. - async fn authenticated_test_client_with_main( - main_session: Arc, - ) -> russh::client::Handle { - // Scoped so the `!Send` ThreadRng is dropped before the first await. - let host_key = { - let mut rng = rand::rng(); - PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") - }; - let mut server_config = russh::server::Config { - auth_rejection_time: Duration::from_millis(1), - ..Default::default() - }; - server_config.keys.push(host_key); - - let handler = SshHandler::new( - forwarding_test_policy(), - ResolvedWorkspace::default(), - None, - None, - None, - ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), - HashMap::new(), - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::NetworkOnly, - main_session, - ); - - let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); - tokio::spawn(async move { - if let Ok(session) = - russh::server::run_stream(Arc::new(server_config), server_stream, handler).await - { - let _ = session.await; - } - }); - - let mut client = russh::client::connect_stream( - Arc::new(russh::client::Config::default()), - client_stream, - AcceptAnyServerKey, - ) - .await - .expect("SSH handshake should complete over the duplex"); - - let auth = client - .authenticate_none("sandbox") - .await - .expect("auth_none should not error"); - assert!( - matches!(auth, russh::client::AuthResult::Success), - "sandbox SSH server accepts the none auth method" - ); - - client - } - - async fn authenticated_test_client() -> russh::client::Handle { - authenticated_test_client_with_main(MainSession::inert()).await - } - - #[tokio::test] - async fn abrupt_transport_drop_releases_main_input_lease() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should acquire canonical input lease"); - - drop(channel); - drop(client); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.acquire_input().is_ok() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("handler drop should release canonical input lease"); - } - - #[tokio::test] - async fn main_attachment_closes_naturally_after_terminal_delivery() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let mut channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should register its attachment"); - - assert!(main_session.finish(7, false).await); - main_session.mark_terminal_reported(); - - let exit_status = tokio::time::timeout(Duration::from_secs(1), async { - let mut exit_status = None; - loop { - match channel.wait().await { - Some(russh::ChannelMsg::ExitStatus { - exit_status: status, - }) => { - exit_status = Some(status); - } - Some(russh::ChannelMsg::Close) => break exit_status, - None => panic!("main channel ended without a close message"), - Some(_) => {} - } - } - }) - .await - .expect("main channel should deliver its exit status"); - assert_eq!(exit_status, Some(7)); - drop(channel); - drop(client); - - tokio::time::timeout( - Duration::from_secs(1), - main_session.wait_for_terminal_attachments(), - ) - .await - .expect("peer channel close should release terminal delivery"); - } - - #[tokio::test] - async fn main_subsystem_applies_initial_pty_dimensions() { - let (main_session, _slave) = MainSession::terminal_for_test(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_pty(true, "xterm-256color", 200, 60, 1600, 900, &[]) - .await - .expect("request PTY"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.terminal_size_for_test() == (200, 60) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should apply the initial PTY dimensions"); - } - - #[tokio::test] - async fn direct_tcpip_rejects_non_loopback_destination() { - let client = authenticated_test_client().await; - - let err = client - .channel_open_direct_tcpip("10.0.0.1", 80, "127.0.0.1", 0) - .await - .expect_err("forwarding to a non-loopback host must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_rejects_port_above_tcp_range() { - let client = authenticated_test_client().await; - - // 65_537 truncates to port 1 when cast to u16, so the guard has to - // reject it before the cast rather than forward to a privileged port. - let err = client - .channel_open_direct_tcpip("127.0.0.1", 65_537, "127.0.0.1", 0) - .await - .expect_err("a port outside the TCP range must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_forwards_to_loopback_listener() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind loopback echo listener"); - let port = listener.local_addr().expect("listener address").port(); - tokio::spawn(async move { - if let Ok((mut socket, _)) = listener.accept().await { - let mut buf = [0u8; 64]; - if let Ok(n) = socket.read(&mut buf).await - && n > 0 - { - let _ = socket.write_all(&buf[..n]).await; - } - } - }); - - let client = authenticated_test_client().await; - let channel = client - .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) - .await - .expect("forwarding to a loopback listener must be allowed"); - - let mut stream = channel.into_stream(); - stream.write_all(b"ping").await.expect("write to channel"); - - let mut echoed = [0u8; 4]; - tokio::time::timeout(Duration::from_secs(10), stream.read_exact(&mut echoed)) - .await - .expect("relayed response should arrive before the timeout") - .expect("read from channel"); - assert_eq!(&echoed, b"ping", "bytes round-trip through the tunnel"); - } } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 98a3c0497b..2772095532 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -11,8 +11,6 @@ //! selection — it has no protocol awareness of the bytes flowing through. use std::net::IpAddr; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; @@ -23,6 +21,7 @@ use openshell_core::proto::{ RelayOpenResult, ReportMainProcessExitRequest, SupervisorHeartbeat, SupervisorHello, SupervisorMessage, TcpRelayTarget, gateway_message, relay_open, supervisor_message, }; +use openshell_isolation_interface::contract::{BoundaryPortForward, LoopbackTarget}; use openshell_ocsf::{ ActivityId, ConnectionInfo, Endpoint, NetworkActivityBuilder, OcsfEvent, SandboxContext, SeverityId, StatusId, ocsf_emit, @@ -33,7 +32,6 @@ use tokio_stream::StreamExt; use tracing::{debug, warn}; use openshell_core::grpc_client; -use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::transport_errors::is_expected_transport_close_status; const INITIAL_BACKOFF: Duration = Duration::from_secs(1); @@ -278,31 +276,59 @@ pub fn spawn( endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, ) -> tokio::task::JoinHandle<()> { + spawn_with_readiness( + endpoint, + sandbox_id, + ssh_socket_path, + port_forward, + expected_ssh_peer_pid, + terminating, + instance_id, + ) + .0 +} + +/// Spawn the supervisor session and expose when the gateway has accepted it. +pub fn spawn_with_readiness( + endpoint: String, + sandbox_id: String, + ssh_socket_path: std::path::PathBuf, + port_forward: Arc, + expected_ssh_peer_pid: Option, + terminating: Arc, + instance_id: String, +) -> ( + tokio::task::JoinHandle<()>, + tokio::sync::watch::Receiver, +) { + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); let config = SessionConfig { endpoint, sandbox_id, ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, terminating, instance_id, + ready_tx, }; - tokio::spawn(run_session_loop(config)) + (tokio::spawn(run_session_loop(config)), ready_rx) } struct SessionConfig { endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, + ready_tx: tokio::sync::watch::Sender, } async fn run_session_loop(config: SessionConfig) { @@ -314,6 +340,7 @@ async fn run_session_loop(config: SessionConfig) { match run_single_session(&config).await { Ok(()) => { + config.ready_tx.send_replace(false); let event = session_closed_event( openshell_ocsf::ctx::ctx(), &config.endpoint, @@ -323,6 +350,7 @@ async fn run_session_loop(config: SessionConfig) { break; } Err(e) => { + config.ready_tx.send_replace(false); let event = session_failed_event( openshell_ocsf::ctx::ctx(), &config.endpoint, @@ -392,6 +420,8 @@ async fn run_single_session( heartbeat_secs, ); ocsf_emit!(event); + config.ready_tx.send_replace(true); + // Main loop: receive gateway messages + send heartbeats. let mut heartbeat_interval = tokio::time::interval(Duration::from_secs(u64::from(heartbeat_secs))); @@ -411,7 +441,7 @@ async fn run_single_session( let context = GatewayMessageContext { sandbox_id: &config.sandbox_id, ssh_socket_path: &config.ssh_socket_path, - netns_fd: config.netns_fd, + port_forward: &config.port_forward, expected_ssh_peer_pid: config.expected_ssh_peer_pid, channel: &channel, tx: &tx, @@ -479,7 +509,7 @@ pub async fn finalize_main_process_exit( struct GatewayMessageContext<'a> { sandbox_id: &'a str, ssh_socket_path: &'a std::path::Path, - netns_fd: Option, + port_forward: &'a Arc, expected_ssh_peer_pid: Option, channel: &'a grpc_client::AuthedChannel, tx: &'a mpsc::Sender, @@ -498,7 +528,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let channel = context.channel.clone(); let ssh_socket_path = context.ssh_socket_path.to_path_buf(); let tx = context.tx.clone(); - let netns_fd = context.netns_fd; + let port_forward = context.port_forward.clone(); let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); @@ -510,7 +540,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< match handle_relay_open( relay_open, &ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, channel, tx, @@ -567,7 +597,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< async fn handle_relay_open( relay_open: RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, channel: grpc_client::AuthedChannel, tx: mpsc::Sender, @@ -577,7 +607,7 @@ async fn handle_relay_open( let target = match open_target( &relay_open, ssh_socket_path, - netns_fd, + &port_forward, expected_ssh_peer_pid, ) .await @@ -722,11 +752,11 @@ async fn send_relay_open_result( async fn open_target( relay_open: &RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: &Arc, expected_ssh_peer_pid: Option, ) -> Result, Box> { match relay_open.target.as_ref() { - Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, netns_fd).await, + Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, port_forward).await, Some(relay_open::Target::Ssh(_)) | None => { let runtime_path = crate::unix_socket::runtime_path(ssh_socket_path); let stream = tokio::net::UnixStream::connect(runtime_path.as_ref()).await?; @@ -747,59 +777,26 @@ async fn open_target( async fn open_tcp_target( target: &TcpRelayTarget, - netns_fd: Option, + port_forward: &Arc, ) -> Result, Box> { let host = normalize_tcp_target_host(target)?; let port = u16::try_from(target.port).map_err(|_| "tcp target port must fit in u16")?; - let stream = connect_tcp_target(host, port, netns_fd).await?; + // `normalize_tcp_target_host` returns a loopback IP string; parse it and let + // `LoopbackTarget::new` re-validate before connecting. + let ip: IpAddr = host + .parse() + .map_err(|_| "tcp target host must be a loopback IP")?; + let target = LoopbackTarget::new(ip, port) + .map_err(|e| -> Box { e.to_string().into() })?; + // Connect through the sandbox-owned loopback-forward interface. The + // supervisor session remains independent of the driver's transport. + let stream = port_forward + .connect(target) + .await + .map_err(|e| -> Box { e.to_string().into() })?; Ok(Box::new(stream)) } -#[cfg(target_os = "linux")] -async fn connect_tcp_target( - host: String, - port: u16, - netns_fd: Option, -) -> Result> { - if let Some(fd) = netns_fd { - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect((host.as_str(), port)) - })(); - let _ = tx.send(result); - }); - - let stream = rx - .await - .map_err(|_| "netns tcp connect thread panicked")??; - stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - -#[cfg(not(target_os = "linux"))] -async fn connect_tcp_target( - host: String, - port: u16, - _netns_fd: Option, -) -> Result> { - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - #[cfg(test)] fn validate_tcp_target(target: &TcpRelayTarget) -> Result<(), String> { normalize_tcp_target_host(target).map(|_| ()) @@ -839,20 +836,6 @@ mod target_tests { } } - /// Regression test: the TCP relay connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_tcp_target_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_tcp_target(addr.ip().to_string(), addr.port(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[test] fn tcp_target_allows_loopback_hosts() { validate_tcp_target(&tcp("127.0.0.1", 8080)).expect("ipv4 loopback"); @@ -895,6 +878,23 @@ mod target_tests { mod ocsf_event_tests { use super::*; + #[cfg(target_os = "linux")] + struct UnusedPortForward; + + #[cfg(target_os = "linux")] + #[async_trait::async_trait] + impl BoundaryPortForward for UnusedPortForward { + async fn connect( + &self, + _target: LoopbackTarget, + ) -> Result< + openshell_isolation_interface::contract::BoundaryDuplexStream, + openshell_isolation_interface::contract::BackendError, + > { + unreachable!("SSH relay does not use loopback port forwarding") + } + } + fn ctx() -> SandboxContext { SandboxContext { sandbox_id: "sbx-1".into(), @@ -1135,7 +1135,11 @@ mod ocsf_event_tests { }); let relay = ssh_relay_open("peer-check"); - let trusted = open_target(&relay, &socket, None, Some(std::process::id())) + // The SSH relay path does not use the port-forward (that is the TCP + // target path); connect from the supervisor's own namespace. + let port_forward: Arc = Arc::new(UnusedPortForward); + + let trusted = open_target(&relay, &socket, &port_forward, Some(std::process::id())) .await .expect("matching peer PID should be accepted"); drop(trusted); @@ -1143,7 +1147,7 @@ mod ocsf_event_tests { let Err(err) = open_target( &relay, &socket, - None, + &port_forward, Some(std::process::id().saturating_add(1)), ) .await diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml new file mode 100644 index 0000000000..f4129e4f84 --- /dev/null +++ b/crates/openshell-supervisor/Cargo.toml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-supervisor" +description = "OpenShell policy and workload supervisor" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "openshell-supervisor" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-ocsf = { path = "../openshell-ocsf" } +openshell-policy = { path = "../openshell-policy" } +openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } +openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } +openshell-supervisor-process = { path = "../openshell-supervisor-process" } + +clap = { workspace = true } +miette = { workspace = true } +nix = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } +rustls = { workspace = true } +rustix = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true, features = ["channel", "tls-native-roots"] } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true } + +[features] +default = ["telemetry", "bundled-ca-roots"] +system-ca-roots = ["telemetry"] +defaults-without-telemetry = ["bundled-ca-roots"] +telemetry = ["openshell-core/telemetry"] +bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] + +[dev-dependencies] +futures = { workspace = true } +temp-env = "0.3" +tempfile = "3" +tokio-tungstenite = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-supervisor/src/activity_aggregator.rs b/crates/openshell-supervisor/src/activity_aggregator.rs new file mode 100644 index 0000000000..33605c1df9 --- /dev/null +++ b/crates/openshell-supervisor/src/activity_aggregator.rs @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Anonymous supervised network activity counter aggregation. +//! +//! Producer-side types (`ActivityEvent`, `ActivitySender`, +//! `ACTIVITY_EVENT_QUEUE_CAPACITY`, `try_record_activity`) live in +//! `openshell_core::activity` so the supervisor leaves can emit without +//! depending on the orchestrator. This module hosts the aggregator that +//! runs orchestrator-side and flushes summaries to the gateway. + +use std::collections::HashMap; +use std::future::Future; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +pub use openshell_core::activity::ActivityEvent; + +const ACTIVITY_FLUSH_QUEUE_CAPACITY: usize = 1; +pub const DEFAULT_ACTIVITY_FLUSH_INTERVAL_SECS: u64 = 10; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FlushableActivitySummary { + pub network_activity_count: u32, + pub denied_action_count: u32, + pub denials_by_group: Vec<(String, u32)>, +} + +pub struct ActivityAggregator { + rx: mpsc::Receiver, + network_activity_count: u32, + denied_action_count: u32, + denials_by_group: HashMap, + flush_interval_secs: u64, +} + +impl ActivityAggregator { + pub fn new(rx: mpsc::Receiver, flush_interval_secs: u64) -> Self { + Self { + rx, + network_activity_count: 0, + denied_action_count: 0, + denials_by_group: HashMap::new(), + flush_interval_secs, + } + } + + /// `ready_gate` is checked before each flush. When it returns `false`, + /// the drain is skipped and events stay in the buffer until the next tick. + pub async fn run(mut self, flush_callback: F, ready_gate: G) + where + F: Fn(FlushableActivitySummary) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + G: Fn() -> bool, + { + let (flush_tx, mut flush_rx) = + mpsc::channel::(ACTIVITY_FLUSH_QUEUE_CAPACITY); + tokio::spawn(async move { + while let Some(summary) = flush_rx.recv().await { + flush_callback(summary).await; + } + }); + + let mut flush_interval = + tokio::time::interval(std::time::Duration::from_secs(self.flush_interval_secs)); + flush_interval.tick().await; + + loop { + tokio::select! { + event = self.rx.recv() => { + if let Some(event) = event { + self.ingest(event); + } else { + if self.network_activity_count > 0 { + if ready_gate() { + if let Some(summary) = self.drain() { + queue_flush_summary(&flush_tx, summary); + } + } else { + warn!( + count = self.network_activity_count, + "ActivityAggregator: dropping unflushed events, workspace not yet known" + ); + } + } + debug!("ActivityAggregator: channel closed, exiting"); + return; + } + } + _ = flush_interval.tick() => { + if ready_gate() + && let Some(summary) = self.drain() + { + debug!( + count = summary.network_activity_count, + denied = summary.denied_action_count, + "ActivityAggregator: flushing anonymous activity summary" + ); + queue_flush_summary(&flush_tx, summary); + } + } + } + } + } + + fn ingest(&mut self, event: ActivityEvent) { + self.network_activity_count = self.network_activity_count.saturating_add(1); + if event.denied { + self.denied_action_count = self.denied_action_count.saturating_add(1); + let group = sanitize_deny_group(event.deny_group).to_string(); + let count = self.denials_by_group.entry(group).or_default(); + *count = count.saturating_add(1); + } + } + + fn drain(&mut self) -> Option { + if self.network_activity_count == 0 { + return None; + } + let mut denials_by_group: Vec<(String, u32)> = self.denials_by_group.drain().collect(); + denials_by_group.sort_by(|left, right| left.0.cmp(&right.0)); + let summary = FlushableActivitySummary { + network_activity_count: self.network_activity_count, + denied_action_count: self.denied_action_count, + denials_by_group, + }; + self.network_activity_count = 0; + self.denied_action_count = 0; + Some(summary) + } +} + +pub fn activity_flush_interval_secs_from_env(value: Option<&str>) -> u64 { + value + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_ACTIVITY_FLUSH_INTERVAL_SECS) +} + +fn queue_flush_summary( + tx: &mpsc::Sender, + summary: FlushableActivitySummary, +) -> bool { + tx.try_send(summary).is_ok() +} + +pub fn sanitize_deny_group(raw: &str) -> &'static str { + match raw { + "connect_policy" | "connect" | "l4_deny" => "connect_policy", + "forward_policy" | "forward" => "forward_policy", + "l7_policy" | "l7" | "l7_deny" | "forward-l7-deny" => "l7_policy", + "l7_parse_rejection" | "parse_rejection" => "l7_parse_rejection", + "ssrf" => "ssrf", + "bypass" => "bypass", + "policy_stale" => "policy_stale", + _ => "unknown", + } +} + +#[cfg(test)] +fn denial_rate_pct(network_activity_count: u32, denied_action_count: u32) -> f64 { + if network_activity_count == 0 { + return 0.0; + } + ((f64::from(denied_action_count) / f64::from(network_activity_count)) * 100.0).clamp(0.0, 100.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_float_eq(actual: f64, expected: f64) { + assert!((actual - expected).abs() <= f64::EPSILON); + } + + #[test] + fn deny_group_sanitization_uses_allowlist() { + assert_eq!(sanitize_deny_group("connect"), "connect_policy"); + assert_eq!(sanitize_deny_group("forward-l7-deny"), "l7_policy"); + assert_eq!(sanitize_deny_group("host=example.test/path"), "unknown"); + assert_eq!(sanitize_deny_group("acme.internal:443"), "unknown"); + assert_eq!( + sanitize_deny_group("binary=/usr/local/bin/private"), + "unknown" + ); + } + + #[test] + fn denial_rate_handles_zero_and_clamps() { + assert_float_eq(denial_rate_pct(0, 10), 0.0); + assert_float_eq(denial_rate_pct(4, 1), 25.0); + assert_float_eq(denial_rate_pct(4, 10), 100.0); + } + + #[test] + fn flush_summary_drops_when_queue_is_full() { + let (tx, _rx) = mpsc::channel(1); + let summary = FlushableActivitySummary { + network_activity_count: 1, + denied_action_count: 0, + denials_by_group: Vec::new(), + }; + + assert!(queue_flush_summary(&tx, summary.clone())); + assert!(!queue_flush_summary(&tx, summary)); + } + + #[test] + fn activity_flush_interval_uses_positive_values_only() { + assert_eq!( + activity_flush_interval_secs_from_env(None), + DEFAULT_ACTIVITY_FLUSH_INTERVAL_SECS + ); + assert_eq!( + activity_flush_interval_secs_from_env(Some("not-a-number")), + DEFAULT_ACTIVITY_FLUSH_INTERVAL_SECS + ); + assert_eq!( + activity_flush_interval_secs_from_env(Some("0")), + DEFAULT_ACTIVITY_FLUSH_INTERVAL_SECS + ); + assert_eq!(activity_flush_interval_secs_from_env(Some("5")), 5); + } +} diff --git a/crates/openshell-supervisor/src/denial_aggregator.rs b/crates/openshell-supervisor/src/denial_aggregator.rs new file mode 100644 index 0000000000..d80d28db50 --- /dev/null +++ b/crates/openshell-supervisor/src/denial_aggregator.rs @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor denial aggregator — collects and deduplicates proxy deny events. +//! +//! The proxy emits a [`DenialEvent`] each time a connection or request is +//! denied. The [`DenialAggregator`] receives these events via an MPSC channel, +//! deduplicates them by `(host, port, binary)` key, and maintains running +//! counters. Periodically, the aggregator flushes accumulated summaries +//! upstream to the gateway via `SubmitPolicyAnalysis`. + +use std::collections::HashMap; +use std::future::Future; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +use openshell_core::denial::DenialEvent; + +/// Aggregated denial summary keyed by `(host, port, binary)`. +#[derive(Debug, Clone)] +struct AggregatedDenial { + host: String, + port: u16, + binary: String, + ancestors: Vec, + deny_reason: String, + denial_stage: String, + first_seen_ms: i64, + last_seen_ms: i64, + count: u32, + sample_cmdlines: Vec, + l7_samples: Vec, +} + +/// A single L7 request sample for aggregation. +#[derive(Debug, Clone)] +struct L7Sample { + method: String, + path: String, + count: u32, +} + +/// The denial aggregator collects proxy deny events and periodically flushes +/// summaries. It is designed to be spawned as a background tokio task. +pub struct DenialAggregator { + rx: mpsc::UnboundedReceiver, + /// Accumulated denials keyed by `(host, port, binary)`. + summaries: HashMap<(String, u16, String), AggregatedDenial>, + /// Flush interval in seconds. + flush_interval_secs: u64, +} + +impl DenialAggregator { + /// Create a new aggregator that reads from the given channel. + pub fn new(rx: mpsc::UnboundedReceiver, flush_interval_secs: u64) -> Self { + Self { + rx, + summaries: HashMap::new(), + flush_interval_secs, + } + } + + /// Run the aggregator loop. This consumes `self` and runs until the + /// channel is closed (all senders are dropped). + /// + /// `flush_callback` is called periodically with the accumulated summaries. + /// In production this calls `SubmitPolicyAnalysis` on the gateway. + /// + /// `ready_gate` is checked before each flush. When it returns `false`, + /// the drain is skipped and events stay in the buffer until the next tick. + pub async fn run(mut self, flush_callback: F, ready_gate: G) + where + F: Fn(Vec) -> Fut, + Fut: Future, + G: Fn() -> bool, + { + let mut flush_interval = + tokio::time::interval(std::time::Duration::from_secs(self.flush_interval_secs)); + // Don't fire immediately on first tick. + flush_interval.tick().await; + + loop { + tokio::select! { + event = self.rx.recv() => { + if let Some(evt) = event { + self.ingest(evt); + } else { + // Channel closed; do a final flush and exit. + if !self.summaries.is_empty() { + if ready_gate() { + let batch = self.drain(); + flush_callback(batch).await; + } else { + warn!( + count = self.summaries.len(), + "DenialAggregator: dropping unflushed summaries, workspace not yet known" + ); + } + } + debug!("DenialAggregator: channel closed, exiting"); + return; + } + } + _ = flush_interval.tick() => { + if ready_gate() && !self.summaries.is_empty() { + let batch = self.drain(); + debug!(count = batch.len(), "DenialAggregator: flushing summaries"); + flush_callback(batch).await; + } + } + } + } + } + + /// Ingest a single denial event, merging into existing summary or creating + /// a new one. + fn ingest(&mut self, event: DenialEvent) { + let now_ms = openshell_core::time::now_ms(); + let key = (event.host.clone(), event.port, event.binary.clone()); + + let entry = self + .summaries + .entry(key) + .or_insert_with(|| AggregatedDenial { + host: event.host.clone(), + port: event.port, + binary: event.binary.clone(), + ancestors: event.ancestors.clone(), + deny_reason: event.deny_reason.clone(), + denial_stage: event.denial_stage.clone(), + first_seen_ms: now_ms, + last_seen_ms: now_ms, + count: 0, + sample_cmdlines: Vec::new(), + l7_samples: Vec::new(), + }); + + entry.count += 1; + entry.last_seen_ms = now_ms; + + // Merge L7 samples. + if let (Some(method), Some(path)) = (&event.l7_method, &event.l7_path) { + if let Some(sample) = entry + .l7_samples + .iter_mut() + .find(|s| s.method == *method && s.path == *path) + { + sample.count += 1; + } else if entry.l7_samples.len() < 20 { + entry.l7_samples.push(L7Sample { + method: method.clone(), + path: path.clone(), + count: 1, + }); + } + } + } + + /// Drain all accumulated summaries into a flushable batch. + fn drain(&mut self) -> Vec { + self.summaries + .drain() + .map(|(_, v)| FlushableDenialSummary { + host: v.host, + port: v.port, + binary: v.binary, + ancestors: v.ancestors, + deny_reason: v.deny_reason, + denial_stage: v.denial_stage, + first_seen_ms: v.first_seen_ms, + last_seen_ms: v.last_seen_ms, + count: v.count, + sample_cmdlines: v.sample_cmdlines, + l7_samples: v + .l7_samples + .into_iter() + .map(|s| FlushableL7Sample { + method: s.method, + path: s.path, + count: s.count, + }) + .collect(), + }) + .collect() + } +} + +/// A denial summary ready to be sent to the gateway. +#[derive(Debug, Clone)] +pub struct FlushableDenialSummary { + pub host: String, + pub port: u16, + pub binary: String, + pub ancestors: Vec, + pub deny_reason: String, + pub denial_stage: String, + pub first_seen_ms: i64, + pub last_seen_ms: i64, + pub count: u32, + pub sample_cmdlines: Vec, + pub l7_samples: Vec, +} + +/// L7 request sample in flushable form. +#[derive(Debug, Clone)] +pub struct FlushableL7Sample { + pub method: String, + pub path: String, + pub count: u32, +} diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs new file mode 100644 index 0000000000..e31fbf6313 --- /dev/null +++ b/crates/openshell-supervisor/src/lib.rs @@ -0,0 +1,5602 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `OpenShell` supervisor library. +//! +//! This crate provides process sandboxing and monitoring capabilities. + +// `defaults-without-telemetry` is an alias for the default feature set minus +// `telemetry`, not a switch that turns telemetry off. Cargo cannot subtract a +// default feature, so adding it on top of the defaults would otherwise produce +// a telemetry-on build that reads as telemetry-free. Fail the build instead. +#[cfg(all(feature = "telemetry", feature = "defaults-without-telemetry"))] +compile_error!( + "features `telemetry` and `defaults-without-telemetry` are mutually exclusive; \ + build a telemetry-free supervisor with `--no-default-features --features defaults-without-telemetry`" +); + +mod activity_aggregator; +mod denial_aggregator; +mod mechanistic_mapper; + +use miette::{IntoDiagnostic, Result, WrapErr}; +use std::future::Future; +use std::io::Write as _; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::time::Duration; +use tracing::{debug, info, warn}; + +use openshell_core::PolicyValidationFailureMode; + +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, + DetectionFindingBuilder, DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, + StateId, StatusId, ocsf_emit, +}; + +// --------------------------------------------------------------------------- +// OCSF Context +// --------------------------------------------------------------------------- +// +// The following log sites intentionally remain as plain `tracing` macros +// and are NOT migrated to OCSF builders: +// +// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) +// - Transient "about to do X" events where the result is logged separately +// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") +// - Internal SSH channel warnings (unknown channel, PTY resize failures) +// - Denial flush telemetry (the individual denials are already OCSF events) +// - Status reporting failures (sync to gateway, non-actionable) +// - Route refresh interval validation warnings +// +// These are operational plumbing that don't represent security decisions, +// policy changes, or observable sandbox behavior worth structuring. +// --------------------------------------------------------------------------- + +/// Re-export the process-wide OCSF sandbox context getter. +/// +/// The singleton lives in `openshell-ocsf` so both supervisor leaves can +/// reach it without depending on `openshell-sandbox`. Initialised once during +/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; + +async fn retain_remote_access_plane( + proxy_exited: impl Future, + shutdown_requested: impl Future, +) -> Result<()> { + tokio::pin!(proxy_exited); + tokio::pin!(shutdown_requested); + tokio::select! { + () = &mut proxy_exited => Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )), + () = &mut shutdown_requested => Ok(()), + } +} + +async fn completion_phase_or_shutdown(phase: F, mut shutdown: Pin<&mut S>) -> bool +where + F: Future, + S: Future + ?Sized, +{ + tokio::pin!(phase); + tokio::select! { + () = &mut phase => false, + () = &mut shutdown => true, + } +} + +struct ControlReadiness { + task: tokio::task::JoinHandle<()>, + path: std::path::PathBuf, +} + +impl ControlReadiness { + fn start( + path: std::path::PathBuf, + mut session_readiness: Option>, + ) -> Result { + if session_readiness + .as_ref() + .is_some_and(|readiness| !*readiness.borrow()) + { + return Err(miette::miette!( + "supervisor session is not ready when starting health listener" + )); + } + prepare_control_readiness_path(&path)?; + let listener = tokio::net::UnixListener::bind(&path) + .into_diagnostic() + .wrap_err_with(|| format!("bind supervisor readiness socket on {}", path.display()))?; + let task_path = path.clone(); + let task = tokio::spawn(async move { + let mut listener = Some(listener); + loop { + let session_unready = session_readiness + .as_ref() + .is_some_and(|readiness| !*readiness.borrow()); + if listener.is_none() || session_unready { + if session_unready { + listener.take(); + let _ = std::fs::remove_file(&task_path); + let Some(readiness) = session_readiness.as_mut() else { + break; + }; + if readiness.wait_for(|ready| *ready).await.is_err() { + break; + } + } + match prepare_control_readiness_path(&task_path).and_then(|()| { + tokio::net::UnixListener::bind(&task_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "rebind supervisor readiness socket on {}", + task_path.display() + ) + }) + }) { + Ok(rebound) => listener = Some(rebound), + Err(error) => { + tracing::warn!(%error, "control-mode readiness rebind failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + } + continue; + } + + let Some(active_listener) = listener.as_ref() else { + continue; + }; + if let Some(readiness) = session_readiness.as_mut() { + tokio::select! { + accepted = active_listener.accept() => match accepted { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + }, + changed = readiness.changed() => { + if changed.is_err() { + break; + } + } + } + } else { + match active_listener.accept().await { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + } + let _ = std::fs::remove_file(&task_path); + }); + Ok(Self { task, path }) + } +} + +#[cfg(unix)] +fn prepare_control_readiness_path(path: &std::path::Path) -> Result<()> { + use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; + + if !path.is_absolute() { + return Err(miette::miette!( + "supervisor readiness socket path must be absolute" + )); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("create readiness directory {}", parent.display()))?; + } + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + if !metadata.file_type().is_socket() + || metadata.uid() != rustix::process::getuid().as_raw() + { + return Err(miette::miette!( + "refusing unsafe existing readiness path {}", + path.display() + )); + } + std::fs::remove_file(path) + .into_diagnostic() + .wrap_err_with(|| format!("remove stale readiness socket {}", path.display()))?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .into_diagnostic() + .wrap_err_with(|| format!("inspect readiness path {}", path.display())); + } + } + Ok(()) +} + +impl Drop for ControlReadiness { + fn drop(&mut self) { + self.task.abort(); + let _ = std::fs::remove_file(&self.path); + } +} + +/// Check whether the live supervisor owns its private readiness socket. +#[cfg(unix)] +pub fn check_control_readiness(path: &std::path::Path) -> Result<()> { + if !path.is_absolute() { + return Err(miette::miette!("health socket path must be absolute")); + } + std::os::unix::net::UnixStream::connect(path) + .into_diagnostic() + .wrap_err_with(|| format!("connect supervisor readiness socket {}", path.display()))?; + Ok(()) +} + +/// Health subcommands are unsupported on non-Unix hosts. +#[cfg(not(unix))] +pub fn check_control_readiness(_path: &std::path::Path) -> Result<()> { + Err(miette::miette!( + "supervisor readiness sockets require a Unix host" + )) +} + +#[cfg(unix)] +async fn wait_for_control_shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).expect("install control SIGTERM handler"); + let mut sigint = signal(SignalKind::interrupt()).expect("install control SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => {} + _ = sigint.recv() => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_control_shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; +} + +use openshell_core::denial::DenialEvent; +use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_supervisor_network::opa::OpaEngine; +use openshell_supervisor_network::proxy::ProxyHandle; +use openshell_supervisor_process::skills; +use tokio::sync::mpsc::UnboundedSender; +use tokio::time::timeout; + +fn shared_ssh_socket_from_env() -> bool { + std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) + .is_ok_and(|value| shared_ssh_socket_value(&value)) +} + +fn shared_ssh_socket_value(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") +} + +/// Run a command in the sandbox. +/// +/// # Errors +/// +/// Returns an error if the command fails to start or encounters a fatal error. +#[allow( + clippy::too_many_arguments, + clippy::implicit_hasher, + clippy::similar_names, + clippy::fn_params_excessive_bools +)] +pub async fn run_sandbox( + command: Vec, + workdir: Option, + timeout_secs: u64, + interactive: bool, + await_main_process_attachment: bool, + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, + ssh_socket_path: Option, + health_socket_path: Option, + inference_routes: Option, + ocsf_enabled: Arc, + upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: openshell_isolation_interface::contract::TopologyDescriptor, + admitted_isolation_backend: Option, + main_exit_marker: Option, +) -> Result { + // An empty command is the versioned scratch-sandbox sentinel. The + // external supervisor cannot inspect the workload filesystem, so preserve + // it for openshell-sandbox to resolve against the agent image. + let (program, args) = command.split_first().map_or_else( + || (String::new(), Vec::new()), + |(program, args)| (program.clone(), args.to_vec()), + ); + + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; the boundary mediation source carries + // workload-side connection metadata. + { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + + if !openshell_ocsf::ctx::set_ctx(SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + }) { + debug!("OCSF context already initialized, keeping existing"); + } + } + + // Extension credentials are owned by this supervisor and shared by every + // gateway connection it opens, so the middleware registry's bearer slots + // and the policy poll loop that rotates them stay the same objects. + let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); + + // Load policy and initialize OPA engine + let openshell_endpoint_for_proxy = openshell_endpoint.clone(); + let sandbox_name_for_agg = sandbox.clone(); + let ( + policy, + opa_engine, + retained_proto, + middleware_registry_status, + loaded_policy_origin, + initial_agent_proposals_enabled, + initial_extension_authentication_enabled, + ) = load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + &extension_credentials, + ) + .await?; + + // Normalize the active driver's identity contract once, while both the + // policy and launched image filesystem are available. Kubernetes and + // OpenShift retain their authoritative numeric pair; Docker fills only + // omitted policy fields from OCI Config.User. A remote boundary resolves + // identity in its own filesystem instead; control must not interpret + // guest account data against the host's /etc/passwd and /etc/group. + let workspace = workdir; + + let provider_credentials = { + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + static_credential_bindings, + non_secret_environment_keys, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + result.static_credential_bindings, + result.non_secret_environment_keys, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message(format!( + "Failed to fetch provider environment; no provider credentials are active: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + Vec::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + Vec::new(), + ) + }; + + let dynamic_credentials_fallback = dynamic_credentials.clone(); + match ProviderCredentialState::from_bound_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + static_credential_bindings, + non_secret_environment_keys, + ) { + Ok(credentials) => credentials, + Err(error) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message(format!( + "Rejected provider environment bindings; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" + )) + .build() + ); + ProviderCredentialState::from_environment( + provider_env_revision, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + dynamic_credentials_fallback, + ) + } + } + }; + + if credential_gating_unavailable( + &loaded_policy_origin, + provider_credentials.resolver().is_some(), + true, + ) { + report_credential_gating_unavailable(); + } + + // Canonical-process overrides are deliberately applied only to the main + // child. Keep the provider snapshot pristine for later exec/editor/SFTP + // children launched by the sandbox. + + // Shared agent-proposals feature flag. Seed from the same initial settings + // snapshot that produced the policy so networking and process setup agree + // before the poll loop starts reconciling later changes. + let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); + + // Shared PID: set after process spawn so the proxy can look up + // the entrypoint process's /proc/net/tcp for identity binding. + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + + // A separated topology uses the shared authenticated boundary protocol. + // The admitted backend name is resolved independently of the protected + // descriptor, and generic supervisor code never imports a driver crate. + let admitted_backend_name = admitted_isolation_backend.ok_or_else(|| { + miette::miette!("protected topology supplied without an admitted isolation backend") + })?; + let topology: openshell_isolation_interface::boundary_protocol::BoundaryTopology = + serde_json::from_slice(&topology_descriptor.payload) + .map_err(|error| miette::miette!("decode boundary topology: {error}"))?; + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let backend: Arc = Arc::new( + openshell_isolation_interface::remote::RemoteIsolationBackend::new( + admitted_backend_name.clone(), + ca_file_paths.clone(), + provider_credentials.clone(), + ), + ); + let mut registry = openshell_isolation_interface::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(topology_descriptor, &admitted_backend_name) + .map_err(|error| miette::miette!(error.to_string()))?; + let context = openshell_isolation_interface::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation_interface::AgentSpec { + program, + args, + workdir: workspace, + timeout_secs, + interactive, + }, + identity: topology.workload_identity, + }; + let bound = backend + .attach(verified, context) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary attached"); + let remote_boundary = (bound, admitted_backend_name, ca_file_paths); + + let transparent_tcp_capable = true; + let transparent_tcp_substrate_ready = true; + // The denial channel is owned by the orchestrator: the proxy (in the + // networking leaf) and the bypass monitor (in the process leaf) both + // produce DenialEvents that the denial aggregator (orchestrator-side) + // consumes via the matching receiver. Both leaves are pure producers; + // the orchestrator owns the consumer task spawned below. + let (denial_tx, denial_rx): (Option>, _) = if sandbox_id.is_some() + { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + + // Anonymous activity channel: same orchestrator-owned pattern as the + // denial channel. The proxy and the bypass monitor both emit per-event + // activity records; the orchestrator-side aggregator drains, sanitizes, + // and flushes anonymous summaries to the gateway. + let (activity_tx, activity_rx) = if sandbox_id.is_some() { + let (tx, rx) = + tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + + // Workspace watch: the policy poll loop learns the workspace from + // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local + // API read the current value so proposals target the correct workspace. + let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); + + let remote_network_source = remote_boundary.0.network_mediation_source(); + let remote_dns_source = remote_boundary.0.dns_mediation_source(); + let remote_host_gateway_ip = remote_boundary.0.host_gateway_ip(); + + let mut networking = Some( + openshell_supervisor_network::run::run_networking( + &policy, + None, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + // The sandbox supplies already-resolved identities across the + // boundary. The host supervisor cannot inspect its mount or PID + // namespace, so waiting for a host-visible entrypoint PID would + // unnecessarily delay DNS and network readiness. + false, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + remote_host_gateway_ip, + #[cfg(target_os = "linux")] + None, + Some(remote_network_source), + remote_dns_source, + ) + .await?, + ); + + let remote_ready = { + let (bound, backend_name, ca_file_paths) = remote_boundary; + ca_file_paths + .lock() + .map_err(|_| miette::miette!("boundary CA path lock is poisoned"))? + .clone_from( + &networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + ); + let ready = bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary enforcement confirmed"); + (ready, backend_name) + }; + + // Spawn the denial-aggregator flush task. The aggregator drains proxy + // denial events, batches them, and ships summaries to the gateway via + // `SubmitPolicyAnalysis`. + if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall + // back to the ID when the name isn't set. + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + + let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); + let denial_workspace_gate = workspace_rx.clone(); + let denial_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + |summaries| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = denial_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_proposals_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summaries, + ) + .await + { + warn!(error = %e, "Failed to flush denial summaries to gateway"); + } + } + }, + move || !denial_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn the activity-aggregator flush task. The aggregator drains + // anonymous activity events from the proxy, sanitizes deny groups, + // and ships periodic summaries to the gateway. + if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( + std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") + .ok() + .as_deref(), + ); + + let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); + let activity_workspace_gate = workspace_rx.clone(); + let activity_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + move |summary| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = activity_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_activity_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summary, + ) + .await + { + warn!(error = %e, "Failed to flush activity summary to gateway"); + } + } + }, + move || !activity_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn background policy poll task (gRPC mode only). + if let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) { + let poll_id = id.to_string(); + let poll_endpoint = endpoint.to_string(); + let poll_engine = engine.clone(); + let poll_ocsf_enabled = ocsf_enabled.clone(); + let poll_pid = entrypoint_pid.clone(); + let poll_provider_credentials = provider_credentials.clone(); + let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); + let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let poll_ctx = PolicyPollLoopContext { + endpoint: poll_endpoint, + sandbox_id: poll_id, + opa_engine: poll_engine, + loaded_policy_origin, + entrypoint_pid: poll_pid, + interval_secs: poll_interval_secs, + ocsf_enabled: poll_ocsf_enabled, + provider_credentials: poll_provider_credentials, + policy_local_ctx: poll_policy_local, + agent_proposals: agent_proposals.clone(), + middleware_registry_status, + workspace_tx, + extension_credentials: extension_credentials.clone(), + extension_authentication_enabled: initial_extension_authentication_enabled, + middleware_connector: default_middleware_connector(), + transparent_tcp: TransparentTcpReloadState { + capable: transparent_tcp_capable, + substrate_ready: transparent_tcp_substrate_ready, + }, + }; + + tokio::spawn(async move { + if let Err(e) = run_policy_poll_loop(poll_ctx).await { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .message(format!("Policy poll loop exited with error: {e}")) + .build() + ); + } + }); + } + + let proxy_exited: Pin + Send>> = if let Some(rx) = networking + .as_mut() + .and_then(|n| n.proxy.as_mut()) + .and_then(ProxyHandle::take_exit_receiver) + { + Box::pin(async { + let _ = rx.await; + }) + } else { + Box::pin(std::future::pending()) + }; + tokio::pin!(proxy_exited); + + let (confirmed, backend_name) = remote_ready; + let exit_code = { + let running = confirmed + .into_boundary() + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary agent started"); + let agent = running.agent(); + let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path.as_deref(), + shared_ssh_socket_from_env(), + networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + running.exec(), + running.port_forward(), + agent.clone(), + ) + .await?; + info!(backend = %backend_name, "Control-mode access plane started"); + let mut control_readiness = if let Some(path) = health_socket_path { + Some(ControlReadiness::start( + path, + boundary_access.session_readiness(), + )?) + } else { + None + }; + let instance_id = boundary_access.instance_id().to_string(); + let wait_agent = agent.clone(); + let shutdown_requested = wait_for_control_shutdown_signal(); + tokio::pin!(shutdown_requested); + let wait = async move { + wait_agent + .wait() + .await + .map(|status| match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => { + code + } + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + ) => 128_i32.saturating_add(signal), + }) + .map_err(|error| miette::miette!(error.to_string())) + }; + let (exit_code, mut retain_access) = tokio::select! { + result = wait => (result?, true), + () = &mut proxy_exited => { + let _ = agent.terminate().await; + return Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )); + } + () = &mut shutdown_requested => { + let _ = agent + .signal(openshell_isolation_interface::contract::BoundarySignal::Term) + .await; + let status = if let Ok(result) = timeout(Duration::from_secs(5), agent.wait()).await { + result + } else { + let _ = agent.terminate().await; + agent.wait().await + } + .map_err(|error| miette::miette!(error.to_string()))?; + let exit_code = match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + }; + (exit_code, false) + } + }; + if !retain_access { + control_readiness.take(); + } + boundary_access + .publish_main_exit(exit_code, await_main_process_attachment) + .await; + // `shutdown_requested` has already completed when shutdown won the + // lifecycle select above and must not be polled again. + let mut completion_cancelled = !retain_access; + if retain_access && let Some(marker) = main_exit_marker.as_deref() { + persist_main_exit_marker(marker, exit_code) + .into_diagnostic() + .wrap_err("persist canonical-process completion marker")?; + } + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let report = openshell_supervisor_process::delegated::report_main_process_exit( + endpoint, + id, + &instance_id, + exit_code, + ); + completion_cancelled = + completion_phase_or_shutdown(report, shutdown_requested.as_mut()).await; + } + if !completion_cancelled { + let drain = boundary_access.drain_main_terminal_delivery(); + completion_cancelled = + completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; + } + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( + endpoint, + id, + &instance_id, + ); + completion_cancelled = + completion_phase_or_shutdown(finalize, shutdown_requested.as_mut()).await; + } + if completion_cancelled { + retain_access = false; + control_readiness.take(); + } + if retain_access { + info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); + retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; + } + drop(control_readiness); + drop(running); + drop(boundary_access); + exit_code + }; + + // Drop networking explicitly so proxy tasks tear down before we return. + drop(networking); + + Ok(exit_code) +} + +fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no parent: {}", path.display()), + ) + })?; + let name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no file name: {}", path.display()), + ) + })?; + let temporary = parent.join(format!( + ".{}.tmp-{}", + name.to_string_lossy(), + std::process::id() + )); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + writeln!(file, "exit_code={exit_code}")?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all() +} + +/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. +async fn flush_proposals_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summaries: Vec, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialSummary, L7RequestSample}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summaries: Vec = summaries + .into_iter() + .map(|s| DenialSummary { + sandbox_id: String::new(), + host: s.host, + port: u32::from(s.port), + binary: s.binary, + ancestors: s.ancestors, + deny_reason: s.deny_reason, + first_seen_ms: s.first_seen_ms, + last_seen_ms: s.last_seen_ms, + count: s.count, + suppressed_count: 0, + total_count: s.count, + sample_cmdlines: s.sample_cmdlines, + binary_sha256: String::new(), + persistent: false, + denial_stage: s.denial_stage, + l7_request_samples: s + .l7_samples + .into_iter() + .map(|l| L7RequestSample { + method: l.method, + path: l.path, + decision: "deny".to_string(), + count: l.count, + }) + .collect(), + l7_inspection_active: false, + }) + .collect(); + + // Run the mechanistic mapper sandbox-side to generate proposals. + // The gateway is a thin persistence + validation layer — it never + // generates proposals itself. + let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); + + info!( + sandbox_name = %sandbox_name, + summaries = proto_summaries.len(), + proposals = proposals.len(), + "Flushed denial analysis to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + proto_summaries, + proposals, + Vec::new(), + "mechanistic", + ) + .await?; + + Ok(()) +} + +/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. +async fn flush_activity_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summary: activity_aggregator::FlushableActivitySummary, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summary = NetworkActivitySummary { + network_activity_count: summary.network_activity_count, + denied_action_count: summary.denied_action_count, + denials_by_group: summary + .denials_by_group + .into_iter() + .map(|(group, count)| DenialGroupCount { + deny_group: group, + denied_count: count, + }) + .collect(), + }; + + info!( + sandbox_name = %sandbox_name, + network_activity_count = proto_summary.network_activity_count, + denied_action_count = proto_summary.denied_action_count, + "Flushed activity summary to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + Vec::new(), + Vec::new(), + vec![proto_summary], + "activity", + ) + .await?; + + Ok(()) +} + +// ============================================================================ +// Baseline filesystem path enrichment +// ============================================================================ + +/// Minimum read-only paths required for a proxy-mode sandbox child process to +/// function: dynamic linker, shared libraries, DNS resolution, CA certs, +/// Python venv, openshell logs, process info, and random bytes. +/// +/// `/proc` and `/dev/urandom` are included here for the same reasons they +/// appear in `restrictive_default_policy()`: virtually every process needs +/// them. Before the Landlock per-path fix (#677) these were effectively free +/// because a missing path silently disabled the entire ruleset; now they must +/// be explicit. +const PROXY_BASELINE_READ_ONLY: &[&str] = &[ + "/usr", + "/lib", + "/etc", + "/app", + "/var/log", + "/proc", + "/dev/urandom", +]; + +/// Minimum read-write paths required for a proxy-mode sandbox child process. +/// The active workspace is granted separately through `include_workdir`. +// `/dev/null` is opened by common child-process launchers when they construct +// piped or discarded stdio. Without it, tools such as uv report EACCES while +// probing an otherwise executable interpreter under an explicit filesystem +// policy. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp", "/dev/null"]; + +/// GPU read-only paths. +/// +/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced +/// socket at init time. If the directory exists but Landlock denies traversal +/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` +/// even though the daemon is optional. Only read/traversal access is needed. +/// +/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, +/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` +/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may +/// not be covered by the parent-directory Landlock rule when the mount crosses +/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal +/// is permitted regardless of Landlock's cross-mount behaviour. +const GPU_BASELINE_READ_ONLY: &[&str] = &[ + "/run/nvidia-persistenced", + "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory +]; + +/// GPU read-write paths (static). +/// +/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, +/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native +/// Linux. Landlock restricts `open(2)` on device files even when DAC allows +/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. +/// These devices do not exist on WSL2 and will be skipped by the existence +/// check in `enrich_proto_baseline_paths()`. +/// +/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver +/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects +/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux +/// and will be skipped there by the existence check. +/// +/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` +/// to set thread names. Without write access, `cuInit()` returns error 304. +/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to +/// inodes and child processes have different procfs inodes than the parent. +/// +/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by +/// `enumerate_gpu_device_nodes()` since the count varies. +const GPU_BASELINE_READ_WRITE: &[&str] = &[ + "/dev/nvidiactl", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia-modeset", + "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) + "/proc", +]; + +/// Returns true if GPU devices are present in the container. +/// +/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and +/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these +/// depending on the host kernel; the other will not exist. +fn has_gpu_devices() -> bool { + std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() +} + +/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). +fn enumerate_gpu_device_nodes() -> Vec { + let mut paths = Vec::new(); + if let Ok(entries) = std::fs::read_dir("/dev") { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(suffix) = name.strip_prefix("nvidia") { + if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { + continue; + } + paths.push(entry.path().to_string_lossy().into_owned()); + } + } + } + paths +} + +fn push_unique(paths: &mut Vec, path: String) { + if !paths.iter().any(|p| p == &path) { + paths.push(path); + } +} + +fn collect_baseline_enrichment_paths( + include_proxy: bool, + include_gpu: bool, + gpu_device_nodes: Vec, +) -> (Vec, Vec) { + let mut ro = Vec::new(); + let mut rw = Vec::new(); + + if include_proxy { + for &path in PROXY_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in PROXY_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + } + + if include_gpu { + for &path in GPU_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in GPU_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + for path in gpu_device_nodes { + push_unique(&mut rw, path); + } + } + + // A path promoted to read_write (e.g. /proc for GPU) should not also + // appear in read_only — Landlock handles the overlap correctly but the + // duplicate is confusing when inspecting the effective policy. + ro.retain(|p| !rw.contains(p)); + + (ro, rw) +} + +fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { + let include_gpu = has_gpu_devices(); + let gpu_device_nodes = if include_gpu { + enumerate_gpu_device_nodes() + } else { + Vec::new() + }; + collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) +} + +/// Collect all active baseline paths for tests and diagnostics. +/// Returns `(read_only, read_write)` as owned `String` vecs. +#[cfg(test)] +fn baseline_enrichment_paths() -> (Vec, Vec) { + active_baseline_enrichment_paths(true) +} + +fn enrich_proto_baseline_paths_with( + proto: &mut openshell_core::proto::SandboxPolicy, + ro: &[String], + rw: &[String], + path_exists: F, +) -> bool +where + F: Fn(&str) -> bool, +{ + if ro.is_empty() && rw.is_empty() { + return false; + } + + let fs = proto + .filesystem + .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { + include_workdir: true, + ..Default::default() + }); + + let mut modified = false; + for path in ro { + if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { + if !path_exists(path) { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + fs.read_only.push(path.clone()); + modified = true; + } + } + for path in rw { + if fs.read_write.iter().any(|p| p == path) { + continue; + } + if !path_exists(path) { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + if fs.read_only.iter().any(|p| p == path) { + if path == "/proc" { + info!( + path, + "Promoting /proc from read-only to read-write for GPU runtime compatibility" + ); + fs.read_only.retain(|p| p != path); + fs.read_write.push(path.clone()); + modified = true; + } + continue; + } + fs.read_write.push(path.clone()); + modified = true; + } + + modified +} + +/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths +/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if +/// missing; user-specified paths are never removed. +/// +/// Returns `true` if the policy was modified (caller may want to sync back). +fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); + + // Baseline paths are system-injected, not user-specified. Skip paths + // that do not exist in this container image to avoid noisy warnings from + // Landlock and, more critically, to prevent a single missing baseline + // path from abandoning the entire Landlock ruleset under best-effort + // mode (see issue #664). + let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { + std::path::Path::new(path).exists() + }); + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } + + modified +} + +fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + openshell_policy::strip_provider_rule_names(proto) +} + +fn proto_sync_payload_for_enriched_policy( + proto: &openshell_core::proto::SandboxPolicy, + enriched: bool, +) -> Option { + if !enriched { + return None; + } + + let mut sync_policy = proto.clone(); + strip_proto_provider_policy_entries(&mut sync_policy); + Some(sync_policy) +} + +/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem +/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the +/// local-file code path where no proto is available. +fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { + let (ro, rw) = + active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); + if ro.is_empty() && rw.is_empty() { + return; + } + + let mut modified = false; + for path in &ro { + let p = std::path::PathBuf::from(path); + if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { + if !p.exists() { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_only.push(p); + modified = true; + } + } + for path in &rw { + let p = std::path::PathBuf::from(path); + if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { + continue; + } + if !p.exists() { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_write.push(p); + modified = true; + } + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod baseline_tests { + use super::*; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; + use std::path::PathBuf; + + #[test] + fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { + // When GPU devices are present, /proc is promoted to read_write + // (CUDA needs to write /proc//task//comm). It should + // NOT also appear in read_only. + if !has_gpu_devices() { + // Can't test GPU dedup without GPU devices; skip silently. + return; + } + let (ro, rw) = baseline_enrichment_paths(); + assert!( + rw.contains(&"/proc".to_string()), + "/proc should be in read_write when GPU is present" + ); + assert!( + !ro.contains(&"/proc".to_string()), + "/proc should NOT be in read_only when it is already in read_write" + ); + } + + #[test] + fn proc_in_read_only_without_gpu() { + if has_gpu_devices() { + // On a GPU host we can't test the non-GPU path; skip silently. + return; + } + let (ro, _rw) = baseline_enrichment_paths(); + assert!( + ro.contains(&"/proc".to_string()), + "/proc should be in read_only when GPU is not present" + ); + } + + #[test] + fn baseline_read_write_does_not_hardcode_sandbox() { + let (_ro, rw) = baseline_enrichment_paths(); + assert!(rw.contains(&"/tmp".to_string())); + assert!(rw.contains(&"/dev/null".to_string())); + assert!(!rw.contains(&"/sandbox".to_string())); + } + + #[test] + fn enumerate_gpu_device_nodes_skips_bare_nvidia() { + // "nvidia" (without a trailing digit) is a valid /dev entry on some + // systems but is not a per-GPU device node. The enumerator must + // not match it. + let nodes = enumerate_gpu_device_nodes(); + assert!( + !nodes.contains(&"/dev/nvidia".to_string()), + "bare /dev/nvidia should not be enumerated: {nodes:?}" + ); + } + + #[test] + fn no_duplicate_paths_in_baseline() { + let (ro, rw) = baseline_enrichment_paths(); + // No path should appear in both lists. + for path in &ro { + assert!( + !rw.contains(path), + "path {path} appears in both read_only and read_write" + ); + } + } + + #[test] + fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + read_only: vec!["/tmp".to_string()], + read_write: vec![], + include_workdir: false, + }); + policy.network_policies.insert( + "test".into(), + openshell_core::proto::NetworkPolicyRule { + name: "test-rule".into(), + endpoints: vec![openshell_core::proto::NetworkEndpoint { + host: "example.com".into(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + + enrich_proto_baseline_paths(&mut policy); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + filesystem.read_only.contains(&"/tmp".to_string()), + "explicit read_only baseline path should be preserved" + ); + assert!( + !filesystem.read_write.contains(&"/tmp".to_string()), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + assert!(strip_proto_provider_policy_entries(&mut policy)); + assert!( + !policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(policy.network_policies.contains_key("sandbox_only")); + assert!(!strip_proto_provider_policy_entries(&mut policy)); + } + + #[test] + fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + + assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "provider-derived rules alone must not trigger sync or mutate runtime policy" + ); + } + + #[test] + fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + runtime_policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) + .expect("enrichment should create a sync payload"); + + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "runtime policy must retain provider-derived rules for OPA input" + ); + assert!( + !sync_policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(sync_policy.network_policies.contains_key("sandbox_only")); + } + + #[test] + fn proto_gpu_enrichment_promotes_proc_without_network_policy() { + let mut policy = openshell_policy::restrictive_default_policy(); + assert!( + policy.network_policies.is_empty(), + "regression setup must exercise the no-network default path" + ); + let (ro, rw) = + collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); + + let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { + matches!(path, "/proc" | "/dev/nvidia0") + }); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + enriched, + "GPU enrichment should not require network policies" + ); + assert!( + filesystem.read_write.contains(&"/dev/nvidia0".to_string()), + "GPU enrichment should add enumerated device nodes without network policies" + ); + assert!( + !filesystem.read_only.contains(&"/proc".to_string()), + "GPU enrichment should remove /proc from read_only" + ); + assert!( + filesystem.read_write.contains(&"/proc".to_string()), + "GPU enrichment should promote /proc to read_write" + ); + } + + #[test] + fn gpu_baseline_read_write_contains_dxg() { + // /dev/dxg must be present so WSL2 sandboxes get the Landlock + // read-write rule for the CDI-injected DXG device. The existence + // check in enrich_proto_baseline_paths() skips it on native Linux. + assert!( + GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), + "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" + ); + } + + #[test] + fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![PathBuf::from("/tmp")], + read_write: vec![], + include_workdir: false, + }, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }; + + enrich_sandbox_baseline_paths(&mut policy); + + assert!( + policy.filesystem.read_only.contains(&PathBuf::from("/tmp")), + "explicit read_only baseline path should be preserved" + ); + assert!( + !policy + .filesystem + .read_write + .contains(&PathBuf::from("/tmp")), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn gpu_baseline_read_only_contains_usr_lib_wsl() { + // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library + // bind-mounts are accessible under Landlock. Skipped on native Linux. + assert!( + GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), + "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" + ); + } + + #[test] + fn has_gpu_devices_reflects_dxg_or_nvidiactl() { + // Verify the OR logic: result must match the manual disjunction of + // the two path checks. Passes in all environments. + let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); + let dxg = std::path::Path::new("/dev/dxg").exists(); + assert_eq!( + has_gpu_devices(), + nvidiactl || dxg, + "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" + ); + } +} + +/// Returns `true` if the error is transient and worth retrying. +/// +/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If +/// found, only the gRPC codes that represent transient failures are retryable. +/// If no `tonic::Status` is present (e.g. a raw connection error), assume the +/// failure is transient. +fn is_retryable_error(err: &miette::Report) -> bool { + let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); + while let Some(e) = source { + if let Some(status) = e.downcast_ref::() { + return matches!( + status.code(), + tonic::Code::Unavailable + | tonic::Code::DeadlineExceeded + | tonic::Code::ResourceExhausted + | tonic::Code::Aborted + | tonic::Code::Internal + | tonic::Code::Unknown + ); + } + source = e.source(); + } + true +} + +/// Retry a gRPC operation with exponential backoff (capped at 4 s). +/// +/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, +/// `PERMISSION_DENIED`) are returned immediately without retrying. +async fn grpc_retry(op_name: &str, f: F) -> Result +where + F: Fn() -> Fut, + Fut: Future>, +{ + let mut last_err = None; + for attempt in 1..=5u32 { + match f().await { + Ok(val) => return Ok(val), + Err(e) => { + if !is_retryable_error(&e) { + return Err(e); + } + if attempt < 5 { + warn!( + attempt, + max_attempts = 5, + error = %e, + "{op_name} failed, retrying" + ); + let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); + tokio::time::sleep(backoff).await; + } + last_err = Some(e); + } + } + } + Err(miette::miette!( + "{op_name} failed after 5 attempts: {}", + last_err.expect("loop executed at least once") + )) +} + +/// Load sandbox policy from local files or gRPC. +/// +/// Priority: +/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files +/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC +/// 3. If the server returns no policy, discover from disk or use restrictive default +/// 4. Otherwise, return an error +/// +/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto +/// policy. The proto is retained so the OPA engine can be rebuilt with symlink +/// resolution after the container entrypoint starts. +async fn load_policy( + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, + extension_credentials: &openshell_extension_core::ExtensionCredentialStore, +) -> Result<( + SandboxPolicy, + Option>, + Option, + MiddlewareRegistryStatus, + LoadedPolicyOrigin, + bool, + bool, +)> { + // File mode: load OPA engine from rego rules + YAML data (dev override) + if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "loading") + .unmapped("policy_rules", serde_json::json!(policy_file)) + .unmapped("policy_data", serde_json::json!(data_file)) + .message(format!( + "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" + )) + .build()); + let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { + openshell_supervisor_middleware_builtins::validate_config(implementation, config) + .map_err(|error| error.to_string()) + }; + let engine = OpaEngine::from_files_with_middleware_config( + std::path::Path::new(policy_file), + std::path::Path::new(data_file), + Some(&validate_middleware_config), + )?; + let middleware_registry = + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + engine.replace_middleware_registry(middleware_registry)?; + let config = engine.query_sandbox_config()?; + let mut policy = SandboxPolicy { + version: 1, + filesystem: config.filesystem, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: config.landlock, + process: config.process, + }; + enrich_sandbox_baseline_paths(&mut policy); + // File mode has no operator-registered middleware to connect. + return Ok(( + policy, + Some(Arc::new(engine)), + None, + MiddlewareRegistryStatus::Synchronized, + LoadedPolicyOrigin::LocalOverride, + false, + false, + )); + } + + // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data + if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + info!( + sandbox_id = %id, + endpoint = %endpoint, + "Fetching sandbox policy via gRPC" + ); + let mut snapshot = grpc_retry("Policy fetch", || { + openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) + }) + .await?; + + let mut proto_policy = if let Some(p) = snapshot.policy.clone() { + p + } else { + // No policy configured on the server. Discover from disk or + // fall back to the restrictive default, then sync to the + // gateway so it becomes the authoritative baseline. + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "discovery") + .message("Server returned no policy; attempting local discovery") + .build() + ); + let mut discovered = discover_policy_from_disk_or_default(); + // Enrich before syncing so the gateway baseline includes + // baseline paths from the start. + enrich_proto_baseline_paths(&mut discovered); + strip_proto_provider_policy_entries(&mut discovered); + let sandbox = sandbox.as_deref().ok_or_else(|| { + miette::miette!( + "Cannot sync discovered policy: sandbox not available.\n\ + Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." + ) + })?; + + // Sync and re-fetch over a single connection to avoid extra + // TLS handshakes. + let ws = snapshot.workspace.clone(); + snapshot = grpc_retry("Policy discovery sync", || { + openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox, + &discovered, + &ws, + ) + }) + .await?; + snapshot.policy.clone().ok_or_else(|| { + miette::miette!("Server still returned no policy after sync — this is a bug") + })? + }; + + // True only while `snapshot` describes the exact policy that will be + // constructed below. If enrichment cannot be synced and re-fetched, + // the policy remains enforceable but cannot be acknowledged by + // inferred structural equality. + let mut policy_bound_to_snapshot = true; + + // Ensure baseline filesystem paths are present for proxy-mode + // sandboxes. If the policy was enriched, sync the updated version + // back to the gateway so users can see the effective policy. + let enriched = enrich_proto_baseline_paths(&mut proto_policy); + let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); + if let Some(sync_policy) = sync_policy { + if let Some(sandbox_name) = sandbox.as_deref() { + match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox_name, + &sync_policy, + &snapshot.workspace, + ) + .await + { + Ok(canonical) => { + if let Some(policy) = canonical.policy.clone() { + proto_policy = policy; + snapshot = canonical; + } else { + policy_bound_to_snapshot = false; + warn!( + "Gateway returned no policy after enrichment sync; initial revision will be reconciled" + ); + } + } + Err(e) => { + policy_bound_to_snapshot = false; + warn!( + error = %e, + "Failed to sync enriched policy back to gateway; initial revision will be reconciled" + ); + } + } + } else { + policy_bound_to_snapshot = false; + } + } + + let mut loaded_policy_revision = + policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); + + // Build OPA engine from baked-in rules + typed proto data. + // In cluster mode, proxy networking is always enabled so OPA is + // always required for allow/deny decisions. + // The initial load uses pid=0 (no symlink resolution) because the + // container hasn't started yet. After the entrypoint spawns, the + // engine is rebuilt with the real PID for symlink resolution. + info!("Creating OPA engine from proto policy data"); + let mut has_last_valid_policy = true; + let engine = match OpaEngine::from_proto(&proto_policy) { + Ok(engine) => Arc::new(engine), + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + let validation_error = e.to_string(); + let candidate_version = snapshot.version; + let candidate_hash = snapshot.policy_hash.clone(); + // There is no in-memory last-known-good generation during + // startup, so both configured modes necessarily fail closed. + // Load the restrictive default atomically and keep the + // rejected revision unacknowledged for poll reconciliation. + has_last_valid_policy = false; + proto_policy = openshell_policy::restrictive_default_policy(); + let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); + let disposition = apply_policy_validation_failure( + &engine, + snapshot.policy_validation_failure_mode, + has_last_valid_policy, + candidate_version, + &validation_error, + )?; + emit_policy_validation_failure( + &disposition, + candidate_version, + &candidate_hash, + &validation_error, + ); + loaded_policy_revision = None; + engine + } + }; + + // Install the in-process catalog before any external connection can + // fail. A newly started sandbox must always be able to resolve built-in + // bindings, even while operator-run services are unavailable. + install_builtin_middleware_registry(&engine).await?; + + // Connect operator-registered middleware services. A connect/describe + // failure keeps the built-in registry active so each request's + // `on_error` policy governs matched traffic. The policy poll loop + // retries the install without waiting for a config change. + let middleware_services = snapshot.supervisor_middleware_services.clone(); + let middleware_registry_status = if middleware_services.is_empty() { + MiddlewareRegistryStatus::Synchronized + } else if let Err(error) = grpc_retry("Middleware connect", || { + let middleware_services = middleware_services.clone(); + let extension_credentials = extension_credentials.clone(); + let extension_authentication_enabled = snapshot.extension_authentication_enabled; + async move { + let credentials = if extension_authentication_enabled { + // Share the supervisor's store so the slots installed here + // are the ones the policy poll loop later rotates in place. + openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( + endpoint, + extension_credentials, + ) + .await? + .refresh_extension_credentials(&middleware_services) + .await? + } else { + std::collections::HashMap::new() + }; + connect_middleware_registry( + &middleware_services, + &MiddlewareAuthentication { + credentials, + enabled: extension_authentication_enabled, + }, + ) + .await + } + }) + .await + .and_then(|registry| engine.replace_middleware_registry(registry)) + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(middleware_services.len()) + ) + .message(format!( + "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" + )) + .build() + ); + MiddlewareRegistryStatus::NeedsReconciliation + } else { + MiddlewareRegistryStatus::Synchronized + }; + let opa_engine = Some(engine); + + let policy = match SandboxPolicy::try_from(proto_policy.clone()) { + Ok(policy) => policy, + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + return Err(e); + } + }; + return Ok(( + policy, + opa_engine, + Some(proto_policy), + middleware_registry_status, + LoadedPolicyOrigin::Gateway { + revision: loaded_policy_revision, + has_last_valid_policy, + }, + agent_proposals_enabled_from_settings(&snapshot.settings), + snapshot.extension_authentication_enabled, + )); + } + + // No policy source available + Err(miette::miette!( + "Sandbox policy required. Provide one of:\n\ + - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ + - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" + )) +} + +/// Try to discover a sandbox policy from the well-known disk path, falling +/// back to the legacy path, then to the hardcoded restrictive default. +fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { + let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); + if primary.exists() { + return discover_policy_from_path(primary); + } + let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); + if legacy.exists() { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "legacy_path", + serde_json::json!(legacy.display().to_string()) + ) + .unmapped("new_path", serde_json::json!(primary.display().to_string())) + .message(format!( + "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", + legacy.display(), + primary.display() + )) + .build() + ); + return discover_policy_from_path(legacy); + } + discover_policy_from_path(primary) +} + +/// Try to read a sandbox policy YAML from `path`, falling back to the +/// hardcoded restrictive default if the file is missing or invalid. +fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { + use openshell_policy::{ + parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, + }; + + let Ok(yaml) = std::fs::read_to_string(path) else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "default") + .message(format!( + "No policy file on disk, using restrictive default [path:{}]", + path.display() + )) + .build() + ); + return restrictive_default_policy(); + }; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Loaded sandbox policy from container disk [path:{}]", + path.display() + )) + .build() + ); + match parse_sandbox_policy(&yaml) { + Ok(policy) => { + // Validate the disk-loaded policy for safety. + if let Err(violations) = validate_sandbox_policy(&policy) { + let messages: Vec = violations.iter().map(ToString::to_string).collect(); + ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Medium) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .finding_info( + FindingInfo::new( + "unsafe-disk-policy", + "Unsafe Disk Policy Content", + ) + .with_desc(&format!( + "Disk policy at {} contains unsafe content: {}", + path.display(), + messages.join("; "), + )), + ) + .message(format!( + "Disk policy contains unsafe content, using restrictive default [path:{}]", + path.display() + )) + .build()); + return restrictive_default_policy(); + } + policy + } + Err(e) => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "fallback") + .message(format!( + "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", + path.display() + )) + .build()); + restrictive_default_policy() + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MiddlewareRegistryStatus { + Synchronized, + NeedsReconciliation, +} + +#[derive(Debug)] +enum GatewayRuntimeReloadError { + PolicyValidation(miette::Report), + TransparentTcpPrerequisite(miette::Report), + MiddlewareRegistry(miette::Report), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GatewayRuntimeFailureClass { + PolicyValidation, + TransparentTcpPrerequisite, + MiddlewareRegistry, +} + +impl GatewayRuntimeReloadError { + fn class(&self) -> GatewayRuntimeFailureClass { + match self { + Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, + Self::TransparentTcpPrerequisite(_) => { + GatewayRuntimeFailureClass::TransparentTcpPrerequisite + } + Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct FailedRuntimeRevision { + config_revision: u64, + policy_hash: String, + failure_class: GatewayRuntimeFailureClass, +} + +impl FailedRuntimeRevision { + fn new(config_revision: u64, policy_hash: &str, failure: &GatewayRuntimeReloadError) -> Self { + Self { + config_revision, + policy_hash: policy_hash.to_string(), + failure_class: failure.class(), + } + } +} + +struct MiddlewareReloadContext<'a> { + desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], + authentication: &'a MiddlewareAuthentication, + registry_changed: bool, + connector: &'a MiddlewareConnector, +} + +async fn reload_gateway_policy_runtime( + engine: &OpaEngine, + policy: Option<&openshell_core::proto::SandboxPolicy>, + entrypoint_pid: u32, + middleware: MiddlewareReloadContext<'_>, + transparent_tcp: TransparentTcpReloadState, +) -> std::result::Result<(), GatewayRuntimeReloadError> { + if let Some(policy) = policy + && policy_contains_explicit_tcp(policy) + { + if !transparent_tcp.capable { + return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( + miette::miette!( + "candidate policy introduces protocol: tcp, but the runtime does not advertise transparent TCP support; previous policy remains active" + ), + )); + } + if !transparent_tcp.substrate_ready { + return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( + miette::miette!( + "candidate policy introduces protocol: tcp, but this sandbox started without the transparent TCP substrate; recreate the sandbox to enable TCP; previous policy remains active" + ), + )); + } + } + match policy { + Some(policy) if middleware.registry_changed => { + let registry = (middleware.connector)( + middleware.desired_services.to_vec(), + middleware.authentication.clone(), + ) + .await + .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; + engine + .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) + .map_err(GatewayRuntimeReloadError::PolicyValidation) + } + // Policy-only change: the installed registry already matches the + // delivered service set, so swap the engine alone. This must not + // require middleware reachability. + Some(policy) => engine + .reload_from_proto_with_pid(policy, entrypoint_pid) + .map_err(GatewayRuntimeReloadError::PolicyValidation), + None => Err(GatewayRuntimeReloadError::PolicyValidation( + miette::miette!("runtime reload requires a policy payload but none was returned"), + )), + } +} + +fn policy_contains_explicit_tcp(policy: &openshell_core::proto::SandboxPolicy) -> bool { + policy.network_policies.values().any(|rule| { + rule.endpoints + .iter() + .any(|endpoint| endpoint.protocol.eq_ignore_ascii_case("tcp")) + }) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct TransparentTcpReloadState { + capable: bool, + substrate_ready: bool, +} + +/// True when the installed middleware registry no longer matches the desired +/// service set and must be rebuilt (reconnecting every delivered service). +/// +/// A policy-only change never requires a rebuild: middleware configs were +/// validated at gateway admission and the installed registry's manifests +/// already cover the unchanged service set, so requiring the services to be +/// reachable would only let a middleware outage block the policy update. +fn middleware_registry_needs_rebuild( + registry_status: MiddlewareRegistryStatus, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> bool { + registry_status == MiddlewareRegistryStatus::NeedsReconciliation + || current_services != desired_services +} + +fn gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy: bool, + current_policy_hash: &str, + desired_policy_hash: &str, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + registry_status: MiddlewareRegistryStatus, +) -> bool { + reloads_gateway_policy + && (current_policy_hash != desired_policy_hash + || middleware_registry_needs_rebuild( + registry_status, + current_services, + desired_services, + )) +} + +/// Identity returned with the exact policy snapshot used to construct OPA. +#[derive(Clone, Debug, PartialEq, Eq)] +struct LoadedPolicyRevision { + version: u32, + policy_hash: String, + config_revision: u64, + policy_source: openshell_core::proto::PolicySource, +} + +/// Identifies where the policy currently loaded into OPA came from. +/// +/// A missing gateway revision means the policy was loaded from the gateway but +/// could not be bound to an authoritative snapshot (for example, enrichment +/// sync failed). That state must reconcile on the first successful poll. A +/// local-file override is different: gateway policy revisions are observed for +/// settings/provider refreshes but must never replace the explicit local OPA +/// policy. +#[derive(Clone, Debug, PartialEq, Eq)] +enum LoadedPolicyOrigin { + LocalOverride, + Gateway { + revision: Option, + has_last_valid_policy: bool, + }, +} + +impl LoadedPolicyOrigin { + fn allows_gateway_policy_reload(&self) -> bool { + matches!(self, Self::Gateway { .. }) + } + + fn has_last_valid_policy(&self) -> bool { + match self { + Self::LocalOverride => true, + Self::Gateway { + has_last_valid_policy, + .. + } => *has_last_valid_policy, + } + } +} + +impl LoadedPolicyRevision { + fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { + Self { + version: snapshot.version, + policy_hash: snapshot.policy_hash.clone(), + config_revision: snapshot.config_revision, + policy_source: snapshot.policy_source, + } + } +} + +/// A sandbox-scoped policy revision that was constructed successfully at +/// startup and must be acknowledged to the gateway exactly once. +#[derive(Clone, Debug, PartialEq, Eq)] +struct InitialPolicyAck { + version: u32, + policy_hash: String, + config_revision: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PolicyStatusUpdate { + version: u32, + loaded: bool, + error: String, + success_event: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum PolicyStatusSuccessEvent { + InitialAcknowledgement { policy_hash: String }, + UnchangedAcknowledgement { policy_hash: String }, +} + +impl PolicyStatusUpdate { + fn initial_loaded(ack: &InitialPolicyAck) -> Self { + Self { + version: ack.version, + loaded: true, + error: String::new(), + success_event: Some(PolicyStatusSuccessEvent::InitialAcknowledgement { + policy_hash: ack.policy_hash.clone(), + }), + } + } + + fn loaded(version: u32) -> Self { + Self { + version, + loaded: true, + error: String::new(), + success_event: None, + } + } + + fn unchanged_loaded(version: u32, policy_hash: String) -> Self { + Self { + version, + loaded: true, + error: String::new(), + success_event: Some(PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash }), + } + } + + fn failed(version: u32, error: String) -> Self { + Self { + version, + loaded: false, + error, + success_event: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum InitialPollDisposition { + Acknowledge(InitialPolicyAck), + Reconcile, + TrackOnly, +} + +/// Determine whether the initially loaded policy corresponds to an +/// authoritative sandbox-scoped revision that must be acknowledged. +/// +/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose +/// captured gateway identity matches the current version and hash. Global +/// policies, local-file development policies, version zero, and changed +/// identities yield `None`, so those paths never emit a sandbox-revision +/// acknowledgement. +fn initial_policy_ack_candidate( + loaded: Option<&LoadedPolicyRevision>, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> Option { + let loaded = loaded?; + if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox + || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox + { + return None; + } + if loaded.version == 0 || canonical.version == 0 { + return None; + } + if loaded.version != canonical.version + || loaded.policy_hash != canonical.policy_hash + || canonical.config_revision < loaded.config_revision + { + return None; + } + Some(InitialPolicyAck { + version: loaded.version, + policy_hash: loaded.policy_hash.clone(), + config_revision: canonical.config_revision, + }) +} + +fn initial_poll_disposition( + origin: &LoadedPolicyOrigin, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> InitialPollDisposition { + match origin { + LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, + LoadedPolicyOrigin::Gateway { revision, .. } => { + initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( + InitialPollDisposition::Reconcile, + InitialPollDisposition::Acknowledge, + ) + } + } +} + +fn unchanged_policy_revision_candidate( + reloads_gateway_policy: bool, + recovering_rejected_policy: bool, + current_policy_version: u32, + current_policy_hash: &str, + result: &openshell_core::grpc_client::SettingsPollResult, +) -> Option { + (reloads_gateway_policy + && !recovering_rejected_policy + && !current_policy_hash.is_empty() + && result.policy_source == openshell_core::proto::PolicySource::Sandbox + && result.version > current_policy_version + && result.policy_hash == current_policy_hash) + .then_some(result.version) +} + +fn unchanged_policy_revision_ready_to_ack( + candidate: Option, + policy_runtime_changed: bool, + policy_runtime_reconciled: bool, +) -> Option { + candidate.filter(|_| !policy_runtime_changed || policy_runtime_reconciled) +} + +/// Whether the credential-provenance gates cannot apply to the loaded policy. +/// +/// The gateway derives `provider_credentialed` and deliberately keeps it out of +/// the policy YAML schema, so a local-file policy never carries it and never +/// will: gateway revisions are observed for settings and providers but must not +/// replace the local OPA policy. Provider credentials still arrive from the +/// gateway on that path, so the raw-tunnel and WebSocket binary-frame refusals +/// have nothing to match on. The request-body backstop is unaffected because it +/// keys off the secret resolver rather than endpoint provenance. +fn credential_gating_unavailable( + origin: &LoadedPolicyOrigin, + has_resolver: bool, + network_enabled: bool, +) -> bool { + network_enabled && has_resolver && matches!(origin, LoadedPolicyOrigin::LocalOverride) +} + +/// Report that credential provenance is unavailable for the loaded policy. +/// +/// Carries no credential name, host, or value: the finding states which +/// controls are inactive, nothing about what they would have protected. +fn report_credential_gating_unavailable() { + ocsf_emit!( + DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::High) + .confidence(ConfidenceId::High) + .is_alert(true) + .finding_info( + FindingInfo::new( + "credential-gating-unavailable", + "Credential Provenance Unavailable", + ) + .with_desc( + "Provider credentials are injected, but the loaded policy comes from local \ + files and carries no gateway-derived credential provenance. Uninspected \ + credentialed tunnels and WebSocket binary frames are not refused. Load \ + policy from the gateway to enable these controls." + ), + ) + .evidence_pairs(&[ + ("policy_source", "local-override"), + ("uninspected_connect_gate", "inactive"), + ("websocket_binary_gate", "inactive"), + ("request_body_backstop", "active"), + ]) + .remediation( + "Remove the local policy override so the gateway-delivered effective policy \ + applies, or detach provider credentials from this sandbox." + ) + .message( + "Credential provenance unavailable for local-file policy; uninspected credential gates inactive" + ) + .build() + ); +} + +/// Deliver policy status updates independently from policy reconciliation. +/// +/// The channel is FIFO, so a delayed older status can never arrive after a +/// newer status and move the gateway's active version backward. Delivery uses +/// the existing bounded retry, but failures never delay policy enforcement. +#[tonic::async_trait] +trait PolicyGatewayClient: Clone + Send + Sync + 'static { + async fn poll_settings( + &self, + sandbox_id: &str, + ) -> Result; + + async fn report_policy_status( + &self, + sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()>; + + async fn refresh_installed_extension_credentials(&self) -> Result<()> { + Ok(()) + } + + async fn extension_credentials_for( + &self, + _services: &[openshell_core::proto::SupervisorMiddlewareService], + ) -> Result> { + Ok(std::collections::HashMap::new()) + } + + fn workspace(&self) -> String; +} + +#[tonic::async_trait] +impl PolicyGatewayClient for openshell_core::grpc_client::CachedOpenShellClient { + async fn poll_settings( + &self, + sandbox_id: &str, + ) -> Result { + self.poll_settings(sandbox_id).await + } + + async fn report_policy_status( + &self, + sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()> { + self.report_policy_status(sandbox_id, version, loaded, error) + .await + } + + async fn refresh_installed_extension_credentials(&self) -> Result<()> { + self.refresh_installed_extension_credentials().await + } + + async fn extension_credentials_for( + &self, + services: &[openshell_core::proto::SupervisorMiddlewareService], + ) -> Result> { + self.extension_credentials_for(services).await + } + + fn workspace(&self) -> String { + self.workspace() + } +} + +async fn run_policy_status_reporter( + client: C, + sandbox_id: String, + mut updates: tokio::sync::mpsc::UnboundedReceiver, +) { + 'updates: while let Some(update) = updates.recv().await { + let operation = if matches!( + update.success_event, + Some(PolicyStatusSuccessEvent::InitialAcknowledgement { .. }) + ) { + "Initial policy acknowledgement" + } else { + "Policy status report" + }; + let mut attempt = 1_u32; + loop { + let sandbox_id = sandbox_id.clone(); + let error = update.error.clone(); + let client = client.clone(); + match client + .report_policy_status(&sandbox_id, update.version, update.loaded, &error) + .await + { + Ok(()) => break, + Err(error) if is_retryable_error(&error) => { + let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); + warn!( + %error, + attempt, + version = update.version, + loaded = update.loaded, + retry_in_secs = backoff.as_secs(), + "{operation} failed transiently; retaining ordered update" + ); + tokio::time::sleep(backoff).await; + attempt = attempt.saturating_add(1); + } + Err(error) => { + warn!( + %error, + version = update.version, + loaded = update.loaded, + "Discarding terminal policy status update" + ); + continue 'updates; + } + } + } + + if let Some(event) = update.success_event { + let (policy_hash, message) = match event { + PolicyStatusSuccessEvent::InitialAcknowledgement { policy_hash } => ( + policy_hash, + format!( + "Acknowledged initial policy revision as loaded [version:{}]", + update.version + ), + ), + PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash } => ( + policy_hash, + format!( + "Acknowledged unchanged policy revision as loaded [version:{}]", + update.version + ), + ), + }; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("version", serde_json::json!(update.version)) + .unmapped("policy_hash", serde_json::json!(policy_hash)) + .message(message) + .build() + ); + } + } +} + +fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { + let version = update.version; + if let Err(error) = sender.send(update) { + warn!( + %error, + version, + "Policy status reporter unavailable during shutdown" + ); + } +} + +/// Best-effort `FAILED` acknowledgement when initial policy construction or +/// conversion fails. +/// +/// Uses the revision identity captured with the policy that failed to build, +/// and preserves the original construction error as the reported message. A +/// delivery failure here is swallowed so it can never mask that error. +async fn report_initial_policy_failure( + endpoint: &str, + sandbox_id: &str, + revision: Option<&LoadedPolicyRevision>, + error: &miette::Report, +) { + let Some(revision) = revision.filter(|revision| { + revision.version > 0 + && revision.policy_source == openshell_core::proto::PolicySource::Sandbox + }) else { + return; + }; + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + Ok(client) => client, + Err(e) => { + warn!(error = %e, "Failed to connect to report initial policy failure"); + return; + } + }; + let message = error.to_string(); + if let Err(e) = grpc_retry("Initial policy failure report", || { + let client = client.clone(); + let message = message.clone(); + async move { + client + .report_policy_status(sandbox_id, revision.version, false, &message) + .await + } + }) + .await + { + warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); + } +} + +/// Background loop that polls the server for policy updates. +/// +/// When a new version is detected, attempts to reload the OPA engine via +/// `reload_from_proto_with_pid()`. Reports load success/failure back to the +/// server. On failure, the previous engine is untouched (LKG behavior). +/// +/// When the entrypoint PID is available, policy reloads include symlink +/// resolution for binary paths via the container filesystem. +struct PolicyPollLoopContext { + endpoint: String, + sandbox_id: String, + opa_engine: Arc, + /// Source of the policy currently loaded into OPA. This distinguishes an + /// explicit local-file override from an unbound gateway revision so the + /// former is never replaced by policy polling. + loaded_policy_origin: LoadedPolicyOrigin, + entrypoint_pid: Arc, + interval_secs: u64, + ocsf_enabled: Arc, + provider_credentials: ProviderCredentialState, + policy_local_ctx: Option>, + agent_proposals: AgentProposals, + middleware_registry_status: MiddlewareRegistryStatus, + workspace_tx: tokio::sync::watch::Sender, + extension_credentials: openshell_extension_core::ExtensionCredentialStore, + extension_authentication_enabled: bool, + middleware_connector: MiddlewareConnector, + /// Immutable driver capability and startup substrate state. + transparent_tcp: TransparentTcpReloadState, +} + +type MiddlewareConnector = Arc< + dyn Fn( + Vec, + MiddlewareAuthentication, + ) -> Pin< + Box< + dyn std::future::Future< + Output = Result, + > + Send, + >, + > + Send + + Sync, +>; + +#[derive(Clone, Default)] +struct MiddlewareAuthentication { + credentials: std::collections::HashMap, + enabled: bool, +} + +fn default_middleware_connector() -> MiddlewareConnector { + Arc::new(|services, authentication| { + Box::pin(async move { connect_middleware_registry(&services, &authentication).await }) + }) +} + +async fn connect_middleware_registry( + services: &[openshell_core::proto::SupervisorMiddlewareService], + authentication: &MiddlewareAuthentication, +) -> Result { + if authentication.enabled { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services_authenticated( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + &authentication.credentials, + ) + .await + } else { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + ) + .await + } +} + +async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + opa_engine.replace_middleware_registry(registry) +} + +/// Wait the configured poll interval, but never past the point at which an +/// installed extension credential must be rotated. +fn next_poll_delay( + store: &openshell_extension_core::ExtensionCredentialStore, + interval: Duration, +) -> Duration { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }); + store.next_refresh_delay(interval, now_ms) +} + +/// Drop credentials for services no longer in the installed registry. +/// +/// Call only after a registry swap succeeds, so a failed candidate cannot +/// invalidate the last-known-good clients. +fn retain_extension_credentials( + store: &openshell_extension_core::ExtensionCredentialStore, + installed: &[openshell_core::proto::SupervisorMiddlewareService], + extension_authentication_enabled: bool, +) { + let retained = if extension_authentication_enabled { + installed + .iter() + .map(|service| service.name.as_str()) + .collect() + } else { + std::collections::HashSet::default() + }; + store.retain(&retained); +} + +struct MiddlewareRegistryReconciliation<'a> { + desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], + authentication: MiddlewareAuthentication, + registry_changed: bool, + extension_credentials: &'a openshell_extension_core::ExtensionCredentialStore, + current_services: &'a mut Vec, + status: &'a mut MiddlewareRegistryStatus, +} + +async fn reconcile_middleware_registry( + opa_engine: &OpaEngine, + middleware_connector: &MiddlewareConnector, + reconciliation: MiddlewareRegistryReconciliation<'_>, +) { + if !reconciliation.registry_changed { + return; + } + + match middleware_connector( + reconciliation.desired_services.to_vec(), + reconciliation.authentication.clone(), + ) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + retain_extension_credentials( + reconciliation.extension_credentials, + reconciliation.desired_services, + reconciliation.authentication.enabled, + ); + reconciliation.current_services.clear(); + reconciliation + .current_services + .extend_from_slice(reconciliation.desired_services); + *reconciliation.status = MiddlewareRegistryStatus::Synchronized; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(reconciliation.current_services.len()) + ) + .message(format!( + "Supervisor middleware registry reloaded [service_count:{}]", + reconciliation.current_services.len() + )) + .build() + ); + } + Err(error) => { + // Emit only on the transition into the failed state to avoid + // repeating the same finding on every poll during an outage. + if *reconciliation.status == MiddlewareRegistryStatus::Synchronized { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + } + *reconciliation.status = MiddlewareRegistryStatus::NeedsReconciliation; + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode, + mode: PolicyValidationFailureMode, + previous_policy_active: bool, + active_generation: u64, +} + +struct RejectedPolicyGeneration { + version: u32, + policy_hash: String, + validation_error: String, + configured_mode: PolicyValidationFailureMode, +} + +enum GatewayRuntimeFailureDisposition { + PolicyRejected { + error: String, + disposition: PolicyValidationFailureDisposition, + }, + MiddlewareUnavailable { + error: String, + }, + TransparentTcpExpansionRejected { + error: String, + active_generation: u64, + }, +} + +fn apply_gateway_runtime_reload_failure( + engine: &OpaEngine, + failure: GatewayRuntimeReloadError, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, +) -> Result { + match failure { + GatewayRuntimeReloadError::PolicyValidation(error) => { + let error = error.to_string(); + let disposition = apply_policy_validation_failure( + engine, + configured_mode, + has_last_valid_policy, + version, + &error, + )?; + Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) + } + GatewayRuntimeReloadError::TransparentTcpPrerequisite(error) => Ok( + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + error: error.to_string(), + active_generation: engine.current_generation(), + }, + ), + GatewayRuntimeReloadError::MiddlewareRegistry(error) => { + Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { + error: error.to_string(), + }) + } + } +} + +fn emit_transparent_tcp_expansion_rejection( + version: u32, + policy_hash: &str, + active_generation: u64, + error: &str, +) { + let message = format!( + "Transparent TCP policy expansion rejected; previous policy IS active [version:{version} active_generation:{active_generation} error:{error}]" + ); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Enabled, "retained_previous_policy") + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped("previous_policy_active", serde_json::json!(true)) + .unmapped("active_generation", serde_json::json!(active_generation)) + .unmapped("validation_error", serde_json::json!(error)) + .message(message) + .build() + ); +} + +fn apply_policy_validation_failure( + engine: &OpaEngine, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, + error: &str, +) -> Result { + let mode = if has_last_valid_policy { + configured_mode + } else { + PolicyValidationFailureMode::FailClosed + }; + match mode { + PolicyValidationFailureMode::FailClosed => { + let reason = format!( + "policy validation failed; fail-closed quarantine is active; candidate version {version} rejected: {error}" + ); + let active_generation = engine.enter_fail_closed(reason)?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: false, + active_generation, + }) + } + PolicyValidationFailureMode::RetainLastValid => { + let active_generation = engine.exit_fail_closed()?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: true, + active_generation, + }) + } + } +} + +fn policy_validation_failure_events( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) -> [OcsfEvent; 2] { + let previous_policy_state = if disposition.previous_policy_active { + "IS active" + } else { + "IS NOT active" + }; + let state = if disposition.previous_policy_active { + (StateId::Enabled, "retained_last_valid") + } else { + (StateId::Disabled, "fail_closed") + }; + let message = format!( + "Policy validation failed; configured_mode={} effective_mode={}; previous policy {previous_policy_state} [version:{version} active_generation:{} error:{error}]", + disposition.configured_mode.as_str(), + disposition.mode.as_str(), + disposition.active_generation, + ); + let finding_uid = format!("policy-validation-failed-{version}"); + let version_string = version.to_string(); + let config = ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(state.0, state.1) + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped( + "validation_failure_mode", + serde_json::json!(disposition.mode.as_str()), + ) + .unmapped( + "configured_validation_failure_mode", + serde_json::json!(disposition.configured_mode.as_str()), + ) + .unmapped( + "previous_policy_active", + serde_json::json!(disposition.previous_policy_active), + ) + .unmapped( + "active_generation", + serde_json::json!(disposition.active_generation), + ) + .unmapped("validation_error", serde_json::json!(error)) + .message(message.clone()) + .build(); + let finding = DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .is_alert(true) + .finding_info( + FindingInfo::new(&finding_uid, "Invalid policy generation rejected").with_desc(error), + ) + .evidence_pairs(&[ + ("candidate_version", &version_string), + ("candidate_policy_hash", policy_hash), + ("validation_failure_mode", disposition.mode.as_str()), + ( + "configured_validation_failure_mode", + disposition.configured_mode.as_str(), + ), + ( + "previous_policy_active", + if disposition.previous_policy_active { + "true" + } else { + "false" + }, + ), + ]) + .remediation("Submit a valid, unambiguous policy generation") + .message(message) + .build(); + [config, finding] +} + +fn emit_policy_validation_failure( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) { + for event in policy_validation_failure_events(disposition, version, policy_hash, error) { + ocsf_emit!(event); + } +} + +async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { + let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( + &ctx.endpoint, + ctx.extension_credentials.clone(), + ) + .await?; + run_policy_poll_loop_with_client(ctx, client).await +} + +async fn run_policy_poll_loop_with_client( + ctx: PolicyPollLoopContext, + client: C, +) -> Result<()> { + use openshell_core::proto::PolicySource; + use std::sync::atomic::Ordering; + + let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(run_policy_status_reporter( + client.clone(), + ctx.sandbox_id.clone(), + status_receiver, + )); + + let mut current_config_revision: u64 = 0; + let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; + let mut current_policy_version: u32 = 0; + let mut current_policy_hash = String::new(); + let mut current_middleware_services = Vec::new(); + let mut current_extension_authentication_enabled = ctx.extension_authentication_enabled; + let mut middleware_registry_status = ctx.middleware_registry_status; + let mut current_settings: std::collections::HashMap< + String, + openshell_core::proto::EffectiveSetting, + > = std::collections::HashMap::new(); + let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); + let mut last_failed_runtime_revision: Option = None; + let mut rejected_policy_generation: Option = None; + let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); + + // A first poll that does not match the policy already loaded into OPA must + // pass through the normal reconciliation path immediately. It must never + // seed the applied-state trackers before OPA actually loads it. + let mut pending_result = None; + + // Initialize revision from the first poll and acknowledge the initial + // policy revision the supervisor actually loaded. A mismatched result is + // reconciled below instead of being recorded as already applied. + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { + InitialPollDisposition::Acknowledge(candidate) => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(candidate.config_revision), + skills::install_static_skills, + ); + current_config_revision = candidate.config_revision; + current_policy_version = candidate.version; + current_policy_hash.clone_from(&candidate.policy_hash); + current_middleware_services = result.supervisor_middleware_services; + current_extension_authentication_enabled = + result.extension_authentication_enabled; + current_settings = result.settings; + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::initial_loaded(&candidate), + ); + debug!( + config_revision = current_config_revision, + "Settings poll: initial policy matches loaded revision" + ); + } + InitialPollDisposition::Reconcile => pending_result = Some(result), + InitialPollDisposition::TrackOnly => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(result.config_revision), + skills::install_static_skills, + ); + current_config_revision = result.config_revision; + current_policy_hash = result.policy_hash.clone(); + current_middleware_services = result.supervisor_middleware_services; + current_extension_authentication_enabled = + result.extension_authentication_enabled; + current_settings = result.settings; + debug!( + config_revision = current_config_revision, + "Settings poll: tracking gateway config while preserving local policy override" + ); + } + } + } + Err(e) => { + warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); + } + } + + let interval = Duration::from_secs(ctx.interval_secs); + loop { + let result = if let Some(result) = pending_result.take() { + result + } else { + tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await; + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + result + } + Err(e) => { + debug!(error = %e, "Settings poll: server unreachable, will retry"); + if current_extension_authentication_enabled + && let Err(refresh_error) = + client.refresh_installed_extension_credentials().await + { + warn!( + error = %refresh_error, + "Settings poll: extension credential refresh failed while configuration was unavailable" + ); + } + continue; + } + } + }; + + // Reuse installed per-service credentials, rotating only when one is + // missing or due. Rotation happens on the existing gateway channel and + // updates slots in place, so it is independent of config revision and + // registry equality. + let middleware_credentials = if result.extension_authentication_enabled { + match client + .extension_credentials_for(&result.supervisor_middleware_services) + .await + { + Ok(credentials) => credentials, + Err(error) => { + warn!(error = %error, "Settings poll: extension credential refresh failed"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; + + let config_changed = result.config_revision != current_config_revision; + let provider_env_changed = result.provider_env_revision != current_provider_env_revision; + let policy_changed = result.policy_hash != current_policy_hash; + let extension_authentication_changed = + current_extension_authentication_enabled != result.extension_authentication_enabled; + let middleware_registry_changed = extension_authentication_changed + || middleware_registry_needs_rebuild( + middleware_registry_status, + ¤t_middleware_services, + &result.supervisor_middleware_services, + ); + // A valid candidate may intentionally restore byte-for-byte policy + // content that was active before a rejected update. Its hash then + // equals `current_policy_hash`, but the runtime is still quarantined + // and must reload (or it would remain deny-all indefinitely). + let recovering_rejected_policy = reloads_gateway_policy + && rejected_policy_generation + .as_ref() + .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); + let policy_runtime_changed = recovering_rejected_policy + || extension_authentication_changed + || gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); + // Recovery already has its own acknowledgement path below. Giving it + // precedence here prevents a restored last-known-good policy from + // also being acknowledged as an ordinary same-hash revision. + let unchanged_policy_revision = unchanged_policy_revision_candidate( + reloads_gateway_policy, + recovering_rejected_policy, + current_policy_version, + ¤t_policy_hash, + &result, + ); + let mut policy_runtime_reconciled = false; + + // A local policy override is not coupled to the gateway policy + // snapshot, so its service registry can still be reconciled alone. + // Gateway policy snapshots, however, must install policy and registry + // as one generation below. + if !reloads_gateway_policy { + reconcile_middleware_registry( + &ctx.opa_engine, + &ctx.middleware_connector, + MiddlewareRegistryReconciliation { + desired_services: &result.supervisor_middleware_services, + authentication: MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + extension_credentials: &ctx.extension_credentials, + current_services: &mut current_middleware_services, + status: &mut middleware_registry_status, + }, + ) + .await; + if middleware_registry_status == MiddlewareRegistryStatus::Synchronized { + current_extension_authentication_enabled = result.extension_authentication_enabled; + } + } + + if !config_changed + && !provider_env_changed + && !policy_runtime_changed + && unchanged_policy_revision.is_none() + { + continue; + } + + if config_changed || provider_env_changed { + // Log which settings changed. + log_setting_changes(¤t_settings, &result.settings); + + // A posture change after a rejected update takes effect immediately. + // The compiled last-known-good engine remains available beneath a + // fail-closed quarantine, so an explicit retain_last_valid selection + // can reactivate it without accepting any part of the invalid policy. + if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { + let mode = result.policy_validation_failure_mode; + if mode != rejected.configured_mode { + let disposition = apply_policy_validation_failure( + &ctx.opa_engine, + mode, + has_last_valid_policy, + rejected.version, + &rejected.validation_error, + )?; + emit_policy_validation_failure( + &disposition, + rejected.version, + &rejected.policy_hash, + &rejected.validation_error, + ); + rejected.configured_mode = mode; + } + } + + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "detected") + .unmapped("old_config_revision", serde_json::json!(current_config_revision)) + .unmapped("new_config_revision", serde_json::json!(result.config_revision)) + .unmapped("policy_changed", serde_json::json!(policy_changed)) + .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) + .message(format!( + "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", + result.config_revision + )) + .build()); + } + + if provider_env_changed { + match openshell_core::grpc_client::fetch_provider_environment( + &ctx.endpoint, + &ctx.sandbox_id, + ) + .await + { + Ok(env_result) => { + let provider_env_revision = env_result.provider_env_revision; + let install_result = ctx.provider_credentials.install_bound_environment( + provider_env_revision, + env_result.environment, + env_result.credential_expires_at_ms, + env_result.dynamic_credentials, + env_result.static_credential_bindings, + env_result.non_secret_environment_keys, + ); + if let Err(error) = install_result { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message(format!( + "Rejected provider environment refresh; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" + )) + .build() + ); + } else { + let env_count = + ctx.provider_credentials.child_env_with_gcp_resolved().len(); + current_provider_env_revision = provider_env_revision; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "provider_env_revision", + serde_json::json!(provider_env_revision) + ) + .message(format!( + "Provider environment refreshed [revision:{provider_env_revision} env_count:{env_count}]" + )) + .build() + ); + } + } + Err(e) => { + ctx.provider_credentials + .revoke_static_provider_environment(result.provider_env_revision); + warn!( + error = %e, + provider_env_revision = result.provider_env_revision, + "Settings poll: failed to refresh provider environment; static provider credentials were revoked; previous dynamic token grants remain active" + ); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message( + "Provider environment refresh failed; static provider credentials were revoked; previous dynamic token grants remain active" + ) + .build() + ); + } + } + } + + if policy_runtime_changed { + let pid = ctx.entrypoint_pid.load(Ordering::Acquire); + let runtime_result = reload_gateway_policy_runtime( + &ctx.opa_engine, + result.policy.as_ref(), + pid, + MiddlewareReloadContext { + desired_services: &result.supervisor_middleware_services, + authentication: &MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + connector: &ctx.middleware_connector, + }, + ctx.transparent_tcp, + ) + .await; + + match runtime_result { + Ok(()) => { + policy_runtime_reconciled = true; + let policy = result + .policy + .as_ref() + .expect("successful runtime reload requires a policy payload"); + has_last_valid_policy = true; + rejected_policy_generation = None; + if policy_changed { + if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { + policy_local_ctx.set_current_policy(policy.clone()).await; + } + if result.global_policy_version > 0 { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .unmapped("global_version", serde_json::json!(result.global_policy_version)) + .message(format!( + "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", + result.policy_hash, + result.global_policy_version + )) + .build()); + } else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + } + if result.version > 0 && result.policy_source == PolicySource::Sandbox { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + current_policy_version = result.version; + } + } else if recovering_rejected_policy + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully and fail-closed quarantine cleared [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + current_policy_version = result.version; + } + + if middleware_registry_changed { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(result.supervisor_middleware_services.len()) + ) + .message(format!( + "Supervisor policy runtime reloaded atomically [service_count:{}]", + result.supervisor_middleware_services.len() + )) + .build()); + } + + current_policy_hash.clone_from(&result.policy_hash); + current_middleware_services.clone_from(&result.supervisor_middleware_services); + current_extension_authentication_enabled = + result.extension_authentication_enabled; + retain_extension_credentials( + &ctx.extension_credentials, + &result.supervisor_middleware_services, + result.extension_authentication_enabled, + ); + middleware_registry_status = MiddlewareRegistryStatus::Synchronized; + last_failed_runtime_revision = None; + } + Err(failure) => { + let failed_revision = FailedRuntimeRevision::new( + result.config_revision, + &result.policy_hash, + &failure, + ); + if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + let failure_mode = result.policy_validation_failure_mode; + match apply_gateway_runtime_reload_failure( + &ctx.opa_engine, + failure, + failure_mode, + has_last_valid_policy, + result.version, + )? { + GatewayRuntimeFailureDisposition::PolicyRejected { + error, + disposition, + } => { + emit_policy_validation_failure( + &disposition, + result.version, + &result.policy_hash, + &error, + ); + rejected_policy_generation = Some(RejectedPolicyGeneration { + version: result.version, + policy_hash: result.policy_hash.clone(), + validation_error: error.clone(), + configured_mode: failure_mode, + }); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, error), + ); + } + } + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error } => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .unmapped("version", serde_json::json!(result.version)) + .unmapped("error", serde_json::json!(&error)) + .unmapped("previous_policy_active", serde_json::json!(true)) + .message(format!( + "Supervisor middleware registry unavailable, keeping last-known-good policy runtime active [version:{} error:{error}]", + result.version + )) + .build()); + } + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + error, + active_generation, + } => { + emit_transparent_tcp_expansion_rejection( + result.version, + &result.policy_hash, + active_generation, + &error, + ); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, error), + ); + } + } + } + } + last_failed_runtime_revision = Some(failed_revision); + // Nothing was installed, so the registry status still + // describes the live registry. The retry is driven by the + // persisting hash/service-set mismatch (or an existing + // NeedsReconciliation), not by degrading the status here. + } + } + } + + if let Some(version) = unchanged_policy_revision_ready_to_ack( + unchanged_policy_revision, + policy_runtime_changed, + policy_runtime_reconciled, + ) { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::unchanged_loaded(version, result.policy_hash.clone()), + ); + current_policy_version = version; + } + + // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + + // Apply the agent-proposals feature toggle. On a false→true transition + // we lazily install the skill so a sandbox that started with the flag + // off picks up the surface without a recreate. We never uninstall on + // a true→false transition: stale skill content on disk is harmless + // because route_request and agent_next_steps both gate on the live + // shared flag, so the agent that reads the skill will see 404s and an + // empty `next_steps` array regardless. + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "settings poll", + Some(result.config_revision), + skills::install_static_skills, + ); + + current_config_revision = result.config_revision; + if !reloads_gateway_policy { + current_policy_hash = result.policy_hash; + } + current_settings = result.settings; + } +} + +fn apply_ocsf_json_setting( + enabled: &AtomicBool, + settings: &std::collections::HashMap, +) { + use std::sync::atomic::Ordering; + + let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); + let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); + if new_ocsf != prev_ocsf { + info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); + } +} + +/// Extract a bool value from an effective setting, if present. +fn extract_bool_setting( + settings: &std::collections::HashMap, + key: &str, +) -> Option { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|v| match v { + setting_value::Value::BoolValue(b) => Some(*b), + _ => None, + }) +} + +fn agent_proposals_enabled_from_settings( + settings: &std::collections::HashMap, +) -> bool { + extract_bool_setting( + settings, + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, + ) + .unwrap_or(false) +} + +fn apply_agent_proposals_enabled( + agent_proposals: &AgentProposals, + enabled: bool, + source: &'static str, + config_revision: Option, + install_static_skills: impl FnOnce() -> Result, +) { + let previously_enabled = agent_proposals.swap_enabled(enabled); + if enabled == previously_enabled { + return; + } + + info!( + agent_policy_proposals_enabled = enabled, + source, config_revision, "agent-driven policy proposals toggled" + ); + + if enabled && !previously_enabled { + match install_static_skills() { + Ok(installed) => info!( + path = %installed.policy_advisor.display(), + "Installed sandbox agent skill on toggle-on" + ), + Err(error) => warn!( + error = %error, + "Failed to install sandbox agent skill on toggle-on" + ), + } + } +} + +/// Log individual setting changes between two snapshots. +fn log_setting_changes( + old: &std::collections::HashMap, + new: &std::collections::HashMap, +) { + for (key, new_es) in new { + let new_val = format_setting_value(new_es); + match old.get(key) { + Some(old_es) => { + let old_val = format_setting_value(old_es); + if old_val != new_val { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "updated") + .unmapped("key", serde_json::json!(key)) + .unmapped("old", serde_json::json!(old_val.clone())) + .unmapped("new", serde_json::json!(new_val.clone())) + .message(format!( + "Setting changed [key:{key} old:{old_val} new:{new_val}]" + )) + .build() + ); + } + } + None => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enabled") + .unmapped("key", serde_json::json!(key)) + .unmapped("value", serde_json::json!(new_val.clone())) + .message(format!("Setting added [key:{key} value:{new_val}]")) + .build() + ); + } + } + } + for key in old.keys() { + if !new.contains_key(key) { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Disabled, "disabled") + .unmapped("key", serde_json::json!(key)) + .message(format!("Setting removed [key:{key}]")) + .build() + ); + } + } +} + +/// Format an `EffectiveSetting` value for log display. +fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { + use openshell_core::proto::setting_value; + match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { + None => "".to_string(), + Some(setting_value::Value::StringValue(v)) => v.clone(), + Some(setting_value::Value::BoolValue(v)) => v.to_string(), + Some(setting_value::Value::IntValue(v)) => v.to_string(), + Some(setting_value::Value::BytesValue(_)) => "".to_string(), + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { + openshell_core::proto::EffectiveSetting { + value: Some(openshell_core::proto::SettingValue { + value: Some(openshell_core::proto::setting_value::Value::BoolValue( + value, + )), + }), + scope: openshell_core::proto::SettingScope::Global.into(), + } + } + + #[test] + fn shared_ssh_socket_setting_is_explicit() { + assert!(shared_ssh_socket_value("1")); + assert!(shared_ssh_socket_value("true")); + assert!(shared_ssh_socket_value("TRUE")); + assert!(!shared_ssh_socket_value("0")); + assert!(!shared_ssh_socket_value("yes")); + } + + #[tokio::test] + async fn control_readiness_exists_only_while_guard_is_live() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("health.sock"); + let readiness = + ControlReadiness::start(path.clone(), None).expect("start readiness listener"); + check_control_readiness(&path).expect("running supervisor accepts readiness probes"); + + drop(readiness); + tokio::task::yield_now().await; + assert!(check_control_readiness(&path).is_err()); + } + + #[tokio::test] + async fn control_readiness_tracks_supervisor_session() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("health.sock"); + let (session_tx, session_rx) = tokio::sync::watch::channel(true); + let _readiness = ControlReadiness::start(path.clone(), Some(session_rx)) + .expect("start readiness listener"); + check_control_readiness(&path).expect("accepted session is ready"); + + session_tx.send_replace(false); + timeout(Duration::from_secs(1), async { + while check_control_readiness(&path).is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("lost session removes readiness socket"); + + session_tx.send_replace(true); + timeout(Duration::from_secs(1), async { + while check_control_readiness(&path).is_err() { + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement session restores readiness socket"); + } + + #[test] + fn control_readiness_rejects_relative_path() { + let error = prepare_control_readiness_path(std::path::Path::new("health.sock")) + .expect_err("relative readiness path must be rejected"); + assert!(error.to_string().contains("must be absolute")); + } + + #[test] + fn main_exit_marker_atomically_replaces_previous_value() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("main-exited"); + std::fs::write(&marker, b"stale\n").unwrap(); + + persist_main_exit_marker(&marker, 23).unwrap(); + + assert_eq!(std::fs::read_to_string(&marker).unwrap(), "exit_code=23\n"); + assert!( + !directory + .path() + .join(format!(".main-exited.tmp-{}", std::process::id())) + .exists() + ); + } + + #[tokio::test] + async fn remote_access_plane_outlives_main_completion_until_teardown() { + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let retained = retain_remote_access_plane(std::future::pending(), async { + let _ = shutdown_rx.await; + }); + tokio::pin!(retained); + + assert!( + timeout(Duration::from_millis(10), &mut retained) + .await + .is_err(), + "access plane must remain live after canonical process completion" + ); + shutdown_tx.send(()).expect("request teardown"); + timeout(Duration::from_secs(1), &mut retained) + .await + .expect("teardown should release retained access plane") + .expect("clean teardown"); + } + + #[tokio::test] + async fn completion_retry_phase_is_cancelled_by_shutdown() { + let mut shutdown = Box::pin(std::future::ready(())); + assert!( + completion_phase_or_shutdown(std::future::pending(), shutdown.as_mut()).await, + "shutdown must cancel an indefinitely retrying completion phase" + ); + } + + #[test] + fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { + let agent_proposals = AgentProposals::default(); + let installs = AtomicUsize::new(0); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(!agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + } + + #[test] + fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { + let enabled = AtomicBool::new(false); + let mut settings = std::collections::HashMap::new(); + settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(enabled.load(Ordering::Relaxed)); + } + + #[test] + fn apply_ocsf_json_setting_disables_when_setting_is_unset() { + let enabled = AtomicBool::new(true); + let settings = std::collections::HashMap::new(); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(!enabled.load(Ordering::Relaxed)); + } + + #[test] + fn agent_proposals_setting_enables_from_initial_settings_snapshot() { + let mut settings = std::collections::HashMap::new(); + settings.insert( + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), + effective_bool(true), + ); + + assert!(agent_proposals_enabled_from_settings(&settings)); + } + + #[test] + fn agent_proposals_setting_defaults_false_when_unset() { + let settings = std::collections::HashMap::new(); + + assert!(!agent_proposals_enabled_from_settings(&settings)); + } + + // ---- Policy disk discovery tests ---- + + #[test] + fn discover_policy_from_nonexistent_path_returns_restrictive_default() { + let path = std::path::Path::new("/nonexistent/policy.yaml"); + let policy = discover_policy_from_path(path); + // Restrictive default has no network policies. + assert!(policy.network_policies.is_empty()); + // It keeps filesystem restrictions while leaving identity to the + // active compute driver. + assert!(policy.filesystem.is_some()); + assert!(policy.process.is_none()); + } + + #[test] + fn discover_policy_from_valid_yaml_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +filesystem_policy: + include_workdir: false + read_only: + - /usr + read_write: + - /tmp +network_policies: + test: + name: test + endpoints: + - { host: example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + assert_eq!(policy.network_policies.len(), 1); + assert!(policy.network_policies.contains_key("test")); + let fs = policy.filesystem.unwrap(); + assert!(!fs.include_workdir); + } + + #[test] + fn discover_policy_from_invalid_yaml_returns_restrictive_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default. + assert!(policy.network_policies.is_empty()); + assert!(policy.filesystem.is_some()); + } + + #[test] + fn discover_policy_from_unsafe_yaml_falls_back_to_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +process: + run_as_user: root + run_as_group: root +filesystem_policy: + include_workdir: true + read_only: + - /usr + read_write: + - /tmp +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default because of root user. + assert!(policy.process.is_none()); + } + + #[test] + fn discover_policy_restrictive_default_blocks_network() { + // In cluster mode we keep proxy mode enabled so `inference.local` + // can always be routed through proxy/OPA controls. + let proto = openshell_policy::restrictive_default_policy(); + let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); + assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); + } + + // ---- Initial policy acknowledgement tests ---- + + fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::restrictive_default_policy() + } + + fn proto_tcp_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::parse_sandbox_policy( + r#" +version: 1 +network_policies: + redis: + name: redis + endpoints: + - host: redis.example.com + port: 6379 + protocol: tcp + binaries: + - path: /usr/bin/redis-cli +"#, + ) + .expect("parse TCP policy") + } + + fn settings_poll_result( + policy: Option, + version: u32, + source: openshell_core::proto::PolicySource, + ) -> openshell_core::grpc_client::SettingsPollResult { + openshell_core::grpc_client::SettingsPollResult { + policy, + version, + policy_hash: format!("hash-v{version}"), + config_revision: u64::from(version) * 100, + policy_source: source, + settings: std::collections::HashMap::new(), + global_policy_version: 0, + provider_env_revision: 0, + supervisor_middleware_services: Vec::new(), + workspace: String::new(), + policy_validation_failure_mode: PolicyValidationFailureMode::default(), + extension_authentication_enabled: false, + } + } + + #[derive(Clone)] + struct ScriptedPolicyGateway { + polls: Arc< + tokio::sync::Mutex< + tokio::sync::mpsc::UnboundedReceiver< + openshell_core::grpc_client::SettingsPollResult, + >, + >, + >, + reports: UnboundedSender<(u32, bool, String)>, + } + + #[tonic::async_trait] + impl PolicyGatewayClient for ScriptedPolicyGateway { + async fn poll_settings( + &self, + _sandbox_id: &str, + ) -> Result { + self.polls + .lock() + .await + .recv() + .await + .ok_or_else(|| miette::miette!("scripted policy poll channel closed")) + } + + async fn report_policy_status( + &self, + _sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()> { + self.reports + .send((version, loaded, error.to_string())) + .map_err(|_| miette::miette!("scripted policy report channel closed")) + } + + fn workspace(&self) -> String { + "test-workspace".to_string() + } + } + + #[derive(Clone)] + struct CredentialRejectingPolicyGateway { + inner: ScriptedPolicyGateway, + credential_requests: Arc, + } + + #[tonic::async_trait] + impl PolicyGatewayClient for CredentialRejectingPolicyGateway { + async fn poll_settings( + &self, + sandbox_id: &str, + ) -> Result { + self.inner.poll_settings(sandbox_id).await + } + + async fn report_policy_status( + &self, + sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()> { + self.inner + .report_policy_status(sandbox_id, version, loaded, error) + .await + } + + async fn extension_credentials_for( + &self, + _services: &[openshell_core::proto::SupervisorMiddlewareService], + ) -> Result> + { + self.credential_requests.fetch_add(1, Ordering::SeqCst); + Err(miette::miette!( + "gateway extension authentication is unavailable" + )) + } + + fn workspace(&self) -> String { + self.inner.workspace() + } + } + + fn scripted_policy_gateway() -> ( + ScriptedPolicyGateway, + UnboundedSender, + tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, + ) { + let (poll_tx, poll_rx) = tokio::sync::mpsc::unbounded_channel(); + let (report_tx, report_rx) = tokio::sync::mpsc::unbounded_channel(); + ( + ScriptedPolicyGateway { + polls: Arc::new(tokio::sync::Mutex::new(poll_rx)), + reports: report_tx, + }, + poll_tx, + report_rx, + ) + } + + fn policy_poll_test_context( + opa_engine: Arc, + loaded_policy_origin: LoadedPolicyOrigin, + middleware_connector: MiddlewareConnector, + ) -> PolicyPollLoopContext { + let (workspace_tx, _workspace_rx) = tokio::sync::watch::channel(String::new()); + PolicyPollLoopContext { + endpoint: String::new(), + sandbox_id: "sandbox-test".to_string(), + opa_engine, + loaded_policy_origin, + entrypoint_pid: Arc::new(AtomicU32::new(0)), + interval_secs: 0, + ocsf_enabled: Arc::new(AtomicBool::new(false)), + provider_credentials: ProviderCredentialState::from_child_env_snapshot( + 0, + std::collections::HashMap::new(), + ), + policy_local_ctx: None, + agent_proposals: AgentProposals::default(), + middleware_registry_status: MiddlewareRegistryStatus::Synchronized, + workspace_tx, + extension_credentials: openshell_extension_core::ExtensionCredentialStore::new(), + extension_authentication_enabled: false, + middleware_connector, + transparent_tcp: TransparentTcpReloadState::default(), + } + } + + async fn expect_policy_report( + reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, + version: u32, + ) { + let report = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("policy report timed out") + .expect("policy reporter stopped"); + assert_eq!(report, (version, true, String::new())); + } + + async fn expect_no_policy_report( + reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, + ) { + assert!( + timeout(Duration::from_millis(50), reports.recv()) + .await + .is_err(), + "unexpected policy status report" + ); + } + + #[tokio::test] + async fn same_hash_poll_revision_is_acknowledged_once_without_opa_reload() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + + polls.send(v2.clone()).unwrap(); + expect_policy_report(&mut reports, 2).await; + polls.send(v2).unwrap(); + expect_no_policy_report(&mut reports).await; + + assert_eq!( + engine.current_generation(), + 0, + "same-hash acknowledgement must not reload OPA" + ); + handle.abort(); + } + + #[tokio::test] + async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { + let v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let v2 = settings_poll_result( + Some(proto_tcp_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let active_generation = engine.current_generation(); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.transparent_tcp = TransparentTcpReloadState { + capable: true, + substrate_ready: false, + }; + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + let report = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("TCP rejection report timed out") + .expect("policy reporter stopped"); + + assert_eq!(report.0, 2); + assert!(!report.1); + assert!(report.2.contains("recreate the sandbox"), "{}", report.2); + assert!(report.2.contains("previous policy remains active")); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + handle.abort(); + } + + #[tokio::test] + async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + v2.supervisor_middleware_services = + vec![openshell_core::proto::SupervisorMiddlewareService { + name: "scripted-guard".to_string(), + grpc_endpoint: "http://scripted.invalid".to_string(), + ..Default::default() + }]; + + let connector_attempts = Arc::new(AtomicUsize::new(0)); + let (attempt_tx, mut attempt_rx) = tokio::sync::mpsc::unbounded_channel(); + let middleware_connector: MiddlewareConnector = { + let connector_attempts = connector_attempts.clone(); + Arc::new(move |_services, _authentication| { + let attempt = connector_attempts.fetch_add(1, Ordering::SeqCst) + 1; + attempt_tx.send(attempt).unwrap(); + Box::pin(async move { + if attempt == 1 { + Err(miette::miette!("scripted middleware connection failure")) + } else { + connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await + } + }) + }) + }; + + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + middleware_connector, + ); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + + polls.send(v2.clone()).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), attempt_rx.recv()) + .await + .unwrap(), + Some(1) + ); + expect_no_policy_report(&mut reports).await; + assert_eq!(engine.current_generation(), 0); + + polls.send(v2.clone()).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), attempt_rx.recv()) + .await + .unwrap(), + Some(2) + ); + expect_policy_report(&mut reports, 2).await; + assert_eq!(engine.current_generation(), 1); + + polls.send(v2).unwrap(); + expect_no_policy_report(&mut reports).await; + assert_eq!(connector_attempts.load(Ordering::SeqCst), 2); + handle.abort(); + } + + #[tokio::test] + async fn no_signer_capability_uses_legacy_middleware_connector_without_credentials() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + v2.supervisor_middleware_services = + vec![openshell_core::proto::SupervisorMiddlewareService { + name: "legacy-guard".to_string(), + grpc_endpoint: "http://legacy.invalid".to_string(), + ..Default::default() + }]; + assert!(!v2.extension_authentication_enabled); + + let (inner, polls, mut reports) = scripted_policy_gateway(); + let credential_requests = Arc::new(AtomicUsize::new(0)); + let client = CredentialRejectingPolicyGateway { + inner, + credential_requests: credential_requests.clone(), + }; + let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); + let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { + connector_tx + .send((authentication.credentials.len(), authentication.enabled)) + .unwrap(); + Box::pin(async move { + connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await + }) + }); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + connector, + ); + + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), connector_rx.recv()) + .await + .unwrap(), + Some((0, false)) + ); + expect_policy_report(&mut reports, 2).await; + assert_eq!(credential_requests.load(Ordering::SeqCst), 0); + handle.abort(); + } + + #[tokio::test] + async fn enabled_extension_authentication_keeps_credential_failure_fail_closed() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + v2.extension_authentication_enabled = true; + v2.supervisor_middleware_services = + vec![openshell_core::proto::SupervisorMiddlewareService { + name: "authenticated-guard".to_string(), + grpc_endpoint: "https://guard.invalid".to_string(), + ..Default::default() + }]; + + let (inner, polls, mut reports) = scripted_policy_gateway(); + let credential_requests = Arc::new(AtomicUsize::new(0)); + let client = CredentialRejectingPolicyGateway { + inner, + credential_requests: credential_requests.clone(), + }; + let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); + let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { + connector_tx + .send((authentication.credentials.len(), authentication.enabled)) + .unwrap(); + Box::pin(async move { + if authentication.enabled && authentication.credentials.is_empty() { + Err(miette::miette!( + "missing authenticated middleware credential" + )) + } else { + connect_middleware_registry(&[], &authentication).await + } + }) + }); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + connector, + ); + + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), connector_rx.recv()) + .await + .unwrap(), + Some((0, true)) + ); + expect_no_policy_report(&mut reports).await; + assert_eq!(credential_requests.load(Ordering::SeqCst), 1); + handle.abort(); + } + + async fn assert_poll_does_not_use_same_hash_acknowledgement( + initial: openshell_core::grpc_client::SettingsPollResult, + next: openshell_core::grpc_client::SettingsPollResult, + origin: LoadedPolicyOrigin, + initial_report: Option, + ) { + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let ctx = policy_poll_test_context(engine.clone(), origin, default_middleware_connector()); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(initial).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + + if let Some(version) = initial_report { + expect_policy_report(&mut reports, version).await; + } else { + expect_no_policy_report(&mut reports).await; + } + + polls.send(next).unwrap(); + expect_no_policy_report(&mut reports).await; + assert_eq!( + engine.current_generation(), + 0, + "negative same-hash scope must not reload OPA" + ); + handle.abort(); + } + + #[tokio::test] + async fn same_hash_ack_poll_loop_rejects_local_global_empty_equal_and_older_scopes() { + let mut sandbox_v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + sandbox_v1.policy_hash = "same-policy".to_string(); + let loaded_v1 = LoadedPolicyRevision::from_snapshot(&sandbox_v1); + let mut sandbox_v2 = sandbox_v1.clone(); + sandbox_v2.version = 2; + sandbox_v2.config_revision = 200; + + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v1.clone(), + sandbox_v2.clone(), + LoadedPolicyOrigin::LocalOverride, + None, + ) + .await; + + let mut global_v2 = sandbox_v2.clone(); + global_v2.policy_source = openshell_core::proto::PolicySource::Global; + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v1.clone(), + global_v2, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_v1.clone()), + has_last_valid_policy: true, + }, + Some(1), + ) + .await; + + let mut empty_v1 = sandbox_v1.clone(); + empty_v1.policy_hash.clear(); + let empty_loaded = LoadedPolicyRevision::from_snapshot(&empty_v1); + let mut empty_v2 = sandbox_v2.clone(); + empty_v2.policy_hash.clear(); + assert_poll_does_not_use_same_hash_acknowledgement( + empty_v1, + empty_v2, + LoadedPolicyOrigin::Gateway { + revision: Some(empty_loaded), + has_last_valid_policy: true, + }, + Some(1), + ) + .await; + + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v1.clone(), + sandbox_v1.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_v1.clone()), + has_last_valid_policy: true, + }, + Some(1), + ) + .await; + + let loaded_v2 = LoadedPolicyRevision::from_snapshot(&sandbox_v2); + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v2, + sandbox_v1, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_v2), + has_last_valid_policy: true, + }, + Some(2), + ) + .await; + } + + #[tokio::test] + async fn changed_hash_poll_uses_normal_opa_reload_and_status_path() { + let v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let v2 = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + expect_policy_report(&mut reports, 2).await; + assert_eq!( + engine.current_generation(), + 1, + "changed policy content must still reload OPA" + ); + handle.abort(); + } + + #[tokio::test] + async fn failed_external_startup_registry_build_preserves_installed_builtins() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let builtins_generation = engine.current_generation(); + assert_eq!(builtins_generation, 1); + + let invalid_external = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_payload_bytes: 1024, + ..Default::default() + }; + connect_middleware_registry(&[invalid_external], &MiddlewareAuthentication::default()) + .await + .expect_err("unavailable external service must not replace built-ins"); + + assert_eq!(engine.current_generation(), builtins_generation); + } + + #[tokio::test] + async fn unavailable_middleware_reload_keeps_last_known_good_runtime_active() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let active_generation = engine.current_generation(); + let unavailable_service = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_payload_bytes: 1024, + ..Default::default() + }; + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_policy_fixture()), + 0, + MiddlewareReloadContext { + desired_services: &[unavailable_service], + authentication: &MiddlewareAuthentication::default(), + registry_changed: true, + connector: &default_middleware_connector(), + }, + TransparentTcpReloadState::default(), + ) + .await + .expect_err("unavailable middleware must fail candidate preparation"); + let disposition = apply_gateway_runtime_reload_failure( + &engine, + failure, + PolicyValidationFailureMode::FailClosed, + true, + 2, + ) + .expect("middleware failure handling must succeed"); + + assert!(matches!( + disposition, + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } + )); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[tokio::test] + async fn tcp_policy_reload_without_startup_substrate_is_rejected_and_keeps_previous_policy() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + let active_generation = engine.current_generation(); + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_tcp_policy_fixture()), + 0, + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, + TransparentTcpReloadState { + capable: true, + substrate_ready: false, + }, + ) + .await + .expect_err("TCP expansion must require startup substrate"); + let disposition = apply_gateway_runtime_reload_failure( + &engine, + failure, + PolicyValidationFailureMode::FailClosed, + true, + 2, + ) + .expect("runtime prerequisite failure handling must succeed"); + + assert!(matches!( + disposition, + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + active_generation: generation, + .. + } if generation == active_generation + )); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[tokio::test] + async fn tcp_policy_reload_on_unsupported_runtime_is_rejected() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_tcp_policy_fixture()), + 0, + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, + TransparentTcpReloadState::default(), + ) + .await + .expect_err("unsupported runtime must reject TCP expansion"); + + assert!(matches!( + failure, + GatewayRuntimeReloadError::TransparentTcpPrerequisite(_) + )); + assert_eq!(engine.current_generation(), 0); + } + + #[test] + fn policy_rejection_after_middleware_outage_is_not_deduplicated() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let middleware_failure = GatewayRuntimeReloadError::MiddlewareRegistry(miette::miette!( + "middleware service unavailable" + )); + let first_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &middleware_failure); + let middleware_disposition = apply_gateway_runtime_reload_failure( + &engine, + middleware_failure, + PolicyValidationFailureMode::FailClosed, + true, + 7, + ) + .unwrap(); + + assert!(matches!( + middleware_disposition, + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } + )); + assert!(engine.fail_closed_reason().is_none()); + + let policy_failure = GatewayRuntimeReloadError::PolicyValidation(miette::miette!( + "conflicting endpoint metadata" + )); + let second_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &policy_failure); + assert_ne!( + first_failure, second_failure, + "a changed failure class for the same candidate must be handled" + ); + + let policy_disposition = apply_gateway_runtime_reload_failure( + &engine, + policy_failure, + PolicyValidationFailureMode::FailClosed, + true, + 7, + ) + .unwrap(); + assert!(matches!( + policy_disposition, + GatewayRuntimeFailureDisposition::PolicyRejected { .. } + )); + assert!(engine.fail_closed_reason().is_some()); + } + + #[test] + fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { + let services = Vec::new(); + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + } + + #[test] + fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &no_services, + &no_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &no_services, + &desired_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + false, + "local-policy", + "hash-v2", + &no_services, + &desired_services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + } + + #[test] + fn policy_only_change_does_not_rebuild_middleware_registry() { + let services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + // The runtime must reconcile, but the registry (and therefore + // middleware reachability) is not part of that reconciliation. + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &services, + &services, + )); + } + + #[test] + fn registry_rebuild_requires_service_set_change_or_degraded_registry() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &no_services, + &desired_services, + )); + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::NeedsReconciliation, + &desired_services, + &desired_services, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &desired_services, + &desired_services, + )); + } + + #[test] + fn initial_ack_candidate_matches_sandbox_revision() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) + .expect("sandbox-sourced matching revision should be acknowledged"); + + assert_eq!(ack.version, 2); + assert_eq!(ack.policy_hash, "hash-v2"); + assert_eq!(ack.config_revision, 200); + } + + #[test] + fn initial_ack_candidate_ignores_global_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Global, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_version_zero() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 0, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_local_file_mode() { + // Local-file mode retains no proto policy, so there is nothing to + // acknowledge to the gateway. + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(None, &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_rejects_mismatched_identity() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let mut newer = proto_policy_fixture(); + newer.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule::default(), + ); + let canonical = + settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); + let canonical = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "hash-provider-change".to_string(), + config_revision: loaded.config_revision + 1, + ..canonical + }; + + assert_eq!( + initial_poll_disposition( + &LoadedPolicyOrigin::Gateway { + revision: Some(loaded), + has_last_valid_policy: true, + }, + &canonical, + ), + InitialPollDisposition::Reconcile + ); + } + + #[test] + fn initial_poll_tracks_local_override_without_reconciliation() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert_eq!( + initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), + InitialPollDisposition::TrackOnly + ); + assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); + } + + #[test] + fn initial_poll_reconciles_unbound_gateway_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let origin = LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }; + + assert_eq!( + initial_poll_disposition(&origin, &canonical), + InitialPollDisposition::Reconcile + ); + assert!(origin.allows_gateway_policy_reload()); + } + + #[test] + fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { + let sandbox_result = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "same-policy".to_string(), + ..settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ) + }; + + assert_eq!( + unchanged_policy_revision_candidate(true, false, 1, "same-policy", &sandbox_result), + Some(2) + ); + assert_eq!( + unchanged_policy_revision_candidate(true, false, 2, "same-policy", &sandbox_result), + None + ); + assert_eq!( + unchanged_policy_revision_candidate( + true, + false, + 1, + "different-policy", + &sandbox_result, + ), + None + ); + assert_eq!( + unchanged_policy_revision_candidate(false, false, 1, "same-policy", &sandbox_result), + None + ); + assert_eq!( + unchanged_policy_revision_candidate(true, false, 1, "", &sandbox_result), + None + ); + assert_eq!( + unchanged_policy_revision_candidate(true, true, 1, "same-policy", &sandbox_result), + None + ); + + let global_result = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "same-policy".to_string(), + ..settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Global, + ) + }; + assert_eq!( + unchanged_policy_revision_candidate(true, false, 1, "same-policy", &global_result), + None + ); + } + + #[test] + fn unchanged_policy_revision_waits_for_required_runtime_reconciliation() { + assert_eq!( + unchanged_policy_revision_ready_to_ack(Some(2), false, false), + Some(2), + "a same-hash revision needs no OPA reload" + ); + assert_eq!( + unchanged_policy_revision_ready_to_ack(Some(2), true, false), + None, + "failed runtime reconciliation must keep the revision pending" + ); + assert_eq!( + unchanged_policy_revision_ready_to_ack(Some(2), true, true), + Some(2), + "successful runtime reconciliation permits acknowledgement" + ); + assert_eq!( + unchanged_policy_revision_ready_to_ack(None, false, true), + None, + "runtime success cannot manufacture a revision candidate" + ); + } + + #[test] + fn credential_gating_unavailable_for_local_override_with_credentials() { + assert!(credential_gating_unavailable( + &LoadedPolicyOrigin::LocalOverride, + true, + true + )); + } + + #[test] + fn credential_gating_available_without_local_override_or_credentials() { + // A gateway policy is stamped with provenance, so the gates apply. + assert!(!credential_gating_unavailable( + &LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }, + true, + true + )); + // No provider credentials means there is nothing to leak. + assert!(!credential_gating_unavailable( + &LoadedPolicyOrigin::LocalOverride, + false, + true + )); + // Without networking the proxy never evaluates endpoint provenance. + assert!(!credential_gating_unavailable( + &LoadedPolicyOrigin::LocalOverride, + true, + false + )); + } + + #[test] + fn policy_status_outbox_preserves_all_revision_order() { + let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); + for version in 1..=128 { + enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); + } + + for version in 1..=128 { + assert_eq!( + receiver.try_recv().unwrap(), + PolicyStatusUpdate::loaded(version) + ); + } + } + + #[test] + fn settings_snapshot_carries_workspace_for_policy_sync() { + let mut snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + snapshot.workspace = "beta".to_string(); + + let revision = LoadedPolicyRevision::from_snapshot(&snapshot); + assert_eq!(revision.version, 1); + assert_eq!( + snapshot.workspace, "beta", + "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" + ); + } + #[test] + fn fail_closed_validation_failure_deactivates_previous_generation() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(!disposition.previous_policy_active); + assert!(disposition.active_generation > previous_generation); + assert!( + engine + .fail_closed_reason() + .expect("quarantine reason") + .contains("candidate version 7 rejected") + ); + } + + #[test] + fn retain_validation_failure_keeps_previous_generation_active() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let quarantined = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 6, + "conflicting tls metadata", + ) + .unwrap(); + assert!(!quarantined.previous_policy_active); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(disposition.previous_policy_active); + assert!(disposition.active_generation > quarantined.active_generation); + assert!(disposition.active_generation > previous_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[test] + fn retain_validation_failure_without_last_valid_policy_stays_fail_closed() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + false, + 1, + "conflicting tls metadata", + ) + .unwrap(); + + assert_eq!( + disposition.configured_mode, + PolicyValidationFailureMode::RetainLastValid + ); + assert_eq!(disposition.mode, PolicyValidationFailureMode::FailClosed); + assert!(!disposition.previous_policy_active); + assert!(engine.fail_closed_reason().is_some()); + + let [config, _] = policy_validation_failure_events( + &disposition, + 1, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "retain_last_valid" + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + } + + #[test] + fn validation_failure_ocsf_states_whether_previous_policy_is_active() { + let fail_closed = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::FailClosed, + mode: PolicyValidationFailureMode::FailClosed, + previous_policy_active: false, + active_generation: 9, + }; + let [config, finding] = policy_validation_failure_events( + &fail_closed, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["class_uid"], 5019); + assert_eq!(config["status"], "Failure"); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "fail_closed" + ); + assert_eq!(config["unmapped"]["previous_policy_active"], false); + assert_eq!( + config["unmapped"]["validation_error"], + "conflicting tls metadata" + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("error:conflicting tls metadata") + ); + + let finding = finding.to_json().unwrap(); + assert_eq!(finding["class_uid"], 2004); + assert_eq!(finding["action"], "Denied"); + assert_eq!(finding["disposition"], "Blocked"); + + let retained = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::RetainLastValid, + mode: PolicyValidationFailureMode::RetainLastValid, + previous_policy_active: true, + active_generation: 4, + }; + let [config, _] = policy_validation_failure_events( + &retained, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["previous_policy_active"], true); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS active") + ); + } +} diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs new file mode 100644 index 0000000000..a33e652a46 --- /dev/null +++ b/crates/openshell-supervisor/src/main.rs @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `OpenShell` supervisor executable. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_isolation_interface::contract::TopologyDescriptor; +use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; + +const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; +const HEALTH_SUBCOMMAND: &str = "health"; + +#[derive(Parser, Debug)] +#[command(name = "openshell-supervisor health")] +struct HealthArgs { + /// Private supervisor readiness socket. + #[arg(long, env = "OPENSHELL_HEALTH_SOCKET_PATH")] + socket: PathBuf, +} + +#[derive(Parser, Debug)] +#[command(name = "openshell-supervisor")] +#[command(version = openshell_core::VERSION)] +#[command(about = "OpenShell policy and workload supervisor")] +#[allow(clippy::struct_excessive_bools)] +struct Args { + /// Command to execute as the canonical workload process. + #[arg(trailing_var_arg = true)] + command: Vec, + + #[arg(long, short)] + workdir: Option, + + #[arg(long, short, default_value = "0")] + timeout: u64, + + #[arg(long, short = 'i')] + interactive: bool, + + #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] + sandbox_id: Option, + + #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] + sandbox: Option, + + #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] + openshell_endpoint: Option, + + #[arg(long, env = "OPENSHELL_POLICY_RULES")] + policy_rules: Option, + + #[arg(long, env = "OPENSHELL_POLICY_DATA")] + policy_data: Option, + + #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] + ssh_socket_path: Option, + + #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] + inference_routes: Option, + + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, + + /// Create the private readiness socket after boundary and gateway attach. + #[arg(long, env = "OPENSHELL_HEALTH_SOCKET_PATH")] + health_socket_path: Option, + + #[arg(long)] + upstream_proxy: Option, + + /// Driver-pinned TCP dial address for the configured upstream proxy. + #[arg(long)] + upstream_proxy_dial_ip: Option, + + #[arg(long)] + upstream_no_proxy: Option, + + #[arg(long)] + upstream_proxy_auth_file: Option, + + #[arg(long)] + upstream_proxy_auth_allow_insecure: bool, + + #[arg(long)] + upstream_proxy_connect_by_hostname: bool, + + #[arg(long)] + upstream_proxy_ca_bundle: Option, + + #[arg(long)] + topology_backend_name: String, + + #[arg(long)] + topology_payload_file: PathBuf, + + #[arg(long, hide = true)] + main_exit_marker: Option, +} + +fn topology(args: &Args) -> Result { + let payload = std::fs::read(&args.topology_payload_file).map_err(|error| { + miette::miette!( + "read topology payload {}: {error}", + args.topology_payload_file.display() + ) + })?; + Ok(TopologyDescriptor { + backend_name: args.topology_backend_name.clone(), + payload, + }) +} + +fn validate_main_exit_marker(marker: Option<&Path>) -> Result<()> { + if let Some(marker) = marker + && !marker.is_absolute() + { + return Err(miette::miette!( + "--main-exit-marker must be an absolute path" + )); + } + Ok(()) +} + +fn main() -> Result<()> { + let raw_args = std::env::args().collect::>(); + if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .into_diagnostic()?; + return runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; + std::process::exit(exit); + }); + } + if raw_args.get(1).map(String::as_str) == Some(HEALTH_SUBCOMMAND) { + let args = HealthArgs::parse_from(&raw_args[1..]); + return openshell_supervisor::check_control_readiness(&args.socket); + } + + let args = Args::parse(); + validate_main_exit_marker(args.main_exit_marker.as_deref())?; + let topology = topology(&args)?; + + let file_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + (writer, guard) + }); + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .into_diagnostic()?; + + let exit_code = runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = + (&args.sandbox_id, &args.openshell_endpoint) + { + let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( + endpoint.clone(), + sandbox_id.clone(), + ); + let layer = + openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); + Some((layer, handle)) + } else { + None + }; + let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); + let _log_push_handle = log_push_state.map(|(_, handle)| handle); + let ocsf_enabled = Arc::new(AtomicBool::new(false)); + + let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { + let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell-ocsf") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + (layer, guard) + }); + let (jsonl_layer, jsonl_guard) = + jsonl_logging.map_or((None, None), |(layer, guard)| (Some(layer), Some(guard))); + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with( + OcsfShorthandLayer::new(file_writer) + .with_non_ocsf(true) + .with_filter(EnvFilter::new("info")), + ) + .with(jsonl_layer.with_filter(LevelFilter::INFO)) + .with(push_layer.clone()) + .init(); + (Some(file_guard), jsonl_guard) + } else { + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with(push_layer) + .init(); + warn!("Could not open /var/log for log rotation; using stderr-only logging"); + (None, None) + }; + + let workdir = args.workdir.clone(); + let (command, interactive, await_main_process_attachment) = if !args.command.is_empty() { + (args.command, args.interactive, false) + } else if let Ok(json) = std::env::var(openshell_core::sandbox_env::MAIN_PROCESS_SPEC) { + let config = openshell_core::sandbox_env::MainProcessConfig::decode(&json) + .map_err(|error| miette::miette!("{error}"))?; + ( + config.command, + config.tty, + config.await_main_process_attachment, + ) + } else { + let config = openshell_core::sandbox_env::MainProcessConfig::scratch(); + ( + config.command, + config.tty, + config.await_main_process_attachment, + ) + }; + info!(command = ?command, "Starting sandbox supervision"); + + let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { + https_proxy: args.upstream_proxy, + proxy_dial_ip: args.upstream_proxy_dial_ip, + no_proxy: args.upstream_no_proxy, + proxy_auth_file: args.upstream_proxy_auth_file, + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, + proxy_ca_bundle: args.upstream_proxy_ca_bundle, + }; + let admitted_isolation_backend = + std::env::var(openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND).ok(); + + openshell_supervisor::run_sandbox( + command, + workdir, + args.timeout, + interactive, + await_main_process_attachment, + args.sandbox_id, + args.sandbox, + args.openshell_endpoint, + args.policy_rules, + args.policy_data, + args.ssh_socket_path, + args.health_socket_path, + args.inference_routes, + ocsf_enabled, + upstream_proxy_args, + topology, + admitted_isolation_backend, + args.main_exit_marker, + ) + .await + })?; + + std::process::exit(exit_code); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn role_specific_cli_has_no_mode_switch() { + let directory = tempfile::tempdir().expect("temporary topology directory"); + let topology_path = directory.path().join("topology.json"); + std::fs::write(&topology_path, [0]).expect("write topology payload"); + let args = Args::try_parse_from([ + "openshell-supervisor", + "--topology-backend-name", + "test", + "--topology-payload-file", + topology_path.to_str().expect("UTF-8 topology path"), + ]) + .expect("supervisor arguments"); + assert_eq!(topology(&args).expect("topology").payload, vec![0]); + } + + #[test] + fn topology_payload_is_mandatory() { + assert!( + Args::try_parse_from(["openshell-supervisor", "--topology-backend-name", "test"]) + .is_err() + ); + } + + #[test] + fn completion_marker_must_be_absolute() { + assert!(validate_main_exit_marker(Some(Path::new("relative"))).is_err()); + assert!(validate_main_exit_marker(Some(Path::new("/run/openshell/main-exit"))).is_ok()); + } +} diff --git a/crates/openshell-supervisor/src/mechanistic_mapper.rs b/crates/openshell-supervisor/src/mechanistic_mapper.rs new file mode 100644 index 0000000000..186cc68e02 --- /dev/null +++ b/crates/openshell-supervisor/src/mechanistic_mapper.rs @@ -0,0 +1,790 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor policy mapper — deterministically converts denial summaries into +//! draft `NetworkPolicyRule` proposals. +//! +//! This is the "zero-LLM" baseline for policy recommendations. It inspects +//! denial patterns (host, port, binary, frequency) and generates concrete rules +//! that would allow the denied connections, annotated with confidence scores and +//! security notes. +//! +//! The LLM-powered `PolicyAdvisor` (issue #205) wraps and enriches these +//! mechanistic proposals with context-aware rationale and smarter grouping. + +use openshell_core::net::{is_always_blocked_ip, is_internal_ip, is_known_metadata_hostname}; +use openshell_core::proto::{ + DenialSummary, L7Allow, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, PolicyChunk, +}; +use std::collections::HashMap; +use std::net::IpAddr; + +/// Well-known ports that get higher confidence scores. +const WELL_KNOWN_PORTS: &[(u16, &str)] = &[ + (80, "HTTP"), + (443, "HTTPS"), + (8080, "HTTP-alt"), + (8443, "HTTPS-alt"), + (5432, "PostgreSQL"), + (3306, "MySQL"), + (6379, "Redis"), + (27017, "MongoDB"), + (9200, "Elasticsearch"), + (9092, "Kafka"), + (2181, "ZooKeeper"), + (11211, "Memcached"), + (5672, "RabbitMQ"), + (6443, "Kubernetes API"), + (53, "DNS"), + (587, "SMTP"), + (993, "IMAP"), + (995, "POP3"), +]; + +/// Generate draft `PolicyChunk` proposals from denial summaries. +/// +/// Groups denials by `(host, port, binary)`, then for each group generates a +/// `PolicyChunk` with a `NetworkPolicyRule` allowing that endpoint for that +/// single binary. This produces one proposal per binary so each +/// `(sandbox_id, host, port, binary)` maps to exactly one DB row. +/// +/// Proposals never include `allowed_ips`. If the user applies a proposed rule +/// and the host resolves to a private IP, the proxy's SSRF defense will deny +/// the connection. That SSRF denial flows back through the aggregator, and the +/// user can then explicitly add `allowed_ips` to their policy. This two-step +/// flow avoids DNS resolution in the mapper, which would leak the denied +/// hostname via DNS even though the connection was blocked. See #1169. +/// +/// Returns an empty vec if there are no actionable denials. +pub fn generate_proposals(summaries: &[DenialSummary]) -> Vec { + // Group denials by (host, port, binary). + let mut groups: HashMap<(String, u32, String), Vec<&DenialSummary>> = HashMap::new(); + + for summary in summaries { + let binary_key = if summary.binary.is_empty() { + String::new() + } else { + summary.binary.clone() + }; + groups + .entry((summary.host.clone(), summary.port, binary_key)) + .or_default() + .push(summary); + } + + let mut proposals = Vec::new(); + + for ((host, port, binary), denials) in &groups { + let rule_name = generate_rule_name(host, *port); + + let mut total_count: u32 = 0; + let mut first_seen_ms: i64 = i64::MAX; + let mut last_seen_ms: i64 = 0; + let mut is_ssrf = false; + + for denial in denials { + total_count += denial.count; + first_seen_ms = first_seen_ms.min(denial.first_seen_ms); + last_seen_ms = last_seen_ms.max(denial.last_seen_ms); + if denial.denial_stage == "ssrf" { + is_ssrf = true; + } + } + + // Collect L7 request samples across all denials in this group. + let mut l7_methods: HashMap<(String, String), u32> = HashMap::new(); + let mut has_l7 = false; + for denial in denials { + if denial.l7_inspection_active || !denial.l7_request_samples.is_empty() { + has_l7 = true; + } + for sample in &denial.l7_request_samples { + *l7_methods + .entry((sample.method.clone(), sample.path.clone())) + .or_insert(0) += sample.count; + } + } + + // Skip proposals for always-blocked destinations (loopback, + // link-local, unspecified, and known metadata hostnames). These would + // be denied at runtime regardless of policy, producing an infinite + // proposal loop in the TUI. + if is_always_blocked_destination(host) { + tracing::info!( + host, + port, + "Skipped proposal for always-blocked destination \ + (SSRF hardening — loopback/link-local/unspecified/metadata)" + ); + continue; + } + + // Build proposed NetworkPolicyRule. + let l7_rules = build_l7_rules(&l7_methods); + let endpoint = if has_l7 && !l7_rules.is_empty() { + NetworkEndpoint { + host: host.clone(), + port: *port, + ports: vec![*port], + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + rules: l7_rules, + advisor_proposed: true, + ..Default::default() + } + } else { + NetworkEndpoint { + host: host.clone(), + port: *port, + ports: vec![*port], + advisor_proposed: true, + ..Default::default() + } + }; + + let binaries: Vec = if binary.is_empty() { + vec![] + } else { + let mut proposal_binary = NetworkBinary { + path: binary.clone(), + ..Default::default() + }; + // The deprecated harness bit is ignored by policy YAML, but OPA + // maps it to advisor_proposed to preserve the SSRF two-step flow. + #[allow(deprecated)] + { + proposal_binary.harness = true; + } + vec![proposal_binary] + }; + + let proposed_rule = NetworkPolicyRule { + name: rule_name.clone(), + endpoints: vec![endpoint], + binaries, + }; + + // Compute confidence. + #[allow(clippy::cast_possible_truncation)] + let confidence = compute_confidence(total_count, *port as u16, is_ssrf); + + // Generate rationale. + let binary_list = if binary.is_empty() { + "unknown binary".to_string() + } else { + short_binary_name(binary) + }; + + #[allow(clippy::cast_possible_truncation)] + let port_u16 = *port as u16; + let port_name = WELL_KNOWN_PORTS + .iter() + .find(|(p, _)| *p == port_u16) + .map(|(_, name)| format!(" ({name})")) + .unwrap_or_default(); + + // Note: hit_count in the DB accumulates across flush cycles, so we + // don't bake a denial count into the rationale text (it would go stale). + let rationale = if has_l7 && !l7_methods.is_empty() { + let paths: Vec = l7_methods.keys().map(|(m, p)| format!("{m} {p}")).collect(); + format!( + "Allow {binary_list} to connect to {host}:{port}{port_name} \ + with L7 inspection. \ + Allowed paths: {}.", + paths.join(", ") + ) + } else { + format!( + "Allow {binary_list} to connect to \ + {host}:{port}{port_name}." + ) + }; + + // Generate security notes. + #[allow(clippy::cast_possible_truncation)] + let security_notes = generate_security_notes(host, *port as u16, is_ssrf); + + // Determine stage based on denial source. + let stage = denials + .first() + .map_or_else(|| "connect".to_string(), |d| d.denial_stage.clone()); + + proposals.push(PolicyChunk { + id: String::new(), // Assigned by the gateway on persist + status: "pending".to_string(), + rule_name, + proposed_rule: Some(proposed_rule), + rationale, + security_notes, + confidence, + denial_summary_ids: vec![], + created_at_ms: 0, // Set by gateway on persist + decided_at_ms: 0, + stage, + supersedes_chunk_id: String::new(), + hit_count: total_count.cast_signed(), + first_seen_ms, + last_seen_ms, + binary: binary.clone(), + validation_result: String::new(), + rejection_reason: String::new(), + ..Default::default() + }); + } + + // Sort proposals by confidence (highest first). + proposals.sort_by(|a, b| { + b.confidence + .partial_cmp(&a.confidence) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + proposals +} + +/// Generate a rule name that doesn't conflict with existing rules. +/// Generate a deterministic, idempotent rule name from host and port. +/// +/// The same `(host, port)` always produces the same name. DB-level dedup on +/// `(sandbox_id, host, port, binary)` handles collisions — no need to check +/// existing rule names. +fn generate_rule_name(host: &str, port: u32) -> String { + let sanitized = host + .replace(['.', '-'], "_") + .chars() + .filter(|c| c.is_alphanumeric() || *c == '_') + .collect::(); + + format!("allow_{sanitized}_{port}") +} + +/// Compute a confidence score (0.0 to 1.0) for a proposed rule. +fn compute_confidence(total_count: u32, port: u16, is_ssrf: bool) -> f32 { + let mut score: f32 = 0.5; + + // Higher count → higher confidence (the denial is repeatable). + if total_count >= 10 { + score += 0.2; + } else if total_count >= 3 { + score += 0.1; + } + + // Well-known port → higher confidence. + if WELL_KNOWN_PORTS.iter().any(|(p, _)| *p == port) { + score += 0.15; + } + + // SSRF denials are lower confidence (may be legitimate blocking). + if is_ssrf { + score -= 0.2; + } + + score.clamp(0.1, 0.95) +} + +/// Generate security notes for a proposed rule. +fn generate_security_notes(host: &str, port: u16, is_ssrf: bool) -> String { + let mut notes = Vec::new(); + + if is_ssrf { + notes.push( + "This connection was blocked by SSRF protection. \ + Private IP access requires an explicit `allowed_ips` policy entry." + .to_string(), + ); + } + + // Flag destinations that are an internal/private address. Parse the host as + // an IP literal and defer to the canonical RFC-accurate classifier + // (openshell-core net::is_internal_ip) rather than naive string prefixes: + // `starts_with("172.")` wrongly matched 172.0-15 / 172.32-255 (RFC 1918 is + // only 172.16.0.0/12) and missed CGNAT (100.64.0.0/10), IPv6 ULA, etc. The + // "localhost" hostname is not an IP literal, so it is checked separately. + // See #1777. + let resolves_internal = host.parse::().is_ok_and(is_internal_ip); + if resolves_internal || host == "localhost" { + notes.push(format!( + "Destination '{host}' appears to be an internal/private address." + )); + } + + // High port numbers may indicate ephemeral services. + if port > 49152 { + notes.push(format!( + "Port {port} is in the ephemeral range — \ + this may be a temporary service." + )); + } + + // Database ports get extra scrutiny. + let db_ports = [5432, 3306, 6379, 27017, 9200, 11211, 5672]; + if db_ports.contains(&port) { + notes.push(format!( + "Port {port} is a well-known database/service port. \ + Consider restricting with L7 rules or read-only access." + )); + } + + notes.join(" ") +} + +/// Build L7 allow-rules from observed (method, path) samples. +/// +/// Groups paths by HTTP method and generalises path patterns where possible: +/// - `/v1/models/abc123` → `/v1/models/**` (ID-like trailing segments) +/// - `/api/v2/users/42` → `/api/v2/users/*` (numeric trailing segment) +/// +/// Falls back to the exact observed path when no pattern applies. +fn build_l7_rules(samples: &HashMap<(String, String), u32>) -> Vec { + // Deduplicate after generalisation. + let mut seen: std::collections::HashSet<(String, String)> = std::collections::HashSet::new(); + let mut rules = Vec::new(); + + for (method, path) in samples.keys() { + let generalised = generalise_path(path); + let key = (method.clone(), generalised.clone()); + if !seen.insert(key) { + continue; + } + + rules.push(L7Rule { + allow: Some(L7Allow { + method: method.clone(), + path: generalised, + command: String::new(), + query: HashMap::new(), + operation_type: String::new(), + operation_name: String::new(), + fields: Vec::new(), + params: HashMap::new(), + }), + }); + } + + // Sort for deterministic output. + rules.sort_by(|a, b| { + let a = a.allow.as_ref().unwrap(); + let b = b.allow.as_ref().unwrap(); + (&a.method, &a.path).cmp(&(&b.method, &b.path)) + }); + + rules +} + +/// Generalise a URL path for policy rules. +/// +/// Heuristics: +/// - Strip query strings. +/// - If the last segment looks like an ID (hex, UUID, or numeric), replace +/// with `*`. +/// - Preserve all other segments verbatim. +fn generalise_path(raw: &str) -> String { + // Strip query string. + let path = raw.split('?').next().unwrap_or(raw); + + let segments: Vec<&str> = path.split('/').collect(); + if segments.len() <= 1 { + return path.to_string(); + } + + let last = segments.last().unwrap_or(&""); + + // Replace ID-like trailing segments with a wildcard. + if looks_like_id(last) { + let mut out = segments[..segments.len() - 1].join("/"); + out.push_str("/*"); + return out; + } + + path.to_string() +} + +/// Heuristic: does a path segment look like an opaque identifier? +fn looks_like_id(segment: &str) -> bool { + if segment.is_empty() { + return false; + } + // Pure numeric + if segment.chars().all(|c| c.is_ascii_digit()) && segment.len() >= 2 { + return true; + } + // UUID-ish (contains dashes, 32+ hex chars) + let hex_only: String = segment.chars().filter(char::is_ascii_hexdigit).collect(); + if hex_only.len() >= 24 && segment.contains('-') { + return true; + } + // Long hex string (hash, token) + if hex_only.len() >= 16 && segment.len() == hex_only.len() { + return true; + } + false +} + +/// Extract just the binary name from a full path. +fn short_binary_name(path: &str) -> String { + path.rsplit('/').next().unwrap_or(path).to_string() +} + +/// Check if a destination host is always-blocked. +/// +/// For literal IP hosts, checks against [`is_always_blocked_ip`]. +/// For hostnames, checks well-known loopback and cloud metadata names. +/// For other hostnames, returns false (DNS may resolve to anything). +fn is_always_blocked_destination(host: &str) -> bool { + // Check literal IP addresses + if let Ok(ip) = host.parse::() { + return is_always_blocked_ip(ip); + } + // Check well-known loopback hostnames + let host_lc = host.to_lowercase(); + host_lc == "localhost" || host_lc == "localhost." || is_known_metadata_hostname(host) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_rule_name() { + let name = generate_rule_name("example.com", 443); + assert_eq!(name, "allow_example_com_443"); + } + + #[test] + fn test_generate_rule_name_subdomain() { + let name = generate_rule_name("api.github.com", 443); + assert_eq!(name, "allow_api_github_com_443"); + } + + #[test] + fn test_compute_confidence() { + // Well-known port + high count + let conf = compute_confidence(10, 443, false); + assert!(conf > 0.8); + + // SSRF + let conf = compute_confidence(5, 80, true); + assert!(conf < 0.6); + } + + #[test] + fn test_security_notes_ssrf() { + let notes = generate_security_notes("169.254.169.254", 80, true); + assert!(notes.contains("SSRF")); + } + + #[test] + fn test_security_notes_internal_ip_uses_canonical_classifier() { + // RFC 1918 is 172.16.0.0/12 only: the old starts_with("172.") prefix + // wrongly flagged 172.15/172.32 and missed CGNAT (100.64.0.0/10). #1777. + assert!(generate_security_notes("172.16.0.1", 80, false).contains("internal/private")); + assert!(!generate_security_notes("172.15.0.1", 80, false).contains("internal/private")); + assert!(!generate_security_notes("172.32.0.1", 80, false).contains("internal/private")); + assert!(generate_security_notes("100.64.0.1", 80, false).contains("internal/private")); + assert!(generate_security_notes("10.0.0.1", 80, false).contains("internal/private")); + assert!(generate_security_notes("192.168.1.1", 80, false).contains("internal/private")); + assert!(generate_security_notes("127.0.0.1", 80, false).contains("internal/private")); + assert!(generate_security_notes("localhost", 80, false).contains("internal/private")); + assert!(!generate_security_notes("8.8.8.8", 80, false).contains("internal/private")); + // Hostnames that merely start with a private-range prefix must NOT be + // flagged: classification parses an IP literal, not a string prefix. #1824. + assert!(!generate_security_notes("10.example.com", 80, false).contains("internal/private")); + assert!( + !generate_security_notes("172.example.com", 80, false).contains("internal/private") + ); + // IPv6 ULA (fc00::/7, RFC 4193) is internal/private. + assert!(generate_security_notes("fd00::1", 80, false).contains("internal/private")); + } + + #[test] + fn test_generate_proposals_empty() { + let proposals = generate_proposals(&[]); + assert!(proposals.is_empty()); + } + + #[test] + fn test_generate_proposals_basic() { + let summaries = vec![DenialSummary { + sandbox_id: "test".to_string(), + host: "api.example.com".to_string(), + port: 443, + binary: "/usr/bin/curl".to_string(), + ancestors: vec![], + deny_reason: "no matching policy".to_string(), + first_seen_ms: 1000, + last_seen_ms: 2000, + count: 5, + suppressed_count: 0, + total_count: 5, + sample_cmdlines: vec![], + binary_sha256: String::new(), + persistent: false, + denial_stage: "connect".to_string(), + l7_request_samples: vec![], + l7_inspection_active: false, + }]; + + let proposals = generate_proposals(&summaries); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].rule_name, "allow_api_example_com_443"); + assert!(proposals[0].proposed_rule.is_some()); + + let rule = proposals[0].proposed_rule.as_ref().unwrap(); + assert_eq!(rule.endpoints.len(), 1); + assert_eq!(rule.endpoints[0].host, "api.example.com"); + assert_eq!(rule.endpoints[0].port, 443); + assert_eq!(rule.binaries.len(), 1); + assert_eq!(rule.binaries[0].path, "/usr/bin/curl"); + #[allow(deprecated)] + { + assert!(rule.binaries[0].harness); + } + + // No L7 fields when no samples provided. + assert!(rule.endpoints[0].protocol.is_empty()); + assert!(rule.endpoints[0].rules.is_empty()); + + // Proposals never include allowed_ips (two-step approval flow). + assert!(rule.endpoints[0].allowed_ips.is_empty()); + } + + #[test] + fn test_generate_proposals_with_l7_samples() { + use openshell_core::proto::L7RequestSample; + + let summaries = vec![DenialSummary { + sandbox_id: "test".to_string(), + host: "icanhazdadjoke.com".to_string(), + port: 443, + binary: "/usr/bin/python3".to_string(), + ancestors: vec![], + deny_reason: "l7 deny".to_string(), + first_seen_ms: 1000, + last_seen_ms: 2000, + count: 3, + suppressed_count: 0, + total_count: 3, + sample_cmdlines: vec![], + binary_sha256: String::new(), + persistent: false, + denial_stage: "l7_deny".to_string(), + l7_request_samples: vec![ + L7RequestSample { + method: "GET".to_string(), + path: "/".to_string(), + decision: "deny".to_string(), + count: 2, + }, + L7RequestSample { + method: "GET".to_string(), + path: "/j/abc123def456abcd0099".to_string(), + decision: "deny".to_string(), + count: 1, + }, + ], + l7_inspection_active: true, + }]; + + let proposals = generate_proposals(&summaries); + assert_eq!(proposals.len(), 1); + + let rule = proposals[0].proposed_rule.as_ref().unwrap(); + let ep = &rule.endpoints[0]; + + // L7 fields should be set. + assert_eq!(ep.protocol, "rest"); + // tls field is no longer set (auto-detection handles it). + assert!(ep.tls.is_empty()); + assert_eq!(ep.enforcement, "enforce"); + + // Should have L7 rules. + assert!(!ep.rules.is_empty()); + + let paths: Vec<&str> = ep + .rules + .iter() + .filter_map(|r| r.allow.as_ref()) + .map(|a| a.path.as_str()) + .collect(); + assert!(paths.contains(&"/")); + // The /j/abc123def456 path should be generalised to /j/* + assert!(paths.contains(&"/j/*")); + + // Rationale should mention L7. + assert!(proposals[0].rationale.contains("L7")); + } + + // -- is_always_blocked_destination tests ------------------------------------ + + #[test] + fn test_always_blocked_destination_loopback_ip() { + assert!(is_always_blocked_destination("127.0.0.1")); + } + + #[test] + fn test_always_blocked_destination_link_local_ip() { + assert!(is_always_blocked_destination("169.254.169.254")); + } + + #[test] + fn test_always_blocked_destination_unspecified_ip() { + assert!(is_always_blocked_destination("0.0.0.0")); + } + + #[test] + fn test_always_blocked_destination_localhost_hostname() { + assert!(is_always_blocked_destination("localhost")); + assert!(is_always_blocked_destination("LOCALHOST")); + } + + #[test] + fn test_always_blocked_destination_known_metadata_hostname() { + assert!(is_always_blocked_destination("metadata.google.internal")); + assert!(is_always_blocked_destination("METADATA.GOOGLE.INTERNAL.")); + } + + #[test] + fn test_always_blocked_destination_allows_rfc1918() { + assert!(!is_always_blocked_destination("10.0.5.20")); + assert!(!is_always_blocked_destination("192.168.1.1")); + } + + #[test] + fn test_always_blocked_destination_allows_public_hostname() { + assert!(!is_always_blocked_destination("api.github.com")); + } + + // -- generate_proposals: always-blocked filtering tests -------------------- + + #[test] + fn test_generate_proposals_skips_loopback_destination() { + let summaries = vec![DenialSummary { + host: "127.0.0.1".to_string(), + port: 80, + binary: "/usr/bin/curl".to_string(), + count: 5, + first_seen_ms: 1000, + last_seen_ms: 2000, + denial_stage: "ssrf".to_string(), + ..Default::default() + }]; + + let proposals = generate_proposals(&summaries); + assert!( + proposals.is_empty(), + "should skip proposals for loopback: {proposals:?}" + ); + } + + #[test] + fn test_generate_proposals_skips_link_local_destination() { + let summaries = vec![DenialSummary { + host: "169.254.169.254".to_string(), + port: 80, + binary: "/usr/bin/curl".to_string(), + count: 5, + first_seen_ms: 1000, + last_seen_ms: 2000, + denial_stage: "ssrf".to_string(), + ..Default::default() + }]; + + let proposals = generate_proposals(&summaries); + assert!( + proposals.is_empty(), + "should skip proposals for link-local: {proposals:?}" + ); + } + + #[test] + fn test_generate_proposals_skips_known_metadata_hostname() { + let summaries = vec![DenialSummary { + host: "metadata.google.internal".to_string(), + port: 80, + binary: "/usr/bin/curl".to_string(), + count: 5, + first_seen_ms: 1000, + last_seen_ms: 2000, + denial_stage: "ssrf".to_string(), + ..Default::default() + }]; + + let proposals = generate_proposals(&summaries); + assert!( + proposals.is_empty(), + "should skip proposals for metadata hostname: {proposals:?}" + ); + } + + #[test] + fn test_generate_proposals_skips_localhost_hostname() { + let summaries = vec![DenialSummary { + host: "localhost".to_string(), + port: 8080, + binary: "/usr/bin/curl".to_string(), + count: 3, + first_seen_ms: 1000, + last_seen_ms: 2000, + denial_stage: "ssrf".to_string(), + ..Default::default() + }]; + + let proposals = generate_proposals(&summaries); + assert!( + proposals.is_empty(), + "should skip proposals for localhost: {proposals:?}" + ); + } + + #[test] + fn test_generate_proposals_keeps_public_destination() { + let summaries = vec![DenialSummary { + host: "api.github.com".to_string(), + port: 443, + binary: "/usr/bin/curl".to_string(), + count: 5, + first_seen_ms: 1000, + last_seen_ms: 2000, + denial_stage: "connect".to_string(), + ..Default::default() + }]; + + let proposals = generate_proposals(&summaries); + assert_eq!(proposals.len(), 1, "should keep proposals for public host"); + } + + #[test] + fn test_generalise_path() { + // Exact path preserved. + assert_eq!( + generalise_path("/api/breeds/image/random"), + "/api/breeds/image/random" + ); + + // Numeric ID replaced. + assert_eq!(generalise_path("/posts/42"), "/posts/*"); + + // UUID-ish replaced. + assert_eq!( + generalise_path("/chunks/550e8400-e29b-41d4-a716-446655440000"), + "/chunks/*" + ); + + // Query string stripped. + assert_eq!(generalise_path("/json/?fields=status,country"), "/json/"); + + // Short path preserved. + assert_eq!(generalise_path("/"), "/"); + } + + #[test] + fn test_looks_like_id() { + assert!(looks_like_id("42")); + assert!(looks_like_id("550e8400-e29b-41d4-a716-446655440000")); + assert!(looks_like_id("abc123def456abcd")); + assert!(!looks_like_id("random")); + assert!(!looks_like_id("get")); + assert!(!looks_like_id("")); + assert!(!looks_like_id("v1")); + } +} diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index d515fd70b1..1a0df27353 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -3,19 +3,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The static sandbox binary is staged at: -# deploy/docker/.build/prebuilt-binaries//openshell-sandbox +# The static sandbox and supervisor binaries are staged under +# deploy/docker/.build/prebuilt-binaries//. # -# Alpine supplies nftables and iptables for pod-namespace egress enforcement. +# Alpine supplies the trusted helper runtime used by VM guest init for its +# one-shot filesystem and loopback setup. Network enforcement stays outside +# this runtime; the capability-free sandbox never receives nftables or +# iptables tooling. FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 \ + && mkdir -p /openshell-runtime \ + && cp -aL /bin /sbin /lib /usr/bin /usr/sbin /usr/lib /openshell-runtime/ \ + && mkdir -p /openshell-runtime/etc \ + && if [ -d /etc/iproute2 ]; then cp -aL /etc/iproute2 /openshell-runtime/etc/; fi \ + && test -x /openshell-runtime/bin/sh \ + && test -x /openshell-runtime/sbin/ip # Keep the binary root-owned for Podman image-volume mounts and executable by # the Kubernetes network sidecar's non-root proxy UID. COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox +COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-supervisor /openshell-supervisor +# Keep the image default usable by drivers that run the sandbox boundary in +# this image. Split-topology drivers select /openshell-supervisor explicitly. ENTRYPOINT ["/openshell-sandbox"] diff --git a/e2e/rust/tests/bypass_detection.rs b/e2e/rust/tests/bypass_detection.rs index 56415a554b..569e3a60d6 100644 --- a/e2e/rust/tests/bypass_detection.rs +++ b/e2e/rust/tests/bypass_detection.rs @@ -1,13 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Verify that sandbox bypass detection provides fast-fail UX: direct TCP -//! connections that skip the HTTP CONNECT proxy are rejected with -//! ECONNREFUSED (immediate) rather than hanging until a network timeout. +//! Verify that direct TCP bypass attempts fail promptly at the sandbox +//! syscall boundary instead of reaching the runtime's external network. //! //! This test is implementation-agnostic — it validates the observable -//! behavior (fast rejection) regardless of whether the kernel rules are -//! installed via iptables or nftables. +//! behavior rather than a particular packet-filter implementation. #![cfg(feature = "e2e")] @@ -15,13 +13,12 @@ use openshell_e2e::harness::sandbox::SandboxGuard; /// Python script that attempts a raw TCP connect bypassing the proxy. /// -/// `socket.connect()` does not honor HTTP_PROXY — it goes directly through -/// the kernel, hitting the OUTPUT chain REJECT rule. The script reports the -/// outcome and wall-clock time so the test can assert on both. +/// `socket.connect()` does not honor proxy environment variables. The script +/// reports the outcome and wall-clock time so the test can assert that the +/// sandbox's seccomp mediation blocks it before the outer fence is needed. /// /// Target 198.51.100.1 is RFC 5737 TEST-NET-2 — documentation-only address -/// space that will never route. This doesn't matter because the REJECT rule -/// fires in the OUTPUT chain before the packet reaches the network. +/// space that will never route. fn bypass_attempt_script() -> &'static str { r#" import json, socket, time @@ -36,6 +33,8 @@ try: s.close() except ConnectionRefusedError: result = "refused" +except PermissionError: + result = "denied" except socket.timeout: result = "timeout" except OSError as e: @@ -46,8 +45,8 @@ print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=Tru "# } -/// A direct TCP connection bypassing the proxy should be rejected -/// immediately (ECONNREFUSED), not hang until a timeout. +/// A direct TCP connection bypassing supervision should be denied without +/// waiting for the socket's network timeout. #[tokio::test] async fn bypass_attempt_is_rejected_fast() { let guard = SandboxGuard::create(&["--", "python3", "-c", bypass_attempt_script()]) @@ -67,16 +66,14 @@ async fn bypass_attempt_is_rejected_fast() { let elapsed_ms = parsed["elapsed_ms"].as_u64().unwrap(); assert_eq!( - result, "refused", - "expected connection refused (REJECT rule), got '{result}' after {elapsed_ms}ms.\n\ - If 'timeout': REJECT rules may not be installed in the sandbox netns.\n\ + result, "denied", + "expected seccomp mediation to deny the direct connect, got '{result}' after {elapsed_ms}ms.\n\ Full output:\n{}", guard.create_output ); assert!( - elapsed_ms < 3000, - "bypass rejection took {elapsed_ms}ms — expected < 3000ms.\n\ - Fast rejection requires REJECT rules in the sandbox OUTPUT chain." + elapsed_ms < 8000, + "bypass rejection took {elapsed_ms}ms — expected < 8000ms." ); } diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs index 25d5516543..9d4cf0325a 100644 --- a/e2e/rust/tests/credential_gating.rs +++ b/e2e/rust/tests/credential_gating.rs @@ -557,27 +557,13 @@ fn body_client_script(port: u16) -> String { r#" import os import socket -import urllib.parse host = {TEST_HOST:?} port = {port} token = os.environ[{TOKEN_ENV:?}] -proxy_url = next(os.environ[name] for name in - ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - if os.environ.get(name)) -proxy = urllib.parse.urlparse(proxy_url) -with socket.create_connection((proxy.hostname, proxy.port or 80), timeout=10) as sock: +with socket.create_connection((host, port), timeout=10) as sock: target = f"{{host}}:{{port}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode("ascii")) - response = b"" - while b"\r\n\r\n" not in response: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk - if not response.startswith(b"HTTP/1.1 200"): - raise RuntimeError("CONNECT failed") body = ("prefix-" + token + "-suffix").encode("utf-8") request = ( f"POST /token HTTP/1.1\r\nHost: {{target}}\r\n" @@ -606,14 +592,9 @@ import base64 import os import socket import struct -import urllib.parse host = {TEST_HOST:?} port = {port} -proxy_url = next(os.environ[name] for name in - ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - if os.environ.get(name)) -proxy = urllib.parse.urlparse(proxy_url) def recv_until(sock, marker): data = b"" @@ -633,11 +614,8 @@ def recv_exact(sock, size): data += chunk return data -with socket.create_connection((proxy.hostname, proxy.port or 80), timeout=10) as sock: +with socket.create_connection((host, port), timeout=10) as sock: target = f"{{host}}:{{port}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode("ascii")) - if not recv_until(sock, b"\r\n\r\n").startswith(b"HTTP/1.1 200"): - raise RuntimeError("CONNECT failed") key = base64.b64encode(os.urandom(16)).decode("ascii") request = ( f"GET /ws HTTP/1.1\r\nHost: {{target}}\r\n" diff --git a/e2e/rust/tests/forward_proxy_graphql_l7.rs b/e2e/rust/tests/forward_proxy_graphql_l7.rs index bcb2b68052..2a1b06d277 100644 --- a/e2e/rust/tests/forward_proxy_graphql_l7.rs +++ b/e2e/rust/tests/forward_proxy_graphql_l7.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! E2E tests for GraphQL L7 inspection across both proxy entry points. +//! E2E tests for GraphQL L7 inspection through transparent interception. //! //! The upstream server deliberately does not implement GraphQL. `OpenShell` //! parses and enforces GraphQL before forwarding, so any HTTP server that @@ -130,7 +130,7 @@ network_policies: #[tokio::test] #[allow(clippy::too_many_lines)] -async fn graphql_l7_enforces_allow_and_deny_rules_on_forward_and_connect_paths() { +async fn graphql_l7_enforces_high_level_and_raw_transparent_paths() { let server = start_test_server().await.expect("start test server"); let policy = write_graphql_policy(&server.host, server.port).expect("write custom policy"); let policy_path = policy @@ -142,7 +142,6 @@ async fn graphql_l7_enforces_allow_and_deny_rules_on_forward_and_connect_paths() let script = format!( r#" import json -import os import socket import time import urllib.error @@ -231,26 +230,14 @@ def retry_forward_allowed(label, request_fn): time.sleep(0.3) return last_status -def proxy_parts(*names): - proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - -def forward_proxy_parts(): - return proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - -def connect_proxy_parts(): - return proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - def forward_chunked_status(query): - proxy_host, proxy_port = forward_proxy_parts() target = f"{{HOST}}:{{PORT}}" body = json.dumps({{"query": query}}).encode() chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" - with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: + with socket.create_connection((HOST, PORT), timeout=15) as sock: request = ( - f"POST http://{{target}}/graphql HTTP/1.1\r\n" + f"POST /graphql HTTP/1.1\r\n" f"Host: {{target}}\r\n" f"Content-Type: application/json\r\n" f"Transfer-Encoding: chunked\r\n" @@ -297,22 +284,11 @@ def status_code(response, label): DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error -def connect_http_status(label, request): - proxy_host, proxy_port = connect_proxy_parts() - target = f"{{HOST}}:{{PORT}}" - +def raw_http_status(label, request): last_error = None for attempt in range(5): try: - with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: - sock.sendall( - f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode() - ) - connect_response = read_until(sock, b"\r\n\r\n") - connect_code = status_code(connect_response, f"{{label}}_connect") - if connect_code != 200: - return connect_code - + with socket.create_connection((HOST, PORT), timeout=15) as sock: sock.sendall(request) sock.shutdown(socket.SHUT_WR) response = read_until(sock, b"\r\n\r\n") @@ -324,7 +300,7 @@ def connect_http_status(label, request): raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") -def connect_status(query, label): +def raw_status(query, label): target = f"{{HOST}}:{{PORT}}" body = json.dumps({{"query": query}}).encode() @@ -336,9 +312,9 @@ def connect_status(query, label): f"Connection: close\r\n" f"\r\n" ).encode() + body - return connect_http_status(label, request) + return raw_http_status(label, request) -def connect_get_status(query, label): +def raw_get_status(query, label): target = f"{{HOST}}:{{PORT}}" encoded = urllib.parse.urlencode({{"query": query}}) @@ -348,9 +324,9 @@ def connect_get_status(query, label): f"Connection: close\r\n" f"\r\n" ).encode() - return connect_http_status(label, request) + return raw_http_status(label, request) -def connect_duplicate_get_status(): +def raw_duplicate_get_status(): target = f"{{HOST}}:{{PORT}}" safe = urllib.parse.quote_plus(QUERY_VIEWER) unsafe = urllib.parse.quote_plus(MUTATION_DELETE) @@ -361,9 +337,9 @@ def connect_duplicate_get_status(): f"Connection: close\r\n" f"\r\n" ).encode() - return connect_http_status("connect_duplicate_get_denied", request) + return raw_http_status("raw_duplicate_get_denied", request) -def connect_persisted_get_status(hash_value, label): +def raw_persisted_get_status(hash_value, label): target = f"{{HOST}}:{{PORT}}" extensions = json.dumps({{"persistedQuery": {{"version": 1, "sha256Hash": hash_value}}}}) encoded = urllib.parse.urlencode({{"operationName": "Viewer", "extensions": extensions}}) @@ -374,9 +350,9 @@ def connect_persisted_get_status(hash_value, label): f"Connection: close\r\n" f"\r\n" ).encode() - return connect_http_status(label, request) + return raw_http_status(label, request) -def connect_chunked_status(query): +def raw_chunked_status(query): target = f"{{HOST}}:{{PORT}}" body = json.dumps({{"query": query}}).encode() chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" @@ -389,7 +365,7 @@ def connect_chunked_status(query): f"Connection: close\r\n" f"\r\n" ).encode() + chunk - return connect_http_status("connect_chunked_query_allowed", request) + return raw_http_status("raw_chunked_query_allowed", request) results = {{ "forward_query_allowed": retry_forward_allowed("forward_query_allowed", lambda: forward_status(QUERY_VIEWER)), @@ -401,15 +377,15 @@ results = {{ "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), "forward_mutation_allowed": retry_forward_allowed("forward_mutation_allowed", lambda: forward_status(MUTATION_CREATE)), "forward_deny_rule_denied": forward_status(MUTATION_DELETE), - "connect_query_allowed": connect_status(QUERY_VIEWER, "connect_query_allowed"), - "connect_get_query_allowed": connect_get_status(QUERY_VIEWER, "connect_get_query_allowed"), - "connect_duplicate_get_denied": connect_duplicate_get_status(), - "connect_persisted_get_allowed": connect_persisted_get_status("abc123", "connect_persisted_get_allowed"), - "connect_unregistered_persisted_get_denied": connect_persisted_get_status("missing", "connect_unregistered_persisted_get_denied"), - "connect_chunked_query_allowed": connect_chunked_status(QUERY_VIEWER), - "connect_unlisted_field_denied": connect_status(QUERY_REPOSITORY, "connect_unlisted_field_denied"), - "connect_mutation_allowed": connect_status(MUTATION_CREATE, "connect_mutation_allowed"), - "connect_deny_rule_denied": connect_status(MUTATION_DELETE, "connect_deny_rule_denied"), + "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), + "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), + "raw_duplicate_get_denied": raw_duplicate_get_status(), + "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), + "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), + "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), + "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), + "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), + "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), }} results.update(DETAILS) print(json.dumps(results, sort_keys=True)) @@ -432,15 +408,15 @@ print(json.dumps(results, sort_keys=True)) ("forward_unlisted_field_denied", 403), ("forward_mutation_allowed", 200), ("forward_deny_rule_denied", 403), - ("connect_query_allowed", 200), - ("connect_get_query_allowed", 200), - ("connect_duplicate_get_denied", 403), - ("connect_persisted_get_allowed", 200), - ("connect_unregistered_persisted_get_denied", 403), - ("connect_chunked_query_allowed", 200), - ("connect_unlisted_field_denied", 403), - ("connect_mutation_allowed", 200), - ("connect_deny_rule_denied", 403), + ("raw_query_allowed", 200), + ("raw_get_query_allowed", 200), + ("raw_duplicate_get_denied", 403), + ("raw_persisted_get_allowed", 200), + ("raw_unregistered_persisted_get_denied", 403), + ("raw_chunked_query_allowed", 200), + ("raw_unlisted_field_denied", 403), + ("raw_mutation_allowed", 200), + ("raw_deny_rule_denied", 403), ] { let expected_fragment = format!(r#""{key}": {expected}"#); assert!( diff --git a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs index 174e3b6db9..b46dac1313 100644 --- a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs +++ b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! E2E tests for JSON-RPC L7 inspection across both proxy entry points. +//! E2E tests for JSON-RPC L7 inspection through transparent interception. //! //! The upstream server deliberately does not implement JSON-RPC. `OpenShell` //! parses and enforces JSON-RPC before forwarding, so any HTTP server that @@ -187,7 +187,7 @@ network_policies: #[tokio::test] #[allow(clippy::too_many_lines)] -async fn jsonrpc_l7_enforces_method_rules_on_forward_and_connect_paths() { +async fn jsonrpc_l7_enforces_high_level_and_raw_transparent_paths() { let server = start_test_server(RULES_TEST_SERVER_ALIAS) .await .expect("start test server"); @@ -201,25 +201,15 @@ async fn jsonrpc_l7_enforces_method_rules_on_forward_and_connect_paths() { let script = format!( r#" import json -import os import socket import time import urllib.error -import urllib.parse import urllib.request HOST = {host:?} PORT = {port} DETAILS = {{ "debug_target": {{"host": HOST, "port": PORT}}, - "debug_proxy_env": {{ - "http_proxy": os.environ.get("http_proxy"), - "https_proxy": os.environ.get("https_proxy"), - "HTTP_PROXY": os.environ.get("HTTP_PROXY"), - "HTTPS_PROXY": os.environ.get("HTTPS_PROXY"), - "NO_PROXY": os.environ.get("NO_PROXY"), - "no_proxy": os.environ.get("no_proxy"), - }}, }} def text(data): @@ -291,11 +281,6 @@ def post_invalid_json(label): except urllib.error.HTTPError as error: return record_http_error(label, error, text(encoded)) -def proxy_parts(*names): - proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_until(sock, marker): data = b"" while marker not in data: @@ -339,21 +324,11 @@ def record_raw_response(label, response, body=b""): DETAILS[f"{{label}}_body"] = text(body) return code -def connect_http_status(label, request): - proxy_host, proxy_port = proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - target = f"{{HOST}}:{{PORT}}" - +def raw_http_status(label, request): last_error = None for attempt in range(5): try: - with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: - sock.sendall( - f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode() - ) - connect_response = read_until(sock, b"\r\n\r\n") - connect_code = record_raw_response(f"{{label}}_connect", connect_response) - if connect_code != 200: - return connect_code + with socket.create_connection((HOST, PORT), timeout=15) as sock: sock.sendall(request) sock.shutdown(socket.SHUT_WR) response, body = read_response(sock) @@ -365,7 +340,7 @@ def connect_http_status(label, request): raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") -def connect_jsonrpc_status(method, params, label): +def raw_jsonrpc_status(method, params, label): target = f"{{HOST}}:{{PORT}}" body = {{"jsonrpc": "2.0", "id": 1, "method": method}} if params is not None: @@ -379,7 +354,7 @@ def connect_jsonrpc_status(method, params, label): f"Connection: close\r\n" f"\r\n" ).encode() + encoded - return connect_http_status(label, request) + return raw_http_status(label, request) results = {{ # forward proxy — method-only allow rules @@ -406,12 +381,12 @@ results = {{ # forward proxy — invalid JSON body fails closed before generic rules apply "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), - # CONNECT path — representative allowed and denied cases - "connect_method_initialize_allowed": connect_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "connect_method_initialize_allowed"), - "connect_method_tools_list_allowed": connect_jsonrpc_status("tools/list", None, "connect_method_tools_list_allowed"), - "connect_method_tools_call_allowed": connect_jsonrpc_status("tools/call", {{"name": "read_status"}}, "connect_method_tools_call_allowed"), - "connect_method_tools_call_with_unmatched_params_allowed": connect_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "connect_method_tools_call_with_unmatched_params_allowed"), - "connect_method_tools_delete_denied": connect_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "connect_method_tools_delete_denied"), + # raw socket path — representative allowed and denied cases + "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "raw_method_initialize_allowed"), + "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), + "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {{"name": "read_status"}}, "raw_method_tools_call_allowed"), + "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "raw_method_tools_call_with_unmatched_params_allowed"), + "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "raw_method_tools_delete_denied"), }} results.update(DETAILS) print(json.dumps(results, sort_keys=True)) @@ -440,16 +415,13 @@ print(json.dumps(results, sort_keys=True)) ("forward_batch_one_denied", 403), // forward proxy — parse error ("forward_invalid_json_denied", 403), - // CONNECT path — allowed - ("connect_method_initialize_allowed", 200), - ("connect_method_tools_list_allowed", 200), - ("connect_method_tools_call_allowed", 200), - ( - "connect_method_tools_call_with_unmatched_params_allowed", - 200, - ), - // CONNECT path — method denied - ("connect_method_tools_delete_denied", 403), + // raw socket path — allowed + ("raw_method_initialize_allowed", 200), + ("raw_method_tools_list_allowed", 200), + ("raw_method_tools_call_allowed", 200), + ("raw_method_tools_call_with_unmatched_params_allowed", 200), + // raw socket path — method denied + ("raw_method_tools_delete_denied", 403), ] { let expected_fragment = format!(r#""{key}": {expected}"#); assert!( diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs index 7a1e12923a..ce47d3c8b7 100644 --- a/e2e/rust/tests/live_policy_update.rs +++ b/e2e/rust/tests/live_policy_update.rs @@ -32,30 +32,6 @@ use openshell_e2e::harness::output::{extract_field, strip_ansi}; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -#[cfg(feature = "e2e-docker")] -const LOCAL_OVERRIDE_REGO: &str = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../crates/openshell-supervisor-network/data/sandbox-policy.rego" -)); - -#[cfg(feature = "e2e-docker")] -const LOCAL_OVERRIDE_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* -RUN groupadd -g 1000660000 sandbox && \ - useradd -m -u 1000660000 -g sandbox sandbox - -COPY local-policy.rego /etc/openshell/local-policy.rego -COPY local-policy.yaml /etc/openshell/local-policy.yaml - -ENV OPENSHELL_POLICY_RULES=/etc/openshell/local-policy.rego -ENV OPENSHELL_POLICY_DATA=/etc/openshell/local-policy.yaml -ENV OPENSHELL_POLICY_POLL_INTERVAL_SECS=1 - -CMD ["sleep", "infinity"] -"#; - // --------------------------------------------------------------------------- // Policy YAML builders // --------------------------------------------------------------------------- @@ -146,44 +122,6 @@ landlock: Ok(file) } -#[cfg(feature = "e2e-docker")] -fn write_local_override_image() -> Result { - let dir = tempfile::tempdir().map_err(|e| format!("create image context: {e}"))?; - std::fs::write(dir.path().join("Dockerfile"), LOCAL_OVERRIDE_DOCKERFILE) - .map_err(|e| format!("write local override Dockerfile: {e}"))?; - std::fs::write(dir.path().join("local-policy.rego"), LOCAL_OVERRIDE_REGO) - .map_err(|e| format!("write local override Rego policy: {e}"))?; - std::fs::write( - dir.path().join("local-policy.yaml"), - r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /etc - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: {} -", - ) - .map_err(|e| format!("write local override policy data: {e}"))?; - Ok(dir) -} - // --------------------------------------------------------------------------- // CLI helpers // --------------------------------------------------------------------------- @@ -580,119 +518,3 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { guard.cleanup().await; } - -/// An explicit local Rego/data override remains authoritative even when the -/// sandbox has a gateway policy and that policy changes while it is running. -/// Gateway polling must continue for settings and providers without replacing -/// the locally loaded OPA engine. -#[cfg(feature = "e2e-docker")] -#[tokio::test] -async fn local_policy_override_survives_gateway_policy_polls() { - let image_context = write_local_override_image().expect("write local override image"); - let dockerfile = image_context.path().join("Dockerfile"); - let dockerfile = dockerfile - .to_str() - .expect("Dockerfile path should be utf-8"); - - let gateway_policy_a_file = write_policy(&["example.com"]).expect("write gateway policy A"); - let gateway_policy_a_path = gateway_policy_a_file - .path() - .to_str() - .expect("gateway policy A path should be utf-8") - .to_string(); - let gateway_policy_b_file = - write_policy(&["example.com", "api.anthropic.com"]).expect("write gateway policy B"); - let gateway_policy_b_path = gateway_policy_b_file - .path() - .to_str() - .expect("gateway policy B path should be utf-8") - .to_string(); - - let mut guard = SandboxGuard::create_keep_with_args( - &[ - "--name", - "e2e-lcl-pol-ovrd", - "--from", - dockerfile, - "--policy", - &gateway_policy_a_path, - "--no-tty", - ], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create sandbox with local policy override"); - - // Allow several one-second poll intervals. Before the fix, the first poll - // immediately reloaded gateway policy A over the local override. - tokio::time::sleep(std::time::Duration::from_secs(4)).await; - let initial_logs = run_cli(&[ - "logs", - &guard.name, - "-n", - "500", - "--since", - "1m", - "--source", - "sandbox", - ]) - .await; - assert!( - initial_logs.success, - "fetch initial sandbox logs:\n{}", - initial_logs.output - ); - assert!( - initial_logs - .output - .contains("Loading OPA policy engine from local files"), - "sandbox should load the explicit local policy:\n{}", - initial_logs.output - ); - assert!( - !initial_logs.output.contains("Policy reloaded successfully"), - "the first gateway poll must not replace the local policy:\n{}", - initial_logs.output - ); - - let update = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &gateway_policy_b_path, - ]) - .await; - assert!( - update.success, - "publish gateway policy B:\n{}", - update.output - ); - - // A later gateway revision must also remain observational in local mode. - tokio::time::sleep(std::time::Duration::from_secs(4)).await; - let updated_logs = run_cli(&[ - "logs", - &guard.name, - "-n", - "500", - "--since", - "1m", - "--source", - "sandbox", - ]) - .await; - assert!( - updated_logs.success, - "fetch updated sandbox logs:\n{}", - updated_logs.output - ); - assert!( - !updated_logs.output.contains("Policy reloaded successfully"), - "gateway policy updates must not replace the local override:\n{}", - updated_logs.output - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/no_proxy.rs b/e2e/rust/tests/no_proxy.rs index ced4d02d5f..447c408f8b 100644 --- a/e2e/rust/tests/no_proxy.rs +++ b/e2e/rust/tests/no_proxy.rs @@ -5,7 +5,7 @@ use openshell_e2e::harness::sandbox::SandboxGuard; -fn localhost_bypass_script() -> &'static str { +fn localhost_transparent_script() -> &'static str { r#" import json import os @@ -13,11 +13,8 @@ import threading import urllib.request from http.server import BaseHTTPRequestHandler, HTTPServer -expected_no_proxy = '127.0.0.1,localhost,::1' -assert os.environ['HTTP_PROXY'].startswith('http://') -assert os.environ['HTTPS_PROXY'].startswith('http://') -assert os.environ['NO_PROXY'] == expected_no_proxy -assert os.environ['no_proxy'] == expected_no_proxy +for name in ('HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'): + assert name not in os.environ, f'unexpected proxy environment variable: {name}' class Handler(BaseHTTPRequestHandler): def log_message(self, format, *args): @@ -36,7 +33,7 @@ thread.start() try: with urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}', timeout=10) as response: print(json.dumps({ - 'no_proxy': os.environ['NO_PROXY'], + 'proxy_env_absent': True, 'payload': json.loads(response.read().decode()), }), flush=True) finally: @@ -47,16 +44,16 @@ finally: } #[tokio::test] -async fn sandbox_bypasses_proxy_for_localhost_http() { - let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_bypass_script()]) +async fn sandbox_reaches_localhost_without_proxy_environment() { + let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_transparent_script()]) .await - .expect("sandbox create with localhost proxy bypass check"); + .expect("sandbox create with transparent localhost check"); assert!( - guard.create_output.contains( - r#"{"no_proxy": "127.0.0.1,localhost,::1", "payload": {"message": "hello"}}"# - ), - "expected localhost HTTP request to bypass proxy and succeed:\n{}", + guard + .create_output + .contains(r#"{"proxy_env_absent": true, "payload": {"message": "hello"}}"#), + "expected localhost HTTP request to stay local and succeed:\n{}", guard.create_output ); } diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs index 4ba4dbd046..d95841d07f 100644 --- a/e2e/rust/tests/websocket_conformance.rs +++ b/e2e/rust/tests/websocket_conformance.rs @@ -364,7 +364,6 @@ import os import socket import struct import time -import urllib.parse HOST = {host:?} PORT = {port} @@ -413,34 +412,15 @@ def read_frame(sock): payload = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) return first, payload -def proxy_parts(): - names = ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) - if not proxy_url: - raise RuntimeError("proxy environment is not configured") - parsed = urllib.parse.urlparse(proxy_url) - if not parsed.hostname: - raise RuntimeError(f"invalid proxy URL: {{proxy_url!r}}") - return parsed.hostname, parsed.port or 80 - -def proxy_socket_with_retry(host, port, mode, timeout_seconds=20): - proxy_host, proxy_port = proxy_parts() - target = f"{{host}}:{{port}}" +def transparent_socket_with_retry(host, port, timeout_seconds=20): deadline = time.monotonic() + timeout_seconds last_error = None while time.monotonic() < deadline: sock = None try: - sock = socket.create_connection((proxy_host, proxy_port), timeout=5) - if mode == "connect": - request = f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n" - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not (response.startswith("HTTP/1.1 200") or response.startswith("HTTP/1.0 200")): - first_line = response.splitlines()[0] if response else "" - raise RuntimeError(f"proxy CONNECT failed: {{first_line}}") + sock = socket.create_connection((host, port), timeout=5) return sock - except (OSError, RuntimeError) as error: + except OSError as error: if sock is not None: sock.close() last_error = error @@ -449,27 +429,25 @@ def proxy_socket_with_retry(host, port, mode, timeout_seconds=20): token = os.environ[TOKEN_ENV] payload = json.dumps({{"authorization": "Bearer " + token}}, sort_keys=True) -results = {{}} -for mode in ("connect", "forward"): - key = base64.b64encode(os.urandom(16)).decode("ascii") - with proxy_socket_with_retry(HOST, PORT, mode) as sock: - request_target = "/ws" if mode == "connect" else f"http://{{HOST}}:{{PORT}}/ws" - request = ( - f"GET {{request_target}} HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\n" - "Sec-WebSocket-Version: 13\r\n" - "\r\n" - ) - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not response.startswith("HTTP/1.1 101"): - raise RuntimeError(f"{{mode}} websocket upgrade failed: {{response!r}}") - sock.sendall(masked_text_frame(payload)) - _, response_payload = read_frame(sock) - results[mode] = json.loads(response_payload.decode("utf-8")) +key = base64.b64encode(os.urandom(16)).decode("ascii") +with transparent_socket_with_retry(HOST, PORT) as sock: + request = ( + "GET /ws HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {{key}}\r\n" + "Sec-WebSocket-Version: 13\r\n" + "\r\n" + ) + sock.sendall(request.encode("ascii")) + response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") + if not response.startswith("HTTP/1.1 101"): + raise RuntimeError(f"websocket upgrade failed: {{response!r}}") + sock.sendall(masked_text_frame(payload)) + _, response_payload = read_frame(sock) + result = json.loads(response_payload.decode("utf-8")) +results = {{"transparent": result}} print(json.dumps(results, sort_keys=True)) "#, host = host, @@ -479,7 +457,7 @@ print(json.dumps(results, sort_keys=True)) } #[tokio::test] -async fn websocket_text_placeholder_is_rewritten_through_both_adapters() { +async fn websocket_text_placeholder_is_rewritten_transparently() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -522,14 +500,7 @@ async fn websocket_text_placeholder_is_rewritten_through_both_adapters() { assert!( guard .create_output - .contains(r#""connect": {"saw_placeholder": false, "saw_secret": true}"#), - "expected CONNECT upstream to see only the resolved secret marker:\n{}", - guard.create_output - ); - assert!( - guard - .create_output - .contains(r#""forward": {"saw_placeholder": false, "saw_secret": true}"#), + .contains(r#""transparent": {"saw_placeholder": false, "saw_secret": true}"#), "expected upstream to see only the resolved secret marker:\n{}", guard.create_output ); diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 2be6b3f6d5..48e2ae3da1 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -19,7 +19,7 @@ links: Today the supervisor both builds the workload's isolation boundary and applies its network policy. Because the supervisor runs inside the agent container, the privilege needed to build that boundary sits beside the code it confines. This RFC moves boundary construction and process operations behind a pluggable **Isolation Backend**. The supervisor continues to apply approved network policy through network mediation. -The compute driver provisions the workload and trusted components. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. The same lifecycle supports today's in-pod implementation and future delegated implementations without topology-specific supervisor paths. +The compute driver prepares the workload topology and trusted inputs. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. OpenShell packages that role as `openshell-sandbox --mode=control`. When the workload is separated by a container, pod, userspace kernel, or VM boundary, the same binary runs a small trusted counterpart as `openshell-sandbox --mode=boundary`. The boundary mode owns process observation and operations that cannot be implemented portably from outside the boundary; it has no gateway credentials and no policy authority. ## Motivation @@ -33,56 +33,78 @@ All three come from coupling boundary construction to boundary operation. A comm ## Non-goals -- **Implementing a delegated backend.** Each topology requires its own design and implementation. +- **Standardizing a topology's resource API.** Docker, Kubernetes, VM, and other resource mechanisms remain backend-specific. - **Changing authorization.** [RFC 0001](../0001-core-architecture/README.md) owns control-plane and sandbox identity. A delegated backend must still authenticate callers and scope them to one boundary. -- **Standardizing backend-internal component coordination.** A backend may coordinate helper, sidecar, or interception processes behind one lifecycle; how those components cooperate is backend-specific, not contract surface. +- **Standardizing resource-specific provisioning or transport setup.** A driver still decides how to place the binary, create a private Unix socket, authenticated TCP endpoint, or vsock endpoint, and establish kernel-specific egress capture. This RFC standardizes the authenticated control-to-boundary messages carried over that endpoint. - **Changing gateway lifecycle or public status.** This RFC adds no gateway activation operation, public phase, or status API, and it does not define how a boundary's effective isolation model is surfaced to operators. ## Proposal The mental model has three roles: -- The **compute driver** provisions the sandbox instance according to the selected placement of the workload and trusted isolation components. That placement is the **topology**. +- The **compute driver** prepares placement and trusted topology inputs and owns durable resource provisioning, deletion, and reconciliation. That placement is the **topology**. - The **Isolation Backend** establishes and operates the topology-specific controls around the workload. It also routes workload egress to network mediation and provides process operations. - The **logical supervisor** is the trusted control-plane bridge between the gateway and the workload. It drives the backend, handles authorized gateway requests, and applies approved network policy through network mediation. Together, network policy, filesystem isolation, syscall filtering, and sandbox identity form the workload's isolation boundary. The roles above enforce that boundary and may run in one process or across several trusted components. Their placement does not change the contract. -Each active boundary has at most one logical supervisor, which may span multiple coupled processes. The backend routes all workload egress through a per-boundary source, and the supervisor consumes that source. Internal delegation and transport remain topology-private. +Each active boundary has exactly one control role and at most one boundary role. Together they implement one logical supervisor; boundary mode is not an independently authorized supervisor. The control role owns the gateway session, admitted policy, network-policy decisions, and RFC 0012 lifecycle. Boundary mode owns boundary-local process groups, `exec`, signal, wait, PTY, loopback forwarding, binary observation, and egress capture. The transport and physical placement remain driver-owned. [RFC 0001](../0001-core-architecture/README.md) continues to own sandbox authentication and authorization. In this contract, sandbox identity means binding the authenticated sandbox context to the isolation boundary. -Admission selects the sandbox's topology and determines its trusted context. The compute driver sets up the topology and gives the logical supervisor a `TopologyDescriptor` describing what it provisioned. The supervisor uses the descriptor to attach the matching Isolation Backend. The backend prepares the required controls before the agent starts. +Admission selects the sandbox's topology and trusted context. The compute driver gives the logical supervisor a `TopologyDescriptor` for the matching backend. Its opaque payload can identify an existing resource or carry trusted prepared inputs from which `attach` establishes the boundary. The backend prepares the required controls before the agent starts. ```mermaid flowchart TB Gateway["Gateway"] -->|"create sandbox"| Driver["Compute driver"] - subgraph Topology["Driver-provisioned topology (placement varies)"] - Supervisor["Supervisor"] - Backend["Isolation Backend (may coordinate components)"] + subgraph Topology["Admitted topology (placement varies)"] + Control["openshell-sandbox
--mode=control"] + Backend["Remote Isolation Backend"] subgraph Boundary["Isolation boundary"] - Mediator["Network mediation"] + BoundaryAgent["openshell-sandbox
--mode=boundary"] subgraph Execution["Workload execution environment"] Workload["Workload"] end end - - Supervisor -->|"drives contract"| Backend - Backend -->|"establishes and confirms"| Boundary - Backend -.->|"routes all workload egress to"| Mediator - Supervisor -.->|"applies network policy through"| Mediator - Backend -->|"after Ready: makes admitted agent runnable"| Workload - Workload ==>|"only egress"| Mediator + Mediator["Network mediation"] + + Control -->|"drives RFC 0012"| Backend + Backend <-->|"authenticated, versioned
boundary protocol"| BoundaryAgent + BoundaryAgent -->|"start / exec / signal / wait"| Workload + Workload ==>|"captured egress"| BoundaryAgent + BoundaryAgent ==>|"attributed streams"| Mediator + Control -.->|"policy decisions"| Mediator end - Driver -->|"resources + TopologyDescriptor"| Supervisor + Driver -->|"resources + protected configs"| BoundaryAgent + Driver -->|"trusted TopologyDescriptor"| Control + Gateway <-->|"authorized session"| Control Mediator -->|"allowed egress"| Egress["Egress"] ``` -In the in-pod topology, the supervisor drives a backend implemented in the same process. Other topologies may delegate backend operations without changing the supervisor lifecycle. +Co-located deployments may keep the existing in-process backend and omit boundary mode. Separated deployments use the shared remote backend and boundary protocol, so adding a driver changes provisioning and transport selection without adding a topology branch to the control role. + +### Supervisor modes and boundary protocol + +`openshell-sandbox --mode=control` runs outside the untrusted execution environment. It receives the admitted backend name and a protected `TopologyDescriptor`, resolves the backend without fallback, and owns the gateway-facing access plane. It is the only mode that possesses gateway credentials or applies approved network policy. + +Orchestrators may run control mode with `--health-check --health-port `. +The listener defaults to `0.0.0.0`; `--health-bind-ip ` (or +`OPENSHELL_HEALTH_BIND_IP`) selects an explicit IPv4 or IPv6 address. Kubernetes +drivers should populate the environment variable from the Downward API +`status.podIP`, which keeps the probe address family aligned with the pod. +The TCP listener becomes reachable only after `start_agent` has returned a +`RunningBoundary` and the gateway-facing access plane is established. Control +mode drops the listener when that boundary/access-plane lifetime ends, so a +Kubernetes `tcpSocket` readiness probe observes semantic control readiness +rather than mere process liveness. The listener carries no application data. + +`openshell-sandbox --mode=boundary --boundary-config ` runs inside, or immediately adjacent to, the execution environment. The driver supplies the config over a protected filesystem channel. It names one boundary, one private listener, one per-boundary bootstrap credential, and the workload identity. The identity is either a platform-resolved numeric UID/GID pair or an OCI `Config.User` declaration that boundary mode resolves against the workload filesystem. Boundary mode authenticates and scopes every request to that boundary. It never accepts policy or identity claims from workload code and it cannot authorize an operation independently of control mode. -A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver owns the sandbox instance and topology lifecycle. +The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, TLS-authenticated TCP, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. A TCP transport that crosses a shared or operator-managed network must authenticate the server name against a driver-provisioned trust root and encrypt every control and stream request. Network isolation alone is not a confidentiality boundary. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. + +A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver or external orchestrator owns the durable resource lifecycle even when the backend establishes a resource as part of `attach`. ### Contract invariants @@ -91,7 +113,7 @@ Six invariants hold for every boundary: 1. Workload egress is denied except through network mediation for the boundary's lifetime. 2. No untrusted instruction executes until every admitted control applicable to that process is in force. 3. An operation is authorized only when the complete effective policy permits it; network operations are decided through network mediation. There is no silent weakening. -4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the compute driver's provisioned execution environment. +4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the admitted execution environment. 5. Shared infrastructure preserves strict per-boundary lifecycle, policy, identity, enforcement, and cleanup isolation. 6. If the logical supervisor is lost, the boundary remains under its last confirmed enforcement state while supervisor-dependent operations fail closed. Loss of required enforcement ends `Running` and terminates all workload processes within a documented bound; detection and termination may be performed by a trusted node or control-plane actor. Network-mediation unavailability denies outbound connections and never enables direct egress. @@ -99,21 +121,22 @@ Each backend states its termination bound in its implementation documentation. L ### Provisioning -Provisioning runs on the control plane, and three rules hold in every topology: +Provisioning is selected by trusted control-plane configuration, and four rules hold in every topology: -1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` supplied by the compute driver must name that backend, and resolution never falls back to another backend. -2. **The compute driver provisions the topology** and anything the selected backend needs. -3. **The backend establishes standing enforcement before untrusted code runs**, during provisioning or `attach`, depending on the backend. +1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` must name that backend, and resolution never falls back to another backend. +2. **The descriptor supports prepared and existing resources.** A compute driver or orchestrator may identify an existing resource, or it may supply trusted prepared inputs that the backend uses to establish the resource during `attach`. +3. **The compute driver owns the durable resource lifecycle.** It provisions or prepares, deletes, and reconciles the topology independently of logical-supervisor availability. +4. **The backend establishes standing enforcement before untrusted code runs**, during `attach` or `confirm`, depending on the backend. If a topology depends on cluster-scoped coverage or registration, admission verifies that the prerequisite covers the boundary's placement before untrusted code runs. Every topology provides a trusted cleanup path that does not depend on logical-supervisor availability. -A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. After claim or assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend either binds that context to the prepared resource and returns `Bound`, or rejects it as incompatible. Pool creation, claim, reset, release, and recycling remain outside this contract. +A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. It may instead prepare immutable image or disk identities, normalized runtime settings, placement results, or protected references to artifacts and encode those inputs in the descriptor. After assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend atomically establishes or locates the resource, binds that context, and returns `Bound`, or rejects it as incompatible. Attach-time establishment is idempotent on trusted sandbox identity and launch generation. Partial resources are removed or remain labeled for compute-driver reconciliation. Pool creation, claim, reset, release, and recycling remain outside this contract. ### The topology descriptor -The driver supplies a descriptor for every topology admitted to this contract, including in-pod and resources prepared before assignment. The common envelope names the backend and carries an opaque payload. +The compute driver supplies a descriptor for every topology admitted to this contract. The common envelope names the backend and carries an opaque payload. ```rust struct TopologyDescriptor { @@ -125,7 +148,7 @@ struct TopologyDescriptor { `version` is the Isolation Backend interface version. Backend name and version match exactly; this contract does not negotiate compatibility ranges. The descriptor is transport-neutral. Provisioning supplies it to the supervisor before `attach`; how it is transported is topology-specific and outside this contract, and every transport preserves one property: workload-controlled input cannot select or modify the descriptor. -The opaque payload identifies, or gives the backend enough information to resolve, the exact driver-provisioned resource. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. +The opaque payload identifies an existing resource or gives the backend trusted prepared inputs with which to establish the exact resource during `attach`. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. Common verification requires: @@ -133,13 +156,13 @@ Common verification requires: - the descriptor's version is one the supervisor supports, and the resolved backend reports that same version; and - `SandboxContext` is constructed after the control plane assigns the resource to the admitted sandbox, using authenticated control-plane and trusted supervisor state. -The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically binds the provisioned resource to the trusted `SandboxContext` during `attach`. Any failure rejects the sandbox. +The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically establishes or locates the resource and binds it to the trusted `SandboxContext` during `attach`. Separated topologies also carry an opaque map of driver-owned immutable resource claims, such as a container ID, pod UID, or VM generation. Control mode presents those claims during authenticated attachment, and boundary mode compares them with its protected configuration before accepting the policy. Any failure rejects the sandbox. ### The lifecycle The contract does not prescribe enforcement mechanisms; it standardizes how the supervisor drives whichever backend a deployment admits. -A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through a fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. +A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through one fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. ```text attach topology + sandbox context -> Bound -> confirm -> Ready -> start_agent -> Running @@ -167,12 +190,23 @@ struct SandboxContext { #[async_trait] trait BoundBoundary: Send { fn network_mediation_source(&self) -> Arc; + fn dns_mediation_source(&self) -> Option>; + fn host_gateway_ip(&self) -> Option; async fn confirm( self: Box, ) -> Result, BackendError>; } +``` +`host_gateway_ip` is the backend's trusted host-side dial target for the +well-known host-gateway aliases. A backend returns it when the mediation +service runs outside the workload boundary and therefore cannot use the +boundary's resolver view; the supervisor preserves the original hostname for +policy, HTTP, and TLS while dialing the backend-provided address. `None` +leaves host-gateway discovery to the supervisor's local environment. + +```rust #[async_trait] trait ReadyBoundary: Send { async fn start_agent( @@ -190,7 +224,7 @@ trait RunningBoundary: Send + Sync { `AgentSpec` carries the complete admitted agent launch specification, including command, arguments, working directory, timeout, and interactive mode. -`SandboxContext` carries the admitted create-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. +`SandboxContext` carries the admitted launch-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. The states have normative meanings: @@ -200,13 +234,15 @@ The states have normative meanings: `confirm` is the pre-launch commit point. The supervisor calls it only after connecting the boundary's network-mediation source to network mediation. The backend confirms standing enforcement for the concrete boundary and may rely on a trusted provisioning-time or out-of-pod signal tied to that boundary's placement, but not on general placement health alone. -`attach` rejects a resource already bound to an active boundary. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. +`attach` rejects a resource already bound to an active boundary or a conflicting launch generation. An idempotent retry resolves the same compatible inactive resource. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. **Standing enforcement** is established independently of a workload process. **Launch-time controls** must be in force before a process executes its first untrusted instruction. Both `start_agent` and `BoundaryExec::exec` enforce this ordering and preserve the provisioned execution environment. `start_agent` is the sole operation that may make the admitted agent runnable. The backend may create or release the process, but workload-controlled code cannot run before `start_agent` applies the required controls. -`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Any exit of the admitted agent ends `Running`; the backend then terminates every remaining workload process within that environment and rejects further runtime operations, except `wait` as defined below. +A separated boundary retains the complete accepted `attach` and `start_agent` inputs for the lifetime of `Running`. If its control process restarts, the replacement replays `attach`, `confirm`, and `start_agent` with the same authenticated boundary identity, resource claims, policy, and launch inputs. The boundary returns the existing process handle without starting a second workload. Any changed input is denied. A main-process stream has one active control owner; transport closure releases that attachment so the replacement control can attach. Compute drivers fence control replacement so old and new control processes do not overlap (for example, a Kubernetes control Deployment uses `Recreate`). + +`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Exit of the admitted agent produces a stable `wait` result and ends that process generation, but it does not implicitly tear down the boundary. The control role may continue serving terminal output, `exec`, and loopback forwarding within the same confirmed boundary until the compute driver or control role explicitly tears the boundary down. Explicit teardown terminates every remaining workload process and rejects new runtime operations. ### Runtime operations @@ -257,11 +293,32 @@ trait NetworkMediationSource: Send + Sync { struct MediatedConnection { stream: BoundaryDuplexStream, binary_identity: Result, + destination: Option, +} + +#[async_trait] +trait DnsMediationSource: Send + Sync { + async fn accept(&self) -> Result; +} + +struct MediatedDnsQuery { + request: Vec, + transport: DnsTransport, + binary_identity: Result, + response: oneshot::Sender, BackendError>>, } ``` `NetworkMediationSource` supplies outbound connections from one boundary to supervisor-owned network mediation. The backend routes all workload egress through that source and authoritatively associates each connection with the boundary without relying solely on workload-provided data. Capture, transport, placement, and coordination are backend-private. +Explicit-proxy transports leave `destination` absent. Transparent transports +capture the original socket destination and supply it before the supervisor +consumes workload bytes. `DnsMediationSource` carries portless DNS exchanges to +the supervisor-owned policy DNS service. It is optional because explicit-proxy +topologies resolve destinations in the supervisor and do not expose workload +DNS. A backend that advertises transparent networking supplies both sources; +DNS or connection-source failure closes that boundary's egress. + Every topology may use the same supervisor-owned mediation libraries or services; the source does not require a backend-specific policy engine. Shared implementations isolate each boundary's state and enforcement. Failure or teardown of one boundary cannot weaken another. Network-mediation unavailability never enables direct egress. @@ -289,13 +346,13 @@ Binary identity is mandatory conformance: RFC 0002 makes it part of the outbound ### The supervisor sequence -The logical supervisor resolves `backend_name` and version through a trusted implementation registry. Adding a backend adds an implementation and registration, not branches in lifecycle, proxy, SSH, or session code. Delegated transport and coordination remain backend-private. +The logical supervisor resolves `backend_name` and version through a trusted implementation registry. A separated topology selects the reusable remote backend and supplies its standardized endpoint in the opaque descriptor. Adding a driver adds provisioning and endpoint construction in that driver's crate, not branches in lifecycle, proxy, SSH, session, or control-mode code. The supervisor runs the same sequence for every backend: -1. Obtain the `TopologyDescriptor` and trusted `SandboxContext`. +1. Obtain the trusted `TopologyDescriptor` and `SandboxContext` selected by admission. 2. Verify the descriptor and resolve its `backend_name` and version without fallback. -3. Call `attach` to obtain `Bound`. +3. Call `attach` to establish or locate the resource, bind it, and obtain `Bound`. 4. Connect the boundary's `NetworkMediationSource` to network mediation. 5. Call `confirm` to obtain `Ready`, then `start_agent` to obtain `Running`. 6. Use the returned runtime handles for agent wait, `exec`, and port forwarding while network mediation consumes outbound connections. @@ -312,18 +369,20 @@ enum BackendErrorKind { Invalid, Denied, Unavailable, Unsupported, Failed, Termi `Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Unsupported` identifies an optional operation the selected backend does not implement; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back. -A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per provisioned topology. If it does not return `Bound`, the topology is reclaimed rather than reused. +An `Unsupported` error variant reports that the selected backend does not implement an optional contract operation; it maps to the `Unavailable` kind for status purposes and never weakens a mandatory conformance requirement. + +A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per orchestration attempt. Attach-time establishment uses sandbox identity and launch generation as an idempotency key. If the operation does not return `Bound`, the compute driver reclaims the topology rather than reusing an ambiguous resource. Failures resolve as follows: -- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the driver to reclaim the topology; -- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the driver reclaims the topology; +- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the compute driver to reclaim the topology; +- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the compute driver reclaims the topology; - if `exec` or port-forward `connect` fails, the backend terminates any process or closes any connection created by that attempt while the boundary otherwise remains active; - after `Running`, supervisor or enforcement loss follows invariant 6; when enforcement loss ends the agent, `BoundaryProcess::wait` fails with `BackendErrorKind::Terminated` where process-exit observation survives; - network-mediation errors yield no authorized connection and do not by themselves end `Running`; and - retained runtime handles and the network-mediation source reject new operations whenever the boundary ends, except `BoundaryProcess::wait` where the backend can still return its stable result. -Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. +Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. Normal agent exit alone does not end the boundary: `BoundaryProcess::wait` returns the stable exit status while the confirmed access plane remains available until explicit teardown. A retained `wait` result may outlive teardown. ### Topologies @@ -331,11 +390,12 @@ The contract fixes the roles; a topology fixes their placement. Components may b ## Implementation plan -This RFC defines the contract; implementation lands in three phases: +This RFC defines the contract; implementation lands in four phases: 1. **Contract.** Add the common types, descriptor handling, registry, and explicit backend selection from deployment configuration. -2. **Co-located backend.** Implement the co-located backend behind a deployment flag and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through it without changing behavior. -3. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, and failure semantics. Make the co-located backend the default after parity validation. Parity covers the agent, binary identity, SSH, `exec`, and forwarding paths; enablement also closes the in-pod egress gaps pinned in [codebase-grounding.md](./codebase-grounding.md), which parity alone would preserve. +2. **Shared supervisor modes.** Add the versioned boundary protocol, reusable remote backend, and the `control` and `boundary` entrypoints to `openshell-sandbox`. +3. **Driver adoption.** Have VM, Docker, and Kubernetes provision protected boundary configs and topology descriptors in their existing driver crates. No adoption may require a shared-supervisor change; that constraint is the abstraction test. +4. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, compute-driver-owned cleanup, and failure semantics. Existing placements remain outside this contract until their backend is implemented and admitted; they do not claim conformance. Delegated backends remain separate design and implementation work. @@ -343,7 +403,8 @@ Existing placements remain outside this contract until their backend is implemen | Risk | Mitigation | |---|---| -| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep the responsibility boundary explicit: the compute driver owns, provisions, and deprovisions the topology; the backend binds and operates the active boundary. The same component may implement both roles. | +| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep placement, preparation, durable deletion, reconciliation, and endpoint selection in the compute driver; keep enforcement sequencing in control mode and boundary-local observation in boundary mode. Generic supervisor code never imports a concrete driver. | +| Attach-time establishment could leave a partial resource after supervisor failure. | Make establishment idempotent by sandbox launch generation, label resources for compute-driver reconciliation, and do not start untrusted code before `attach` and `confirm` complete. | | Contract conformance could be mistaken for equivalent isolation across topologies. | Treat conformance as behavioral, not as a security-strength rating. Document and validate each topology's actual containment and reject policy it cannot enforce. | | Shared backend or network-mediation components concentrate privilege and failure impact. | Isolate state, connection attribution, enforcement, and control authority per boundary. Failure of one boundary must not weaken another or enable direct egress. | | The mandatory contract may exclude otherwise useful but incomplete backends. | Keep the network-mediation source, binary identity, process control, `exec`, and port forwarding mandatory. An incomplete backend does not claim conformance or silently degrade. | @@ -358,17 +419,17 @@ OpenShell could keep the current in-pod design and add topology-specific supervi Doing nothing avoids a new interface, but retains privileged boundary construction beside the workload. Implementing each delegated topology as a one-off supervisor change moves that privilege for one placement but accretes topology-specific supervisor behavior. The proposed contract instead keeps one supervisor lifecycle while allowing the topology to change. -### Extend the compute-driver contract +### Require every resource to exist before attach -The compute driver could own both provisioning and active-boundary operation. +The compute driver could always create the concrete resource before the supervisor calls `attach`. -This is natural for topologies such as MXC, and the same component may implement both responsibilities. The interfaces remain distinct because they serve different callers and lifecycles: the gateway uses the compute driver to provision and deprovision resources, while the supervisor uses the Isolation Backend to operate an active boundary. Combining them would couple runtime policy, identity, network mediation, and process operations to the gateway-facing driver API. +This remains natural for controller-driven systems such as Kubernetes. Requiring it everywhere prevents a local backend from establishing host listeners and enforcement state before a runtime creates the workload. Allowing a trusted topology descriptor to carry prepared inputs preserves the single `attach` contract while allowing security-sensitive establishment to remain atomic with binding. The compute driver still owns deletion and reconciliation. -### Start with a remote backend service +### Give each driver its own remote control service -The contract could be expressed as a gRPC service or plugin ABI rather than an in-process Rust contract. [RFC 0001](../0001-core-architecture/README.md) chose gRPC for its gateway-facing drivers, so the question applies here. +Each separated driver could define a private gRPC service, guest agent, or runtime-specific plugin ABI behind its `IsolationBackend` implementation. -The callers differ. A gateway driver is a control-plane peer with its own release cycle, while the Isolation Backend is driven by the supervisor that operates the boundary, and the co-located topology needs no transport at all. Starting in-process serves that case directly and lets delegated implementations carry their own transport behind the same interface. A transport-bearing surface is not precluded: it is versioned contract surface, added when a concrete delegated backend requires it. +That would hide transport differences from the Rust trait, but it would duplicate lifecycle, authentication, process streaming, signaling, forwarding, and binary-identity semantics across Docker, Kubernetes, and VM implementations. The proposed versioned boundary protocol standardizes those semantics once. Drivers still choose and provision Unix socket, authenticated TCP, vsock, or adapter transport and bind their own immutable resource claims. ### Standardize topology and capabilities @@ -381,10 +442,11 @@ That would make known deployments explicit, but it would also encode current top - **Driver-backed subsystems (CRI/CNI/CSI).** Kubernetes factors runtime, networking, and storage into pluggable driver contracts so the orchestrator drives one interface while implementations vary. RFC 0001 describes OpenShell's other subsystems the same way; this RFC specifies the one it left open: isolation. - **Istio privilege placement.** Init-sidecar and node-agent modes demonstrate that network setup can move without changing the policy data path. OpenShell keeps its identity-aware proxy. - **CRI exec/attach/port-forward.** `exec` and `connect` follow CRI's `Exec` and `PortForward` shape; lifecycle and network mediation remain OpenShell-specific. +- **[OCI seccomp listener handoff](https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#seccomp).** The runtime specification lets a runtime send a seccomp notification FD and process state to a host Unix listener. It demonstrates why some local enforcement must exist before workload creation and informs attach-time boundary establishment. ## Open questions -None. +None for this revision. New common fields require evidence from a concrete backend and a protocol-version change when compatibility cannot be preserved. ## Appendix: codebase grounding diff --git a/rfc/0012-isolation-backend/topology-matrix.md b/rfc/0012-isolation-backend/topology-matrix.md index 8dc14d84ba..85335be545 100644 --- a/rfc/0012-isolation-backend/topology-matrix.md +++ b/rfc/0012-isolation-backend/topology-matrix.md @@ -6,18 +6,29 @@ kernel; it does not select a deployment or establish conformance. ## Representative placements -| Pattern | Logical supervisor and network-mediation placement | Backend placement | Workload-kernel relationship | Topology status | +| Pattern | Control and network-mediation placement | Boundary-mode placement | Workload-kernel relationship | Topology status | |---|---|---|---|---| -| **Co-located/in-pod** | With the workload | In the supervisor process | Trusted components share the workload's host, guest, or application kernel, depending on the runtime | Placement implemented (original topology) | -| **Same-pod composite** | Spans the workload-local supervisor process and, when used, a network-mediation sidecar | In the workload-local supervisor process | Components share the workload's kernel | Placement implemented (#2076) | -| **Delegated backend components** | With the workload and any delegated mediation component | A node or remote helper establishes some controls behind a workload-local backend | Depends on which trusted components remain with the workload | Placement proposed (#2606) | -| **Driver-hosted/shared service** | With the compute driver or another trusted service; no in-sandbox supervisor process is required | May be co-located with the logical supervisor; one host may operate many isolated boundaries | Depends on the workload runtime | Placement proposed | +| **Co-located/in-pod** | With the workload; the legacy in-process backend may omit boundary mode | Same process when used | Trusted components share the workload's host kernel | Placement implemented (original topology) | +| **Kubernetes proxy pod** | Trusted control pod | Boundary-mode workload entrypoint owning the workload PID and network namespaces | Shared cluster-node kernel; pod security boundaries separate control from workload | Implemented in #3144; requires a conforming NetworkPolicy CNI and trusted namespace | +| **Docker** | Gateway host | Trusted container entrypoint sharing the workload container's PID and network namespaces | Shared host kernel | Implemented in #2965 | +| **MicroVM** | Gateway host | Guest PID 1 | Boundary mode shares the guest kernel; control is kernel-separated | Implemented in #2945 | + +The Kubernetes proxy-pod topology uses the same boundary protocol as Docker and +VM, with per-boundary TLS because the connection traverses the pod network. +Kubernetes-specific code provisions the workload fence, pair labels, boundary +Service, control Deployment, immutable bootstrap Secret, and stable +namespace/Sandbox/Deployment/NetworkPolicy claims. The workload pod has no +direct egress; attributed proxy streams cross the TLS channel and hostname +resolution occurs on the control side. Admission requires an explicitly acknowledged conforming CNI and a +namespace in which untrusted principals cannot create pods, mutate pair labels, +or read bootstrap Secrets. Pod readiness or the existence of a `NetworkPolicy` +object alone does not prove enforcement. ## Durable rules - Every active boundary has one verified descriptor, one trusted - `SandboxContext`, and at most one logical supervisor, which may span multiple - coupled processes. + `SandboxContext`, one control role, and at most one boundary role. Those + processes form one logical supervisor. - Physical processes and listeners may be shared, but lifecycle state, policy, binary identity, enforcement, and cleanup remain isolated per boundary. - Moving a privileged component does not itself provide kernel separation. diff --git a/tasks/rust.toml b/tasks/rust.toml index e62e22b3cf..50eda4118c 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -61,8 +61,8 @@ run = [ # operators to produce these artifacts. "cargo build -p openshell-gateway --bin openshell-gateway --no-default-features --features defaults-without-telemetry", "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway", - "cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features defaults-without-telemetry", - "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox", + "cargo build -p openshell-supervisor --bin openshell-supervisor --no-default-features --features defaults-without-telemetry", + "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-supervisor", ] ["rust:verify:defaults-without-telemetry"] @@ -72,11 +72,11 @@ run = "tasks/scripts/verify-defaults-without-telemetry.sh" ["rust:verify:system-ca-roots"] description = "Verify system CA roots build mode compiles and excludes bundled Mozilla root crates" run = [ - # Check that the sandbox compiles cleanly in system CA roots mode (all + # Check that the supervisor compiles cleanly in system CA roots mode (all # defaults except bundled-ca-roots). - "cargo check -p openshell-sandbox --all-targets --no-default-features --features system-ca-roots", + "cargo check -p openshell-supervisor --all-targets --no-default-features --features system-ca-roots", # Guard: webpki-roots must not appear in the dependency graph. - "bash -c 'if cargo tree -p openshell-sandbox -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then echo \"ERROR: webpki-roots found in system CA roots build\" >&2; exit 1; fi'", + "bash -c 'if cargo tree -p openshell-supervisor -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then echo \"ERROR: webpki-roots found in system CA roots build\" >&2; exit 1; fi'", # Guard: webpki-root-certs must not appear either (webpki-roots re-exports it). - "bash -c 'if cargo tree -p openshell-sandbox -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then echo \"ERROR: webpki-root-certs found in system CA roots build\" >&2; exit 1; fi'", + "bash -c 'if cargo tree -p openshell-supervisor -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then echo \"ERROR: webpki-root-certs found in system CA roots build\" >&2; exit 1; fi'", ] diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index 08ba00e066..5f055bf397 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -44,7 +44,7 @@ required_prebuilt_binaries() { echo "openshell-gateway" ;; supervisor|supervisor-sideload|supervisor-output) - echo "openshell-sandbox" + echo "openshell-sandbox openshell-supervisor" ;; esac } diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index fe4913439a..211864d007 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -109,10 +109,10 @@ components_for_target() { echo "gateway" ;; sandbox|supervisor|supervisor-output) - echo "supervisor" + echo "sandbox supervisor" ;; all) - echo "gateway supervisor" + echo "gateway sandbox supervisor" ;; *) usage @@ -128,11 +128,16 @@ resolve_component() { binary=openshell-gateway target_libc=gnu ;; - supervisor) + sandbox) crate=openshell-sandbox binary=openshell-sandbox target_libc=$(supervisor_libc) ;; + supervisor) + crate=openshell-supervisor + binary=openshell-supervisor + target_libc=$(supervisor_libc) + ;; *) echo "unsupported binary component: $1" >&2 exit 1 @@ -260,7 +265,7 @@ build_component_for_arch() { binary_path="${ROOT}/target/${target}/release/${binary}" if [[ "$component" == "gateway" ]]; then "$SCRIPT_DIR/verify-glibc-symbols.sh" 2.28 "$binary_path" - elif [[ "$component" == "supervisor" ]]; then + else "$SCRIPT_DIR/verify-static-binary.sh" "$binary_path" fi diff --git a/tasks/scripts/verify-defaults-without-telemetry.sh b/tasks/scripts/verify-defaults-without-telemetry.sh index 1fd7e67dff..104523b04b 100755 --- a/tasks/scripts/verify-defaults-without-telemetry.sh +++ b/tasks/scripts/verify-defaults-without-telemetry.sh @@ -23,7 +23,7 @@ set -euo pipefail # `defaults-without-telemetry`. CRATES=( openshell-gateway - openshell-sandbox + openshell-supervisor openshell-driver-vm ) From 2afe2c8da603cfdf010737a9a31d39477e94b99c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 15:38:42 -0700 Subject: [PATCH 02/10] docs(agents): register supervisor runtime Signed-off-by: Drew Newberry --- AGENTS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index d0049c85a3..3d56412929 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,8 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner | | `crates/openshell-conformance-cli/` | Conformance CLI | Distributable `list` and `run` entrypoint for gateway conformance | | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | -| `crates/openshell-sandbox/` | Sandbox runtime | Container supervision, policy-enforced egress routing | +| `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | +| `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | | `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | | `crates/openshell-isolation-interface/` | Isolation backend interface | RFC 0012 `IsolationBackend` trait + types; the supervisor-facing runtime contract for the boundary | | `crates/openshell-policy/` | Policy engine | Filesystem, network, process, and inference constraints | From 4713d1cdb2d5d29035f7baaa12668c82b3fe4268 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 16:57:56 -0700 Subject: [PATCH 03/10] fix(sandbox): harden boundary isolation and lifecycle ownership Signed-off-by: Drew Newberry --- architecture/sandbox.md | 25 +- .../openshell-sandbox/src/accept_interrupt.rs | 262 ++++++++++++++ crates/openshell-sandbox/src/boundary_exec.rs | 36 +- .../openshell-sandbox/src/boundary_server.rs | 341 +++++++----------- crates/openshell-sandbox/src/delegated.rs | 11 +- crates/openshell-sandbox/src/lib.rs | 4 +- crates/openshell-sandbox/src/main.rs | 92 +++-- .../openshell-sandbox/src/network_broker.rs | 173 +++++---- crates/openshell-sandbox/src/process.rs | 45 +-- .../src/sandbox/linux/landlock.rs | 170 +++++++-- 10 files changed, 752 insertions(+), 407 deletions(-) create mode 100644 crates/openshell-sandbox/src/accept_interrupt.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 7f4a5d17a5..32c6518634 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -25,7 +25,8 @@ The compute driver provisions separate protected configurations and one mutually authenticated gRPC connection over a private Unix socket, Kubernetes TCP Service, or VM vsock channel. Independent bidirectional `Exchange` RPCs carry lifecycle, exec, TCP, and forwarding traffic, while one persistent -bidirectional `Mediate` RPC carries multiplexed DNS and UDP traffic. +bidirectional `Mediate` RPC carries multiplexed DNS traffic. General application +UDP is unsupported; UDP DNS remains mediated by the supervisor. NetworkPolicy is an outer reachability fence, not a confidentiality boundary. Each sandbox generation receives a fresh CA and distinct server/client leaves; both endpoints bind the same workload identity and immutable driver resource @@ -48,7 +49,7 @@ replacement from granting authority. attaches to the sandbox, and verifies the driver's generation and evidence. 4. The sandbox installs its seccomp notification broker and Landlock baseline, then reports measured confirmation. The supervisor must accept that evidence - before it sends the launch permit. +before it sends the launch permit. 5. The sandbox starts the canonical process through its single workload launcher. The supervisor starts SSH and registers its gateway session. 6. Exec, signaling, PTY, DNS, TCP, and loopback-forwarding operations cross the @@ -59,6 +60,11 @@ remain available. The confirmed sandbox and supervisor-owned access plane contin to serve policy-authorized exec and loopback forwarding until explicit stop or delete tears down the boundary and terminates any remaining workload processes. +Completed exec output handles can be reclaimed, but execution request IDs remain +reserved for the boundary generation. The sandbox accepts at most 4,096 exec +attempts per generation, then rejects new attempts rather than forgetting replay +protection. A disconnected attachment does not authorize another execution. + ## Isolation Layers OpenShell uses overlapping controls rather than a single sandbox primitive: @@ -74,6 +80,12 @@ OpenShell uses overlapping controls rather than a single sandbox primitive: The supervisor may enrich baseline filesystem allowances for runtime-required paths, such as proxy support files or GPU device paths when a GPU is present. +The mandatory self-protection baseline is separate from optional workload +filesystem policy. It requires Landlock ABI v3, including pathname truncation +protection. Rules cover individually opened root children except `/.openshell`; +the sandbox opens entries relative to a pinned root descriptor without following +symlinks. An image-provided alias cannot grant access to the protected subtree. + ## Network and Inference See [Sandbox Limits](sandbox-limits.md) for the current numeric safety ceilings, @@ -86,7 +98,14 @@ bounded syscall inputs from the notifying task, resolves the calling binary, and blocks external `connect` until the supervisor returns a policy decision and relay stream. Connected data stays on ordinary kernel sockets, so the notification path is limited to socket setup and pointer-bearing operations. -This topology requires Linux 5.19 or newer: the sandbox treats +Blocking listener accepts retain native workload socket flags. A broker-owned +watchdog interrupts an accept when its seccomp notification is cancelled or the +broker stops, including when readiness disappears before the accept syscall. +The sandbox reserves `SIGUSR2` with a non-restarting no-op handler for these +broker threads; startup rejects a conflicting handler. This signal disposition +is process-global kernel state, while registrations and cancellation state are +owned by the broker. Workload exec resets the caught handler to its default. +This topology requires Linux 6.2 or newer for Landlock ABI v3 and treats `SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` as mandatory so cancelled notifications cannot race task-memory writes. diff --git a/crates/openshell-sandbox/src/accept_interrupt.rs b/crates/openshell-sandbox/src/accept_interrupt.rs new file mode 100644 index 0000000000..ef282d021d --- /dev/null +++ b/crates/openshell-sandbox/src/accept_interrupt.rs @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Cancellation for broker-owned blocking accepts without changing workload OFDs. +//! +//! SIGUSR2 is reserved by the sandbox binary. Its process-global disposition is +//! necessarily kernel state, not a global application context. All registration, +//! cancellation and thread ownership state belongs to one broker instance. + +#![allow(unsafe_code)] + +use std::collections::HashMap; +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +const INTERRUPT_SIGNAL: libc::c_int = libc::SIGUSR2; +const INTERRUPT_INTERVAL: Duration = Duration::from_millis(10); + +extern "C" fn interrupt_accept(_: libc::c_int) {} + +fn reserve_signal() -> io::Result<()> { + // SAFETY: both actions are initialized storage. The no-op handler is + // async-signal-safe and deliberately omits SA_RESTART so accept returns EINTR. + unsafe { + let mut previous: libc::sigaction = std::mem::zeroed(); + if libc::sigaction(INTERRUPT_SIGNAL, std::ptr::null(), &raw mut previous) < 0 { + return Err(io::Error::last_os_error()); + } + if previous.sa_sigaction != libc::SIG_DFL + && previous.sa_sigaction != interrupt_accept as *const () as usize + { + return Err(io::Error::other( + "sandbox SIGUSR2 is already reserved by another handler", + )); + } + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = interrupt_accept as *const () as usize; + libc::sigemptyset(&raw mut action.sa_mask); + if libc::sigaction(INTERRUPT_SIGNAL, &raw const action, std::ptr::null_mut()) < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) +} + +#[derive(Default)] +struct State { + workers: Mutex>, + changed: Condvar, + stopped: AtomicBool, +} + +pub struct AcceptMonitor { + state: Arc, + thread: Option>, +} + +impl AcceptMonitor { + pub(crate) fn start(valid: impl Fn(u64) -> bool + Send + 'static) -> io::Result { + reserve_signal()?; + let state = Arc::new(State::default()); + let worker_state = state.clone(); + let thread = std::thread::Builder::new() + .name("openshell-accept-cancellation".into()) + .spawn(move || monitor(&worker_state, valid))?; + Ok(Self { + state, + thread: Some(thread), + }) + } + + pub(crate) fn registrar(&self) -> AcceptRegistrar { + AcceptRegistrar(self.state.clone()) + } +} + +impl Drop for AcceptMonitor { + fn drop(&mut self) { + let workers = lock(&self.state.workers); + self.state.stopped.store(true, Ordering::Release); + self.state.changed.notify_all(); + drop(workers); + // The monitor keeps interrupting registered workers during shutdown. + // Registrations are removed before their threads can exit/reuse IDs. + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +#[derive(Clone)] +pub struct AcceptRegistrar(Arc); + +impl AcceptRegistrar { + pub(crate) fn register(&self, notification_id: u64) -> io::Result { + // SAFETY: this changes only the current broker worker's signal mask. + // Workload launchers do not inherit this mask; exec resets the handler. + let thread = unsafe { + let mut mask: libc::sigset_t = std::mem::zeroed(); + libc::sigemptyset(&raw mut mask); + libc::sigaddset(&raw mut mask, INTERRUPT_SIGNAL); + let error = + libc::pthread_sigmask(libc::SIG_UNBLOCK, &raw const mask, std::ptr::null_mut()); + if error != 0 { + return Err(io::Error::from_raw_os_error(error)); + } + libc::pthread_self() + }; + let mut workers = lock(&self.0.workers); + if self.0.stopped.load(Ordering::Acquire) { + return Err(io::Error::from_raw_os_error(libc::ECANCELED)); + } + if workers.contains_key(¬ification_id) { + return Err(io::Error::other( + "duplicate accept notification registration", + )); + } + workers.insert(notification_id, thread); + self.0.changed.notify_one(); + Ok(AcceptRegistration { + state: self.0.clone(), + notification_id, + }) + } +} + +pub struct AcceptRegistration { + state: Arc, + notification_id: u64, +} + +impl AcceptRegistration { + pub(crate) fn ensure_running(&self) -> io::Result<()> { + if self.state.stopped.load(Ordering::Acquire) { + Err(io::Error::from_raw_os_error(libc::ECANCELED)) + } else { + Ok(()) + } + } +} + +impl Drop for AcceptRegistration { + fn drop(&mut self) { + lock(&self.state.workers).remove(&self.notification_id); + self.state.changed.notify_one(); + } +} + +fn monitor(state: &State, valid: impl Fn(u64) -> bool) { + let mut workers = lock(&state.workers); + loop { + let stopped = state.stopped.load(Ordering::Acquire); + if stopped && workers.is_empty() { + return; + } + for (¬ification_id, &thread) in &*workers { + if stopped || !valid(notification_id) { + // SAFETY: the registration lock pins this live pthread_t. + // Repeated interrupts close the check-to-accept race: a signal + // received before accept cannot leave a later accept stranded. + let _ = unsafe { libc::pthread_kill(thread, INTERRUPT_SIGNAL) }; + } + } + workers = if workers.is_empty() { + state + .changed + .wait(workers) + .unwrap_or_else(std::sync::PoisonError::into_inner) + } else { + state + .changed + .wait_timeout(workers, INTERRUPT_INTERVAL) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0 + }; + } +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{TcpListener, TcpStream}; + use std::os::fd::AsRawFd; + + #[test] + fn cancellation_interrupts_competing_accept_after_readiness_was_consumed() { + let valid = Arc::new(AtomicBool::new(true)); + let monitored = valid.clone(); + let monitor = AcceptMonitor::start(move |_| monitored.load(Ordering::Acquire)).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + // Both contenders could observe this same readable listener. Consume + // its only connection before the second contender actually accepts. + let accepted = listener.accept().unwrap(); + let registrar = monitor.registrar(); + let (ready_tx, ready_rx) = std::sync::mpsc::channel(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let registration = registrar.register(1).unwrap(); + ready_tx.send(()).unwrap(); + // SAFETY: the listener is live and null address outputs are valid. + // Use the syscall directly: std::net retries EINTR internally. + let result = unsafe { + libc::accept4( + listener.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + assert_eq!(result, -1); + let error = io::Error::last_os_error(); + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + drop(registration); + done_tx.send(()).unwrap(); + }); + ready_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + valid.store(false, Ordering::Release); + done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + worker.join().unwrap(); + drop((accepted, client, monitor)); + } + + #[test] + fn shutdown_interrupts_registered_accepts_and_reclaims_the_monitor() { + let monitor = AcceptMonitor::start(|_| true).unwrap(); + let registrar = monitor.registrar(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let (ready_tx, ready_rx) = std::sync::mpsc::channel(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let registration = registrar.register(2).unwrap(); + ready_tx.send(()).unwrap(); + // SAFETY: owned listener and optional null address outputs. + let result = unsafe { + libc::accept4( + listener.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + assert_eq!(result, -1); + assert!(registration.ensure_running().is_err()); + drop(registration); + done_tx.send(()).unwrap(); + }); + ready_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let shutdown = std::thread::spawn(move || drop(monitor)); + done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + worker.join().unwrap(); + shutdown.join().unwrap(); + } +} diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index 5fb9ea5ab7..57d9778a6f 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -30,6 +30,8 @@ pub struct LocalBoundaryExec { provider_credentials: ProviderCredentialState, user_environment: HashMap, runtime: Arc, + #[cfg(target_os = "linux")] + launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, } impl LocalBoundaryExec { @@ -42,6 +44,8 @@ impl LocalBoundaryExec { provider_credentials: ProviderCredentialState, user_environment: HashMap, runtime: Arc, + #[cfg(target_os = "linux")] + launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, ) -> Self { Self { policy, @@ -50,6 +54,8 @@ impl LocalBoundaryExec { provider_credentials, user_environment, runtime, + #[cfg(target_os = "linux")] + launcher, } } @@ -138,8 +144,9 @@ impl LocalBoundaryExec { #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_workload_launcher(command) - .map_err(|error| BackendError::Process(error.to_string()))?; + let mut child = + crate::process::spawn_std_command_with_workload_launcher(&self.launcher, command) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(not(target_os = "linux"))] let mut child = command .spawn() @@ -248,8 +255,9 @@ impl LocalBoundaryExec { #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_workload_launcher(command) - .map_err(|error| BackendError::Process(error.to_string()))?; + let mut child = + crate::process::spawn_std_command_with_workload_launcher(&self.launcher, command) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(not(target_os = "linux"))] let mut child = command .spawn() @@ -495,22 +503,15 @@ impl BoundaryProcess for LocalExecProcess { #[cfg(all(test, target_os = "linux"))] mod tests { use super::*; - use std::sync::Once; use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn executor() -> LocalBoundaryExec { - static LAUNCHER: Once = Once::new(); - LAUNCHER.call_once(|| { - let (launcher, listener) = - openshell_isolation_interface::linux::workload_launcher::start() - .expect("start test workload launcher"); - std::thread::spawn(move || { - while let Ok(notification) = listener.receive() { - let _ = listener.respond_errno(notification.id, libc::EPERM); - } - }); - crate::process::configure_workload_launcher(launcher) - .expect("configure test workload launcher"); + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start test workload launcher"); + std::thread::spawn(move || { + while let Ok(notification) = listener.receive() { + let _ = listener.respond_errno(notification.id, libc::EPERM); + } }); LocalBoundaryExec::new( SandboxPolicy { @@ -530,6 +531,7 @@ mod tests { ), HashMap::new(), crate::boundary_io::BoundaryRuntimeState::new(), + launcher, ) } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 7fcd040662..12ae615478 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -13,12 +13,12 @@ use std::path::Path; #[cfg(target_os = "linux")] mod linux { - use super::Path; use std::fs::File; use std::io::{self, Read, Write}; use std::mem::size_of; use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _, PermissionsExt as _}; + use std::path::Path; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex}; @@ -52,11 +52,10 @@ mod linux { AgentSpecWire, BinaryIdentityWire, BoundaryConfig, BoundaryListener as BoundaryListenerConfig, DnsQueryResultWire, ExecSpecWire, ExitStatusWire, MediationTimingWire, OutputWindowWire, ProcessKindWire, - ProcessSnapshotWire, Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_DNS_ACK, - STREAM_DNS_RESPONSE, STREAM_EXIT, STREAM_NETWORK_DECISION, STREAM_STDERR, STREAM_STDIN, - STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SessionSnapshotWire, SignalWire, - encode_frame, read_frame, read_stream_frame, validate_resource_claims, write_frame, - write_stream_frame, + ProcessSnapshotWire, Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_EXIT, + STREAM_NETWORK_DECISION, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, + SandboxPolicyWire, SessionSnapshotWire, SignalWire, encode_frame, read_frame, + read_stream_frame, validate_resource_claims, write_frame, write_stream_frame, }; const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); @@ -120,8 +119,6 @@ mod linux { .map_err(|error| format!("install sandbox process prelude: {error}"))?; let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() .map_err(|error| format!("start sandbox workload launcher: {error}"))?; - crate::process::configure_workload_launcher(launcher.clone()) - .map_err(|error| format!("configure sandbox workload launcher: {error}"))?; let network_broker = NetworkBroker::start(listener) .map_err(|error| format!("start sandbox network broker: {error}"))?; let process_runtime = tokio::runtime::Builder::new_multi_thread() @@ -135,7 +132,7 @@ mod linux { launcher, qualification, )); - serve(&config.listener, config.multiplexed, runtime) + serve(&config.listener, runtime) } fn make_boundary_nondumpable() -> Result<(), String> { @@ -256,7 +253,7 @@ mod linux { let expected = config .resource_claims .get(claim) - .expect("validated resource-claim file key"); + .ok_or_else(|| format!("resource claim file has no expected value: {claim}"))?; let observed = std::fs::read_to_string(path).map_err(|error| { format!( "read runtime resource claim {claim} from {}: {error}", @@ -348,11 +345,7 @@ mod linux { Ok(()) } - fn serve( - config: &BoundaryListenerConfig, - multiplexed: bool, - runtime: Arc, - ) -> Result<(), String> { + fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { let listener = ControlListener::bind(config) .map_err(|error| format!("bind boundary control listener: {error}"))?; let active_connections = Arc::new(AtomicUsize::new(0)); @@ -381,7 +374,7 @@ mod linux { return; } }; - let result = if multiplexed { + let result = { let stream = match stream.into_tokio() { Ok(stream) => stream, Err(error) => { @@ -392,8 +385,6 @@ mod linux { runtime .process_runtime .block_on(serve_grpc(stream, runtime.clone())) - } else { - serve_one(stream, &runtime) }; if let Err(error) = result { tracing::warn!(%error, "Boundary control session failed: {error}"); @@ -417,7 +408,7 @@ mod linux { tonic::transport::Server::builder() .max_concurrent_streams( u32::try_from(MAX_CONTROL_CONNECTIONS) - .expect("control connection limit fits in HTTP/2 settings"), + .map_err(|error| format!("invalid control connection limit: {error}"))?, ) .initial_stream_window_size(16 * 1024 * 1024) .initial_connection_window_size(16 * 1024 * 1024) @@ -490,7 +481,7 @@ mod linux { let runtime = self.runtime.clone(); tokio::task::spawn_blocking(move || { let stream = ControlStream::Grpc { - stream: Some(stream), + stream, runtime: runtime.process_runtime.clone(), }; if let Err(error) = serve_one(stream, &runtime) { @@ -946,67 +937,6 @@ mod linux { })?; return Ok(()); } - Request::AcceptDns => { - let broker = runtime.network_accept_context()?; - let request_id = request.request_id; - runtime.process_runtime.block_on(async move { - let mut stream = stream.into_tokio()?; - let mut disconnect_probe = [0_u8; 1]; - let pending = tokio::select! { - biased; - read = stream.read(&mut disconnect_probe) => { - match read { - Ok(0) => return Ok(()), - Ok(_) => return Err("control sent data before DNS mediation response".to_string()), - Err(error) => return Err(format!("watch DNS mediation control stream: {error}")), - } - } - pending = broker.accept_dns() => pending - .map_err(|error| format!("accept sandbox DNS query: {error}"))?, - }; - let response = encode_frame(&ResponseEnvelope { - request_id, - response: Response::DnsQuery { - request: pending.request.clone(), - transport: pending.transport, - identity: BinaryIdentityWire::from(pending.identity.clone()), - timing: MediationTimingWire { - notification_to_queue_us: duration_micros( - pending.notification_to_queue, - ), - queue_wait_us: duration_micros(pending.queued_at.elapsed()), - }, - }, - }) - .map_err(|error| format!("encode DNS mediation response: {error}"))?; - stream - .write_all(&response) - .await - .map_err(|error| format!("write DNS mediation response: {error}"))?; - let Some((channel, payload)) = read_stream_frame(&mut stream) - .await - .map_err(|error| format!("read DNS mediation result: {error}"))? - else { - return Err("control disconnected before DNS response".to_string()); - }; - if channel != STREAM_DNS_RESPONSE { - return Err(format!("unexpected DNS response channel {channel}")); - } - let result: DnsQueryResultWire = serde_json::from_slice(&payload) - .map_err(|error| format!("decode DNS mediation result: {error}"))?; - let result = match result { - DnsQueryResultWire::Response(response) => Ok(response), - DnsQueryResultWire::Error(error) => Err(io::Error::other(error)), - }; - pending - .complete(result) - .map_err(|error| format!("complete sandbox DNS query: {error}"))?; - write_stream_frame(&mut stream, STREAM_DNS_ACK, &[]) - .await - .map_err(|error| format!("acknowledge sandbox DNS response: {error}")) - })?; - return Ok(()); - } _ => {} } let response = ResponseEnvelope { @@ -1034,6 +964,10 @@ mod linux { mediation_active: AtomicBool, next_mediation_stream_id: AtomicU64, exec_handles: Mutex>, + /// Never evicted within a boundary generation. Reclaiming process I/O + /// must not make an old command executable again. At capacity, reject + /// new commands instead of silently weakening at-most-once execution. + exec_requests: Mutex>, replay_ledger: Mutex, network_broker: NetworkBroker, workload_launcher: @@ -1052,6 +986,27 @@ mod linux { status: Arc>>, } + #[allow(clippy::result_large_err)] + fn reserve_exec_request( + requests: &mut std::collections::HashSet, + request_id: &str, + ) -> Result<(), Response> { + if requests.contains(request_id) { + return Err(guest_error( + "denied", + "exec request has expired; it cannot be executed again", + )); + } + if requests.len() >= MAX_REPLAY_LEDGER_ENTRIES { + return Err(guest_error( + "unavailable", + "boundary generation exec request limit reached", + )); + } + requests.insert(request_id.to_owned()); + Ok(()) + } + struct StartedExec { process_id: String, terminal: bool, @@ -1206,6 +1161,7 @@ mod linux { mediation_active: AtomicBool::new(false), next_mediation_stream_id: AtomicU64::new(1), exec_handles: Mutex::new(std::collections::HashMap::new()), + exec_requests: Mutex::new(std::collections::HashSet::new()), replay_ledger: Mutex::new(ReplayLedger::default()), network_broker, workload_launcher, @@ -1305,8 +1261,7 @@ mod linux { Request::Exec { .. } | Request::AttachProcess { .. } | Request::PortForward { .. } - | Request::AcceptNetwork - | Request::AcceptDns => { + | Request::AcceptNetwork => { guest_error("invalid", "streaming request used on control path") } }; @@ -1388,6 +1343,10 @@ mod linux { )); } } + { + let mut requests = lock(&self.exec_requests); + reserve_exec_request(&mut requests, request_id)?; + } let session = self .process_runtime .block_on(executor.exec(spec.into())) @@ -1733,7 +1692,7 @@ mod linux { }; let mut state = lock(&self.state); let requested = StartedAgent { - sandbox_id: sandbox_id.clone(), + sandbox_id, spec: spec.clone(), policy: policy.clone(), ca_cert: ca_cert.clone(), @@ -1774,18 +1733,21 @@ mod linux { } let launch = ManagedProcessLaunch { process_id: format!("{}:main:0", self.config.generation), - sandbox_id, spec, policy, provider_env_revision, provider_env, ca_file_paths, }; - let process = - match ManagedProcess::spawn(&self.process_runtime, launch, prepared.clone()) { - Ok(process) => Arc::new(process), - Err(error) => return guest_error("failed", error), - }; + let process = match ManagedProcess::spawn( + &self.process_runtime, + &self.workload_launcher, + launch, + prepared.clone(), + ) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; let process_id = process.process_id(); *lock(&self.started_agent) = Some(requested); *state = RuntimeState::Running(process); @@ -2072,7 +2034,6 @@ mod linux { struct ManagedProcessLaunch { process_id: String, - sandbox_id: String, spec: AgentSpecWire, policy: openshell_core::policy::SandboxPolicy, provider_env_revision: u64, @@ -2101,12 +2062,12 @@ mod linux { impl ManagedProcess { fn spawn( runtime: &tokio::runtime::Handle, + launcher: &openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, launch: ManagedProcessLaunch, _prepared: PreparedBoundary, ) -> Result { let ManagedProcessLaunch { process_id, - sandbox_id, spec, policy, provider_env_revision, @@ -2122,18 +2083,14 @@ mod linux { ); let mut spawned = runtime .block_on(spawn_workload( + launcher, &spec.program, &spec.args, spec.workdir.as_deref(), spec.timeout_secs, spec.interactive, - Some(&sandbox_id), - None, - None, - false, &policy, entrypoint_pid, - None, provider_credentials.clone(), provider_env, ca_file_paths, @@ -2176,12 +2133,14 @@ mod linux { fn wait(&self) -> ProcessExit { let (state, changed) = &*self.exit; let mut exit = lock(state); - while exit.is_none() { + loop { + if let Some(result) = exit.as_ref() { + return result.clone(); + } exit = changed .wait(exit) .unwrap_or_else(std::sync::PoisonError::into_inner); } - exit.as_ref().expect("exit checked above").clone() } fn signal(&self, signal: SignalWire) -> Result<(), String> { @@ -2618,17 +2577,15 @@ mod linux { server_config: Arc, }, Tls { - stream: Option< - Box< - tokio_rustls::server::TlsStream< - openshell_isolation_interface::contract::BoundaryDuplexStream, - >, + stream: Box< + tokio_rustls::server::TlsStream< + openshell_isolation_interface::contract::BoundaryDuplexStream, >, >, runtime: tokio::runtime::Handle, }, Grpc { - stream: Option, + stream: tokio::io::DuplexStream, runtime: tokio::runtime::Handle, }, #[cfg(test)] @@ -2658,45 +2615,36 @@ mod linux { .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) })?; Ok(Self::Tls { - stream: Some(Box::new(stream)), + stream: Box::new(stream), runtime: runtime.clone(), }) } fn set_timeout(&self, timeout: Duration) -> io::Result<()> { let _ = timeout; - if matches!(self, Self::Tls { .. } | Self::Grpc { .. }) { - return Ok(()); - } - if matches!(self, Self::PendingTls { .. }) { - return Err(io::Error::new( + match self { + Self::Tls { .. } | Self::Grpc { .. } => Ok(()), + Self::PendingTls { .. } => Err(io::Error::new( io::ErrorKind::NotConnected, "boundary TLS stream has not completed its handshake", - )); - } - #[cfg(test)] - if let Self::TestUnix(stream) = self { - stream.set_read_timeout(Some(timeout))?; - return stream.set_write_timeout(Some(timeout)); + )), + #[cfg(test)] + Self::TestUnix(stream) => { + stream.set_read_timeout(Some(timeout))?; + stream.set_write_timeout(Some(timeout)) + } } - unreachable!("all established sandbox streams use mutual TLS") } fn into_tokio( self, ) -> Result { match self { - Self::Tls { mut stream, .. } => Ok(stream - .take() - .expect("boundary TLS stream can only be converted once")), + Self::Tls { stream, .. } => Ok(stream), Self::PendingTls { .. } => { Err("boundary TLS stream has not completed its handshake".to_string()) } - Self::Grpc { mut stream, .. } => Ok(Box::new( - stream - .take() - .expect("gRPC boundary stream can only be converted once"), - )), + Self::Grpc { stream, .. } => Ok(Box::new(stream)), #[cfg(test)] Self::TestUnix(stream) => { stream @@ -2714,34 +2662,22 @@ mod linux { fn read(&mut self, buffer: &mut [u8]) -> io::Result { match self { Self::Tls { stream, runtime } => runtime.block_on(async { - tokio::time::timeout( - CONTROL_IO_TIMEOUT, - stream - .as_mut() - .expect("boundary TLS stream must be present") - .read(buffer), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "boundary TLS read timed out") - })? + tokio::time::timeout(CONTROL_IO_TIMEOUT, stream.read(buffer)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS read timed out") + })? }), Self::PendingTls { .. } => Err(io::Error::new( io::ErrorKind::NotConnected, "boundary TLS stream has not completed its handshake", )), Self::Grpc { stream, runtime } => runtime.block_on(async { - tokio::time::timeout( - CONTROL_IO_TIMEOUT, - stream - .as_mut() - .expect("gRPC boundary stream must be present") - .read(buffer), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary read timed out") - })? + tokio::time::timeout(CONTROL_IO_TIMEOUT, stream.read(buffer)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary read timed out") + })? }), #[cfg(test)] Self::TestUnix(stream) => stream.read(buffer), @@ -2753,34 +2689,22 @@ mod linux { fn write(&mut self, buffer: &[u8]) -> io::Result { match self { Self::Tls { stream, runtime } => runtime.block_on(async { - tokio::time::timeout( - CONTROL_IO_TIMEOUT, - stream - .as_mut() - .expect("boundary TLS stream must be present") - .write(buffer), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "boundary TLS write timed out") - })? + tokio::time::timeout(CONTROL_IO_TIMEOUT, stream.write(buffer)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS write timed out") + })? }), Self::PendingTls { .. } => Err(io::Error::new( io::ErrorKind::NotConnected, "boundary TLS stream has not completed its handshake", )), Self::Grpc { stream, runtime } => runtime.block_on(async { - tokio::time::timeout( - CONTROL_IO_TIMEOUT, - stream - .as_mut() - .expect("gRPC boundary stream must be present") - .write(buffer), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary write timed out") - })? + tokio::time::timeout(CONTROL_IO_TIMEOUT, stream.write(buffer)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary write timed out") + })? }), #[cfg(test)] Self::TestUnix(stream) => stream.write(buffer), @@ -2790,34 +2714,22 @@ mod linux { fn flush(&mut self) -> io::Result<()> { match self { Self::Tls { stream, runtime } => runtime.block_on(async { - tokio::time::timeout( - CONTROL_IO_TIMEOUT, - stream - .as_mut() - .expect("boundary TLS stream must be present") - .flush(), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "boundary TLS flush timed out") - })? + tokio::time::timeout(CONTROL_IO_TIMEOUT, stream.flush()) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS flush timed out") + })? }), Self::PendingTls { .. } => Err(io::Error::new( io::ErrorKind::NotConnected, "boundary TLS stream has not completed its handshake", )), Self::Grpc { stream, runtime } => runtime.block_on(async { - tokio::time::timeout( - CONTROL_IO_TIMEOUT, - stream - .as_mut() - .expect("gRPC boundary stream must be present") - .flush(), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary flush timed out") - })? + tokio::time::timeout(CONTROL_IO_TIMEOUT, stream.flush()) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "gRPC boundary flush timed out") + })? }), #[cfg(test)] Self::TestUnix(stream) => stream.flush(), @@ -2832,6 +2744,21 @@ mod linux { BoundaryClientTls, BoundaryServerTls, generate_boundary_mutual_tls_material, }; + #[test] + fn exec_tombstones_outlive_retained_handles_and_fail_closed_at_capacity() { + let mut requests = std::collections::HashSet::new(); + reserve_exec_request(&mut requests, "first").unwrap(); + // Process/I/O retention is deliberately not consulted by this + // ledger: dropping all handles cannot make this ID executable. + assert!(reserve_exec_request(&mut requests, "first").is_err()); + for index in 1..MAX_REPLAY_LEDGER_ENTRIES { + reserve_exec_request(&mut requests, &format!("request-{index}")).unwrap(); + } + assert!(reserve_exec_request(&mut requests, "overflow").is_err()); + assert!(requests.contains("first")); + assert_eq!(requests.len(), MAX_REPLAY_LEDGER_ENTRIES); + } + #[test] fn replay_ledger_evicts_oldest_records_without_disabling_control() { let mut ledger = ReplayLedger::default(); @@ -2937,7 +2864,6 @@ mod linux { control_port: 5500, tls: placeholder_server_tls(), }, - multiplexed: false, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), workload_identity: test_workload_identity(), @@ -3086,8 +3012,6 @@ mod linux { let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() .expect("start test listener"); - crate::process::configure_workload_launcher(launcher.clone()) - .expect("configure test workload launcher"); ( NetworkBroker::start_for_test(listener).expect("start test network broker"), launcher, @@ -3157,7 +3081,6 @@ mod linux { control_port: 5500, tls: placeholder_server_tls(), }, - multiplexed: false, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), workload_identity: test_workload_identity(), @@ -3183,7 +3106,6 @@ mod linux { control_port: 5500, tls: placeholder_server_tls(), }, - multiplexed: false, resource_claims: std::collections::BTreeMap::from([( "kubernetes.pod_uid".to_string(), "pod-uid-a".to_string(), @@ -3224,7 +3146,6 @@ mod linux { address: "127.0.0.1:5500".parse().expect("control address"), tls: placeholder_server_tls(), }, - multiplexed: true, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), workload_identity: test_workload_identity(), @@ -3396,7 +3317,6 @@ mod linux { address: "127.0.0.1:5500".parse().expect("control address"), tls: placeholder_server_tls(), }, - multiplexed: false, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), workload_identity: test_workload_identity(), @@ -3643,9 +3563,9 @@ mod linux { let process = Arc::new( ManagedProcess::spawn( process_runtime.handle(), + &workload_launcher, ManagedProcessLaunch { process_id: "generation-retained:main:0".to_string(), - sandbox_id: "sandbox-retained".to_string(), spec: agent_spec.clone(), policy, provider_env_revision: 0, @@ -3666,7 +3586,6 @@ mod linux { address: "127.0.0.1:5500".parse().expect("control address"), tls: placeholder_server_tls(), }, - multiplexed: false, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), workload_identity: test_workload_identity(), @@ -3811,7 +3730,7 @@ mod linux { .start_exec( &sleep_request.request_id, &sleep_request.payload_digest, - sleep_spec, + sleep_spec.clone(), ) .expect("reattach exec after response loss"); assert_eq!(replayed.process_id, retained_id); @@ -3821,6 +3740,17 @@ mod linux { boundary.signal_exec(&retained_id, SignalWire::Kill), Response::Signaled ); + lock(&boundary.exec_handles).remove(&retained_id); + assert!( + boundary + .start_exec( + &sleep_request.request_id, + &sleep_request.payload_digest, + sleep_spec, + ) + .is_err(), + "an evicted exec request must never start a second process" + ); let deadline = std::time::Instant::now() + Duration::from_secs(5); while !process.has_exited() && std::time::Instant::now() < deadline { @@ -3863,7 +3793,12 @@ mod linux { } #[cfg(target_os = "linux")] -pub use linux::run_boundary; +pub fn run_boundary( + config_path: &Path, + qualification: crate::RuntimeQualification, +) -> Result<(), String> { + linux::run_boundary(config_path, qualification) +} #[cfg(not(target_os = "linux"))] pub fn run_boundary( diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index d8c15535f0..7aa60f6520 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -28,18 +28,14 @@ fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { /// inside its boundary. #[allow(clippy::too_many_arguments, clippy::implicit_hasher)] pub async fn spawn_workload( + launcher: &openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, program: &str, args: &[String], workdir: Option<&str>, timeout_secs: u64, interactive: bool, - _sandbox_id: Option<&str>, - _openshell_endpoint: Option<&str>, - _ssh_socket_path: Option, - _shared_ssh_socket: bool, policy: &SandboxPolicy, entrypoint_pid: Arc, - entrypoint_started_tx: Option>, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -80,10 +76,12 @@ pub async fn spawn_workload( provider_credentials, user_environment, boundary_runtime.clone(), + launcher.clone(), )); #[cfg(target_os = "linux")] let mut handle = ProcessHandle::spawn( + launcher, program, args, &workspace, @@ -105,9 +103,6 @@ pub async fn spawn_workload( )?; entrypoint_pid.store(handle.pid(), Ordering::Release); - if let Some(sender) = entrypoint_started_tx { - let _ = sender.send(handle.pid()); - } let main_session = crate::main_session::MainSession::new(handle.take_io(), handle.pid()); let (terminal, signal_lock) = handle.signaling_state(); boundary_runtime diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index dea0ef1c77..78bc3d2629 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -3,6 +3,8 @@ //! Capability-free in-workload sandbox boundary. +#[cfg(target_os = "linux")] +mod accept_interrupt; pub mod boundary_exec; pub mod boundary_io; mod boundary_server; @@ -16,7 +18,7 @@ pub mod main_session; pub mod managed_children; #[cfg(target_os = "linux")] mod network_broker; -#[cfg(target_os = "linux")] +#[cfg(unix)] pub mod process; mod pty; pub mod sandbox; diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 192c9dec6e..bd61156004 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -108,15 +108,47 @@ fn validate_workspace(_args: &[String]) -> Result<()> { fn run_capability_probe() -> Result<()> { let (qualification, report) = qualify_runtime()?; debug_assert!(qualification.seccomp.notification_round_trip); - println!("{report}"); + println!("{}", serde_json::to_string(&report).into_diagnostic()?); Ok(()) } /// Actively qualify every kernel primitive used by the capability-free /// sandbox. Callers decide whether to emit the resulting diagnostic report. +#[cfg(target_os = "linux")] +#[derive(serde::Serialize)] +#[allow( + clippy::struct_excessive_bools, + reason = "diagnostic report records independent active probes" +)] +struct QualificationReport { + qualified: bool, + uid: u32, + gid: u32, + supplementary_groups: Vec, + capabilities_zero: bool, + no_new_privileges: bool, + sandbox_dumpable: bool, + child_dumpable: bool, + child_core_limit_zero: bool, + same_uid_self_protection: bool, + landlock_abi: u32, + landlock_allow_deny: bool, + seccomp_notification: bool, + seccomp_addfd_send: bool, + task_memory_copy: bool, + connected_send_fast_path: bool, + socket_virtualization: bool, + dns_relay_bind: bool, + udp_dns_round_trip: bool, + tcp_dns_round_trip: bool, + tcp_allow_round_trip: bool, + tcp_deny_round_trip: bool, + wait_killable_recv: bool, +} + #[cfg(target_os = "linux")] #[allow(unsafe_code)] -fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, serde_json::Value)> { +fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, QualificationReport)> { use miette::Context as _; let uid = nix::unistd::geteuid().as_raw(); @@ -168,8 +200,10 @@ fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, serde_j let landlock_abi = openshell_isolation_interface::linux::landlock::abi_version() .into_diagnostic() .wrap_err("Landlock ABI probe")?; - if landlock_abi == 0 { - return Err(miette::miette!("Landlock ABI version is zero")); + if landlock_abi < 3 { + return Err(miette::miette!( + "sandbox self-protection requires Landlock ABI v3 or newer (including truncation), found v{landlock_abi}" + )); } let groups = nix::unistd::getgroups() @@ -177,31 +211,31 @@ fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, serde_j .into_iter() .map(nix::unistd::Gid::as_raw) .collect::>(); - let report = serde_json::json!({ - "qualified": true, - "uid": uid, - "gid": gid, - "supplementary_groups": groups, - "capabilities_zero": true, - "no_new_privileges": true, - "sandbox_dumpable": false, - "child_dumpable": true, - "child_core_limit_zero": true, - "same_uid_self_protection": true, - "landlock_abi": landlock_abi, - "landlock_allow_deny": true, - "seccomp_notification": notification.notification_round_trip(), - "seccomp_addfd_send": notification.addfd_send(), - "task_memory_copy": notification.task_memory_copy(), - "connected_send_fast_path": notification.connected_send_fast_path(), - "socket_virtualization": true, - "dns_relay_bind": true, - "udp_dns_round_trip": true, - "tcp_dns_round_trip": true, - "tcp_allow_round_trip": true, - "tcp_deny_round_trip": true, - "wait_killable_recv": notification.wait_killable_recv, - }); + let report = QualificationReport { + qualified: true, + uid, + gid, + supplementary_groups: groups, + capabilities_zero: true, + no_new_privileges: true, + sandbox_dumpable: false, + child_dumpable: true, + child_core_limit_zero: true, + same_uid_self_protection: true, + landlock_abi, + landlock_allow_deny: true, + seccomp_notification: notification.notification_round_trip(), + seccomp_addfd_send: notification.addfd_send(), + task_memory_copy: notification.task_memory_copy(), + connected_send_fast_path: notification.connected_send_fast_path(), + socket_virtualization: true, + dns_relay_bind: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + wait_killable_recv: notification.wait_killable_recv, + }; let qualification = openshell_sandbox::RuntimeQualification { seccomp: openshell_isolation_interface::contract::SeccompEvidence { new_listener: notification.notification_round_trip(), diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 049ba5a432..4467762b3e 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -158,6 +158,8 @@ struct DnsRelay { #[derive(Clone)] struct NotificationQueues { + accept_registrar: crate::accept_interrupt::AcceptRegistrar, + identity_resolver: ProcfsIdentityResolver, pending: mpsc::Sender, dns_relay: DnsRelay, active_opens: Arc, @@ -167,6 +169,7 @@ struct NotificationQueues { /// Live broker handle retained by the sandbox boundary. #[derive(Clone)] pub struct NetworkBroker { + _accept_monitor: Arc, pending: Arc>>, pending_dns: Arc>>, dns_address: SocketAddr, @@ -191,6 +194,10 @@ impl NetworkBroker { dns_address: SocketAddr, ) -> io::Result { let listener = Arc::new(listener); + let monitor_listener = listener.clone(); + let accept_monitor = Arc::new(crate::accept_interrupt::AcceptMonitor::start(move |id| { + monitor_listener.validate_id(id).is_ok() + })?); let (pending_tx, pending_rx) = mpsc::channel(OPEN_QUEUE_CAPACITY); let (pending_dns_tx, pending_dns_rx) = mpsc::channel(DNS_QUEUE_CAPACITY); let registry = Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)); @@ -199,6 +206,8 @@ impl NetworkBroker { let dns_relay = start_dns_relay(dns_address, pending_dns_tx)?; let dns_address = dns_relay.address; let queues = NotificationQueues { + accept_registrar: accept_monitor.registrar(), + identity_resolver: ProcfsIdentityResolver::for_pid_namespace(), pending: pending_tx, dns_relay, active_opens, @@ -239,6 +248,7 @@ impl NetworkBroker { }) .map_err(|error| io::Error::other(format!("start network broker: {error}")))?; Ok(Self { + _accept_monitor: accept_monitor, pending: Arc::new(tokio::sync::Mutex::new(pending_rx)), pending_dns: Arc::new(tokio::sync::Mutex::new(pending_dns_rx)), dns_address, @@ -458,6 +468,13 @@ fn dispatch_notification( queues: NotificationQueues, ) -> io::Result<()> { let syscall = i64::from(notification.syscall); + if matches!(syscall, libc::SYS_kill | libc::SYS_rt_sigqueueinfo) { + return openshell_isolation_interface::linux::process_signal::mediate_process_signal( + &listener, + notification, + std::process::id(), + ); + } if syscall == libc::SYS_socket { return create_socket(®istry, &listener, notification); } @@ -469,6 +486,7 @@ fn dispatch_notification( queues.pending, &queues.dns_relay, queues.active_opens, + &queues.identity_resolver, ); } if syscall == libc::SYS_bind { @@ -478,13 +496,25 @@ fn dispatch_notification( return listen_socket(®istry, &listener, notification); } if matches!(syscall, libc::SYS_accept | libc::SYS_accept4) { - return accept_socket(registry, listener, notification, queues.active_accepts); + return accept_socket( + registry, + listener, + notification, + queues.active_accepts, + queues.accept_registrar, + ); } if matches!( syscall, libc::SYS_sendto | libc::SYS_sendmsg | libc::SYS_sendmmsg ) { - return classify_send(®istry, &listener, notification, &queues.dns_relay); + return classify_send( + ®istry, + &listener, + notification, + &queues.dns_relay, + &queues.identity_resolver, + ); } if syscall == libc::SYS_getpeername { return get_peer_name(®istry, &listener, notification); @@ -571,6 +601,7 @@ fn connect_socket( pending: mpsc::Sender, dns_relay: &DnsRelay, active_opens: Arc, + identity_resolver: &ProcfsIdentityResolver, ) -> io::Result<()> { let notification_started = Instant::now(); let fd = raw_fd(notification.args[0])?; @@ -641,7 +672,7 @@ fn connect_socket( return listener.respond_value(notification.id, 0); } if destination == dns_relay.address { - let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let identity = identity_resolver.resolve(notification.tid); let mut registry = lock(®istry); let entry = registry.resolve_mut(notification.tid, fd)?; if !matches!( @@ -680,7 +711,7 @@ fn connect_socket( return Err(io::Error::from_raw_os_error(libc::EACCES)); } - let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let identity = identity_resolver.resolve(notification.tid); let (decision_tx, decision_rx) = std::sync::mpsc::sync_channel(1); let (relay_tx, relay_rx) = oneshot::channel(); let slot = acquire_pending_open_slot(&active_opens)?; @@ -891,6 +922,7 @@ fn accept_socket( listener: Arc, notification: Notification, active_accepts: Arc, + accept_registrar: crate::accept_interrupt::AcceptRegistrar, ) -> io::Result<()> { let fd = raw_fd(notification.args[0])?; let flags = if i64::from(notification.syscall) == libc::SYS_accept4 { @@ -924,14 +956,24 @@ fn accept_socket( .name("openshell-local-accept".to_string()) .spawn(move || { let _slot = slot; + let registration = match accept_registrar.register(notification.id) { + Ok(registration) => registration, + Err(error) => { + let _ = worker_listener.respond_errno(notification.id, error_to_errno(&error)); + return; + } + }; if let Err(error) = accept_and_inject( ®istry, &worker_listener, notification, - flags, - listener_inode, - metadata, - source, + AcceptOperation { + flags, + listener_inode, + metadata, + source, + registration, + }, ) { let _ = worker_listener.respond_errno(notification.id, error_to_errno(&error)); } @@ -940,15 +982,27 @@ fn accept_socket( Ok(()) } -fn accept_and_inject( - registry: &Mutex, - listener: &NotificationListener, - notification: Notification, +struct AcceptOperation { flags: i32, listener_inode: u64, metadata: SocketMetadata, source: OwnedFd, + registration: crate::accept_interrupt::AcceptRegistration, +} + +fn accept_and_inject( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, + operation: AcceptOperation, ) -> io::Result<()> { + let AcceptOperation { + flags, + listener_inode, + metadata, + source, + registration, + } = operation; let mut poll = libc::pollfd { fd: source.as_raw_fd(), events: libc::POLLIN, @@ -963,9 +1017,14 @@ fn accept_and_inject( let timeout = if nonblocking { 0 } else { - i32::try_from(ACCEPT_POLL_INTERVAL.as_millis()).expect("accept poll interval fits i32") + i32::try_from(ACCEPT_POLL_INTERVAL.as_millis()).map_err(io::Error::other)? }; + // Readiness may disappear before accept (another accept or an aborted + // connection). The registered watchdog interrupts a blocked syscall when + // its notification dies or the broker shuts down. No workload OFD flags + // are changed, and no worker can outlive its cancellation registration. loop { + registration.ensure_running()?; listener.validate_id(notification.id)?; // SAFETY: poll references one live pollfd for this call. let ready = unsafe { libc::poll(&raw mut poll, 1, timeout) }; @@ -986,8 +1045,8 @@ fn accept_and_inject( } let mut storage = std::mem::MaybeUninit::::zeroed(); - let mut length = libc::socklen_t::try_from(size_of::()) - .expect("sockaddr storage size fits"); + let mut length = + libc::socklen_t::try_from(size_of::()).map_err(io::Error::other)?; // Always keep the broker-side descriptor close-on-exec. ADDFD separately // applies the workload's requested descriptor flag. let accepted_flags = flags | libc::SOCK_CLOEXEC; @@ -1004,6 +1063,10 @@ fn accept_and_inject( if accepted < 0 { return Err(io::Error::last_os_error()); } + // Only the blocking accept phase needs asynchronous interruption. Stop + // monitoring before ADDFD completes the notification, otherwise a normal + // successful response could be mistaken for cancellation during commit. + drop(registration); // SAFETY: successful accept4 returned one newly owned descriptor. let accepted = unsafe { OwnedFd::from_raw_fd(accepted) }; // SAFETY: accept4 initialized the reported prefix of storage. @@ -1071,6 +1134,7 @@ fn classify_send( listener: &NotificationListener, notification: Notification, dns_relay: &DnsRelay, + identity_resolver: &ProcfsIdentityResolver, ) -> io::Result<()> { let fd = raw_fd(notification.args[0])?; let syscall = i64::from(notification.syscall); @@ -1149,7 +1213,7 @@ fn classify_send( .is_some_and(|value| value == dns_relay.address) }) => { - let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let identity = identity_resolver.resolve(notification.tid); let entry = registry.resolve_mut(notification.tid, fd)?; let source_fd = entry.retained_preconnect()?.as_raw_fd(); let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; @@ -1395,13 +1459,12 @@ fn connect_exact(fd: RawFd, address: SocketAddr) -> io::Result<()> { revents: 0, }; // SAFETY: poll points to one live pollfd. - let timeout = i32::try_from(RELAY_CONNECT_TIMEOUT.as_millis()) - .expect("relay timeout fits poll milliseconds"); + let timeout = i32::try_from(RELAY_CONNECT_TIMEOUT.as_millis()).map_err(io::Error::other)?; if unsafe { libc::poll(&raw mut poll, 1, timeout) } <= 0 { return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); } let mut socket_error = 0_i32; - let mut size = libc::socklen_t::try_from(size_of::()).expect("SO_ERROR size fits"); + let mut size = libc::socklen_t::try_from(size_of::()).map_err(io::Error::other)?; // SAFETY: getsockopt writes one i32 into live storage. if unsafe { libc::getsockopt( @@ -1443,8 +1506,8 @@ fn bind_exact(fd: RawFd, address: SocketAddr) -> io::Result<()> { fn socket_local_addr(fd: RawFd) -> io::Result { let mut storage = std::mem::MaybeUninit::::zeroed(); - let mut length = libc::socklen_t::try_from(size_of::()) - .expect("sockaddr storage size fits"); + let mut length = + libc::socklen_t::try_from(size_of::()).map_err(io::Error::other)?; // SAFETY: storage and length are live output buffers. if unsafe { libc::getsockname(fd, storage.as_mut_ptr().cast(), &raw mut length) } < 0 { return Err(io::Error::last_os_error()); @@ -1521,7 +1584,7 @@ fn write_socket_addr( let mut supplied_length = [0_u8; size_of::()]; task_memory::read_exact(tid, length_address, &mut supplied_length)?; let supplied_length = libc::socklen_t::from_ne_bytes(supplied_length); - let (bytes, actual_length) = sockaddr_bytes(value); + let (bytes, actual_length) = sockaddr_bytes(value)?; let copied = usize::try_from(supplied_length) .unwrap_or(0) .min(bytes.len()); @@ -1533,56 +1596,13 @@ fn write_socket_addr( task_memory::write_exact(tid, length_address, &actual_length.to_ne_bytes()) } -fn sockaddr_bytes(address: SocketAddr) -> (Vec, libc::socklen_t) { - match address { - SocketAddr::V4(address) => { - let native = libc::sockaddr_in { - sin_family: libc::sa_family_t::try_from(libc::AF_INET) - .expect("AF_INET fits sa_family_t"), - sin_port: address.port().to_be(), - sin_addr: libc::in_addr { - s_addr: u32::from_ne_bytes(address.ip().octets()), - }, - sin_zero: [0; 8], - }; - // SAFETY: native is plain initialized storage. - let bytes = unsafe { - std::slice::from_raw_parts( - (&raw const native).cast::(), - size_of::(), - ) - }; - ( - bytes.to_vec(), - libc::socklen_t::try_from(size_of::()) - .expect("sockaddr_in size fits socklen_t"), - ) - } - SocketAddr::V6(address) => { - let native = libc::sockaddr_in6 { - sin6_family: libc::sa_family_t::try_from(libc::AF_INET6) - .expect("AF_INET6 fits sa_family_t"), - sin6_port: address.port().to_be(), - sin6_flowinfo: address.flowinfo(), - sin6_addr: libc::in6_addr { - s6_addr: address.ip().octets(), - }, - sin6_scope_id: address.scope_id(), - }; - // SAFETY: native is plain initialized storage. - let bytes = unsafe { - std::slice::from_raw_parts( - (&raw const native).cast::(), - size_of::(), - ) - }; - ( - bytes.to_vec(), - libc::socklen_t::try_from(size_of::()) - .expect("sockaddr_in6 size fits socklen_t"), - ) - } - } +fn sockaddr_bytes(address: SocketAddr) -> io::Result<(Vec, libc::socklen_t)> { + with_sockaddr(address, |native, length| { + let length_usize = usize::try_from(length).map_err(io::Error::other)?; + // SAFETY: with_sockaddr lends fully initialized storage for this call. + let bytes = unsafe { std::slice::from_raw_parts(native.cast::(), length_usize) }; + Ok((bytes.to_vec(), length)) + }) } fn with_sockaddr( @@ -1592,8 +1612,7 @@ fn with_sockaddr( match address { SocketAddr::V4(address) => { let native = libc::sockaddr_in { - sin_family: libc::sa_family_t::try_from(libc::AF_INET) - .expect("AF_INET fits sa_family_t"), + sin_family: libc::sa_family_t::try_from(libc::AF_INET).map_err(io::Error::other)?, sin_port: address.port().to_be(), sin_addr: libc::in_addr { s_addr: u32::from_ne_bytes(address.ip().octets()), @@ -1603,13 +1622,13 @@ fn with_sockaddr( operation( (&raw const native).cast(), libc::socklen_t::try_from(size_of::()) - .expect("sockaddr_in size fits socklen_t"), + .map_err(io::Error::other)?, ) } SocketAddr::V6(address) => { let native = libc::sockaddr_in6 { sin6_family: libc::sa_family_t::try_from(libc::AF_INET6) - .expect("AF_INET6 fits sa_family_t"), + .map_err(io::Error::other)?, sin6_port: address.port().to_be(), sin6_flowinfo: address.flowinfo(), sin6_addr: libc::in6_addr { @@ -1620,7 +1639,7 @@ fn with_sockaddr( operation( (&raw const native).cast(), libc::socklen_t::try_from(size_of::()) - .expect("sockaddr_in6 size fits socklen_t"), + .map_err(io::Error::other)?, ) } } diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 18e6d9dfe8..676ff32053 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -24,8 +24,6 @@ use std::path::Path; use std::path::PathBuf; use std::process::Stdio; use std::sync::Arc; -#[cfg(target_os = "linux")] -use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; use tracing::{debug, info}; @@ -422,33 +420,10 @@ fn validate_capability_bounding_set_clear( } #[cfg(target_os = "linux")] -static WORKLOAD_LAUNCHER: OnceLock< - openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, -> = OnceLock::new(); - -/// Install the sandbox-owned launcher that every later workload spawn must -/// traverse. A second launcher would create a second listener generation and -/// is therefore rejected. -#[cfg(target_os = "linux")] -pub fn configure_workload_launcher( - launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, -) -> std::io::Result<()> { - WORKLOAD_LAUNCHER.set(launcher).map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "workload launcher was already configured", - ) - }) -} - -#[cfg(target_os = "linux")] -pub fn spawn_command_with_workload_launcher(mut cmd: Command) -> std::io::Result { - let launcher = WORKLOAD_LAUNCHER.get().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::NotConnected, - "sandbox workload launcher is not configured", - ) - })?; +pub fn spawn_command_with_workload_launcher( + launcher: &openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + mut cmd: Command, +) -> std::io::Result { let runtime = tokio::runtime::Handle::current(); launcher.execute(move || { let _guard = runtime.enter(); @@ -458,14 +433,9 @@ pub fn spawn_command_with_workload_launcher(mut cmd: Command) -> std::io::Result #[cfg(target_os = "linux")] pub fn spawn_std_command_with_workload_launcher( + launcher: &openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, mut cmd: std::process::Command, ) -> std::io::Result { - let launcher = WORKLOAD_LAUNCHER.get().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::NotConnected, - "sandbox workload launcher is not configured", - ) - })?; launcher.execute(move || cmd.spawn())? } @@ -500,6 +470,7 @@ impl ProcessHandle { #[cfg(target_os = "linux")] #[allow(clippy::too_many_arguments)] pub fn spawn( + launcher: &openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, program: &str, args: &[String], workspace: &ResolvedWorkspace, @@ -509,6 +480,7 @@ impl ProcessHandle { provider_env: &HashMap, ) -> Result { Self::spawn_impl( + launcher, program, args, workspace, @@ -549,6 +521,7 @@ impl ProcessHandle { #[cfg(target_os = "linux")] #[allow(clippy::too_many_arguments)] fn spawn_impl( + launcher: &openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, program: &str, args: &[String], workspace: &ResolvedWorkspace, @@ -674,7 +647,7 @@ impl ProcessHandle { // or interpreter, and is a common failure on images that lack the // requested shell/binary (e.g. bash on Alpine). #[cfg(target_os = "linux")] - let mut child = spawn_command_with_workload_launcher(cmd) + let mut child = spawn_command_with_workload_launcher(launcher, cmd) .into_diagnostic() .wrap_err_with(|| format!("failed to spawn sandbox entrypoint process '{program}'"))?; #[cfg(not(target_os = "linux"))] diff --git a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs index 9d4502dc5f..a20d9b087b 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs @@ -8,11 +8,8 @@ use landlock::{ Ruleset, RulesetAttr, RulesetCreatedAttr, }; use miette::{IntoDiagnostic, Result}; -use openshell_core::policy::{ - FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkPolicy, ProcessPolicy, - SandboxPolicy, -}; -use std::os::fd::AsFd; +use openshell_core::policy::{LandlockCompatibility, SandboxPolicy}; +use std::os::fd::{AsFd, OwnedFd}; use std::path::{Path, PathBuf}; use tracing::debug; @@ -136,43 +133,75 @@ pub fn prepare_current_user( /// hierarchy. Entries the final UID cannot open are already inaccessible and /// are safely omitted by [`PathOpenMode::CurrentUser`]. pub fn prepare_capability_free_baseline() -> Result { - let read_write = capability_free_baseline_paths(Path::new("/"))?; - if read_write.is_empty() { + prepare_capability_free_baseline_at(Path::new("/")) +} + +fn prepare_capability_free_baseline_at(root: &Path) -> Result { + // Unlike optional filesystem policy, self-protection must cover pathname + // truncation as well as opens. Never silently downgrade this ABI requirement. + let abi = ABI::V3; + let access = AccessFs::from_all(abi); + let mut ruleset = Ruleset::default() + .set_compatibility(CompatLevel::HardRequirement) + .handle_access(access) + .into_diagnostic()? + .create() + .into_diagnostic()?; + let entries = capability_free_baseline_entries(root)?; + if entries.is_empty() { return Err(miette::miette!( "capability-free Landlock baseline found no usable root entries" )); } - - let policy = SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy { - read_only: Vec::new(), - read_write, - include_workdir: false, - }, - network: NetworkPolicy::default(), - landlock: LandlockPolicy { - compatibility: LandlockCompatibility::HardRequirement, - }, - process: ProcessPolicy::default(), - }; - prepare_with_path_open_mode(&policy, None, PathOpenMode::CurrentUser)?.ok_or_else(|| { - miette::miette!("capability-free Landlock baseline unexpectedly produced no ruleset") + for (_, fd) in entries { + let allowed = access_for_path_fd(&fd, access, abi)?; + ruleset = ruleset + .add_rule(PathBeneath::new(fd, allowed)) + .into_diagnostic()?; + } + Ok(PreparedRuleset { + ruleset, + compatibility: LandlockCompatibility::HardRequirement, }) } -fn capability_free_baseline_paths(root: &Path) -> Result> { +fn capability_free_baseline_entries(root: &Path) -> Result> { + use rustix::fs::{Mode, OFlags, open, openat}; const PRIVATE_ROOT: &str = ".openshell"; - let mut paths = Vec::new(); + let root_fd = open( + root, + OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + ) + .into_diagnostic()?; + let mut entries = Vec::new(); for entry in std::fs::read_dir(root).into_diagnostic()? { let entry = entry.into_diagnostic()?; - if entry.file_name() != PRIVATE_ROOT { - paths.push(entry.path()); + if entry.file_name() == PRIVATE_ROOT { + continue; + } + // Open relative to the pinned root and classify this exact descriptor. + // O_PATH|O_NOFOLLOW opens a symlink itself, never its target. A root + // alias to `/` or `/.openshell` therefore cannot broaden the allowlist. + let fd = match openat( + &root_fd, + entry.file_name(), + OFlags::PATH | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(fd) => fd, + Err(rustix::io::Errno::NOENT | rustix::io::Errno::ACCESS) => continue, + Err(error) => return Err(error).into_diagnostic(), + }; + let stat = rustix::fs::fstat(&fd).into_diagnostic()?; + if rustix::fs::FileType::from_raw_mode(stat.st_mode) == rustix::fs::FileType::Symlink { + continue; } + entries.push((entry.path(), fd)); } - paths.sort(); - Ok(paths) + entries.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(entries) } fn prepare_with_path_open_mode( @@ -236,7 +265,10 @@ fn prepare_with_path_open_mode( } let total_paths = read_only.len() + read_write.len(); - let abi = ABI::V2; + // Read-only policy must also deny pathname truncation. The mandatory + // baseline already qualifies ABI v3; optional best-effort policy keeps its + // independent compatibility behavior for other callers. + let abi = ABI::V3; openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Informational) @@ -402,7 +434,7 @@ pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { /// files and device nodes in hard-requirement mode. Classifying through the /// same `PathFd` used by the rule avoids a pathname TOCTOU race. fn access_for_path_fd( - path_fd: &PathFd, + path_fd: &impl AsFd, requested_access: BitFlags, abi: ABI, ) -> Result> { @@ -537,7 +569,7 @@ fn compat_level(level: &LandlockCompatibility) -> CompatLevel { #[cfg(test)] mod tests { use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy}; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy}; fn hard_requirement_policy(read_only: Vec, read_write: Vec) -> SandboxPolicy { SandboxPolicy { @@ -579,7 +611,17 @@ mod tests { std::fs::create_dir(root.path().join(name)).unwrap(); } - let paths = capability_free_baseline_paths(root.path()).unwrap(); + std::os::unix::fs::symlink(root.path(), root.path().join("root-alias")).unwrap(); + std::os::unix::fs::symlink( + root.path().join(".openshell"), + root.path().join("private-alias"), + ) + .unwrap(); + let paths: Vec<_> = capability_free_baseline_entries(root.path()) + .unwrap() + .into_iter() + .map(|(path, _)| path) + .collect(); assert_eq!( paths, ["bin", "etc", "sandbox"] @@ -587,6 +629,68 @@ mod tests { .to_vec() ); } + + #[test] + fn capability_free_baseline_denies_alias_reads_and_path_truncation() { + if !matches!(probe_availability(), LandlockAvailability::Available { abi } if abi >= 3) { + return; + } + let root = tempfile::tempdir().unwrap(); + let public = root.path().join("public"); + let private = root.path().join(".openshell"); + std::fs::create_dir(&public).unwrap(); + std::fs::create_dir(&private).unwrap(); + std::fs::write(public.join("sentinel"), b"allowed").unwrap(); + std::fs::write(private.join("secret"), b"protected").unwrap(); + std::os::unix::fs::symlink(root.path(), root.path().join("root-alias")).unwrap(); + std::os::unix::fs::symlink(&private, root.path().join("private-alias")).unwrap(); + let path = root.path().to_path_buf(); + std::thread::spawn(move || { + enforce(prepare_capability_free_baseline_at(&path).unwrap()).unwrap(); + assert_eq!( + std::fs::read(path.join("public/sentinel")).unwrap(), + b"allowed" + ); + for name in [ + ".openshell/secret", + "root-alias/.openshell/secret", + "private-alias/secret", + ] { + let target = path.join(name); + assert_eq!( + std::fs::read(&target).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + let target = std::ffi::CString::new(target.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: target is a live, NUL-terminated path. The syscall + // tests pathname truncation without opening a file first. + #[allow(unsafe_code)] + let result = unsafe { libc::truncate(target.as_ptr(), 0) }; + assert_eq!(result, -1); + assert_eq!( + std::io::Error::last_os_error().kind(), + std::io::ErrorKind::PermissionDenied + ); + } + let policy = hard_requirement_policy(vec![path.join("public")], Vec::new()); + enforce(prepare_current_user(&policy, None).unwrap().unwrap()).unwrap(); + let read_only = + std::ffi::CString::new(path.join("public/sentinel").as_os_str().as_encoded_bytes()) + .unwrap(); + // SAFETY: live NUL-terminated pathname; optional read-only policy + // must handle truncation independently of the protected baseline. + #[allow(unsafe_code)] + let truncated = unsafe { libc::truncate(read_only.as_ptr(), 0) }; + assert_eq!(truncated, -1); + assert_eq!( + std::io::Error::last_os_error().kind(), + std::io::ErrorKind::PermissionDenied + ); + }) + .join() + .unwrap(); + assert_eq!(std::fs::read(private.join("secret")).unwrap(), b"protected"); + } fn tailored_access(path: &Path, requested_access: BitFlags) -> BitFlags { let path_fd = PathFd::new(path).unwrap(); access_for_path_fd(&path_fd, requested_access, ABI::V2).unwrap() From 8c9904e03fdb9bd920c9e6c38cdee2b807ad6086 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:13:01 -0700 Subject: [PATCH 04/10] fix(sandbox): reject private root redirects and adopt typed errors Signed-off-by: Drew Newberry --- architecture/sandbox.md | 3 + .../openshell-sandbox/src/boundary_server.rs | 143 +++++++++++------- .../openshell-sandbox/src/network_broker.rs | 2 +- .../src/sandbox/linux/landlock.rs | 31 ++++ 4 files changed, 123 insertions(+), 56 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 32c6518634..559ce21441 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -85,6 +85,9 @@ filesystem policy. It requires Landlock ABI v3, including pathname truncation protection. Rules cover individually opened root children except `/.openshell`; the sandbox opens entries relative to a pinned root descriptor without following symlinks. An image-provided alias cannot grant access to the protected subtree. +The reserved `/.openshell` root must itself be a real directory if present; +a symlink or non-directory aborts preparation so private child mounts cannot +redirect into an allowed subtree. ## Network and Inference diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 12ae615478..1a34854423 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -49,7 +49,7 @@ mod linux { use tokio_stream::wrappers::ReceiverStream; use openshell_isolation_interface::boundary_protocol::{ - AgentSpecWire, BinaryIdentityWire, BoundaryConfig, + AgentSpecWire, BinaryIdentityWire, BoundaryConfig, BoundaryErrorKind, BoundaryListener as BoundaryListenerConfig, DnsQueryResultWire, ExecSpecWire, ExitStatusWire, MediationTimingWire, OutputWindowWire, ProcessKindWire, ProcessSnapshotWire, Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_EXIT, @@ -603,7 +603,10 @@ mod linux { { Some(MediationLease(runtime.clone())) } else { - response = guest_error("denied", "a mediation session is already active"); + response = guest_error( + BoundaryErrorKind::Denied, + "a mediation session is already active", + ); None } } else { @@ -761,7 +764,7 @@ mod linux { if !runtime.authenticate(&request) { let response = ResponseEnvelope { request_id: request.request_id, - response: guest_error("denied", "control authentication failed"), + response: guest_error(BoundaryErrorKind::Denied, "control authentication failed"), }; return write_frame(&mut stream, &response) .map_err(|error| format!("write control frame: {error}")); @@ -769,7 +772,10 @@ mod linux { if request.validate_payload_digest().is_err() { let response = ResponseEnvelope { request_id: request.request_id, - response: guest_error("denied", "control request payload digest mismatch"), + response: guest_error( + BoundaryErrorKind::Denied, + "control request payload digest mismatch", + ), }; return write_frame(&mut stream, &response) .map_err(|error| format!("write control frame: {error}")); @@ -842,7 +848,7 @@ mod linux { &mut stream, &ResponseEnvelope { request_id: request.request_id, - response: guest_error("failed", error), + response: guest_error(BoundaryErrorKind::Process, error), }, ) .map_err(|error| format!("write port-forward error response: {error}"))?; @@ -993,13 +999,13 @@ mod linux { ) -> Result<(), Response> { if requests.contains(request_id) { return Err(guest_error( - "denied", + BoundaryErrorKind::Denied, "exec request has expired; it cannot be executed again", )); } if requests.len() >= MAX_REPLAY_LEDGER_ENTRIES { return Err(guest_error( - "unavailable", + BoundaryErrorKind::Unavailable, "boundary generation exec request limit reached", )); } @@ -1120,7 +1126,7 @@ mod linux { .is_err() { return Err(guest_error( - "denied", + BoundaryErrorKind::Denied, "process already has a control attachment", )); } @@ -1186,10 +1192,13 @@ mod linux { fn dispatch(&self, envelope: RequestEnvelope) -> Response { if !self.authenticate(&envelope) { - return guest_error("denied", "control authentication failed"); + return guest_error(BoundaryErrorKind::Denied, "control authentication failed"); } if envelope.validate_payload_digest().is_err() { - return guest_error("denied", "control request payload digest mismatch"); + return guest_error( + BoundaryErrorKind::Denied, + "control request payload digest mismatch", + ); } let replayable = envelope.request.is_replayable_mutation(); let mut replay_ledger = replayable.then(|| lock(&self.replay_ledger)); @@ -1201,7 +1210,7 @@ mod linux { record.response.clone() } else { guest_error( - "denied", + BoundaryErrorKind::Denied, "control request ID was reused with a different payload", ) }; @@ -1217,7 +1226,7 @@ mod linux { self.attach(*policy) } else { guest_error( - "denied", + BoundaryErrorKind::Denied, "topology resource claims do not match the boundary configuration", ) } @@ -1255,15 +1264,16 @@ mod linux { rows, } => self.resize_process(&process_id, cols, rows), Request::OpenMediation => self.network_accept_context().map_or_else( - |error| guest_error("unavailable", error), + |error| guest_error(BoundaryErrorKind::Unavailable, error), |_| Response::MediationReady, ), Request::Exec { .. } | Request::AttachProcess { .. } | Request::PortForward { .. } - | Request::AcceptNetwork => { - guest_error("invalid", "streaming request used on control path") - } + | Request::AcceptNetwork => guest_error( + BoundaryErrorKind::Invalid, + "streaming request used on control path", + ), }; if let Some(ledger) = replay_ledger.as_mut() { ledger.insert( @@ -1300,7 +1310,10 @@ mod linux { let executor = { let state = lock(&self.state); let RuntimeState::Running(process) = &*state else { - return Err(guest_error("invalid", "agent process has not been started")); + return Err(guest_error( + BoundaryErrorKind::Invalid, + "agent process has not been started", + )); }; process.boundary_exec() }; @@ -1311,13 +1324,13 @@ mod linux { { if handle.payload_digest != payload_digest { return Err(guest_error( - "denied", + BoundaryErrorKind::Denied, "exec request ID was reused with a different payload", )); } if handle.attached.load(Ordering::Acquire) { return Err(guest_error( - "unavailable", + BoundaryErrorKind::Unavailable, "prior exec attachment is still being released", )); } @@ -1338,7 +1351,7 @@ mod linux { handles.remove(&process_id); } else { return Err(guest_error( - "unavailable", + BoundaryErrorKind::Unavailable, "retained exec process limit reached", )); } @@ -1350,7 +1363,7 @@ mod linux { let session = self .process_runtime .block_on(executor.exec(spec.into())) - .map_err(|error| guest_error("failed", error.to_string()))?; + .map_err(|error| guest_error(BoundaryErrorKind::Process, error.to_string()))?; let process_id = format!( "{}:exec:{}", self.config.generation, @@ -1365,7 +1378,7 @@ mod linux { } = session; let Some(stdin) = stdin else { return Err(guest_error( - "failed", + BoundaryErrorKind::Process, "exec process stdin pipe is unavailable", )); }; @@ -1423,11 +1436,11 @@ mod linux { .get(process_id) .map(|handle| handle.process.clone()); let Some(process) = process else { - return guest_error("invalid", "unknown exec process ID"); + return guest_error(BoundaryErrorKind::Invalid, "unknown exec process ID"); }; match self.process_runtime.block_on(process.signal(signal.into())) { Ok(()) => Response::Signaled, - Err(error) => guest_error("failed", error.to_string()), + Err(error) => guest_error(BoundaryErrorKind::Process, error.to_string()), } } @@ -1435,7 +1448,10 @@ mod linux { if let Ok(process) = self.running_process(process_id) { let session = process.main_session(); if !session.terminal() { - return guest_error("invalid", "agent process has no terminal"); + return guest_error( + BoundaryErrorKind::Invalid, + "agent process has no terminal", + ); } self.process_runtime.block_on(session.resize( u32::from(cols), @@ -1449,11 +1465,11 @@ mod linux { .get(process_id) .and_then(|handle| handle.terminal.clone()); let Some(terminal) = terminal else { - return guest_error("invalid", "exec process has no terminal"); + return guest_error(BoundaryErrorKind::Invalid, "exec process has no terminal"); }; match self.process_runtime.block_on(terminal.resize(cols, rows)) { Ok(()) => Response::Resized, - Err(error) => guest_error("failed", error.to_string()), + Err(error) => guest_error(BoundaryErrorKind::Process, error.to_string()), } } @@ -1498,7 +1514,7 @@ mod linux { let handles = lock(&self.exec_handles); let handle = handles .get(process_id) - .ok_or_else(|| guest_error("invalid", "unknown process ID"))?; + .ok_or_else(|| guest_error(BoundaryErrorKind::Invalid, "unknown process ID"))?; Ok((acquire_exec_attachment(handle)?, handle.terminal.is_some())) } @@ -1519,7 +1535,7 @@ mod linux { RuntimeState::AwaitingAttach => { let prepared = match PreparedBoundary::establish(self.network_broker.clone()) { Ok(prepared) => prepared, - Err(error) => return guest_error("failed", error), + Err(error) => return guest_error(BoundaryErrorKind::Process, error), }; *lock(&self.attached_policy) = Some(policy); *state = RuntimeState::Bound(prepared); @@ -1537,7 +1553,10 @@ mod linux { snapshot: self.session_snapshot(), } } else { - guest_error("denied", "attach policy does not match the bound boundary") + guest_error( + BoundaryErrorKind::Denied, + "attach policy does not match the bound boundary", + ) } } @@ -1595,11 +1614,11 @@ mod linux { match &*state { RuntimeState::Bound(prepared) => { if let Err(error) = prepared.confirm(&self.process_runtime) { - return guest_error("failed", error); + return guest_error(BoundaryErrorKind::Process, error); } let evidence = match self.measure_confirmation_evidence() { Ok(evidence) => evidence, - Err(error) => return guest_error("failed", error), + Err(error) => return guest_error(BoundaryErrorKind::Process, error), }; *state = RuntimeState::Ready(prepared.clone()); Response::Confirmed { @@ -1608,15 +1627,16 @@ mod linux { } RuntimeState::Ready(_) | RuntimeState::Running(_) => { self.measure_confirmation_evidence().map_or_else( - |error| guest_error("failed", error), + |error| guest_error(BoundaryErrorKind::Process, error), |evidence| Response::Confirmed { evidence: Box::new(evidence), }, ) } - RuntimeState::AwaitingAttach => { - guest_error("invalid", "boundary must be attached before confirm") - } + RuntimeState::AwaitingAttach => guest_error( + BoundaryErrorKind::Invalid, + "boundary must be attached before confirm", + ), } } @@ -1688,7 +1708,7 @@ mod linux { ) -> Response { let spec = match resolve_agent_spec(spec) { Ok(spec) => spec, - Err(error) => return guest_error("failed", error), + Err(error) => return guest_error(BoundaryErrorKind::Process, error), }; let mut state = lock(&self.state); let requested = StartedAgent { @@ -1711,17 +1731,20 @@ mod linux { } } else { guest_error( - "denied", + BoundaryErrorKind::Denied, "start_agent inputs do not match the running boundary", ) }; } let RuntimeState::Ready(prepared) = &*state else { - return guest_error("invalid", "boundary must be confirmed before start_agent"); + return guest_error( + BoundaryErrorKind::Invalid, + "boundary must be confirmed before start_agent", + ); }; let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { Ok(paths) => paths, - Err(error) => return guest_error("failed", error), + Err(error) => return guest_error(BoundaryErrorKind::Process, error), }; let mut policy = policy.into(); let driver_identity = DriverIdentity::Resolved { @@ -1729,7 +1752,7 @@ mod linux { gid: self.config.workload_identity.gid, }; if let Err(error) = resolve_process_identity(&mut policy, &driver_identity) { - return guest_error("failed", error.to_string()); + return guest_error(BoundaryErrorKind::Process, error.to_string()); } let launch = ManagedProcessLaunch { process_id: format!("{}:main:0", self.config.generation), @@ -1746,7 +1769,7 @@ mod linux { prepared.clone(), ) { Ok(process) => Arc::new(process), - Err(error) => return guest_error("failed", error), + Err(error) => return guest_error(BoundaryErrorKind::Process, error), }; let process_id = process.process_id(); *lock(&self.started_agent) = Some(requested); @@ -1767,15 +1790,19 @@ mod linux { let state = lock(&self.state); let RuntimeState::Running(process) = &*state else { return guest_error( - "invalid", + BoundaryErrorKind::Invalid, "agent process must be running before provider environment updates", ); }; process.clone() }; - let revision = process + let revision = match process .provider_credentials - .compare_and_install_child_env_snapshot(expected_revision, revision, provider_env); + .compare_and_install_child_env_snapshot(expected_revision, revision, provider_env) + { + Ok(revision) => revision, + Err(error) => return guest_error(BoundaryErrorKind::Process, error.to_string()), + }; Response::ProviderEnvironmentUpdated { revision } } @@ -1786,7 +1813,7 @@ mod linux { }; match process.wait() { Ok(status) => Response::Exited { status }, - Err(error) => guest_error("failed", error), + Err(error) => guest_error(BoundaryErrorKind::Process, error), } } @@ -1797,7 +1824,7 @@ mod linux { }; match process.signal(signal) { Ok(()) => Response::Signaled, - Err(error) => guest_error("terminated", error), + Err(error) => guest_error(BoundaryErrorKind::Terminated, error), } } @@ -1809,7 +1836,7 @@ mod linux { match process.signal(SignalWire::Kill) { Ok(()) => Response::Terminated, Err(_) if process.has_exited() => Response::Terminated, - Err(error) => guest_error("failed", error), + Err(error) => guest_error(BoundaryErrorKind::Process, error), } } @@ -1820,10 +1847,16 @@ mod linux { fn running_process(&self, process_id: &str) -> Result, Response> { let state = lock(&self.state); let RuntimeState::Running(process) = &*state else { - return Err(guest_error("invalid", "agent process has not been started")); + return Err(guest_error( + BoundaryErrorKind::Invalid, + "agent process has not been started", + )); }; if process.process_id() != process_id { - return Err(guest_error("invalid", "unknown process ID")); + return Err(guest_error( + BoundaryErrorKind::Invalid, + "unknown process ID", + )); } Ok(process.clone()) } @@ -2284,9 +2317,9 @@ mod linux { ) } - fn guest_error(kind: &str, message: impl Into) -> Response { + fn guest_error(kind: BoundaryErrorKind, message: impl Into) -> Response { Response::Error { - kind: kind.to_string(), + kind, message: message.into(), } } @@ -3400,7 +3433,7 @@ mod linux { changed.request_id = update.request_id; assert!(matches!( boundary.dispatch(changed), - Response::Error { kind, .. } if kind == "denied" + Response::Error { kind, .. } if kind == BoundaryErrorKind::Denied )); let (first_attachment, _) = boundary @@ -3441,7 +3474,7 @@ mod linux { changed_policy.version += 1; assert!(matches!( boundary.attach(changed_policy.clone()), - Response::Error { kind, .. } if kind == "denied" + Response::Error { kind, .. } if kind == BoundaryErrorKind::Denied )); assert!(matches!( boundary.start_agent( @@ -3453,7 +3486,7 @@ mod linux { 0, std::collections::HashMap::new(), ), - Response::Error { kind, .. } if kind == "denied" + Response::Error { kind, .. } if kind == BoundaryErrorKind::Denied )); let exec_spec = ExecSpecWire { diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 4467762b3e..5948d31b91 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -274,7 +274,7 @@ impl NetworkBroker { .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "DNS broker queue closed")) } - #[cfg(any(test, feature = "perf-harness"))] + #[cfg(test)] pub(crate) fn dns_address(&self) -> SocketAddr { self.dns_address } diff --git a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs index a20d9b087b..d893b828ba 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs @@ -175,6 +175,23 @@ fn capability_free_baseline_entries(root: &Path) -> Result Some(fd), + Err(rustix::io::Errno::NOENT) => None, + Err(error) => { + return Err(miette::miette!( + "private sandbox root must be a real directory: {error}" + )); + } + }; let mut entries = Vec::new(); for entry in std::fs::read_dir(root).into_diagnostic()? { let entry = entry.into_diagnostic()?; @@ -630,6 +647,20 @@ mod tests { ); } + #[test] + fn capability_free_baseline_rejects_private_root_redirect() { + let root = tempfile::tempdir().unwrap(); + let public = root.path().join("public"); + let private = root.path().join(".openshell"); + std::fs::create_dir(&public).unwrap(); + std::fs::write(public.join("secret"), b"private mount contents").unwrap(); + std::os::unix::fs::symlink(&public, &private).unwrap(); + assert!(capability_free_baseline_entries(root.path()).is_err()); + std::fs::remove_file(&private).unwrap(); + std::fs::write(&private, b"not a directory").unwrap(); + assert!(capability_free_baseline_entries(root.path()).is_err()); + } + #[test] fn capability_free_baseline_denies_alias_reads_and_path_truncation() { if !matches!(probe_availability(), LandlockAvailability::Available { abi } if abi >= 3) { From f07e378d8dcbf6b7df27127def4d0b13e989e408 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:22:58 -0700 Subject: [PATCH 05/10] fix(sandbox): preserve accept thread ownership on musl Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/accept_interrupt.rs | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/crates/openshell-sandbox/src/accept_interrupt.rs b/crates/openshell-sandbox/src/accept_interrupt.rs index ef282d021d..86bbf39218 100644 --- a/crates/openshell-sandbox/src/accept_interrupt.rs +++ b/crates/openshell-sandbox/src/accept_interrupt.rs @@ -11,6 +11,8 @@ use std::collections::HashMap; use std::io; +use std::marker::PhantomData; +use std::rc::Rc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; @@ -47,11 +49,21 @@ fn reserve_signal() -> io::Result<()> { #[derive(Default)] struct State { - workers: Mutex>, + workers: Mutex>, changed: Condvar, stopped: AtomicBool, } +// musl represents pthread_t as an opaque pointer, unlike glibc's integer. It +// is only passed back to pthread_kill, never dereferenced by this module. +struct RegisteredThread(libc::pthread_t); + +// SAFETY: POSIX permits signaling a live pthread from another thread. The +// handle is accessed only under State::workers, and the owning worker removes +// its registration under that same mutex before returning. AcceptRegistration +// cannot move to another thread, so its Drop cannot outlive the owning worker. +unsafe impl Send for RegisteredThread {} + pub struct AcceptMonitor { state: Arc, thread: Option>, @@ -117,11 +129,12 @@ impl AcceptRegistrar { "duplicate accept notification registration", )); } - workers.insert(notification_id, thread); + workers.insert(notification_id, RegisteredThread(thread)); self.0.changed.notify_one(); Ok(AcceptRegistration { state: self.0.clone(), notification_id, + owning_thread: PhantomData, }) } } @@ -129,6 +142,9 @@ impl AcceptRegistrar { pub struct AcceptRegistration { state: Arc, notification_id: u64, + // Drop must run on the registering thread before its pthread_t can expire. + // No Rc is allocated; this marker makes the guard neither Send nor Sync. + owning_thread: PhantomData>, } impl AcceptRegistration { @@ -155,12 +171,12 @@ fn monitor(state: &State, valid: impl Fn(u64) -> bool) { if stopped && workers.is_empty() { return; } - for (¬ification_id, &thread) in &*workers { + for (¬ification_id, thread) in &*workers { if stopped || !valid(notification_id) { // SAFETY: the registration lock pins this live pthread_t. // Repeated interrupts close the check-to-accept race: a signal // received before accept cannot leave a later accept stranded. - let _ = unsafe { libc::pthread_kill(thread, INTERRUPT_SIGNAL) }; + let _ = unsafe { libc::pthread_kill(thread.0, INTERRUPT_SIGNAL) }; } } workers = if workers.is_empty() { @@ -190,6 +206,25 @@ mod tests { use std::net::{TcpListener, TcpStream}; use std::os::fd::AsRawFd; + #[test] + fn registrar_crosses_threads_but_registration_ends_before_worker_exit() { + fn assert_send_sync() {} + assert_send_sync::(); + + let monitor = AcceptMonitor::start(|_| true).unwrap(); + let registrar = monitor.registrar(); + std::thread::spawn(move || { + let registration = registrar.register(3).unwrap(); + assert!(registrar.register(3).is_err()); + assert!(lock(®istrar.0.workers).contains_key(&3)); + drop(registration); + assert!(lock(®istrar.0.workers).is_empty()); + }) + .join() + .unwrap(); + assert!(lock(&monitor.state.workers).is_empty()); + } + #[test] fn cancellation_interrupts_competing_accept_after_readiness_was_consumed() { let valid = Arc::new(AtomicBool::new(true)); From 3c6a4d984d7dedb14ef1257b0be01f0cfcbb5080 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:27:02 -0700 Subject: [PATCH 06/10] test(sandbox): isolate credential probes from filtered threads Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/process.rs | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 676ff32053..afb811e565 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -3730,8 +3730,34 @@ mod tests { // ---- Numeric UID tests (Phase 2) ---- + // Even a failing setuid(0) probe synchronizes libc credentials across all + // threads. Other tests own seccomp-notified launcher threads in this same + // process; signaling those while they await their broker can deadlock the + // parallel harness. Re-exec just the credential probe, without those threads. + fn numeric_uid_probe_runs_in_child(test_name: &str) -> bool { + const MARKER: &str = "OPENSHELL_TEST_ISOLATED_NUMERIC_UID_PROBE"; + if std::env::var(MARKER).as_deref() == Ok(test_name) { + return true; + } + let output = std::process::Command::new(std::env::current_exe().expect("test executable")) + .args(["--exact", test_name, "--test-threads=1", "--nocapture"]) + .env(MARKER, test_name) + .output() + .expect("run isolated credential probe"); + assert!( + output.status.success(), + "isolated credential probe failed: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + false + } + #[test] fn drop_privileges_accepts_numeric_uid() { + if !numeric_uid_probe_runs_in_child("process::tests::drop_privileges_accepts_numeric_uid") { + return; + } // When running as non-root, a numeric UID/GID that matches the // current process should succeed without any passwd lookup. if nix::unistd::geteuid().is_root() { @@ -3754,6 +3780,11 @@ mod tests { #[test] fn drop_privileges_numeric_uid_skips_initgroups() { + if !numeric_uid_probe_runs_in_child( + "process::tests::drop_privileges_numeric_uid_skips_initgroups", + ) { + return; + } // When running as non-root with a numeric user but group matches, // initgroups should not be called (guard: target_uid != geteuid()). if nix::unistd::geteuid().is_root() { From 34b2d289e6186b74836c3af1212c9be3724339c3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:41:11 -0700 Subject: [PATCH 07/10] fix(sandbox): return retained exec exit status to independent waiters Signed-off-by: Drew Newberry --- architecture/sandbox.md | 4 ++ .../openshell-sandbox/src/boundary_server.rs | 71 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 559ce21441..2ce35f295b 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -64,6 +64,10 @@ Completed exec output handles can be reclaimed, but execution request IDs remain reserved for the boundary generation. The sandbox accepts at most 4,096 exec attempts per generation, then rejects new attempts rather than forgetting replay protection. A disconnected attachment does not authorize another execution. +While an exec handle is retained, independent waits return its stable exit or +signal status, whether or not an output attachment is open or the main process +has exited. Waiting never holds the exec registry lock, so other operations can +still signal or attach to the process. ## Isolation Layers diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 1a34854423..cbb376c31e 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1807,6 +1807,19 @@ mod linux { } fn wait(&self, process_id: &str) -> Response { + let exec = lock(&self.exec_handles) + .get(process_id) + .map(|handle| handle.process.clone()); + if let Some(process) = exec { + // Wait independently of the output attachment. Retain the + // process, not the registry lock, while its exit is pending. + return match self.process_runtime.block_on(process.wait()) { + Ok(status) => Response::Exited { + status: status.into(), + }, + Err(error) => guest_error(BoundaryErrorKind::Process, error.to_string()), + }; + } let process = match self.running_process(process_id) { Ok(process) => process, Err(response) => return response, @@ -3530,6 +3543,15 @@ mod linux { }); assert_eq!(output, "reconnected"); drop(exec); + for _ in 0..2 { + assert_eq!( + boundary.wait(&exec_id), + Response::Exited { + status: ExitStatusWire::Exited(0), + }, + "exec status must remain available after its output attachment closes" + ); + } let Response::Attached { snapshot } = boundary.attach(policy.clone()) else { panic!("reconnect attach did not return a session snapshot"); }; @@ -3773,6 +3795,15 @@ mod linux { boundary.signal_exec(&retained_id, SignalWire::Kill), Response::Signaled ); + for _ in 0..2 { + assert_eq!( + boundary.wait(&retained_id), + Response::Exited { + status: ExitStatusWire::Signaled(libc::SIGKILL), + }, + "independent waits must preserve a retained exec's signal status" + ); + } lock(&boundary.exec_handles).remove(&retained_id); assert!( boundary @@ -3790,6 +3821,46 @@ mod linux { std::thread::sleep(Duration::from_millis(10)); } assert!(process.has_exited(), "canonical process did not exit"); + assert_eq!( + boundary.wait(&process.process_id()), + Response::Exited { + status: ExitStatusWire::Exited(0), + } + ); + for exit_code in [0, 7] { + let spec = ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), format!("exit {exit_code}")], + env: Vec::new(), + workdir: None, + pty: false, + }; + let request = RequestEnvelope::new( + "sandbox-retained".to_string(), + "a".repeat(32), + Request::Exec { spec: spec.clone() }, + ) + .expect("build exec status request"); + let exec = boundary + .start_exec(&request.request_id, &request.payload_digest, spec) + .expect("start exec after canonical exit"); + for _ in 0..2 { + assert_eq!( + boundary.wait(&exec.process_id), + Response::Exited { + status: ExitStatusWire::Exited(exit_code), + }, + "wait must work independently of attachment consumption and main exit" + ); + } + } + assert!(matches!( + boundary.wait("generation-retained:exec:unknown"), + Response::Error { + kind: BoundaryErrorKind::Invalid, + .. + } + )); let mut session = process_runtime .block_on( From 84be44cdd31b1ec21b0677b2b25c392c22e97d4f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 21:16:29 -0700 Subject: [PATCH 08/10] fix(sandbox): bound network mediation and preserve socket authorization Signed-off-by: Drew Newberry --- architecture/sandbox.md | 23 +- .../openshell-sandbox/src/network_broker.rs | 332 ++++++++++++++---- .../src/policy_dns/runtime.rs | 69 +++- docs/sandboxes/policies.mdx | 6 + 4 files changed, 358 insertions(+), 72 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 2ce35f295b..cb7b32a29a 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -118,10 +118,25 @@ notifications cannot race task-memory writes. DNS uses an exact sandbox-local resolver at `127.0.0.53:53`. The driver sets the nameserver and permits an unprivileged bind to port 53. UDP and TCP DNS requests -are attributed to the calling binary and forwarded through the supervisor; the -kernel delivers replies from the configured nameserver address, including for -strict musl and c-ares resolvers. No proxy environment variable, nftables rule, -or workload network namespace setup is part of enforcement. +are forwarded through the supervisor, which applies hostname-based DNS policy. +DNS sender identity is explicitly unavailable: native writes can come from an +inheriting process or after exec, and neither the connecting binary nor a later +descriptor-owner snapshot proves who sent an already queued query. Consumers +must not use this unavailable identity to grant binary-specific access. TCP +connection authorization still uses decision-time binary identity. + +The sandbox retains only bounded DNS socket-admission records, consumes TCP +records on accept, and reclaims closed UDP records when capacity is reached. +The kernel delivers replies from the configured nameserver address, including +for strict musl and c-ares resolvers. No proxy environment variable, nftables +rule, or workload network namespace setup is part of enforcement. The supervisor +retries failed DNS accepts with backoff, preserving service across a channel +reconnect. + +External TCP opens wait at most 30 seconds for a supervisor decision, then fail +with `ETIMEDOUT` and release their worker quota. An approval is tied to the +original socket identity; replacing the descriptor during policy evaluation +cannot transfer that approval to another socket. The outer fence remains mandatory. If notification handling misses a syscall, loses the supervisor, exceeds a bound, or encounters an unsupported socket diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 5948d31b91..f97a7c8321 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -21,7 +21,7 @@ use openshell_isolation_interface::contract::{ }; use openshell_isolation_interface::linux::seccomp_notify::{Notification, NotificationListener}; use openshell_isolation_interface::linux::socket_registry::{ - InetFamily, InetKind, SocketMetadata, SocketRegistry, SocketState, + InetFamily, InetKind, SocketIdentity, SocketMetadata, SocketRegistry, SocketState, }; use openshell_isolation_interface::linux::task_memory; use tokio::sync::{mpsc, oneshot}; @@ -38,6 +38,7 @@ const DNS_RELAY_ADDRESS: SocketAddr = SocketAddr::V4(std::net::SocketAddrV4::new 53, )); const RELAY_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const NETWORK_DECISION_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug)] struct PendingOpenSlot(Arc); @@ -105,7 +106,6 @@ pub struct PendingTcpOpen { pub(crate) queued_at: Instant, decision: std::sync::mpsc::SyncSender, relay: oneshot::Receiver>, - _slot: PendingOpenSlot, } impl PendingTcpOpen { @@ -152,8 +152,37 @@ impl PendingDnsQuery { #[derive(Clone)] struct DnsRelay { address: SocketAddr, - udp_attribution: Arc>>>, - tcp_attribution: Arc>>>, + udp_admissions: Arc>>, + tcp_admissions: Arc>>, +} + +fn dns_sender_identity() -> Result { + // Native writes can come from an inheriting process or after execve. A + // connect-time identity (or a later procfs holder scan) cannot identify + // the sender of an already queued query. Never assert that it can. + Err(ResolveError::Failed( + "DNS sender identity is unavailable for kernel-driven socket writes".to_string(), + )) +} + +fn register_dns_socket( + admissions: &Mutex>, + peer: SocketAddr, + identity: SocketIdentity, +) -> io::Result<()> { + let mut admissions = lock(admissions); + if admissions.len() >= SOCKET_CAPACITY && !admissions.contains_key(&peer) { + let installed = + openshell_isolation_interface::linux::proc_fd::installed_socket_inodes_excluding( + std::process::id(), + )?; + admissions.retain(|_, socket| installed.contains(&socket.inode)); + if admissions.len() >= SOCKET_CAPACITY { + return Err(io::Error::from_raw_os_error(libc::EMFILE)); + } + } + admissions.insert(peer, identity); + Ok(()) } #[derive(Clone)] @@ -164,6 +193,7 @@ struct NotificationQueues { dns_relay: DnsRelay, active_opens: Arc, active_accepts: Arc, + decision_timeout: Duration, } /// Live broker handle retained by the sandbox boundary. @@ -192,6 +222,14 @@ impl NetworkBroker { fn start_with_dns_address( listener: NotificationListener, dns_address: SocketAddr, + ) -> io::Result { + Self::start_with_decision_timeout(listener, dns_address, NETWORK_DECISION_TIMEOUT) + } + + fn start_with_decision_timeout( + listener: NotificationListener, + dns_address: SocketAddr, + decision_timeout: Duration, ) -> io::Result { let listener = Arc::new(listener); let monitor_listener = listener.clone(); @@ -212,6 +250,7 @@ impl NetworkBroker { dns_relay, active_opens, active_accepts, + decision_timeout, }; let healthy = Arc::new(AtomicBool::new(true)); let broker_healthy = healthy.clone(); @@ -296,13 +335,13 @@ fn start_dns_relay( pending: mpsc::Sender, ) -> io::Result { let (udp, tcp, address) = bind_dns_relay_sockets(address)?; - let udp_attribution = Arc::new(Mutex::new(HashMap::new())); - let tcp_attribution = Arc::new(Mutex::new(HashMap::new())); + let udp_admissions = Arc::new(Mutex::new(HashMap::new())); + let tcp_admissions = Arc::new(Mutex::new(HashMap::new())); let active_workers = Arc::new(AtomicUsize::new(0)); let relay = DnsRelay { address, - udp_attribution: Arc::clone(&udp_attribution), - tcp_attribution: Arc::clone(&tcp_attribution), + udp_admissions: Arc::clone(&udp_admissions), + tcp_admissions: Arc::clone(&tcp_admissions), }; let udp_active_workers = Arc::clone(&active_workers); @@ -312,10 +351,10 @@ fn start_dns_relay( .spawn(move || { let mut request = vec![0_u8; u16::MAX as usize]; while let Ok((length, peer)) = udp.recv_from(&mut request) { - let Some(identity) = lock(&udp_attribution).get(&peer).cloned() else { - tracing::warn!(%peer, "dropping DNS datagram from unattributed socket"); + if !lock(&udp_admissions).contains_key(&peer) { + tracing::warn!(%peer, "dropping DNS datagram from unregistered socket"); continue; - }; + } let Ok(worker_slot) = acquire_pending_dns_slot(&udp_active_workers) else { tracing::warn!(%peer, "dropping DNS datagram because the worker quota is full"); continue; @@ -324,7 +363,7 @@ fn start_dns_relay( let query = PendingDnsQuery { request: request[..length].to_vec(), transport: DnsTransport::Udp, - identity, + identity: dns_sender_identity(), notification_to_queue: Duration::ZERO, queued_at: Instant::now(), response: response_tx, @@ -358,11 +397,12 @@ fn start_dns_relay( }) else { break; }; - let identity = lock(&tcp_attribution).get(&peer).cloned(); - let Some(identity) = identity else { - tracing::warn!(%peer, "dropping DNS stream from unattributed socket"); + // One admission authorizes exactly one accepted TCP stream; + // no peer mapping needs to outlive this accept. + if lock(&tcp_admissions).remove(&peer).is_none() { + tracing::warn!(%peer, "dropping DNS stream from unregistered socket"); continue; - }; + } let Ok(worker_slot) = acquire_pending_dns_slot(&tcp_active_workers) else { tracing::warn!(%peer, "dropping DNS stream because the worker quota is full"); continue; @@ -372,7 +412,7 @@ fn start_dns_relay( .name("openshell-dns-tcp-query".to_string()) .spawn(move || { let _worker_slot = worker_slot; - serve_dns_tcp(stream, identity, tcp_pending); + serve_dns_tcp(stream, tcp_pending); }); } }) @@ -419,11 +459,7 @@ fn pending_try_send( }) } -fn serve_dns_tcp( - mut stream: TcpStream, - identity: Result, - pending: mpsc::Sender, -) { +fn serve_dns_tcp(mut stream: TcpStream, pending: mpsc::Sender) { use std::io::{Read as _, Write as _}; let _ = stream.set_read_timeout(Some(DNS_QUERY_TIMEOUT)); @@ -444,7 +480,7 @@ fn serve_dns_tcp( let query = PendingDnsQuery { request, transport: DnsTransport::Tcp, - identity: identity.clone(), + identity: dns_sender_identity(), notification_to_queue: Duration::ZERO, queued_at: Instant::now(), response: response_tx, @@ -479,15 +515,7 @@ fn dispatch_notification( return create_socket(®istry, &listener, notification); } if syscall == libc::SYS_connect { - return connect_socket( - registry, - listener, - notification, - queues.pending, - &queues.dns_relay, - queues.active_opens, - &queues.identity_resolver, - ); + return connect_socket(registry, listener, notification, queues); } if syscall == libc::SYS_bind { return bind_socket(®istry, &listener, notification); @@ -508,13 +536,7 @@ fn dispatch_notification( syscall, libc::SYS_sendto | libc::SYS_sendmsg | libc::SYS_sendmmsg ) { - return classify_send( - ®istry, - &listener, - notification, - &queues.dns_relay, - &queues.identity_resolver, - ); + return classify_send(®istry, &listener, notification, &queues.dns_relay); } if syscall == libc::SYS_getpeername { return get_peer_name(®istry, &listener, notification); @@ -598,11 +620,16 @@ fn connect_socket( registry: Arc>, listener: Arc, notification: Notification, - pending: mpsc::Sender, - dns_relay: &DnsRelay, - active_opens: Arc, - identity_resolver: &ProcfsIdentityResolver, + queues: NotificationQueues, ) -> io::Result<()> { + let NotificationQueues { + pending, + dns_relay, + active_opens, + identity_resolver, + decision_timeout, + .. + } = queues; let notification_started = Instant::now(); let fd = raw_fd(notification.args[0])?; let address_family = @@ -634,12 +661,12 @@ fn connect_socket( } let destination = read_socket_addr(notification.tid, notification.args[1], notification.args[2])?; - let (kind, socket_cookie, nonblocking) = { + let (kind, socket_identity, nonblocking) = { let registry = lock(®istry); let entry = registry.resolve(notification.tid, fd)?; ( entry.metadata().kind, - entry.identity().cookie, + entry.identity(), entry.metadata().nonblocking, ) }; @@ -672,7 +699,6 @@ fn connect_socket( return listener.respond_value(notification.id, 0); } if destination == dns_relay.address { - let identity = identity_resolver.resolve(notification.tid); let mut registry = lock(®istry); let entry = registry.resolve_mut(notification.tid, fd)?; if !matches!( @@ -683,13 +709,13 @@ fn connect_socket( } let source_fd = entry.retained_preconnect()?.as_raw_fd(); let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; - let attribution = match kind { - InetKind::Tcp => &dns_relay.tcp_attribution, - InetKind::DnsUdp => &dns_relay.udp_attribution, + let admissions = match kind { + InetKind::Tcp => &dns_relay.tcp_admissions, + InetKind::DnsUdp => &dns_relay.udp_admissions, }; - lock(attribution).insert(peer, identity); + register_dns_socket(admissions, peer, entry.identity())?; if let Err(error) = connect_exact(source_fd, destination) { - lock(attribution).remove(&peer); + lock(admissions).remove(&peer); return Err(error); } entry.set_state(match kind { @@ -720,7 +746,7 @@ fn connect_socket( destination, identity, socket: NetworkSocketMetadata { - socket_cookie, + socket_cookie: socket_identity.cookie, nonblocking, process_generation: u64::from(notification.tid), }, @@ -728,7 +754,6 @@ fn connect_socket( queued_at: Instant::now(), decision: decision_tx, relay: relay_rx, - _slot: slot, }) .map_err(|error| match error { mpsc::error::TrySendError::Full(_) => io::Error::from_raw_os_error(libc::EAGAIN), @@ -740,15 +765,24 @@ fn connect_socket( std::thread::Builder::new() .name("openshell-network-open".to_string()) .spawn(move || { - let result = decision_rx.recv().unwrap_or(NetworkOpenResult::Denied { - errno: libc::ECANCELED, - }); + // The worker owns its quota: an unresponsive supervisor must not + // retain a blocked syscall or worker slot indefinitely. + let _slot = slot; + let result = await_network_decision(&decision_rx, decision_timeout); match result { NetworkOpenResult::Denied { errno } => { let _ = worker_listener.respond_errno(notification.id, errno); } NetworkOpenResult::RelayReady => { - match establish_relay(®istry, notification.tid, fd, destination) { + match worker_listener.validate_id(notification.id).and_then(|()| { + establish_relay( + ®istry, + notification.tid, + fd, + socket_identity, + destination, + ) + }) { Ok(stream) => { let result = worker_listener .respond_value(notification.id, 0) @@ -768,6 +802,20 @@ fn connect_socket( Ok(()) } +fn await_network_decision( + decision: &std::sync::mpsc::Receiver, + timeout: Duration, +) -> NetworkOpenResult { + decision + .recv_timeout(timeout) + .unwrap_or_else(|error| NetworkOpenResult::Denied { + errno: match error { + std::sync::mpsc::RecvTimeoutError::Timeout => libc::ETIMEDOUT, + std::sync::mpsc::RecvTimeoutError::Disconnected => libc::ECANCELED, + }, + }) +} + fn ensure_dns_source_bound(fd: RawFd, family: InetFamily) -> io::Result { let mut address = socket_local_addr(fd)?; let loopback = match family { @@ -794,6 +842,7 @@ fn establish_relay( registry: &Mutex, tid: u32, fd: RawFd, + expected_socket: SocketIdentity, destination: SocketAddr, ) -> io::Result { let relay = TcpListener::bind(match destination { @@ -805,6 +854,9 @@ fn establish_relay( let expected_peer = { let mut registry = lock(registry); let entry = registry.resolve_mut(tid, fd)?; + if entry.identity() != expected_socket { + return Err(io::Error::from_raw_os_error(libc::EBADF)); + } connect_exact(entry.retained_preconnect()?.as_raw_fd(), relay_address)?; let expected_peer = socket_local_addr(entry.retained_preconnect()?.as_raw_fd())?; entry.set_state(SocketState::Connected { @@ -1134,7 +1186,6 @@ fn classify_send( listener: &NotificationListener, notification: Notification, dns_relay: &DnsRelay, - identity_resolver: &ProcfsIdentityResolver, ) -> io::Result<()> { let fd = raw_fd(notification.args[0])?; let syscall = i64::from(notification.syscall); @@ -1213,13 +1264,12 @@ fn classify_send( .is_some_and(|value| value == dns_relay.address) }) => { - let identity = identity_resolver.resolve(notification.tid); let entry = registry.resolve_mut(notification.tid, fd)?; let source_fd = entry.retained_preconnect()?.as_raw_fd(); let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; - lock(&dns_relay.udp_attribution).insert(peer, identity); + register_dns_socket(&dns_relay.udp_admissions, peer, entry.identity())?; if let Err(error) = connect_exact(source_fd, dns_relay.address) { - lock(&dns_relay.udp_attribution).remove(&peer); + lock(&dns_relay.udp_admissions).remove(&peer); return Err(error); } for message in &messages { @@ -1665,6 +1715,166 @@ mod tests { use std::io::{Read as _, Write as _}; use std::os::unix::net::{UnixListener, UnixStream}; + #[test] + fn relay_rejects_descriptor_replaced_after_policy_decision() { + let metadata = SocketMetadata { + family: InetFamily::V4, + kind: InetKind::Tcp, + close_on_exec: true, + nonblocking: false, + creator_generation: 1, + }; + let mut registry = SocketRegistry::new(1, 2).unwrap(); + let mut create = || { + // SAFETY: a successful socket call returns a new owned descriptor. + let fd = unsafe { libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0) }; + assert!(fd >= 0); + let socket = unsafe { OwnedFd::from_raw_fd(fd) }; + let installed = duplicate_close_on_exec(fd).unwrap(); + let tentative = registry.stage(socket, metadata).unwrap(); + let identity = registry.commit(tentative).unwrap(); + (installed, identity) + }; + let (original, original_identity) = create(); + let (replacement, replacement_identity) = create(); + // SAFETY: both descriptors are live; replace only the test-owned FD. + assert_eq!( + unsafe { libc::dup2(replacement.as_raw_fd(), original.as_raw_fd()) }, + original.as_raw_fd() + ); + let registry = Mutex::new(registry); + let error = establish_relay( + ®istry, + std::process::id(), + original.as_raw_fd(), + original_identity, + "203.0.113.7:443".parse().unwrap(), + ) + .expect_err("an approval for the old socket must not connect its replacement"); + assert_eq!(error.raw_os_error(), Some(libc::EBADF)); + let registry = lock(®istry); + let entry = registry + .resolve(std::process::id(), original.as_raw_fd()) + .unwrap(); + assert_eq!(entry.identity(), replacement_identity); + assert_eq!(entry.state(), &SocketState::Created); + assert_eq!( + socket_local_addr(replacement.as_raw_fd()).unwrap().port(), + 0 + ); + } + + #[test] + fn external_connect_times_out_when_supervisor_retains_the_decision() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_with_decision_timeout( + listener, + "127.0.0.1:0".parse().unwrap(), + Duration::from_millis(50), + ) + .unwrap(); + let client = std::thread::spawn(move || { + launcher + .execute(|| TcpStream::connect("203.0.113.7:443")) + .unwrap() + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let pending = runtime.block_on(broker.accept()).unwrap(); + // Keep the request alive: channel disconnection must not be what + // releases the workload's blocked connect. + let error = client + .join() + .unwrap() + .expect_err("unanswered connect must time out"); + assert_eq!(error.raw_os_error(), Some(libc::ETIMEDOUT)); + assert!( + runtime + .block_on(pending.complete(NetworkOpenResult::RelayReady)) + .is_err() + ); + } + + #[test] + fn dns_admissions_reclaim_closed_sockets_at_the_bound() { + let admissions = Mutex::new(HashMap::new()); + let stale = SocketIdentity { + listener_generation: 1, + inode: 0, + cookie: 1, + }; + for port in 1..=SOCKET_CAPACITY { + lock(&admissions).insert( + SocketAddr::from(([127, 0, 0, 1], u16::try_from(port).unwrap())), + stale, + ); + } + let peer = "127.0.0.1:50000".parse().unwrap(); + register_dns_socket(&admissions, peer, stale).unwrap(); + assert_eq!(lock(&admissions).len(), 1); + assert_eq!(lock(&admissions).get(&peer), Some(&stale)); + } + + #[test] + fn inherited_dns_socket_after_exec_never_claims_the_connecting_binary() { + use std::process::{Command, Stdio}; + + for transport in [DnsTransport::Udp, DnsTransport::Tcp] { + let (launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start().unwrap(); + let broker = NetworkBroker::start_for_test(listener).unwrap(); + let address = broker.dns_address(); + let child = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result<()> { + let (socket, script): (OwnedFd, &str) = match transport { + DnsTransport::Udp => { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.connect(address)?; + (socket.into(), "printf dns >&0") + } + DnsTransport::Tcp => ( + TcpStream::connect(address)?.into(), + "printf '\\000\\003dns' >&0", + ), + }; + // The socket was connected by this executable. A forked + // child inherits it, execs a different binary, and writes + // without a new connect or a destination-bearing send. + let status = Command::new("/bin/sh") + .args(["-c", script]) + .stdin(Stdio::from(socket)) + .status()?; + if !status.success() { + return Err(io::Error::other("DNS-writing child failed")); + } + Ok(()) + }) + .unwrap() + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let query = runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(5), broker.accept_dns()) + .await + .unwrap() + .unwrap() + }); + assert_eq!(query.transport, transport); + assert!(matches!( + query.identity, + Err(ResolveError::Failed(ref message)) if message.contains("unavailable") + )); + query.complete(Ok(Vec::new())).unwrap(); + child.join().unwrap().unwrap(); + } + } + #[test] fn pending_external_open_slots_are_bounded_and_reusable() { let active = Arc::new(AtomicUsize::new(OPEN_QUEUE_CAPACITY - 1)); diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index 9e866ebb03..63428e4921 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -3,14 +3,14 @@ //! Runtime-owned policy DNS listeners for combined Linux supervisors. -use super::resolver::MAX_DNS_MESSAGE_BYTES; -use super::store::{ResolvedEndpointStore, StoreConfig, SyntheticPools}; -use super::{PolicyDnsService, SocketTrustedResolver, wire}; use crate::opa::OpaEngine; +use crate::policy_dns::resolver::MAX_DNS_MESSAGE_BYTES; +use crate::policy_dns::store::{ResolvedEndpointStore, StoreConfig, SyntheticPools}; +use crate::policy_dns::{PolicyDnsService, SocketTrustedResolver, wire}; use futures::{FutureExt as _, StreamExt as _, stream::FuturesUnordered}; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::net::set_tcp_nodelay_best_effort; -use openshell_isolation_interface::contract::{DnsMediationSource, DnsTransport}; +use openshell_isolation_interface::contract::{DnsMediationSource, DnsTransport, MediatedDnsQuery}; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -23,6 +23,22 @@ const IPV4_EPOCH_WINDOWS: u64 = 1 << (IPV4_POOL_PREFIX - 15); const MAX_MAPPINGS: usize = 1024; const MAX_CONCURRENT_UDP_QUERIES: usize = 64; const MEDIATION_ACCEPT_WINDOW: usize = 32; +const MEDIATION_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100); +const MEDIATION_MAX_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); + +async fn accept_mediated_dns(source: Arc) -> MediatedDnsQuery { + let mut delay = MEDIATION_RETRY_DELAY; + loop { + match source.accept().await { + Ok(query) => return query, + Err(error) => { + tracing::warn!(%error, "mediated DNS accept failed; retrying"); + tokio::time::sleep(delay).await; + delay = delay.saturating_mul(2).min(MEDIATION_MAX_RETRY_DELAY); + } + } + } +} #[derive(Debug, Clone)] pub(crate) struct PolicyDnsRuntimeConfig { @@ -93,14 +109,14 @@ impl PolicyDnsRuntime { let mut accepts = FuturesUnordered::new(); for _ in 0..MEDIATION_ACCEPT_WINDOW { let source = source.clone(); - accepts.push(async move { source.accept().await }.boxed()); + accepts.push(accept_mediated_dns(source).boxed()); } loop { - let Some(Ok(query)) = accepts.next().await else { + let Some(query) = accepts.next().await else { return; }; let source = source.clone(); - accepts.push(async move { source.accept().await }.boxed()); + accepts.push(accept_mediated_dns(source).boxed()); let service = service.clone(); tokio::spawn(async move { let timing = query.timing.clone(); @@ -307,6 +323,45 @@ fn trusted_resolver_from_resolv_conf() -> Result { mod tests { use super::*; + #[tokio::test(start_paused = true)] + async fn mediated_dns_accept_retries_errors_with_backoff() { + use openshell_isolation_interface::contract::{ + BackendError, MediationTiming, ResolveError, + }; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RecoveringSource(AtomicUsize); + + #[async_trait::async_trait] + impl DnsMediationSource for RecoveringSource { + async fn accept(&self) -> std::result::Result { + if self.0.fetch_add(1, Ordering::AcqRel) < 2 { + return Err(BackendError::Unavailable("injected disconnect".into())); + } + let (response, _) = tokio::sync::oneshot::channel(); + Ok(MediatedDnsQuery { + request: b"recovered query".to_vec(), + transport: DnsTransport::Udp, + binary_identity: Err(ResolveError::Failed("unknown sender".into())), + timing: MediationTiming::default(), + response, + }) + } + } + + let source = Arc::new(RecoveringSource(AtomicUsize::new(0))); + let start = tokio::time::Instant::now(); + let query = accept_mediated_dns(source.clone()).await; + assert_eq!(query.request, b"recovered query"); + assert_eq!(source.0.load(Ordering::Acquire), 3); + assert!(start.elapsed() >= MEDIATION_RETRY_DELAY * 3); + // The same source remains usable when the accept window replenishes. + assert_eq!( + accept_mediated_dns(source).await.request, + b"recovered query" + ); + } + #[test] fn production_pool_is_disjoint_from_workload_veth() { let workload: ipnet::IpNet = "10.200.0.0/24".parse().unwrap(); diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 1e57d8cd20..1249620e08 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -72,6 +72,12 @@ When a hot reload changes rules, the supervisor publishes a new policy generatio | `network_policies` | Dynamic | Controls network access for ordinary outbound traffic from the sandbox. Each block has a name, a list of endpoints (host, port, protocol, and optional rules), and a list of binaries allowed to use those endpoints.
Every outbound connection except `https://inference.local` passes through the network supervisor, which queries the [policy engine](/about/how-it-works#core-components) with the destination and calling binary. A connection is allowed only when both match an entry in the same policy block.
For endpoints with `protocol: rest`, the proxy auto-detects TLS and terminates it so each HTTP request can be checked against that endpoint's `rules` (method and path). For endpoints with `protocol: websocket`, the proxy validates the RFC 6455 upgrade and evaluates `GET` rules for the handshake plus either `WEBSOCKET_TEXT` rules for raw client text messages or GraphQL operation rules for GraphQL-over-WebSocket messages. Set `websocket_credential_rewrite: true` only when a WebSocket or REST compatibility endpoint must keep placeholder credentials in sandbox-owned text frames and resolve them at the OpenShell relay boundary.
Endpoints with `protocol: tcp` allow ordinary DNS resolution and native TCP connections without inspecting payloads. Endpoints without `protocol` retain L4 passthrough through an explicit proxy.
If no endpoint matches, the connection is denied. Configure managed inference separately through [Inference Routing](/sandboxes/inference-routing). | | `network_middlewares` | Dynamic | Declares keyed HTTP and WebSocket middleware configs. After network and L7 policy admit a request or upgrade, OpenShell matches each config's host selectors independently and runs matching entries by their unique ascending `order` before credential injection. WebSocket-capable entries continue on complete client text messages. | +In the separate sandbox/supervisor topology, the supervisor mediates DNS by +hostname. DNS sender identity is unavailable because native socket writes can +come from a process that inherited the socket or replaced its executable. +Resolving a name does not authorize a connection: the supervisor still checks +the destination and calling binary when the workload opens TCP traffic. + ## Supervisor Middleware Supervisor middleware can inspect, deny, or replace admitted HTTP request bodies and client WebSocket text messages before provider credentials are injected. Middleware selection is independent of the `network_policies` rule that admitted the traffic: each keyed `network_middlewares` entry matches the destination host through `endpoints.include` and `endpoints.exclude`. From ef53c8101ff52581982d0c58e68a8f9cc0721811 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 21:24:24 -0700 Subject: [PATCH 09/10] fix(sandbox): bound control admission and retire stale mediation Signed-off-by: Drew Newberry --- architecture/sandbox.md | 11 + .../openshell-sandbox/src/boundary_server.rs | 535 ++- .../openshell-sandbox/src/network_broker.rs | 87 +- .../src/sandbox/linux/landlock.rs | 11 - .../src/sandbox/linux/mod.rs | 9 - crates/openshell-sandbox/src/sandbox/mod.rs | 11 +- .../src/child_env.rs | 91 - .../src/identity.rs | 833 ---- .../src/process.rs | 4032 ----------------- .../src/sandbox/linux/landlock.rs | 699 --- .../src/sandbox/linux/mod.rs | 181 - .../src/sandbox/linux/seccomp.rs | 841 ---- .../src/sandbox/mod.rs | 57 - 13 files changed, 561 insertions(+), 6837 deletions(-) delete mode 100644 crates/openshell-supervisor-process/src/child_env.rs delete mode 100644 crates/openshell-supervisor-process/src/identity.rs delete mode 100644 crates/openshell-supervisor-process/src/process.rs delete mode 100644 crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs delete mode 100644 crates/openshell-supervisor-process/src/sandbox/linux/mod.rs delete mode 100644 crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs delete mode 100644 crates/openshell-supervisor-process/src/sandbox/mod.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index cb7b32a29a..14667206e9 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -27,6 +27,17 @@ TCP Service, or VM vsock channel. Independent bidirectional `Exchange` RPCs carry lifecycle, exec, TCP, and forwarding traffic, while one persistent bidirectional `Mediate` RPC carries multiplexed DNS traffic. General application UDP is unsupported; UDP DNS remains mediated by the supervisor. +The sandbox probes HTTP/2 connection liveness every five seconds and closes +connections that miss a ten-second acknowledgement deadline. Closing a +connection cancels its stream bridges before releasing the exclusive DNS +mediation lease; an authenticated replacement waits for release instead of +preempting a live supervisor. Idle healthy connections remain usable. +Unauthenticated TLS handshakes have a separate bounded asynchronous pool and +five-second deadline, never consuming authenticated control slots or threads. +The socket broker reserves the TCP control-listener port against workload +connections, including loopback aliases. Unix control listeners reject workload +descendants using kernel peer credentials and process ancestry, while ordinary +workload loopback and Unix services remain available. NetworkPolicy is an outer reachability fence, not a confidentiality boundary. Each sandbox generation receives a fresh CA and distinct server/client leaves; both endpoints bind the same workload identity and immutable driver resource diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index cbb376c31e..c701279e4c 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -59,6 +59,11 @@ mod linux { }; const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + const CONTROL_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); + const CONTROL_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5); + const CONTROL_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(10); + const MEDIATION_REPLACEMENT_TIMEOUT: Duration = Duration::from_secs(20); + const MAX_PENDING_HANDSHAKES: usize = 32; const MAX_CONTROL_CONNECTIONS: usize = 128; const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096; const MAX_RETAINED_EXEC_PROCESSES: usize = 64; @@ -119,7 +124,11 @@ mod linux { .map_err(|error| format!("install sandbox process prelude: {error}"))?; let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() .map_err(|error| format!("start sandbox workload launcher: {error}"))?; - let network_broker = NetworkBroker::start(listener) + let protected_control_port = match &config.listener { + BoundaryListenerConfig::TlsTcp { address, .. } => Some(address.port()), + BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::Vsock { .. } => None, + }; + let network_broker = NetworkBroker::start(listener, protected_control_port) .map_err(|error| format!("start sandbox network broker: {error}"))?; let process_runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -349,6 +358,7 @@ mod linux { let listener = ControlListener::bind(config) .map_err(|error| format!("bind boundary control listener: {error}"))?; let active_connections = Arc::new(AtomicUsize::new(0)); + let pending_handshakes = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING_HANDSHAKES)); tracing::info!(?config, "Boundary control listener ready"); loop { if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { @@ -357,41 +367,31 @@ mod linux { } match listener.accept() { Ok(stream) => { - let Some(slot) = acquire_control_connection_slot(&active_connections) else { - tracing::warn!( - limit = MAX_CONTROL_CONNECTIONS, - "Boundary control connection limit reached" - ); + // Unauthenticated sockets use only a bounded async TLS + // task, never an OS thread or an authenticated session slot. + let Ok(pending) = pending_handshakes.clone().try_acquire_owned() else { continue; }; + let active_connections = active_connections.clone(); let runtime = runtime.clone(); - std::thread::spawn(move || { - let _slot = slot; - let stream = match stream.establish(&runtime.process_runtime) { - Ok(stream) => stream, - Err(error) => { - tracing::warn!(%error, "Boundary control transport handshake failed"); - return; + runtime.process_runtime.spawn({ + let runtime = runtime.clone(); + async move { + if let Err(error) = serve_control_connection( + stream, + runtime, + pending, + active_connections, + ) + .await + { + tracing::debug!(%error, "Boundary control connection ended"); } - }; - let result = { - let stream = match stream.into_tokio() { - Ok(stream) => stream, - Err(error) => { - tracing::warn!(%error, "prepare boundary gRPC session"); - return; - } - }; - runtime - .process_runtime - .block_on(serve_grpc(stream, runtime.clone())) - }; - if let Err(error) = result { - tracing::warn!(%error, "Boundary control session failed: {error}"); } }); } Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) if error.kind() == io::ErrorKind::PermissionDenied => {} Err(error) if error.kind() == io::ErrorKind::WouldBlock => { std::thread::sleep(Duration::from_millis(10)); } @@ -400,12 +400,39 @@ mod linux { } } + async fn serve_control_connection( + stream: ControlStream, + runtime: Arc, + pending: tokio::sync::OwnedSemaphorePermit, + active_connections: Arc, + ) -> Result<(), String> { + let stream = stream + .establish_async(&runtime.process_runtime) + .await + .map_err(|error| format!("authenticate boundary transport: {error}"))?; + drop(pending); + let Some(_slot) = acquire_control_connection_slot(&active_connections) else { + return Err("authenticated control connection limit reached".to_string()); + }; + serve_grpc(stream.into_tokio()?, runtime).await + } + async fn serve_grpc( stream: openshell_isolation_interface::contract::BoundaryDuplexStream, runtime: Arc, ) -> Result<(), String> { - let incoming = tokio_stream::iter([Ok::<_, io::Error>(GrpcServerIo(stream))]); + let (connection_alive, connection_closed) = tokio::sync::watch::channel(()); + let incoming = tokio_stream::StreamExt::chain( + tokio_stream::iter([Ok::<_, io::Error>(GrpcServerIo { + stream, + _connection_alive: connection_alive, + })]), + tokio_stream::pending(), + ); + let mut shutdown = connection_closed.clone(); tonic::transport::Server::builder() + .http2_keepalive_interval(Some(CONTROL_KEEPALIVE_INTERVAL)) + .http2_keepalive_timeout(Some(CONTROL_KEEPALIVE_TIMEOUT)) .max_concurrent_streams( u32::try_from(MAX_CONTROL_CONNECTIONS) .map_err(|error| format!("invalid control connection limit: {error}"))?, @@ -413,16 +440,26 @@ mod linux { .initial_stream_window_size(16 * 1024 * 1024) .initial_connection_window_size(16 * 1024 * 1024) .add_service( - IsolationBoundaryServer::new(GrpcBoundaryService { runtime }) - .max_decoding_message_size(64 * 1024) - .max_encoding_message_size(64 * 1024), + IsolationBoundaryServer::new(GrpcBoundaryService { + runtime, + connection_closed, + }) + .max_decoding_message_size(64 * 1024) + .max_encoding_message_size(64 * 1024), ) - .serve_with_incoming(incoming) + .serve_with_incoming_shutdown(incoming, async move { + let _ = shutdown.changed().await; + }) .await .map_err(|error| format!("serve boundary gRPC connection: {error}")) } - struct GrpcServerIo(openshell_isolation_interface::contract::BoundaryDuplexStream); + struct GrpcServerIo { + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + // Dropping the actual HTTP/2 transport stops all detached stream + // bridges, including on keepalive failure or task cancellation. + _connection_alive: tokio::sync::watch::Sender<()>, + } impl tokio::io::AsyncRead for GrpcServerIo { fn poll_read( @@ -430,7 +467,7 @@ mod linux { context: &mut Context<'_>, buffer: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { - Pin::new(&mut self.0).poll_read(context, buffer) + Pin::new(&mut self.stream).poll_read(context, buffer) } } @@ -440,18 +477,18 @@ mod linux { context: &mut Context<'_>, buffer: &[u8], ) -> Poll> { - Pin::new(&mut self.0).poll_write(context, buffer) + Pin::new(&mut self.stream).poll_write(context, buffer) } fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.0).poll_flush(context) + Pin::new(&mut self.stream).poll_flush(context) } fn poll_shutdown( mut self: Pin<&mut Self>, context: &mut Context<'_>, ) -> Poll> { - Pin::new(&mut self.0).poll_shutdown(context) + Pin::new(&mut self.stream).poll_shutdown(context) } } @@ -464,6 +501,7 @@ mod linux { #[derive(Clone)] struct GrpcBoundaryService { runtime: Arc, + connection_closed: tokio::sync::watch::Receiver<()>, } type GrpcResponseStream = ReceiverStream>; @@ -477,7 +515,8 @@ mod linux { &self, request: tonic::Request>, ) -> Result, tonic::Status> { - let (stream, response) = bridge_grpc_server_stream(request.into_inner()); + let (stream, response) = + bridge_grpc_server_stream(request.into_inner(), self.connection_closed.clone()); let runtime = self.runtime.clone(); tokio::task::spawn_blocking(move || { let stream = ControlStream::Grpc { @@ -495,7 +534,8 @@ mod linux { &self, request: tonic::Request>, ) -> Result, tonic::Status> { - let (stream, response) = bridge_grpc_server_stream(request.into_inner()); + let (stream, response) = + bridge_grpc_server_stream(request.into_inner(), self.connection_closed.clone()); let runtime = self.runtime.clone(); tokio::spawn(async move { if let Err(error) = serve_persistent_mediation(stream, runtime).await { @@ -508,12 +548,17 @@ mod linux { fn bridge_grpc_server_stream( mut inbound: tonic::Streaming, + connection_closed: tokio::sync::watch::Receiver<()>, ) -> (tokio::io::DuplexStream, GrpcResponseStream) { let (application, bridge) = tokio::io::duplex(256 * 1024); let (mut reader, mut writer) = tokio::io::split(bridge); let (outbound, outbound_rx) = tokio::sync::mpsc::channel::>(64); + let mut inbound_closed = connection_closed.clone(); tokio::spawn(async move { + tokio::select! { + _ = inbound_closed.changed() => {}, + () = async { loop { match inbound.message().await { Ok(Some(chunk)) => { @@ -531,8 +576,14 @@ mod linux { } } } + } => {}, + } }); + let mut outbound_closed = connection_closed; tokio::spawn(async move { + tokio::select! { + _ = outbound_closed.changed() => {}, + () = async { let mut buffer = vec![0_u8; 16 * 1024]; loop { let read = match reader.read(&mut buffer).await { @@ -555,6 +606,8 @@ mod linux { return; } } + } => {}, + } }); (application, ReceiverStream::new(outbound_rx)) } @@ -574,41 +627,38 @@ mod linux { >, >; - struct MediationLease(Arc); - - impl Drop for MediationLease { - fn drop(&mut self) { - self.0.mediation_active.store(false, Ordering::Release); - } - } - async fn serve_persistent_mediation( mut stream: tokio::io::DuplexStream, runtime: Arc, ) -> Result<(), String> { - let request: RequestEnvelope = - openshell_isolation_interface::boundary_protocol::read_frame_async(&mut stream) - .await - .map_err(|error| format!("read mediation attach: {error}"))?; + let request: RequestEnvelope = tokio::time::timeout( + CONTROL_IO_TIMEOUT, + openshell_isolation_interface::boundary_protocol::read_frame_async(&mut stream), + ) + .await + .map_err(|_| "mediation attach timed out".to_string())? + .map_err(|error| format!("read mediation attach: {error}"))?; let request_id = request.request_id.clone(); if !matches!(request.request, Request::OpenMediation) { return Err("persistent mediation stream omitted OpenMediation".to_string()); } let mut response = runtime.dispatch(request); let lease = if matches!(response, Response::MediationReady) { - if runtime - .mediation_active - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - Some(MediationLease(runtime.clone())) - } else { - response = guest_error( - BoundaryErrorKind::Denied, - "a mediation session is already active", - ); - None - } + tokio::time::timeout( + MEDIATION_REPLACEMENT_TIMEOUT, + runtime.mediation_active.lock(), + ) + .await + .map_or_else( + |_| { + response = guest_error( + BoundaryErrorKind::Denied, + "a mediation session is already active", + ); + None + }, + Some, + ) } else { None }; @@ -629,7 +679,7 @@ mod linux { return Ok(()); }; let broker = runtime.network_accept_context()?; - run_boundary_mediation(stream, runtime, broker).await + run_boundary_mediation(stream, runtime.clone(), broker).await } async fn run_boundary_mediation( @@ -967,7 +1017,7 @@ mod linux { /// any launch input or start a second workload. started_agent: Mutex>, next_exec_id: AtomicU64, - mediation_active: AtomicBool, + mediation_active: tokio::sync::Mutex<()>, next_mediation_stream_id: AtomicU64, exec_handles: Mutex>, /// Never evicted within a boundary generation. Reclaiming process I/O @@ -1164,7 +1214,7 @@ mod linux { attached_policy: Mutex::new(None), started_agent: Mutex::new(None), next_exec_id: AtomicU64::new(1), - mediation_active: AtomicBool::new(false), + mediation_active: tokio::sync::Mutex::new(()), next_mediation_stream_id: AtomicU64::new(1), exec_handles: Mutex::new(std::collections::HashMap::new()), exec_requests: Mutex::new(std::collections::HashSet::new()), @@ -2480,6 +2530,7 @@ mod linux { server_config, } => { let (stream, _) = listener.accept()?; + reject_workload_unix_peer(&stream)?; Ok(ControlStream::PendingTls { stream: PlainControlStream::Unix(stream), server_config: server_config.clone(), @@ -2502,6 +2553,74 @@ mod linux { } } + fn reject_workload_unix_peer(stream: &std::os::unix::net::UnixStream) -> io::Result<()> { + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = + libc::socklen_t::try_from(size_of::()).map_err(io::Error::other)?; + // SAFETY: both output pointers reference initialized storage of the + // declared length, and stream owns the connected Unix descriptor. + if unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&raw mut credentials).cast(), + &raw mut length, + ) + } != 0 + { + return Err(io::Error::last_os_error()); + } + let peer = u32::try_from(credentials.pid) + .map_err(|_| io::Error::from_raw_os_error(libc::EACCES))?; + // Linux reports PID zero for a peer outside our PID namespace. Such a + // peer still must authenticate with the per-sandbox mTLS certificate. + if peer != 0 + && peer != std::process::id() + && is_process_descendant(peer, std::process::id()) + .map_err(|_| io::Error::from_raw_os_error(libc::EACCES))? + { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + Ok(()) + } + + fn is_process_descendant(mut process: u32, ancestor: u32) -> io::Result { + // Drivers run the sandbox as workload PID 1, so orphaned descendants + // reparent to it and cannot escape this check by double-forking. + // Read kernel-owned ancestry, never workload-supplied paths or UIDs. + // If a peer exits during inspection, fail closed for that connection. + for _ in 0..1024 { + if process == ancestor { + return Ok(true); + } + if process == 0 { + return Ok(false); + } + let stat = std::fs::read_to_string(format!("/proc/{process}/stat"))?; + let parent = stat + .rsplit_once(')') + .and_then(|(_, fields)| fields.split_whitespace().nth(1)) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "missing peer process parent") + })? + .parse::() + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + if parent == process { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "cyclic peer process ancestry", + )); + } + process = parent; + } + Err(io::Error::from_raw_os_error(libc::EACCES)) + } + fn remove_owned_stale_control_socket(socket_path: &Path) -> io::Result<()> { let metadata = match std::fs::symlink_metadata(socket_path) { Ok(metadata) => metadata, @@ -2639,7 +2758,12 @@ mod linux { } impl ControlStream { + #[cfg(test)] fn establish(self, runtime: &tokio::runtime::Handle) -> io::Result { + runtime.block_on(self.establish_async(runtime)) + } + + async fn establish_async(self, runtime: &tokio::runtime::Handle) -> io::Result { let Self::PendingTls { stream, server_config, @@ -2652,14 +2776,14 @@ mod linux { stream.into_tokio()? }; let acceptor = tokio_rustls::TlsAcceptor::from(server_config); - let stream = runtime.block_on(async { - tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)) + let stream = { + tokio::time::timeout(CONTROL_HANDSHAKE_TIMEOUT, acceptor.accept(stream)) .await .map_err(|_| { io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out") })? .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) - })?; + }?; Ok(Self::Tls { stream: Box::new(stream), runtime: runtime.clone(), @@ -2996,7 +3120,7 @@ mod linux { } #[test] - fn control_connection_slots_bound_unauthenticated_threads() { + fn control_connection_slots_bound_authenticated_sessions() { let active = Arc::new(AtomicUsize::new(MAX_CONTROL_CONNECTIONS - 1)); let slot = acquire_control_connection_slot(&active).expect("last available slot"); assert!(acquire_control_connection_slot(&active).is_none()); @@ -3004,6 +3128,273 @@ mod linux { assert_eq!(active.load(Ordering::Acquire), MAX_CONTROL_CONNECTIONS - 1); } + #[test] + fn unix_control_rejects_workload_descendants_before_admission() { + const CHILD_SOCKET: &str = "OPENSHELL_TEST_CONTROL_PEER_SOCKET"; + if let Some(path) = std::env::var_os(CHILD_SOCKET) { + let mut stream = std::os::unix::net::UnixStream::connect(path).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + assert_eq!(stream.read(&mut [0_u8; 1]).unwrap(), 0); + return; + } + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("control.sock"); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "boundary_server::linux::tests::unix_control_rejects_workload_descendants_before_admission", "--nocapture"]) + .env(CHILD_SOCKET, &path).spawn().unwrap(); + let (stream, _) = listener.accept().unwrap(); + assert_eq!( + reject_workload_unix_peer(&stream) + .unwrap_err() + .raw_os_error(), + Some(libc::EACCES) + ); + drop(stream); + assert!(child.wait().unwrap().success()); + assert!(!is_process_descendant(std::process::id(), child.id()).unwrap()); + // Trusted same-process connections and external ancestors remain + // eligible for mTLS; we do not equate same UID with workload trust. + let client = std::os::unix::net::UnixStream::connect(&path).unwrap(); + let (stream, _) = listener.accept().unwrap(); + reject_workload_unix_peer(&stream).unwrap(); + drop(client); + } + + fn availability_test_runtime() -> Arc { + let (broker, launcher) = test_network_broker(); + Arc::new(BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "availability".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "a".repeat(32), + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().unwrap(), + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }, + tokio::runtime::Handle::current(), + broker, + launcher, + test_runtime_qualification(), + )) + } + + #[tokio::test(flavor = "multi_thread")] + async fn idle_uds_and_tcp_handshakes_do_not_consume_authenticated_slots() { + let directory = tempfile::tempdir().unwrap(); + let (tls, _) = stage_test_tls(directory.path(), "pending"); + let server_config = Arc::new(load_tls_server_config(&tls).unwrap()); + let runtime = availability_test_runtime(); + let active = Arc::new(AtomicUsize::new(0)); + let pending = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING_HANDSHAKES)); + let mut clients: Vec> = Vec::new(); + let mut tasks = tokio::task::JoinSet::new(); + for index in 0..MAX_PENDING_HANDSHAKES { + let stream = if index % 2 == 0 { + let (server, client) = std::os::unix::net::UnixStream::pair().unwrap(); + clients.push(Box::new(client)); + PlainControlStream::Unix(server) + } else { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + clients.push(Box::new( + std::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap(), + )); + PlainControlStream::Tcp(listener.accept().unwrap().0) + }; + let permit = pending.clone().try_acquire_owned().unwrap(); + tasks.spawn(serve_control_connection( + ControlStream::PendingTls { + stream, + server_config: server_config.clone(), + }, + runtime.clone(), + permit, + active.clone(), + )); + } + assert!(pending.clone().try_acquire_owned().is_err()); + assert_eq!(active.load(Ordering::Acquire), 0); + tokio::time::timeout(CONTROL_HANDSHAKE_TIMEOUT + Duration::from_secs(2), async { + while let Some(result) = tasks.join_next().await { + assert!(result.unwrap().unwrap_err().contains("timed out")); + } + }) + .await + .unwrap(); + assert_eq!(pending.available_permits(), MAX_PENDING_HANDSHAKES); + assert_eq!(active.load(Ordering::Acquire), 0); + drop(clients); + } + + #[tokio::test(flavor = "multi_thread")] + async fn grpc_blackhole_expires_connection_and_releases_mediation_lease() { + let runtime = availability_test_runtime(); + let server_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let server_address = server_listener.local_addr().unwrap(); + let server_runtime = runtime.clone(); + let server = tokio::spawn(async move { + let (stream, _) = server_listener.accept().await.unwrap(); + serve_grpc(Box::new(stream), server_runtime).await + }); + let proxy_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_address = proxy_listener.local_addr().unwrap(); + let (blackhole, stop_forwarding) = tokio::sync::oneshot::channel::<()>(); + let proxy = tokio::spawn(async move { + let (mut downstream, _) = proxy_listener.accept().await.unwrap(); + let mut upstream = tokio::net::TcpStream::connect(server_address) + .await + .unwrap(); + tokio::select! { + _ = stop_forwarding => {}, + _ = tokio::io::copy_bidirectional(&mut upstream, &mut downstream) => panic!("proxy closed before blackhole"), + } + // Keep both sockets open without forwarding PING or ACK: this + // models a silently dropped Kubernetes TCP path, not FIN/RST. + std::future::pending::<()>().await; + drop((upstream, downstream)); + }); + let channel = + tonic::transport::Endpoint::from_shared(format!("http://{proxy_address}")) + .unwrap() + .connect() + .await + .unwrap(); + let (sender, receiver) = tokio::sync::mpsc::channel(4); + let request = RequestEnvelope::new( + "availability".to_string(), + "a".repeat(32), + Request::OpenMediation, + ) + .unwrap(); + sender + .send(BoundaryChunk { + data: encode_frame(&request).unwrap(), + }) + .await + .unwrap(); + let mut response = IsolationBoundaryClient::new(channel) + .mediate(ReceiverStream::new(receiver)) + .await + .unwrap() + .into_inner(); + assert!(response.message().await.unwrap().is_some()); + tokio::time::sleep(CONTROL_KEEPALIVE_INTERVAL + Duration::from_secs(1)).await; + assert!( + !server.is_finished(), + "healthy idle session must survive keepalive" + ); + assert!(runtime.mediation_active.try_lock().is_err()); + blackhole.send(()).unwrap(); + tokio::time::timeout( + CONTROL_KEEPALIVE_INTERVAL + CONTROL_KEEPALIVE_TIMEOUT + Duration::from_secs(3), + server, + ) + .await + .expect("blackholed HTTP/2 connection must expire") + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while runtime.mediation_active.try_lock().is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("connection teardown must stop all bridges and release lease"); + let (mut replacement, task) = request_test_mediation(runtime, &"a".repeat(32)).await; + let ready: ResponseEnvelope = + openshell_isolation_interface::boundary_protocol::read_frame_async( + &mut replacement, + ) + .await + .unwrap(); + assert!(matches!(ready.response, Response::MediationReady)); + drop(replacement); + task.await.unwrap().unwrap(); + drop(sender); + proxy.abort(); + } + + async fn request_test_mediation( + runtime: Arc, + token: &str, + ) -> ( + tokio::io::DuplexStream, + tokio::task::JoinHandle>, + ) { + let (mut client, server) = tokio::io::duplex(4096); + let task = tokio::spawn(serve_persistent_mediation(server, runtime)); + let envelope = RequestEnvelope::new( + "availability".to_string(), + token.to_string(), + Request::OpenMediation, + ) + .unwrap(); + client + .write_all(&encode_frame(&envelope).unwrap()) + .await + .unwrap(); + (client, task) + } + + #[tokio::test(flavor = "multi_thread")] + async fn mediation_replacement_waits_for_lease_and_rejects_bad_authentication() { + let runtime = availability_test_runtime(); + let (mut first, first_task) = + request_test_mediation(runtime.clone(), &"a".repeat(32)).await; + let ready: ResponseEnvelope = + openshell_isolation_interface::boundary_protocol::read_frame_async(&mut first) + .await + .unwrap(); + assert!(matches!(ready.response, Response::MediationReady)); + let (mut denied, denied_task) = + request_test_mediation(runtime.clone(), &"b".repeat(32)).await; + let response: ResponseEnvelope = + openshell_isolation_interface::boundary_protocol::read_frame_async(&mut denied) + .await + .unwrap(); + assert!(matches!( + response.response, + Response::Error { + kind: BoundaryErrorKind::Denied, + .. + } + )); + denied_task.await.unwrap().unwrap(); + let (mut replacement, replacement_task) = + request_test_mediation(runtime.clone(), &"a".repeat(32)).await; + assert!( + tokio::time::timeout(Duration::from_millis(50), replacement.read_u8()) + .await + .is_err(), + "a live lease cannot be preempted" + ); + drop(first); + first_task.await.unwrap().unwrap(); + let ready: ResponseEnvelope = tokio::time::timeout( + Duration::from_secs(1), + openshell_isolation_interface::boundary_protocol::read_frame_async( + &mut replacement, + ), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!(ready.response, Response::MediationReady)); + assert!(runtime.mediation_active.try_lock().is_err()); + drop(replacement); + replacement_task.await.unwrap().unwrap(); + assert!(runtime.mediation_active.try_lock().is_ok()); + } + fn test_workload_identity() -> ResolvedWorkloadIdentity { let mut supplementary_gids = nix::unistd::getgroups() .unwrap() diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index f97a7c8321..549cc509ae 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -187,6 +187,7 @@ fn register_dns_socket( #[derive(Clone)] struct NotificationQueues { + protected_control_port: Option, accept_registrar: crate::accept_interrupt::AcceptRegistrar, identity_resolver: ProcfsIdentityResolver, pending: mpsc::Sender, @@ -207,8 +208,11 @@ pub struct NetworkBroker { } impl NetworkBroker { - pub(crate) fn start(listener: NotificationListener) -> io::Result { - Self::start_with_dns_address(listener, DNS_RELAY_ADDRESS) + pub(crate) fn start( + listener: NotificationListener, + protected_control_port: Option, + ) -> io::Result { + Self::start_with_dns_address(listener, DNS_RELAY_ADDRESS, protected_control_port) } #[cfg(any(test, feature = "perf-harness"))] @@ -216,19 +220,27 @@ impl NetworkBroker { Self::start_with_dns_address( listener, SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + None, ) } fn start_with_dns_address( listener: NotificationListener, dns_address: SocketAddr, + protected_control_port: Option, ) -> io::Result { - Self::start_with_decision_timeout(listener, dns_address, NETWORK_DECISION_TIMEOUT) + Self::start_with_decision_timeout( + listener, + dns_address, + protected_control_port, + NETWORK_DECISION_TIMEOUT, + ) } fn start_with_decision_timeout( listener: NotificationListener, dns_address: SocketAddr, + protected_control_port: Option, decision_timeout: Duration, ) -> io::Result { let listener = Arc::new(listener); @@ -244,6 +256,7 @@ impl NetworkBroker { let dns_relay = start_dns_relay(dns_address, pending_dns_tx)?; let dns_address = dns_relay.address; let queues = NotificationQueues { + protected_control_port, accept_registrar: accept_monitor.registrar(), identity_resolver: ProcfsIdentityResolver::for_pid_namespace(), pending: pending_tx, @@ -616,6 +629,19 @@ fn create_socket( Ok(()) } +fn reject_protected_control_destination( + destination: SocketAddr, + protected_port: Option, +) -> io::Result<()> { + // Reserve the listener's port across loopback aliases, IPv4-mapped IPv6, + // and wildcard Pod listeners. A workload must never reach its control + // endpoint, including through a supervisor-authorized external relay. + if protected_port == Some(destination.port()) { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + Ok(()) +} + fn connect_socket( registry: Arc>, listener: Arc, @@ -628,6 +654,7 @@ fn connect_socket( active_opens, identity_resolver, decision_timeout, + protected_control_port, .. } = queues; let notification_started = Instant::now(); @@ -661,6 +688,7 @@ fn connect_socket( } let destination = read_socket_addr(notification.tid, notification.args[1], notification.args[2])?; + reject_protected_control_destination(destination, protected_control_port)?; let (kind, socket_identity, nonblocking) = { let registry = lock(®istry); let entry = registry.resolve(notification.tid, fd)?; @@ -1771,6 +1799,7 @@ mod tests { let broker = NetworkBroker::start_with_decision_timeout( listener, "127.0.0.1:0".parse().unwrap(), + None, Duration::from_millis(50), ) .unwrap(); @@ -1875,6 +1904,58 @@ mod tests { } } + #[test] + fn protected_control_port_rejects_loopback_aliases_and_pod_addresses() { + for address in [ + "127.0.0.1:7443", + "127.0.0.2:7443", + "[::1]:7443", + "[::ffff:127.0.0.1]:7443", + "10.42.0.8:7443", + ] { + assert_eq!( + reject_protected_control_destination(address.parse().unwrap(), Some(7443)) + .unwrap_err() + .raw_os_error(), + Some(libc::EACCES) + ); + } + assert!( + reject_protected_control_destination("127.0.0.1:8080".parse().unwrap(), Some(7443)) + .is_ok() + ); + } + + #[test] + fn workload_cannot_fill_control_listener_with_loopback_connections() { + let control = TcpListener::bind("127.0.0.1:0").unwrap(); + control.set_nonblocking(true).unwrap(); + let address = control.local_addr().unwrap(); + let (launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start().unwrap(); + let _broker = NetworkBroker::start_with_dns_address( + listener, + "127.0.0.1:0".parse().unwrap(), + Some(address.port()), + ) + .unwrap(); + launcher + .execute(move || -> io::Result<()> { + for _ in 0..160 { + let error = + TcpStream::connect(address).expect_err("control port must be unreachable"); + assert_eq!(error.raw_os_error(), Some(libc::EACCES)); + } + Ok(()) + }) + .unwrap() + .unwrap(); + assert_eq!( + control.accept().unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + } + #[test] fn pending_external_open_slots_are_bounded_and_reusable() { let active = Arc::new(AtomicUsize::new(OPEN_QUEUE_CAPACITY - 1)); diff --git a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs index d893b828ba..c18b633e75 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs @@ -434,17 +434,6 @@ pub fn enforce(prepared: PreparedRuleset) -> Result<()> { Ok(()) } -/// Legacy single-phase apply. Kept for non-Linux platforms and tests. -/// On Linux, callers should use [`prepare`] + [`enforce`] for correct -/// privilege ordering. -#[allow(dead_code)] // Retained for backward compat; live callers use prepare+enforce. -pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { - if let Some(prepared) = prepare(policy, workdir)? { - enforce(prepared)?; - } - Ok(()) -} - /// Tailor a rule's access mask to the inode referenced by its already-open FD. /// /// Landlock directory-only rights such as `ReadDir` are invalid for regular diff --git a/crates/openshell-sandbox/src/sandbox/linux/mod.rs b/crates/openshell-sandbox/src/sandbox/linux/mod.rs index 523d33bd0c..3f0084450e 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/mod.rs @@ -102,15 +102,6 @@ pub fn apply_supervisor_prelude() -> Result<()> { seccomp::apply_supervisor_prelude() } -/// Legacy single-phase apply. Kept for backward compatibility. -/// New callers should use [`prepare`] + [`enforce`] for correct privilege ordering. -#[allow(dead_code)] // Retained for backward compat; live callers use prepare+enforce. -pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { - landlock::apply(policy, workdir)?; - seccomp::apply(policy)?; - Ok(()) -} - /// Probe Landlock availability and emit OCSF logs from the parent process. /// /// This must be called **before** `pre_exec` / `fork()` so that the OCSF events diff --git a/crates/openshell-sandbox/src/sandbox/mod.rs b/crates/openshell-sandbox/src/sandbox/mod.rs index ff44f8ba10..bcdb800f79 100644 --- a/crates/openshell-sandbox/src/sandbox/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/mod.rs @@ -4,6 +4,7 @@ //! Platform sandboxing implementation. use miette::Result; +#[cfg(not(target_os = "linux"))] use openshell_core::policy::SandboxPolicy; #[cfg(target_os = "linux")] @@ -16,15 +17,9 @@ pub mod linux; /// Returns an error if the sandbox cannot be applied. // On Linux the spawn path uses `prepare`+`enforce` directly; this single-phase // apply is only invoked from the non-Linux spawn_impl. -#[cfg_attr(target_os = "linux", allow(dead_code))] -#[cfg_attr(not(target_os = "linux"), allow(clippy::unnecessary_wraps))] +#[cfg(not(target_os = "linux"))] +#[allow(clippy::unnecessary_wraps)] pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { - #[cfg(target_os = "linux")] - { - linux::apply(policy, workdir) - } - - #[cfg(not(target_os = "linux"))] { let _ = (policy, workdir); openshell_ocsf::ocsf_emit!( diff --git a/crates/openshell-supervisor-process/src/child_env.rs b/crates/openshell-supervisor-process/src/child_env.rs deleted file mode 100644 index 32eecbee35..0000000000 --- a/crates/openshell-supervisor-process/src/child_env.rs +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::path::Path; - -const LOCAL_NO_PROXY: &str = "127.0.0.1,localhost,::1"; - -pub fn proxy_env_vars(proxy_url: &str) -> [(&'static str, String); 9] { - [ - ("ALL_PROXY", proxy_url.to_owned()), - ("HTTP_PROXY", proxy_url.to_owned()), - ("HTTPS_PROXY", proxy_url.to_owned()), - ("NO_PROXY", LOCAL_NO_PROXY.to_owned()), - ("http_proxy", proxy_url.to_owned()), - ("https_proxy", proxy_url.to_owned()), - ("no_proxy", LOCAL_NO_PROXY.to_owned()), - ("grpc_proxy", proxy_url.to_owned()), - // Node.js only honors HTTP(S)_PROXY for built-in fetch/http clients when - // proxy support is explicitly enabled at process startup. - ("NODE_USE_ENV_PROXY", "1".to_owned()), - ] -} - -pub fn tls_env_vars( - ca_cert_path: &Path, - combined_bundle_path: &Path, -) -> [(&'static str, String); 6] { - let ca_cert_path = ca_cert_path.display().to_string(); - let combined_bundle_path = combined_bundle_path.display().to_string(); - [ - ("NODE_EXTRA_CA_CERTS", ca_cert_path.clone()), - ("DENO_CERT", ca_cert_path), - ("SSL_CERT_FILE", combined_bundle_path.clone()), - ("REQUESTS_CA_BUNDLE", combined_bundle_path.clone()), - ("CURL_CA_BUNDLE", combined_bundle_path.clone()), - // Ubuntu Noble's git links against libcurl-gnutls, which ignores SSL_CERT_FILE. - // git reads GIT_SSL_CAINFO (or http.sslCAInfo) to locate the CA bundle. - ("GIT_SSL_CAINFO", combined_bundle_path), - ] -} - -#[cfg(test)] -mod tests { - use super::*; - use std::process::Command; - use std::process::Stdio; - - #[test] - fn apply_proxy_env_includes_node_proxy_opt_in_and_local_bypass() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()); - - for (key, value) in proxy_env_vars("http://10.200.0.1:3128") { - cmd.env(key, value); - } - - let output = cmd.output().expect("spawn env"); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - - assert!(stdout.contains("HTTP_PROXY=http://10.200.0.1:3128")); - assert!(stdout.contains("NO_PROXY=127.0.0.1,localhost,::1")); - assert!(stdout.contains("NODE_USE_ENV_PROXY=1")); - assert!(stdout.contains("no_proxy=127.0.0.1,localhost,::1")); - } - - #[test] - fn apply_tls_env_sets_node_and_bundle_paths() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()); - - let ca_cert_path = Path::new("/etc/openshell-tls/openshell-ca.pem"); - let combined_bundle_path = Path::new("/etc/openshell-tls/ca-bundle.pem"); - for (key, value) in tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - - let output = cmd.output().expect("spawn env"); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - - assert!(stdout.contains("NODE_EXTRA_CA_CERTS=/etc/openshell-tls/openshell-ca.pem")); - assert!(stdout.contains("DENO_CERT=/etc/openshell-tls/openshell-ca.pem")); - assert!(stdout.contains("SSL_CERT_FILE=/etc/openshell-tls/ca-bundle.pem")); - assert!(stdout.contains("REQUESTS_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); - assert!(stdout.contains("CURL_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); - assert!(stdout.contains("GIT_SSL_CAINFO=/etc/openshell-tls/ca-bundle.pem")); - } -} diff --git a/crates/openshell-supervisor-process/src/identity.rs b/crates/openshell-supervisor-process/src/identity.rs deleted file mode 100644 index df79a4137d..0000000000 --- a/crates/openshell-supervisor-process/src/identity.rs +++ /dev/null @@ -1,833 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver identity normalization and OCI `USER` resolution. - -use crate::process::ResolvedProcessIdentity; -use miette::{IntoDiagnostic, Result}; -use openshell_core::policy::SandboxPolicy; -use std::fs::{File, OpenOptions}; -use std::io::Read; -use std::os::unix::fs::OpenOptionsExt; -use std::path::Path; - -const PASSWD_PATH: &str = "/etc/passwd"; -const GROUP_PATH: &str = "/etc/group"; -const MAX_ACCOUNT_FILE_SIZE: u64 = 1024 * 1024; -const MAX_ACCOUNT_LINE_SIZE: usize = 8 * 1024; -const MAX_ACCOUNT_FIELD_SIZE: usize = 1024; - -/// Identity input selected by the active compute driver. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DriverIdentity { - /// Platform-selected identity used by Kubernetes and `OpenShift`. - Resolved { uid: u32, gid: u32 }, - /// Raw OCI `Config.User` selected by Docker and Podman. - OciUser { declaration: String }, - /// Drivers with no authoritative identity metadata. - None, -} - -impl DriverIdentity { - /// Normalize the protected driver environment into one identity variant. - pub fn from_env() -> Result { - let oci_user = optional_utf8_env(openshell_core::sandbox_env::OCI_IMAGE_USER)?; - let uid = optional_nonempty_utf8_env(openshell_core::sandbox_env::SANDBOX_UID)?; - let gid = optional_nonempty_utf8_env(openshell_core::sandbox_env::SANDBOX_GID)?; - Self::from_values(oci_user, uid, gid) - } - - fn from_values( - oci_user: Option, - uid: Option, - gid: Option, - ) -> Result { - // Resolved-identity drivers explicitly clear the OCI declaration so - // an image-baked or user-supplied value cannot select the OCI path. - // Preserve an empty declaration when no resolved pair is present: - // Docker and Podman use that state to reject images without USER. - let oci_user = if oci_user.as_deref() == Some("") && (uid.is_some() || gid.is_some()) { - None - } else { - oci_user - }; - - match (oci_user, uid, gid) { - (Some(declaration), None, None) => Ok(Self::OciUser { declaration }), - (None, Some(uid), Some(gid)) => { - let uid = uid.parse::().ok().filter(|uid| { - (openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(uid) - }); - let gid = gid.parse::().ok().filter(|gid| { - (openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(gid) - }); - let (Some(uid), Some(gid)) = (uid, gid) else { - return Err(miette::miette!( - "driver UID/GID must be numeric identities in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID - )); - }; - Ok(Self::Resolved { uid, gid }) - } - (None, None, None) => Ok(Self::None), - (Some(_), _, _) => Err(miette::miette!( - "{} conflicts with non-empty {}/{} driver identity", - openshell_core::sandbox_env::OCI_IMAGE_USER, - openshell_core::sandbox_env::SANDBOX_UID, - openshell_core::sandbox_env::SANDBOX_GID - )), - (None, _, _) => Err(miette::miette!( - "{} and {} must be supplied together", - openshell_core::sandbox_env::SANDBOX_UID, - openshell_core::sandbox_env::SANDBOX_GID - )), - } - } -} - -/// Apply a driver identity before any workload child becomes reachable. -pub fn resolve_process_identity( - policy: &mut SandboxPolicy, - driver_identity: &DriverIdentity, -) -> Result { - match driver_identity { - DriverIdentity::Resolved { uid, gid } => { - policy.process.run_as_user = Some(uid.to_string()); - policy.process.run_as_group = Some(gid.to_string()); - // Kubernetes/OpenShift already supply numeric policy values and - // retain their existing privilege-drop path. - Ok(ResolvedProcessIdentity::default()) - } - DriverIdentity::OciUser { declaration } => resolve_oci_process_identity_at( - policy, - declaration, - Path::new(PASSWD_PATH), - Path::new(GROUP_PATH), - ), - DriverIdentity::None => { - // VM/offline drivers retain the pre-OCI per-field fallback. A - // partial policy must never leave the omitted component at the - // root supervisor identity. - if policy - .process - .run_as_user - .as_deref() - .is_none_or(str::is_empty) - { - policy.process.run_as_user = Some("sandbox".into()); - } - if policy - .process - .run_as_group - .as_deref() - .is_none_or(str::is_empty) - { - policy.process.run_as_group = Some("sandbox".into()); - } - Ok(ResolvedProcessIdentity::default()) - } - } -} - -#[allow(clippy::similar_names)] -fn resolve_oci_process_identity_at( - policy: &mut SandboxPolicy, - declaration: &str, - passwd_path: &Path, - group_path: &Path, -) -> Result { - let explicit_user = policy - .process - .run_as_user - .as_deref() - .is_some_and(|value| !value.is_empty()); - let explicit_group = policy - .process - .run_as_group - .as_deref() - .is_some_and(|value| !value.is_empty()); - - if explicit_user && explicit_group { - return Ok(ResolvedProcessIdentity::default()); - } - - let (oci_user, oci_group) = split_oci_declaration(declaration); - let needs_primary_gid = !explicit_group && oci_group.is_none(); - let resolved_user = if !explicit_user || needs_primary_gid { - Some(resolve_required_oci_user( - oci_user, - passwd_path, - declaration, - needs_primary_gid, - )?) - } else { - None - }; - - let oci_uid = if explicit_user { - None - } else { - Some( - resolved_user - .as_ref() - .expect("omitted OCI user must have been resolved") - .0, - ) - }; - - if !explicit_user { - policy.process.run_as_user = Some(oci_user.to_string()); - } - - let oci_gid = if explicit_group { - None - } else { - let (group_value, gid) = match oci_group { - Some(group) if !group.is_empty() => { - let gid = validate_oci_group(group, group_path, declaration)?; - (group.to_string(), gid) - } - Some(_) => { - return Err(miette::miette!( - "OCI USER '{declaration}' has an empty group component" - )); - } - None => { - let gid = resolved_user - .and_then(|(_, primary_gid)| primary_gid) - .ok_or_else(|| { - miette::miette!( - "OCI USER '{declaration}' uses a numeric UID without an explicit group, \ - but /etc/passwd has no matching primary GID" - ) - })?; - (gid.to_string(), gid) - } - }; - policy.process.run_as_group = Some(group_value); - Some(gid) - }; - - Ok(ResolvedProcessIdentity::new(oci_uid, oci_gid)) -} - -fn split_oci_declaration(declaration: &str) -> (&str, Option<&str>) { - declaration - .split_once(':') - .map_or((declaration, None), |(user, group)| (user, Some(group))) -} - -fn resolve_required_oci_user( - user: &str, - passwd_path: &Path, - declaration: &str, - require_primary_gid: bool, -) -> Result<(u32, Option)> { - if user.is_empty() { - return Err(miette::miette!( - "OCI USER is required because run_as_user is omitted" - )); - } - validate_component(user, "OCI user")?; - if user == "root" { - return Err(miette::miette!("OCI USER '{declaration}' selects root")); - } - if let Ok(uid) = user.parse::() { - if uid == 0 { - return Err(miette::miette!("OCI USER '{declaration}' selects UID 0")); - } - let primary_gid = if require_primary_gid { - find_passwd_by_uid(passwd_path, uid)?.map(|entry| entry.gid) - } else { - None - }; - if primary_gid == Some(0) { - return Err(miette::miette!( - "OCI USER '{declaration}' resolves to prohibited primary GID 0" - )); - } - return Ok((uid, primary_gid)); - } - let entry = find_passwd_by_name(passwd_path, user)? - .ok_or_else(|| miette::miette!("OCI USER name '{user}' was not found in /etc/passwd"))?; - if entry.uid == 0 { - return Err(miette::miette!( - "OCI USER '{declaration}' resolves to prohibited UID 0" - )); - } - if require_primary_gid && entry.gid == 0 { - return Err(miette::miette!( - "OCI USER '{declaration}' resolves to prohibited primary GID 0" - )); - } - Ok((entry.uid, require_primary_gid.then_some(entry.gid))) -} - -fn validate_oci_group(value: &str, group_path: &Path, declaration: &str) -> Result { - validate_component(value, "OCI group")?; - if value == "root" { - return Err(miette::miette!( - "OCI USER '{declaration}' selects root group" - )); - } - let gid = if let Ok(gid) = value.parse::() { - gid - } else { - find_group_by_name(group_path, value)? - .ok_or_else(|| miette::miette!("OCI group '{value}' was not found in /etc/group"))? - .gid - }; - if gid == 0 { - return Err(miette::miette!( - "OCI USER '{declaration}' resolves to prohibited GID 0" - )); - } - Ok(gid) -} - -fn validate_component(value: &str, kind: &str) -> Result<()> { - if value.is_empty() - || value.len() > MAX_ACCOUNT_FIELD_SIZE - || value.trim() != value - || value.chars().any(|ch| ch.is_control() || ch == ':') - { - return Err(miette::miette!("{kind} component '{value}' is malformed")); - } - Ok(()) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct PasswdEntry { - uid: u32, - gid: u32, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct GroupEntry { - gid: u32, -} - -fn find_passwd_by_name(path: &Path, name: &str) -> Result> { - find_unique(path, |fields| { - (fields.first().copied() == Some(name)).then(|| parse_passwd(fields)) - }) -} - -fn find_passwd_by_uid(path: &Path, uid: u32) -> Result> { - find_unique(path, |fields| { - fields - .get(2) - .and_then(|value| value.parse::().ok()) - .filter(|candidate| *candidate == uid) - .map(|_| parse_passwd(fields)) - }) -} - -fn find_group_by_name(path: &Path, name: &str) -> Result> { - find_unique(path, |fields| { - (fields.first().copied() == Some(name)).then(|| parse_group(fields)) - }) -} - -/// Resolve supplementary groups declared for an OCI named user without -/// consulting NSS. Numeric OCI users have no trustworthy group-membership -/// name and therefore receive no supplementary groups. -pub fn resolve_oci_supplementary_gids(declaration: &str, primary_gid: u32) -> Result> { - resolve_oci_supplementary_gids_at(declaration, primary_gid, Path::new(GROUP_PATH)) -} - -fn resolve_oci_supplementary_gids_at( - declaration: &str, - primary_gid: u32, - group_path: &Path, -) -> Result> { - let (user, _) = split_oci_declaration(declaration); - validate_component(user, "OCI user")?; - if user.parse::().is_ok() { - return Ok(Vec::new()); - } - - let content = read_account_file(group_path)?; - let mut gids = vec![primary_gid]; - for line in content.lines() { - if line.is_empty() || line.starts_with('#') { - continue; - } - if line.len() > MAX_ACCOUNT_LINE_SIZE { - return Err(miette::miette!( - "account file '{}' contains an oversized line", - group_path.display() - )); - } - let fields = line.split(':').collect::>(); - if fields.len() != 4 - || fields - .iter() - .any(|field| field.len() > MAX_ACCOUNT_FIELD_SIZE) - { - return Err(miette::miette!( - "group membership entry in '{}' is malformed", - group_path.display() - )); - } - if !fields[3].split(',').any(|member| member == user) { - continue; - } - let gid = fields[2].parse::().map_err(|_| { - miette::miette!( - "group membership GID in '{}' is malformed", - group_path.display() - ) - })?; - if gid == 0 { - return Err(miette::miette!( - "OCI user '{user}' is a member of prohibited GID 0" - )); - } - gids.push(gid); - } - gids.sort_unstable(); - gids.dedup(); - Ok(gids) -} - -fn find_unique( - path: &Path, - mut select: impl FnMut(&[&str]) -> Option>, -) -> Result> { - let content = read_account_file(path)?; - let mut found = None; - for line in content.lines() { - if line.is_empty() || line.starts_with('#') { - continue; - } - if line.len() > MAX_ACCOUNT_LINE_SIZE { - return Err(miette::miette!( - "account file '{}' contains an oversized line", - path.display() - )); - } - let fields = line.split(':').collect::>(); - if fields - .iter() - .any(|field| field.len() > MAX_ACCOUNT_FIELD_SIZE) - { - return Err(miette::miette!( - "account file '{}' contains an oversized field", - path.display() - )); - } - let Some(candidate) = select(&fields) else { - continue; - }; - let candidate = candidate?; - if found.replace(candidate).is_some() { - return Err(miette::miette!( - "account identity is ambiguous in '{}'", - path.display() - )); - } - } - Ok(found) -} - -fn parse_passwd(fields: &[&str]) -> Result { - if fields.len() != 7 { - return Err(miette::miette!("matching /etc/passwd entry is malformed")); - } - Ok(PasswdEntry { - uid: fields[2] - .parse() - .map_err(|_| miette::miette!("matching /etc/passwd UID is malformed"))?, - gid: fields[3] - .parse() - .map_err(|_| miette::miette!("matching /etc/passwd GID is malformed"))?, - }) -} - -fn parse_group(fields: &[&str]) -> Result { - if fields.len() != 4 { - return Err(miette::miette!("matching /etc/group entry is malformed")); - } - Ok(GroupEntry { - gid: fields[2] - .parse() - .map_err(|_| miette::miette!("matching /etc/group GID is malformed"))?, - }) -} - -fn read_account_file(path: &Path) -> Result { - let mut options = OpenOptions::new(); - options - .read(true) - .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); - let mut file = options - .open(path) - .into_diagnostic() - .map_err(|error| miette::miette!("failed to open '{}': {error}", path.display()))?; - validate_account_file(&file, path)?; - - let mut bytes = Vec::new(); - file.by_ref() - .take(MAX_ACCOUNT_FILE_SIZE + 1) - .read_to_end(&mut bytes) - .into_diagnostic()?; - if bytes.len() as u64 > MAX_ACCOUNT_FILE_SIZE { - return Err(miette::miette!( - "account file '{}' exceeds {MAX_ACCOUNT_FILE_SIZE} bytes", - path.display() - )); - } - String::from_utf8(bytes) - .map_err(|_| miette::miette!("account file '{}' is not valid UTF-8", path.display())) -} - -fn validate_account_file(file: &File, path: &Path) -> Result<()> { - let metadata = file.metadata().into_diagnostic()?; - if !metadata.is_file() { - return Err(miette::miette!( - "account path '{}' is not a regular file", - path.display() - )); - } - if metadata.len() > MAX_ACCOUNT_FILE_SIZE { - return Err(miette::miette!( - "account file '{}' exceeds {MAX_ACCOUNT_FILE_SIZE} bytes", - path.display() - )); - } - Ok(()) -} - -fn optional_utf8_env(name: &str) -> Result> { - std::env::var_os(name) - .map(|value| { - value - .into_string() - .map_err(|_| miette::miette!("{name} is not valid UTF-8")) - }) - .transpose() -} - -fn optional_nonempty_utf8_env(name: &str) -> Result> { - Ok(optional_utf8_env(name)?.filter(|value| !value.is_empty())) -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::policy::SandboxPolicy; - use std::fs; - use tempfile::tempdir; - - fn account_files( - passwd: &str, - group: &str, - ) -> (tempfile::TempDir, std::path::PathBuf, std::path::PathBuf) { - let dir = tempdir().unwrap(); - let passwd_path = dir.path().join("passwd"); - let group_path = dir.path().join("group"); - fs::write(&passwd_path, passwd).unwrap(); - fs::write(&group_path, group).unwrap(); - (dir, passwd_path, group_path) - } - - fn policy(user: Option<&str>, group: Option<&str>) -> SandboxPolicy { - let mut policy = SandboxPolicy { - version: 1, - filesystem: openshell_core::policy::FilesystemPolicy::default(), - network: openshell_core::policy::NetworkPolicy::default(), - landlock: openshell_core::policy::LandlockPolicy::default(), - process: openshell_core::policy::ProcessPolicy::default(), - }; - policy.process.run_as_user = user.map(str::to_string); - policy.process.run_as_group = group.map(str::to_string); - policy - } - - #[test] - fn per_field_policy_precedence_resolves_complete_pair() { - let (_dir, passwd, group) = account_files( - "app:x:1234:1235::/home/app:/bin/sh\nsandbox:x:2000:2001::/sandbox:/bin/sh\n", - "staff:x:1235:\nsandbox:x:2001:\n", - ); - let cases = [ - ( - Some("2000"), - Some("2001"), - "root", - "2000", - "2001", - None, - None, - ), - ( - Some("2000"), - None, - "app:staff", - "2000", - "staff", - None, - Some(1235), - ), - ( - None, - Some("2001"), - "app:root", - "app", - "2001", - Some(1234), - None, - ), - ( - None, - None, - "app:staff", - "app", - "staff", - Some(1234), - Some(1235), - ), - (None, None, "app", "app", "1235", Some(1234), Some(1235)), - ]; - for ( - user, - group_name, - declaration, - expected_user, - expected_group, - resolved_uid, - resolved_gid, - ) in cases - { - let mut policy = policy(user, group_name); - let resolved = - resolve_oci_process_identity_at(&mut policy, declaration, &passwd, &group).unwrap(); - assert_eq!(policy.process.run_as_user.as_deref(), Some(expected_user)); - assert_eq!(policy.process.run_as_group.as_deref(), Some(expected_group)); - assert_eq!(resolved.uid(), resolved_uid); - assert_eq!(resolved.gid(), resolved_gid); - } - } - - #[test] - fn numeric_pair_does_not_require_account_entries() { - let dir = tempdir().unwrap(); - let passwd = dir.path().join("missing-passwd"); - let group = dir.path().join("missing-group"); - let mut policy = policy(None, None); - let resolved = - resolve_oci_process_identity_at(&mut policy, "1234:1235", &passwd, &group).unwrap(); - assert_eq!(policy.process.run_as_user.as_deref(), Some("1234")); - assert_eq!(policy.process.run_as_group.as_deref(), Some("1235")); - assert_eq!( - resolved, - ResolvedProcessIdentity::new(Some(1234), Some(1235)) - ); - } - - #[test] - fn explicit_identity_is_preserved_without_inspecting_oci_or_accounts() { - let dir = tempdir().unwrap(); - let mut policy = policy(Some("sandbox"), Some("sandbox")); - - let resolved = resolve_oci_process_identity_at( - &mut policy, - "root:root", - &dir.path().join("missing-passwd"), - &dir.path().join("missing-group"), - ) - .unwrap(); - - assert_eq!(policy.process.run_as_user.as_deref(), Some("sandbox")); - assert_eq!(policy.process.run_as_group.as_deref(), Some("sandbox")); - assert_eq!(resolved, ResolvedProcessIdentity::default()); - } - - #[test] - fn driver_identity_inputs_are_mutually_exclusive_and_complete() { - assert_eq!( - DriverIdentity::from_values(Some("app".into()), None, None).unwrap(), - DriverIdentity::OciUser { - declaration: "app".into() - } - ); - assert_eq!( - DriverIdentity::from_values(None, Some("1234".into()), Some("1235".into())).unwrap(), - DriverIdentity::Resolved { - uid: 1234, - gid: 1235 - } - ); - assert_eq!( - DriverIdentity::from_values(None, Some("500".into()), Some("30".into())).unwrap(), - DriverIdentity::Resolved { uid: 500, gid: 30 } - ); - assert_eq!( - DriverIdentity::from_values( - Some(String::new()), - Some("1234".into()), - Some("1235".into()) - ) - .unwrap(), - DriverIdentity::Resolved { - uid: 1234, - gid: 1235 - } - ); - assert_eq!( - DriverIdentity::from_values(Some(String::new()), None, None).unwrap(), - DriverIdentity::OciUser { - declaration: String::new() - } - ); - assert_eq!( - DriverIdentity::from_values(None, None, None).unwrap(), - DriverIdentity::None - ); - assert!( - DriverIdentity::from_values( - Some("app".into()), - Some("1234".into()), - Some("1235".into()) - ) - .is_err() - ); - assert!(DriverIdentity::from_values(None, Some("1234".into()), None).is_err()); - } - - #[test] - fn no_driver_identity_completes_partial_policy_with_sandbox() { - let cases = [ - (None, Some("staff"), "sandbox", "staff"), - (Some("app"), None, "app", "sandbox"), - (None, None, "sandbox", "sandbox"), - (Some("app"), Some("staff"), "app", "staff"), - ]; - - for (user, group, expected_user, expected_group) in cases { - let mut policy = policy(user, group); - let resolved = resolve_process_identity(&mut policy, &DriverIdentity::None).unwrap(); - - assert_eq!(policy.process.run_as_user.as_deref(), Some(expected_user)); - assert_eq!(policy.process.run_as_group.as_deref(), Some(expected_group)); - assert_eq!(resolved, ResolvedProcessIdentity::default()); - } - } - - #[test] - fn numeric_uid_uses_passwd_primary_gid() { - let (_dir, passwd, group) = account_files("app:x:1234:4321::/home/app:/bin/sh\n", ""); - let mut policy = policy(None, None); - let resolved = - resolve_oci_process_identity_at(&mut policy, "1234", &passwd, &group).unwrap(); - assert_eq!(policy.process.run_as_group.as_deref(), Some("4321")); - assert_eq!( - resolved, - ResolvedProcessIdentity::new(Some(1234), Some(4321)) - ); - } - - #[test] - fn named_oci_user_resolves_bounded_supplementary_groups() { - let (_dir, _passwd, group) = account_files( - "", - "primary:x:1235:\nvideo:x:44:app,other\naudio:x:63:other\nrender:x:107:app\n", - ); - - let gids = resolve_oci_supplementary_gids_at("app:primary", 1235, &group).unwrap(); - assert_eq!(gids, vec![44, 107, 1235]); - } - - #[test] - fn numeric_oci_user_has_no_named_supplementary_groups() { - let dir = tempdir().unwrap(); - let missing_group = dir.path().join("missing-group"); - - let gids = resolve_oci_supplementary_gids_at("1234:1235", 1235, &missing_group).unwrap(); - assert!(gids.is_empty()); - } - - #[test] - fn oci_supplementary_membership_rejects_root_group() { - let (_dir, _passwd, group) = account_files("", "root:x:0:app\n"); - - let error = - resolve_oci_supplementary_gids_at("app", 1235, &group).expect_err("GID 0 must fail"); - assert!(error.to_string().contains("prohibited GID 0")); - } - - #[test] - fn missing_unknown_ambiguous_and_root_identities_fail() { - let (_dir, passwd, group) = account_files( - "app:x:1234:1235::/home/app:/bin/sh\napp:x:2234:2235::/home/app2:/bin/sh\n", - "staff:x:1235:\nstaff:x:2235:\n", - ); - for declaration in ["", "unknown", "app", "9999", "0:1235", "1234:0"] { - let mut policy = policy(None, None); - assert!( - resolve_oci_process_identity_at(&mut policy, declaration, &passwd, &group).is_err(), - "{declaration:?} unexpectedly resolved" - ); - } - } - - #[test] - fn selected_component_is_validated_independently() { - let (_dir, passwd, group) = - account_files("app:x:1234:1235::/home/app:/bin/sh\n", "staff:x:1235:\n"); - - let mut explicit_user = policy(Some("1234"), None); - let resolved = - resolve_oci_process_identity_at(&mut explicit_user, "root:staff", &passwd, &group) - .unwrap(); - assert_eq!(explicit_user.process.run_as_user.as_deref(), Some("1234")); - assert_eq!(explicit_user.process.run_as_group.as_deref(), Some("staff")); - assert_eq!(resolved, ResolvedProcessIdentity::new(None, Some(1235))); - - let mut explicit_group = policy(None, Some("1235")); - let resolved = - resolve_oci_process_identity_at(&mut explicit_group, "app:root", &passwd, &group) - .unwrap(); - assert_eq!(explicit_group.process.run_as_user.as_deref(), Some("app")); - assert_eq!(explicit_group.process.run_as_group.as_deref(), Some("1235")); - assert_eq!(resolved, ResolvedProcessIdentity::new(Some(1234), None)); - } - - #[test] - fn named_oci_components_mapping_to_root_are_rejected() { - let (_dir, passwd, group) = account_files( - "root_alias:x:0:1235::/root:/bin/sh\napp:x:1234:1235::/home/app:/bin/sh\n", - "root_alias:x:0:\nstaff:x:1235:\n", - ); - - let mut root_user = policy(None, None); - assert!( - resolve_oci_process_identity_at(&mut root_user, "root_alias:staff", &passwd, &group) - .is_err() - ); - - let mut root_group = policy(None, None); - assert!( - resolve_oci_process_identity_at(&mut root_group, "app:root_alias", &passwd, &group) - .is_err() - ); - } - - #[cfg(unix)] - #[test] - fn account_file_symlinks_are_rejected() { - use std::os::unix::fs::symlink; - - let (_dir, passwd, group) = - account_files("app:x:1234:1235::/home/app:/bin/sh\n", "staff:x:1235:\n"); - let link = passwd.with_file_name("passwd-link"); - symlink(&passwd, &link).unwrap(); - - let mut policy = policy(None, None); - assert!(resolve_oci_process_identity_at(&mut policy, "app:staff", &link, &group).is_err()); - } -} diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs deleted file mode 100644 index 0ab1dd3187..0000000000 --- a/crates/openshell-supervisor-process/src/process.rs +++ /dev/null @@ -1,4032 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Process management and signal handling. - -use crate::child_env; -#[cfg(target_os = "linux")] -use crate::managed_children; -#[cfg(target_os = "linux")] -use crate::netns::NetworkNamespace; -use crate::sandbox; -#[cfg(target_os = "linux")] -use miette::WrapErr; -use miette::{IntoDiagnostic, Result}; -use nix::sys::signal::{self, Signal}; -use nix::unistd::{Gid, Group, Pid, Uid, User}; -use openshell_core::policy::{NetworkMode, SandboxPolicy}; -use std::collections::HashMap; -use std::ffi::CString; -#[cfg(unix)] -use std::os::fd::AsRawFd; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; -#[cfg(target_os = "linux")] -use std::os::unix::ffi::OsStrExt; -#[cfg(unix)] -use std::os::unix::fs::{MetadataExt, PermissionsExt}; -#[cfg(any(test, unix))] -use std::path::Path; -use std::path::PathBuf; -use std::process::Stdio; -#[cfg(target_os = "linux")] -use std::sync::OnceLock; -#[cfg(target_os = "linux")] -use std::sync::mpsc; -use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; -use tracing::{debug, info}; - -// `libc::TIOCSCTTY` and the request parameter accepted by `ioctl` vary across -// glibc, musl, and BSD targets. The conversion is a no-op on some targets but -// is required on others. -#[cfg(unix)] -#[allow(unsafe_code, clippy::useless_conversion)] -fn set_controlling_tty(fd: libc::c_int) -> std::io::Result<()> { - if unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) } < 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) -} - -/// Process/filesystem enforcement performed by the process supervisor. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProcessEnforcementMode { - /// Preserve the existing supervisor behavior: prepare filesystem policy, - /// drop privileges, and apply Landlock/seccomp to workload processes. - Full, - /// Preserve process launch and SSH/session behavior, but skip controls - /// that require root or extra Linux capabilities. Kubernetes sidecar mode - /// uses this when network policy is enforced by the network sidecar. - NetworkOnly, -} - -/// Numeric identity components resolved once from driver-owned metadata. -/// -/// A component is `None` when the corresponding policy field was explicit and -/// must continue through the existing policy identity path. OCI-derived -/// components are carried numerically so later filesystem setup and direct/SSH -/// privilege drops cannot resolve them differently through NSS. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct ResolvedProcessIdentity { - uid: Option, - gid: Option, -} - -impl ResolvedProcessIdentity { - #[must_use] - pub const fn new(uid: Option, gid: Option) -> Self { - Self { uid, gid } - } - - #[must_use] - pub const fn uid(self) -> Option { - self.uid - } - - #[must_use] - pub const fn gid(self) -> Option { - self.gid - } - - /// Whether at least one process identity component came from OCI `USER`. - /// - /// Platform-resolved identities are written directly into the policy and - /// return the default value, so this is specific to Docker/Podman OCI - /// fallback without adding another driver contract. - #[must_use] - pub const fn uses_oci_user_fallback(self) -> bool { - self.uid.is_some() || self.gid.is_some() - } -} - -/// Resolved process workspace and its child-environment semantics. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ResolvedWorkspace { - root: Option, - use_as_home: bool, -} - -impl ResolvedWorkspace { - #[must_use] - pub fn new(root: Option, use_as_home: bool) -> Self { - Self { root, use_as_home } - } - - #[must_use] - pub fn root(&self) -> Option<&str> { - self.root.as_deref() - } - - #[must_use] - pub fn owned_root(&self) -> Option { - self.root.clone() - } - - #[must_use] - pub fn home(&self) -> Option<&str> { - self.use_as_home.then(|| self.root()).flatten() - } -} - -impl ProcessEnforcementMode { - #[must_use] - pub const fn uses_privileged_process_setup(self) -> bool { - matches!(self, Self::Full) - } - - #[must_use] - pub const fn enforces_child_sandbox(self) -> bool { - matches!(self, Self::Full | Self::NetworkOnly) - } -} - -#[cfg(target_os = "linux")] -pub(crate) fn prepare_child_sandbox( - policy: &SandboxPolicy, - workdir: Option<&str>, - enforcement_mode: ProcessEnforcementMode, -) -> Result> { - if !enforcement_mode.enforces_child_sandbox() { - return Ok(None); - } - - let prepared = if enforcement_mode.uses_privileged_process_setup() { - sandbox::linux::prepare(policy, workdir) - } else { - sandbox::linux::prepare_current_user(policy, workdir) - }?; - Ok(Some(prepared)) -} - -const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ - openshell_core::sandbox_env::OCI_IMAGE_USER, - openshell_core::sandbox_env::SANDBOX_UID, - openshell_core::sandbox_env::SANDBOX_GID, - openshell_core::sandbox_env::SANDBOX_TOKEN, - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, - openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, - openshell_core::sandbox_env::TLS_CA, - openshell_core::sandbox_env::TLS_CERT, - openshell_core::sandbox_env::TLS_KEY, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, -]; - -pub fn is_supervisor_only_env_var(key: &str) -> bool { - SUPERVISOR_ONLY_ENV_VARS.contains(&key) -} - -fn strip_supervisor_only_env(cmd: &mut Command) { - for key in SUPERVISOR_ONLY_ENV_VARS { - cmd.env_remove(key); - } -} - -fn inject_provider_env(cmd: &mut Command, provider_env: &HashMap) { - for (key, value) in provider_env { - if is_supervisor_only_env_var(key) { - continue; - } - cmd.env(key, value); - } -} - -/// Derive the child USER and HOME from the policy's sandbox identity. -/// -/// Name-based identities use their passwd entry. Numeric identities have no -/// reliable passwd entry, so their workspace remains the portable fallback. -pub(crate) fn session_user_and_home( - policy: &SandboxPolicy, - workdir_home: Option<&str>, -) -> (String, String) { - let (user, default_home) = match policy.process.run_as_user.as_deref() { - Some(user) if !user.is_empty() => { - if user.parse::().is_ok() { - (user.to_string(), "/sandbox".to_string()) - } else { - let home = User::from_name(user).ok().flatten().map_or_else( - || format!("/home/{user}"), - |entry| entry.dir.to_string_lossy().into_owned(), - ); - (user.to_string(), home) - } - } - _ => ("sandbox".to_string(), "/sandbox".to_string()), - }; - let home = workdir_home.map_or(default_home, str::to_string); - (user, home) -} - -fn apply_canonical_process_environment( - cmd: &mut Command, - policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, - interactive: bool, - user_environment: &HashMap, -) { - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); - // Resolve a shell present in the sandbox image (minimal images such as - // Alpine ship only `/bin/sh`, not bash). Runs in the supervisor. - let shell = openshell_core::shell::detect_login_shell(); - - for (key, value) in [ - ("HOME", session_home.as_str()), - ("USER", session_user.as_str()), - ("SHELL", shell.as_str()), - ( - "TERM", - if interactive { - "xterm-256color" - } else { - "dumb" - }, - ), - ] { - if !user_environment.contains_key(key) { - cmd.env(key, value); - } - } -} - -fn configured_user_environment() -> HashMap { - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default() -} - -#[cfg(unix)] -pub fn harden_child_process() -> Result<()> { - use rustix::process::{Resource, Rlimit, setrlimit}; - - setrlimit( - Resource::Core, - Rlimit { - current: Some(0), - maximum: Some(0), - }, - ) - .map_err(|e| miette::miette!("Failed to disable core dumps: {e}"))?; - - #[cfg(target_os = "linux")] - { - use rustix::process::{DumpableBehavior, set_dumpable_behavior}; - set_dumpable_behavior(DumpableBehavior::NotDumpable) - .map_err(|e| miette::miette!("Failed to set PR_SET_DUMPABLE=0: {e}"))?; - } - - Ok(()) -} - -#[cfg(target_os = "linux")] -const CGROUP_PIDS_MAX_PATH: &str = "/sys/fs/cgroup/pids.max"; - -#[cfg(target_os = "linux")] -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum RuntimePidLimitStatus { - Limited(u64), - Unlimited, - Unavailable(String), -} - -#[cfg(target_os = "linux")] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RuntimePidLimitMode { - Warn, - Require, -} - -#[cfg(target_os = "linux")] -pub fn check_runtime_pid_limit(mode: RuntimePidLimitMode) -> Result<()> { - check_runtime_pid_limit_status(runtime_pid_limit_status(), mode) -} - -#[cfg(target_os = "linux")] -fn check_runtime_pid_limit_status( - status: RuntimePidLimitStatus, - mode: RuntimePidLimitMode, -) -> Result<()> { - match status { - RuntimePidLimitStatus::Limited(limit) => { - debug!(pids_max = limit, "runtime PID limit detected"); - Ok(()) - } - RuntimePidLimitStatus::Unlimited => { - let message = "runtime cgroup pids.max is unlimited; configure the compute driver or container runtime to enforce a PID limit"; - if matches!(mode, RuntimePidLimitMode::Require) { - Err(miette::miette!(message)) - } else { - tracing::warn!("{message}"); - Ok(()) - } - } - RuntimePidLimitStatus::Unavailable(reason) => { - let message = format!( - "runtime cgroup pids.max is unavailable ({reason}); configure the compute driver or container runtime to enforce a PID limit" - ); - if matches!(mode, RuntimePidLimitMode::Require) { - Err(miette::miette!(message)) - } else { - tracing::warn!("{message}"); - Ok(()) - } - } - } -} - -#[cfg(target_os = "linux")] -fn runtime_pid_limit_status() -> RuntimePidLimitStatus { - match std::fs::read_to_string(CGROUP_PIDS_MAX_PATH) { - Ok(contents) => parse_pids_max(&contents), - Err(err) => RuntimePidLimitStatus::Unavailable(err.to_string()), - } -} - -#[cfg(target_os = "linux")] -fn parse_pids_max(contents: &str) -> RuntimePidLimitStatus { - let raw = contents.trim(); - if raw.eq_ignore_ascii_case("max") { - return RuntimePidLimitStatus::Unlimited; - } - match raw.parse::() { - Ok(limit) => RuntimePidLimitStatus::Limited(limit), - Err(err) => { - RuntimePidLimitStatus::Unavailable(format!("invalid pids.max value {raw:?}: {err}")) - } - } -} - -#[cfg(target_os = "linux")] -fn drop_capability_bounding_set() -> Result<()> { - let clear_result = capctl::caps::bounding::clear(); - let remaining = capctl::caps::bounding::probe(); - - validate_capability_bounding_set_clear( - clear_result, - remaining, - capctl::caps::bounding::clear_unknown, - ) -} - -#[cfg(target_os = "linux")] -fn validate_capability_bounding_set_clear( - clear_result: capctl::Result<()>, - remaining: capctl::caps::CapSet, - clear_unknown: impl FnOnce() -> capctl::Result<()>, -) -> Result<()> { - match clear_result { - Ok(()) if remaining.is_empty() => Ok(()), - Ok(()) => Err(miette::miette!( - "Failed to clear child capability bounding set: capabilities remain raised: {remaining:?}" - )), - Err(err) if err.code() == libc::EPERM && remaining.is_empty() => match clear_unknown() { - Ok(()) => { - debug!( - "CAP_SETPCAP is unavailable, but the child capability bounding set is already empty" - ); - Ok(()) - } - Err(unknown_err) => Err(miette::miette!( - "Failed to clear unknown child capability bounding set entries: {unknown_err}" - )), - }, - Err(err) => Err(miette::miette!( - "Failed to clear child capability bounding set: {err}" - )), - } -} - -// Pins the pre-seccomp child mount namespace where supervisor identity sockets -// are shadowed. Children enter it with setns before dropping privileges. -#[cfg(target_os = "linux")] -static SUPERVISOR_IDENTITY_MOUNT_NS: OnceLock> = - OnceLock::new(); - -#[cfg(target_os = "linux")] -pub struct SupervisorIdentityMountNamespace { - spawn_tx: mpsc::Sender, -} - -#[cfg(target_os = "linux")] -type SupervisorIdentityNsRef = &'static SupervisorIdentityMountNamespace; -#[cfg(target_os = "linux")] -type SupervisorIdentitySpawnJob = Box; - -#[cfg(target_os = "linux")] -impl SupervisorIdentityMountNamespace { - fn from_socket_path(socket_path: &str) -> Result> { - let Some(target) = supervisor_identity_mount_target(socket_path)? else { - return Ok(None); - }; - Ok(Some(Self { - spawn_tx: start_supervisor_identity_spawn_worker(target)?, - })) - } -} - -#[cfg(target_os = "linux")] -pub fn prepare_supervisor_identity_mount_namespace_from_env() -> Result<()> { - if SUPERVISOR_IDENTITY_MOUNT_NS.get().is_some() { - return Ok(()); - } - - let Some((_env_name, socket_path)) = supervisor_identity_socket_path_from_env() else { - let _ = SUPERVISOR_IDENTITY_MOUNT_NS.set(None); - return Ok(()); - }; - let namespace = SupervisorIdentityMountNamespace::from_socket_path(&socket_path)?; - let _ = SUPERVISOR_IDENTITY_MOUNT_NS.set(namespace); - Ok(()) -} - -#[cfg(target_os = "linux")] -pub fn supervisor_identity_mount_from_env() -> Result> { - let Some(namespace) = SUPERVISOR_IDENTITY_MOUNT_NS.get() else { - if supervisor_identity_socket_path_from_env().is_some() { - return Err(miette::miette!( - "supervisor identity mount namespace was not prepared before startup hardening" - )); - } - return Ok(None); - }; - Ok(namespace.as_ref()) -} - -#[cfg(target_os = "linux")] -pub fn spawn_command_with_supervisor_identity_namespace( - mut cmd: Command, -) -> std::io::Result { - let namespace = supervisor_identity_mount_from_env() - .map_err(|err| std::io::Error::other(err.to_string()))?; - let Some(namespace) = namespace else { - return cmd.spawn(); - }; - namespace.spawn_tokio_command(cmd) -} - -#[cfg(target_os = "linux")] -pub fn spawn_std_command_with_supervisor_identity_namespace( - mut cmd: std::process::Command, -) -> std::io::Result { - let namespace = supervisor_identity_mount_from_env() - .map_err(|err| std::io::Error::other(err.to_string()))?; - let Some(namespace) = namespace else { - return cmd.spawn(); - }; - namespace.spawn_std_command(cmd) -} - -#[cfg(target_os = "linux")] -impl SupervisorIdentityMountNamespace { - fn spawn_tokio_command(&self, mut cmd: Command) -> std::io::Result { - let (result_tx, result_rx) = mpsc::channel(); - let handle = tokio::runtime::Handle::current(); - self.spawn_tx - .send(Box::new(move || { - let _guard = handle.enter(); - let _ = result_tx.send(cmd.spawn()); - })) - .map_err(|_| std::io::Error::other("supervisor identity spawn worker stopped"))?; - result_rx - .recv() - .map_err(|_| std::io::Error::other("supervisor identity spawn worker dropped result"))? - } - - fn spawn_std_command( - &self, - mut cmd: std::process::Command, - ) -> std::io::Result { - let (result_tx, result_rx) = mpsc::channel(); - self.spawn_tx - .send(Box::new(move || { - let _ = result_tx.send(cmd.spawn()); - })) - .map_err(|_| std::io::Error::other("supervisor identity spawn worker stopped"))?; - result_rx - .recv() - .map_err(|_| std::io::Error::other("supervisor identity spawn worker dropped result"))? - } -} - -#[cfg(target_os = "linux")] -fn start_supervisor_identity_spawn_worker( - target: PathBuf, -) -> Result> { - let (spawn_tx, spawn_rx) = mpsc::channel::(); - let (ready_tx, ready_rx) = mpsc::channel::>(); - std::thread::Builder::new() - .name("openshell-identity-spawn".into()) - .spawn(move || { - let setup = (|| -> std::io::Result<()> { - private_mount_namespace()?; - let target = - cstring_path(&target).map_err(|err| std::io::Error::other(err.to_string()))?; - mount_empty_tmpfs(&target) - })(); - let ready = match &setup { - Ok(()) => Ok(()), - Err(err) => Err(std::io::Error::new( - err.kind(), - format!("supervisor identity setup failed: {err}"), - )), - }; - let _ = ready_tx.send(ready); - if setup.is_err() { - return; - } - while let Ok(job) = spawn_rx.recv() { - job(); - } - }) - .map_err(|err| miette::miette!("failed to spawn supervisor identity worker: {err}"))?; - ready_rx - .recv() - .map_err(|err| miette::miette!("supervisor identity worker did not start: {err}"))? - .map_err(|err| miette::miette!("{err}"))?; - Ok(spawn_tx) -} - -#[cfg(target_os = "linux")] -fn supervisor_identity_socket_path_from_env() -> Option<(&'static str, String)> { - std::env::var(openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) - .ok() - .filter(|socket_path| !socket_path.trim().is_empty()) - .map(|socket_path| { - ( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - socket_path, - ) - }) -} - -#[cfg(any(test, target_os = "linux"))] -fn supervisor_identity_mount_target(socket_path: &str) -> Result> { - let trimmed = socket_path.trim(); - if trimmed.is_empty() { - return Ok(None); - } - if trimmed.starts_with("tcp:") { - return Ok(None); - } - let path = trimmed.strip_prefix("unix:").unwrap_or(trimmed); - let path = Path::new(path); - if !path.is_absolute() { - return Err(miette::miette!( - "{} must be an absolute UNIX socket path", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - )); - } - let Some(parent) = path.parent() else { - return Err(miette::miette!( - "{} has no parent directory", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - )); - }; - if parent == Path::new("/") { - return Err(miette::miette!( - "{} must live below a dedicated directory, not directly under /", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - )); - } - if is_shared_root_mount_shadow(parent) { - return Err(miette::miette!( - "{} must live below a dedicated subdirectory; refusing to hide shared directory {}", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - parent.display() - )); - } - Ok(Some(parent.to_path_buf())) -} - -#[cfg(any(test, target_os = "linux"))] -fn is_shared_root_mount_shadow(parent: &Path) -> bool { - matches!(parent.to_str(), Some("/run" | "/var" | "/tmp" | "/etc")) -} - -#[cfg(target_os = "linux")] -fn cstring_path(path: &Path) -> Result { - CString::new(path.as_os_str().as_bytes()) - .map_err(|_| miette::miette!("path contains an interior NUL byte: {}", path.display())) -} - -#[cfg(target_os = "linux")] -fn private_mount_namespace() -> std::io::Result<()> { - #[allow(unsafe_code)] - let rc = unsafe { libc::unshare(libc::CLONE_NEWNS) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - - #[allow(unsafe_code)] - let rc = unsafe { - let flags: libc::c_ulong = libc::MS_REC | libc::MS_PRIVATE; - libc::mount( - std::ptr::null(), - c"/".as_ptr(), - std::ptr::null(), - flags, - std::ptr::null(), - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) -} - -#[cfg(target_os = "linux")] -fn mount_empty_tmpfs(target: &CString) -> std::io::Result<()> { - #[allow(unsafe_code)] - let rc = unsafe { - let flags: libc::c_ulong = - libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC | libc::MS_RDONLY; - libc::mount( - c"tmpfs".as_ptr(), - target.as_ptr(), - c"tmpfs".as_ptr(), - flags, - c"mode=0555,size=4k".as_ptr().cast(), - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) -} - -/// Handle to a running process. -pub struct ProcessHandle { - child: Child, - pid: u32, - io: Option, -} - -/// Supervisor-owned canonical-process I/O. These handles outlive individual -/// SSH attachments and are consumed by the main-session multiplexer. -pub enum ProcessIo { - Pty(std::fs::File), - Pipes { - stdin: ChildStdin, - stdout: ChildStdout, - stderr: ChildStderr, - }, -} - -impl ProcessHandle { - /// Spawn a new process. - /// - /// # Errors - /// - /// Returns an error if the process fails to start. - #[cfg(target_os = "linux")] - #[allow(clippy::too_many_arguments)] - pub fn spawn( - program: &str, - args: &[String], - workspace: &ResolvedWorkspace, - interactive: bool, - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - netns: Option<&NetworkNamespace>, - ca_paths: Option<&(PathBuf, PathBuf)>, - provider_env: &HashMap, - ) -> Result { - Self::spawn_impl( - program, - args, - workspace, - interactive, - policy, - resolved_identity, - enforcement_mode, - netns.and_then(NetworkNamespace::ns_fd), - ca_paths, - provider_env, - ) - } - - /// Spawn a new process (non-Linux platforms). - /// - /// # Errors - /// - /// Returns an error if the process fails to start. - #[cfg(not(target_os = "linux"))] - #[allow(clippy::too_many_arguments)] - pub fn spawn( - program: &str, - args: &[String], - workspace: &ResolvedWorkspace, - interactive: bool, - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - ca_paths: Option<&(PathBuf, PathBuf)>, - provider_env: &HashMap, - ) -> Result { - Self::spawn_impl( - program, - args, - workspace, - interactive, - policy, - resolved_identity, - enforcement_mode, - ca_paths, - provider_env, - ) - } - - #[cfg(target_os = "linux")] - #[allow(clippy::too_many_arguments)] - fn spawn_impl( - program: &str, - args: &[String], - workspace: &ResolvedWorkspace, - interactive: bool, - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - netns_fd: Option, - ca_paths: Option<&(PathBuf, PathBuf)>, - provider_env: &HashMap, - ) -> Result { - let mut cmd = Command::new(program); - cmd.args(args) - .kill_on_drop(true) - .env(openshell_core::sandbox_env::SANDBOX, "1"); - - let mut pty_master = None; - let mut terminal_slave_fd = None; - if interactive { - let winsize = nix::pty::Winsize { - ws_row: 24, - ws_col: 80, - ws_xpixel: 0, - ws_ypixel: 0, - }; - let pty = nix::pty::openpty(Some(&winsize), None).into_diagnostic()?; - let master = std::fs::File::from(pty.master); - let slave = std::fs::File::from(pty.slave); - terminal_slave_fd = Some(slave.as_raw_fd()); - cmd.stdin(slave.try_clone().into_diagnostic()?) - .stdout(slave.try_clone().into_diagnostic()?) - .stderr(slave); - pty_master = Some(master); - } else { - cmd.stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - } - - // Strip supervisor-only identity material from the entrypoint's - // inherited environment. The entrypoint drops to the sandbox user - // before `exec`; without this strip, sandbox code could recover - // supervisor credentials from its inherited environment. - strip_supervisor_only_env(&mut cmd); - - inject_provider_env(&mut cmd, provider_env); - apply_canonical_process_environment( - &mut cmd, - policy, - workspace, - interactive, - &configured_user_environment(), - ); - - if let Some(dir) = workspace.root() { - cmd.current_dir(dir); - } - - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - // When using network namespace, set proxy URL to the veth host IP - if netns_fd.is_some() { - // The proxy is on 10.200.0.1:3128 (or configured port) - let port = proxy.http_addr.map_or(3128, |addr| addr.port()); - let proxy_url = format!("http://10.200.0.1:{port}"); - // Both uppercase and lowercase variants: curl/wget use uppercase, - // gRPC C-core (libgrpc) checks lowercase http_proxy/https_proxy. - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } else if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } - } - - // Set TLS trust store env vars so sandbox processes trust the ephemeral CA - if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { - for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - } - - // Probe Landlock availability and emit OCSF logs from the parent - // process where the tracing subscriber is functional. The child's - // pre_exec context cannot reliably emit structured logs. - #[cfg(target_os = "linux")] - if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - } - - // Phase 1: Prepare Landlock ruleset by opening PathFds. - // In full mode this runs before drop_privileges() so root-only paths - // can be opened. In sidecar network-only mode the container already - // runs as the sandbox UID, so inaccessible paths are unavailable to - // the workload and best-effort compatibility skips them. - #[cfg(target_os = "linux")] - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; - // Set up process group for signal handling (non-interactive mode only). - // In interactive mode, we inherit the parent's process group to maintain - // proper terminal control for shells and interactive programs. - // SAFETY: pre_exec runs after fork but before exec in the child process. - // setpgid and setns are async-signal-safe and safe to call in this context. - { - let policy = policy.clone(); - // Wrap in Option so we can .take() it out of the FnMut closure. - // pre_exec is only called once (after fork, before exec). - #[cfg(target_os = "linux")] - let mut prepared_sandbox = prepared_sandbox; - #[allow(unsafe_code)] - unsafe { - cmd.pre_exec(move || { - if let Some(slave_fd) = terminal_slave_fd { - if libc::setsid() < 0 { - return Err(std::io::Error::last_os_error()); - } - set_controlling_tty(slave_fd)?; - } else if libc::setpgid(0, 0) < 0 { - return Err(std::io::Error::last_os_error()); - } - - // Enter network namespace before applying other restrictions. - if let Some(fd) = netns_fd { - let result = libc::setns(fd, libc::CLONE_NEWNET); - if result != 0 { - return Err(std::io::Error::other(format!( - "failed to enter network namespace: {}", - std::io::Error::last_os_error() - ))); - } - } - - // Drop privileges. initgroups/setgid/setuid need access to - // /etc/group and /etc/passwd which would be blocked if - // Landlock were already enforced. - if enforcement_mode.uses_privileged_process_setup() { - drop_privileges_with_identity(&policy, resolved_identity) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - - harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; - - // Phase 2 (as unprivileged user): Enforce the prepared - // Landlock ruleset via restrict_self() + apply seccomp. - // restrict_self() does not require root. - #[cfg(target_os = "linux")] - if let Some(prepared) = prepared_sandbox.take() { - sandbox::linux::enforce(prepared) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - - Ok(()) - }); - } - } - - // Name the program in the error: a bare "No such file or directory" - // here is otherwise indistinguishable from a missing working directory - // or interpreter, and is a common failure on images that lack the - // requested shell/binary (e.g. bash on Alpine). - #[cfg(target_os = "linux")] - let mut child = spawn_command_with_supervisor_identity_namespace(cmd) - .into_diagnostic() - .wrap_err_with(|| format!("failed to spawn sandbox entrypoint process '{program}'"))?; - #[cfg(not(target_os = "linux"))] - let mut child = cmd - .spawn() - .into_diagnostic() - .wrap_err_with(|| format!("failed to spawn sandbox entrypoint process '{program}'"))?; - let pid = child.id().unwrap_or(0); - managed_children::register(pid); - - let io = if let Some(master) = pty_master { - ProcessIo::Pty(master) - } else { - ProcessIo::Pipes { - stdin: child.stdin.take().expect("canonical stdin must be piped"), - stdout: child.stdout.take().expect("canonical stdout must be piped"), - stderr: child.stderr.take().expect("canonical stderr must be piped"), - } - }; - - debug!(pid, program, "Process spawned"); - - Ok(Self { - child, - pid, - io: Some(io), - }) - } - - #[cfg(not(target_os = "linux"))] - #[allow(clippy::too_many_arguments)] - fn spawn_impl( - program: &str, - args: &[String], - workspace: &ResolvedWorkspace, - interactive: bool, - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - ca_paths: Option<&(PathBuf, PathBuf)>, - provider_env: &HashMap, - ) -> Result { - let mut cmd = Command::new(program); - cmd.args(args) - .kill_on_drop(true) - .env(openshell_core::sandbox_env::SANDBOX, "1"); - - let mut pty_master = None; - let mut terminal_slave_fd = None; - #[cfg(unix)] - if interactive { - let winsize = nix::pty::Winsize { - ws_row: 24, - ws_col: 80, - ws_xpixel: 0, - ws_ypixel: 0, - }; - let pty = nix::pty::openpty(Some(&winsize), None).into_diagnostic()?; - let master = std::fs::File::from(pty.master); - let slave = std::fs::File::from(pty.slave); - terminal_slave_fd = Some(slave.as_raw_fd()); - cmd.stdin(slave.try_clone().into_diagnostic()?) - .stdout(slave.try_clone().into_diagnostic()?) - .stderr(slave); - pty_master = Some(master); - } - if !interactive { - cmd.stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - } - - // Strip supervisor-only identity material from the entrypoint's - // inherited environment. - strip_supervisor_only_env(&mut cmd); - - inject_provider_env(&mut cmd, provider_env); - apply_canonical_process_environment( - &mut cmd, - policy, - workspace, - interactive, - &configured_user_environment(), - ); - - if let Some(dir) = workspace.root() { - cmd.current_dir(dir); - } - - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } - } - - // Set TLS trust store env vars so sandbox processes trust the ephemeral CA - if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { - for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - } - - // Create a dedicated session for PTY children and a dedicated process - // group for pipe children so attachment signals target only the - // canonical workload tree. - // SAFETY: pre_exec runs after fork but before exec in the child process. - // setpgid is async-signal-safe and safe to call in this context. - #[cfg(unix)] - { - let policy = policy.clone(); - let workdir = workspace.owned_root(); - #[allow(unsafe_code)] - unsafe { - cmd.pre_exec(move || { - if let Some(slave_fd) = terminal_slave_fd { - if libc::setsid() < 0 { - return Err(std::io::Error::last_os_error()); - } - set_controlling_tty(slave_fd)?; - } else if libc::setpgid(0, 0) < 0 { - return Err(std::io::Error::last_os_error()); - } - - // Drop privileges before applying sandbox restrictions. - // initgroups/setgid/setuid need access to /etc/group and /etc/passwd - // which may be blocked by Landlock. - if enforcement_mode.uses_privileged_process_setup() { - drop_privileges_with_identity(&policy, resolved_identity) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - - harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; - - if enforcement_mode.enforces_child_sandbox() { - sandbox::apply(&policy, workdir.as_deref()) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - - Ok(()) - }); - } - } - - let mut child = cmd.spawn().into_diagnostic()?; - let pid = child.id().unwrap_or(0); - #[cfg(target_os = "linux")] - managed_children::register(pid); - - debug!(pid, program, "Process spawned"); - - let io = if let Some(master) = pty_master { - ProcessIo::Pty(master) - } else { - ProcessIo::Pipes { - stdin: child.stdin.take().expect("canonical stdin must be piped"), - stdout: child.stdout.take().expect("canonical stdout must be piped"), - stderr: child.stderr.take().expect("canonical stderr must be piped"), - } - }; - - Ok(Self { - child, - pid, - io: Some(io), - }) - } - - /// Get the process ID. - #[must_use] - pub const fn pid(&self) -> u32 { - self.pid - } - - /// Transfer retained stdio to the main-session multiplexer. - pub fn take_io(&mut self) -> ProcessIo { - self.io.take().expect("canonical process I/O already taken") - } - - /// Wait for the process to exit. - /// - /// # Errors - /// - /// Returns an error if waiting fails. - pub async fn wait(&mut self) -> std::io::Result { - let status = self.child.wait().await; - #[cfg(target_os = "linux")] - managed_children::unregister(self.pid); - let status = status?; - Ok(ProcessStatus::from(status)) - } - - /// Observe an already-terminated child without blocking. - pub fn try_wait(&mut self) -> std::io::Result> { - let status = self.child.try_wait()?; - if status.is_some() { - #[cfg(target_os = "linux")] - managed_children::unregister(self.pid); - } - Ok(status.map(ProcessStatus::from)) - } - - /// Send a signal to the process. - /// - /// # Errors - /// - /// Returns an error if the signal cannot be sent. - pub fn signal(&self, sig: Signal) -> Result<()> { - let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); - signal::kill(Pid::from_raw(pid), sig).into_diagnostic() - } - - /// Kill the process. - /// - /// # Errors - /// - /// Returns an error if the process cannot be killed. - pub fn kill(&mut self) -> Result<()> { - // First try SIGTERM - if let Err(e) = self.signal(Signal::SIGTERM) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(openshell_ocsf::ActivityId::Close) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .message(format!("Failed to send SIGTERM: {e}")) - .build() - ); - } - - // Give the process a moment to terminate gracefully - std::thread::sleep(std::time::Duration::from_millis(100)); - - // Force kill if still running - if let Some(id) = self.child.id() { - debug!(pid = id, "Sending SIGKILL"); - let pid = i32::try_from(id).unwrap_or(i32::MAX); - let _ = signal::kill(Pid::from_raw(pid), Signal::SIGKILL); - } - - Ok(()) - } -} - -impl Drop for ProcessHandle { - fn drop(&mut self) { - #[cfg(target_os = "linux")] - managed_children::unregister(self.pid); - } -} - -/// Validate the configured process user. -/// -/// Numeric identities do not require a passwd entry. The legacy explicit -/// `"sandbox"` identity and other names must resolve in `/etc/passwd`. -#[cfg(unix)] -pub fn validate_sandbox_user(policy: &SandboxPolicy) -> Result<()> { - let identity = policy.process.run_as_user.as_deref().unwrap_or("sandbox"); - - if let Ok(uid) = identity.parse::() { - if !(MIN_SANDBOX_UID..=MAX_SANDBOX_UID).contains(&uid) { - return Err(miette::miette!( - "process user UID must be in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" - )); - } - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "validated") - .message(format!( - "Accepted numeric UID {identity} (no passwd entry required)" - )) - .build() - ); - return Ok(()); - } - - // Legacy explicit "sandbox" name — must exist in /etc/passwd. - if identity == "sandbox" { - match User::from_name("sandbox") { - Ok(Some(_)) => { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "validated") - .message("Validated 'sandbox' user exists in image") - .build() - ); - } - Ok(None) => { - return Err(miette::miette!( - "explicit process user 'sandbox' was not found in the image" - )); - } - Err(e) => { - return Err(miette::miette!("failed to look up 'sandbox' user: {e}")); - } - } - } else if !identity.is_empty() { - // Other names are supported by local/offline policy paths and must - // resolve before privilege dropping. - match User::from_name(identity) { - Ok(Some(_)) => { - tracing::warn!(identity, "named process user accepted via passwd entry"); - } - Ok(None) => { - return Err(miette::miette!( - "unrecognized sandbox identity '{identity}'; \ - expected 'sandbox' or a numeric UID in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" - )); - } - Err(e) => { - return Err(miette::miette!( - "failed to look up identity '{identity}': {e}" - )); - } - } - } - - Ok(()) -} - -/// Validate that the configured sandbox group identity is acceptable. -/// -/// Mirrors [`validate_sandbox_user`] for the group dimension. -#[cfg(unix)] -pub fn validate_sandbox_group(policy: &SandboxPolicy) -> Result<()> { - let identity = policy.process.run_as_group.as_deref().unwrap_or("sandbox"); - - if let Ok(gid) = identity.parse::() { - if !(MIN_SANDBOX_UID..=MAX_SANDBOX_UID).contains(&gid) { - return Err(miette::miette!( - "process group GID must be in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" - )); - } - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "validated") - .message(format!( - "Accepted numeric GID {identity} (no group entry required)" - )) - .build() - ); - return Ok(()); - } - - if identity == "sandbox" { - match Group::from_name("sandbox") { - Ok(Some(_)) => { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "validated") - .message("Validated 'sandbox' group exists in image") - .build() - ); - } - Ok(None) => { - return Err(miette::miette!( - "explicit process group 'sandbox' was not found in the image" - )); - } - Err(e) => { - return Err(miette::miette!("failed to look up 'sandbox' group: {e}")); - } - } - } else if !identity.is_empty() { - match Group::from_name(identity) { - Ok(Some(_)) => { - tracing::warn!(identity, "named process group accepted via group entry"); - } - Ok(None) => { - return Err(miette::miette!( - "unrecognized sandbox group identity '{identity}'; \ - expected 'sandbox' or a numeric GID in range [{MIN_SANDBOX_UID}, {MAX_SANDBOX_UID}]" - )); - } - Err(e) => { - return Err(miette::miette!( - "failed to look up group identity '{identity}': {e}" - )); - } - } - } - - Ok(()) -} - -#[cfg(unix)] -pub fn validate_sandbox_user_with_identity( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, -) -> Result<()> { - let Some(uid) = resolved_identity.uid() else { - return validate_sandbox_user(policy); - }; - if uid == 0 { - return Err(miette::miette!("process user must not select UID 0")); - } - Ok(()) -} - -#[cfg(unix)] -pub fn validate_sandbox_group_with_identity( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, -) -> Result<()> { - let Some(gid) = resolved_identity.gid() else { - return validate_sandbox_group(policy); - }; - if gid == 0 { - return Err(miette::miette!("process group must not select GID 0")); - } - Ok(()) -} - -pub use openshell_policy::{MAX_SANDBOX_UID, MIN_SANDBOX_UID}; - -/// Prepare a `read_write` path for the sandboxed process. -/// -/// Returns `true` when the path was created by the supervisor and therefore -/// still needs to be chowned to the sandbox user/group. Existing paths keep -/// their image-defined ownership. -#[cfg(unix)] -fn prepare_read_write_path(path: &Path) -> Result { - // SECURITY: use symlink_metadata (lstat) to inspect each path *before* - // calling chown. chown follows symlinks, so a malicious container image - // could place a symlink (e.g. /sandbox -> /etc/shadow) to trick the - // root supervisor into transferring ownership of arbitrary files. - // The TOCTOU window between lstat and chown is not exploitable because - // no untrusted process is running yet (the child has not been forked). - if let Ok(meta) = std::fs::symlink_metadata(path) { - if meta.file_type().is_symlink() { - return Err(miette::miette!( - "read_write path '{}' is a symlink — refusing to chown (potential privilege escalation)", - path.display() - )); - } - - debug!( - path = %path.display(), - "Preserving ownership for existing read_write path" - ); - Ok(false) - } else { - debug!(path = %path.display(), "Creating read_write directory"); - std::fs::create_dir_all(path).into_diagnostic()?; - Ok(true) - } -} - -/// Update `/etc/passwd` and `/etc/group` so the "sandbox" user/group entries -/// match the driver-injected UID/GID from environment variables. -/// -/// When `OPENSHELL_SANDBOX_UID` is set, the image-baked "sandbox" entry may -/// have a different UID. Updating the files ensures `whoami`, `id`, `ls -l`, -/// SSH sessions, and `initgroups` resolve the sandbox identity correctly. -/// If no "sandbox" entry exists, one is appended. -#[cfg(unix)] -pub fn update_sandbox_passwd_entries() -> Result<()> { - let uid_str = match std::env::var(openshell_core::sandbox_env::SANDBOX_UID) { - Ok(v) if !v.is_empty() => v, - _ => return Ok(()), - }; - let gid_str = match std::env::var(openshell_core::sandbox_env::SANDBOX_GID) { - Ok(v) if !v.is_empty() => v, - _ => uid_str.clone(), - }; - - let _: u32 = uid_str - .parse() - .map_err(|e| miette::miette!("invalid OPENSHELL_SANDBOX_UID '{uid_str}': {e}"))?; - let _: u32 = gid_str - .parse() - .map_err(|e| miette::miette!("invalid OPENSHELL_SANDBOX_GID '{gid_str}': {e}"))?; - - update_passwd_file(&uid_str, &gid_str)?; - update_group_file(&gid_str)?; - - info!( - uid = %uid_str, - gid = %gid_str, - "Updated /etc/passwd and /etc/group for sandbox identity" - ); - Ok(()) -} - -/// Rewrite the `sandbox` line in `/etc/passwd` with the given UID/GID, -/// or append a new entry if none exists. -#[cfg(unix)] -fn update_passwd_file(uid: &str, gid: &str) -> Result<()> { - rewrite_passwd_at(Path::new("/etc/passwd"), uid, gid) -} - -/// Rewrite the `sandbox` line in `/etc/group` with the given GID, -/// or append a new entry if none exists. -#[cfg(unix)] -fn update_group_file(gid: &str) -> Result<()> { - rewrite_group_at(Path::new("/etc/group"), gid) -} - -#[cfg(unix)] -fn rewrite_passwd_at(path: &Path, uid: &str, gid: &str) -> Result<()> { - let content = std::fs::read_to_string(path).into_diagnostic()?; - - let mut found = false; - let mut lines: Vec = content - .lines() - .map(|line| { - if line.starts_with("sandbox:") { - found = true; - let fields: Vec<&str> = line.split(':').collect(); - if let [name, pass, _, _, gecos, home, shell, ..] = fields.as_slice() { - format!("{name}:{pass}:{uid}:{gid}:{gecos}:{home}:{shell}") - } else { - line.to_string() - } - } else { - line.to_string() - } - }) - .collect(); - - if !found { - lines.push(format!("sandbox:x:{uid}:{gid}::/sandbox:/bin/sh")); - } - - let mut output = lines.join("\n"); - if content.ends_with('\n') || !found { - output.push('\n'); - } - - std::fs::write(path, output).into_diagnostic()?; - Ok(()) -} - -#[cfg(unix)] -fn rewrite_group_at(path: &Path, gid: &str) -> Result<()> { - let content = std::fs::read_to_string(path).into_diagnostic()?; - - let mut found = false; - let mut lines: Vec = content - .lines() - .map(|line| { - if line.starts_with("sandbox:") { - found = true; - let fields: Vec<&str> = line.split(':').collect(); - if let [name, pass, _, members, ..] = fields.as_slice() { - format!("{name}:{pass}:{gid}:{members}") - } else { - line.to_string() - } - } else { - line.to_string() - } - }) - .collect(); - - if !found { - lines.push(format!("sandbox:x:{gid}:")); - } - - let mut output = lines.join("\n"); - if content.ends_with('\n') || !found { - output.push('\n'); - } - - std::fs::write(path, output).into_diagnostic()?; - Ok(()) -} - -/// Recursively chown a directory tree to the given UID/GID. -/// -/// This retains the Kubernetes/OpenShift workspace reconciliation from before -/// OCI image identity fallback. Symlinks are skipped, and read-only nested -/// mounts are not traversed. -#[cfg(unix)] -fn chown_sandbox_home(root: &Path, uid: Option, gid: Option) -> Result<()> { - let meta = std::fs::symlink_metadata(root).into_diagnostic()?; - if meta.file_type().is_symlink() { - return Err(miette::miette!( - "path '{}' is a symlink — refusing to chown (potential privilege escalation)", - root.display() - )); - } - - nix::unistd::chown(root, uid, gid).into_diagnostic()?; - - if meta.is_dir() { - chown_children(root, uid, gid, &nix::unistd::chown)?; - } - - Ok(()) -} - -#[cfg(unix)] -fn prepare_oci_workspace( - root: &Path, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], -) -> Result<()> { - prepare_oci_workspace_with(root, uid, gid, supplementary_gids, &nix::unistd::chown) -} - -/// Validate that selecting an image-provided OCI workdir does not grant the -/// sandbox identity any filesystem authority it lacked in the immutable image. -/// -/// Every path component must be a real directory (never a symlink), every -/// parent must already be traversable, and the final directory must already be -/// writable and traversable. No ownership or mode bits are changed. -#[cfg(unix)] -pub fn validate_oci_workspace( - root: &Path, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], -) -> Result<()> { - let components = validated_workspace_components(root, false)?; - let mut current = PathBuf::from("/"); - validate_workspace_component(¤t, uid, gid, supplementary_gids, false)?; - let last_component = components.len().saturating_sub(1); - for (index, component) in components.into_iter().enumerate() { - current.push(component); - validate_workspace_component( - ¤t, - uid, - gid, - supplementary_gids, - index == last_component, - )?; - } - Ok(()) -} - -/// Validate an image-provided workdir in a clean copy of the supervisor so the -/// main process retains the root authority needed for subsequent setup. -#[cfg(target_os = "linux")] -fn validate_oci_workspace_in_subprocess( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - workdir: &Path, -) -> Result<()> { - use std::os::unix::process::CommandExt; - - let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; - let uid = uid.ok_or_else(|| miette::miette!("workspace validator UID is unresolved"))?; - let gid = gid.ok_or_else(|| miette::miette!("workspace validator GID is unresolved"))?; - let groups = supplementary_gids - .iter() - .map(|group| group.as_raw()) - .collect::>(); - let executable = std::env::current_exe().into_diagnostic()?; - let mut command = std::process::Command::new(executable); - command - .arg("validate-workspace") - .arg("--workdir") - .arg(workdir) - .arg("--expected-uid") - .arg(uid.to_string()) - .arg("--expected-gid") - .arg(gid.to_string()) - .env_clear() - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()); - - // `pre_exec` runs after fork and before exec. These direct credential - // syscalls are async-signal-safe and affect only the one-shot child. - #[allow(unsafe_code)] - unsafe { - command.pre_exec(move || { - if libc::setgroups(groups.len(), groups.as_ptr()) != 0 - || libc::setgid(gid.as_raw()) != 0 - || libc::setuid(uid.as_raw()) != 0 - { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - }); - } - - let output = command.output().into_diagnostic()?; - if output.status.success() { - return Ok(()); - } - - let diagnostic = String::from_utf8_lossy(&output.stderr); - let diagnostic = diagnostic.trim(); - if diagnostic.is_empty() { - return Err(miette::miette!( - "image workspace validation failed with status {}", - output.status - )); - } - Err(miette::miette!( - "image workspace validation failed: {diagnostic}" - )) -} - -#[cfg(unix)] -fn validate_workspace_component( - path: &Path, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], - is_workspace: bool, -) -> Result<()> { - let metadata = std::fs::symlink_metadata(path).map_err(|error| { - if error.kind() == std::io::ErrorKind::NotFound { - miette::miette!( - "image workspace path component '{}' does not exist", - path.display() - ) - } else { - miette::miette!( - "failed to inspect image workspace path component '{}': {error}", - path.display() - ) - } - })?; - if metadata.file_type().is_symlink() { - return Err(miette::miette!( - "workspace path component '{}' is a symlink — refusing to follow it", - path.display() - )); - } - if !metadata.is_dir() { - return Err(miette::miette!( - "workspace path component '{}' is not a directory", - path.display() - )); - } - let required = if is_workspace { 0o3 } else { 0o1 }; - if !identity_has_permissions(&metadata, uid, gid, supplementary_gids, required) { - let requirement = if is_workspace { - "writable and traversable" - } else { - "traversable" - }; - return Err(miette::miette!( - "workspace path component '{}' is not {requirement} by the sandbox identity in the image", - path.display() - )); - } - Ok(()) -} - -#[cfg(target_os = "linux")] -pub fn validate_oci_workspace_as_effective_identity(root: &Path) -> Result<()> { - use rustix::fs::{Access, AtFlags, FileType, Mode, OFlags}; - - let components = validated_workspace_components(root, false)?; - let open_flags = OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; - let mut current_path = PathBuf::from("/"); - let mut current_fd = rustix::fs::open("/", open_flags, Mode::empty()).into_diagnostic()?; - rustix::fs::accessat( - ¤t_fd, - ".", - Access::EXEC_OK, - AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, - ) - .map_err(|error| { - miette::miette!( - "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", - current_path.display() - ) - })?; - - let last_component = components.len().saturating_sub(1); - for (index, component) in components.into_iter().enumerate() { - current_path.push(&component); - let stat = rustix::fs::statat(¤t_fd, &component, AtFlags::SYMLINK_NOFOLLOW).map_err( - |error| { - if error == rustix::io::Errno::NOENT { - miette::miette!( - "image workspace path component '{}' does not exist", - current_path.display() - ) - } else { - miette::miette!( - "failed to inspect image workspace path component '{}': {error}", - current_path.display() - ) - } - }, - )?; - let file_type = FileType::from_raw_mode(stat.st_mode); - if file_type.is_symlink() { - return Err(miette::miette!( - "workspace path component '{}' is a symlink — refusing to follow it", - current_path.display() - )); - } - if !file_type.is_dir() { - return Err(miette::miette!( - "workspace path component '{}' is not a directory", - current_path.display() - )); - } - - let is_workspace = index == last_component; - rustix::fs::accessat( - ¤t_fd, - &component, - Access::EXEC_OK, - AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, - ) - .map_err(|error| { - miette::miette!( - "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", - current_path.display() - ) - })?; - - let next_fd = rustix::fs::openat(¤t_fd, &component, open_flags, Mode::empty()) - .map_err(|error| { - miette::miette!( - "failed to open image workspace path component '{}': {error}", - current_path.display() - ) - })?; - if is_workspace { - validate_effective_workspace_write(&next_fd, ¤t_path)?; - } - current_fd = next_fd; - } - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn validate_effective_workspace_write(fd: &impl std::os::fd::AsFd, path: &Path) -> Result<()> { - use rustix::fs::{AtFlags, Mode, OFlags}; - - let mode = Mode::RUSR | Mode::WUSR; - let tmpfile_flags = OFlags::TMPFILE | OFlags::WRONLY | OFlags::CLOEXEC; - match rustix::fs::openat(fd, ".", tmpfile_flags, mode) { - Ok(_probe) => return Ok(()), - Err(rustix::io::Errno::INVAL | rustix::io::Errno::ISDIR | rustix::io::Errno::NOTSUP) => {} - Err(error) => { - return Err(miette::miette!( - "workspace path component '{}' is not writable by the sandbox identity in the image: {error}", - path.display() - )); - } - } - - // Some filesystems do not implement O_TMPFILE. Fall back to a short-lived, - // no-follow entry. A collision fails closed after bounded retries. - let create_flags = - OFlags::CREATE | OFlags::EXCL | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC; - for attempt in 0..16 { - let name = format!(".openshell-workdir-probe-{}-{attempt}", std::process::id()); - match rustix::fs::openat(fd, &name, create_flags, mode) { - Ok(_probe) => { - rustix::fs::unlinkat(fd, &name, AtFlags::empty()).map_err(|error| { - miette::miette!( - "workspace write probe cleanup failed for '{}': {error}", - path.display() - ) - })?; - return Ok(()); - } - Err(rustix::io::Errno::EXIST) => {} - Err(error) => { - return Err(miette::miette!( - "workspace path component '{}' is not writable by the sandbox identity in the image: {error}", - path.display() - )); - } - } - } - - Err(miette::miette!( - "workspace write probe could not allocate a unique entry in '{}'", - path.display() - )) -} - -/// Prepare only the resolved `OpenShell` workspace directory itself. -/// -/// Image-provided children retain their declared ownership. This avoids -/// crossing symlinks or user-provided nested mounts. -#[cfg(unix)] -fn prepare_oci_workspace_with( - root: &Path, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], - do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, -) -> Result<()> { - let components = validated_workspace_components(root, true)?; - - let last_component = components.len().saturating_sub(1); - let mut current = PathBuf::from("/"); - for (index, component) in components.into_iter().enumerate() { - current.push(component); - match std::fs::symlink_metadata(¤t) { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(miette::miette!( - "workspace path component '{}' is a symlink — refusing to follow it", - current.display() - )); - } - Ok(metadata) if !metadata.is_dir() => { - return Err(miette::miette!( - "workspace path component '{}' is not a directory", - current.display() - )); - } - Ok(metadata) => { - if index != last_component - && !identity_can_traverse(&metadata, uid, gid, supplementary_gids) - { - return Err(miette::miette!( - "workspace parent '{}' is not traversable by the sandbox identity", - current.display() - )); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - std::fs::create_dir(¤t).into_diagnostic()?; - std::fs::set_permissions(¤t, std::fs::Permissions::from_mode(0o755)) - .into_diagnostic()?; - } - Err(error) => return Err(error).into_diagnostic(), - } - } - - do_chown(root, uid, gid).into_diagnostic()?; - - let metadata = std::fs::symlink_metadata(root).into_diagnostic()?; - let mode = metadata.permissions().mode() & 0o7777; - if mode & 0o300 != 0o300 { - std::fs::set_permissions(root, std::fs::Permissions::from_mode(mode | 0o300)) - .into_diagnostic()?; - } - Ok(()) -} - -#[cfg(unix)] -fn validated_workspace_components( - root: &Path, - allow_managed_fallback: bool, -) -> Result> { - let root_str = root - .to_str() - .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; - let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) - .map_err(|error| miette::miette!(error))?; - if Path::new(&validated_root) != root - || (!allow_managed_fallback - && validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) - { - return Err(miette::miette!( - "workspace path '{}' must be a normalized absolute {}path", - root.display(), - if allow_managed_fallback { - "non-root " - } else { - "non-fallback " - } - )); - } - - root.components() - .skip(1) - .map(|component| match component { - std::path::Component::Normal(component) => Ok(component.to_os_string()), - _ => Err(miette::miette!( - "workspace path '{}' must be normalized", - root.display() - )), - }) - .collect() -} - -#[cfg(unix)] -fn identity_can_traverse( - metadata: &std::fs::Metadata, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], -) -> bool { - identity_has_permissions(metadata, uid, gid, supplementary_gids, 0o1) -} - -#[cfg(unix)] -fn identity_has_permissions( - metadata: &std::fs::Metadata, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], - required: u32, -) -> bool { - let user_id = uid.unwrap_or_else(nix::unistd::geteuid).as_raw(); - if user_id == 0 { - return true; - } - - let group_id = gid.unwrap_or_else(nix::unistd::getegid).as_raw(); - let mode = metadata.permissions().mode(); - if metadata.uid() == user_id { - mode & (required << 6) == required << 6 - } else if metadata.gid() == group_id - || supplementary_gids - .iter() - .any(|supplementary_gid| supplementary_gid.as_raw() == metadata.gid()) - { - mode & (required << 3) == required << 3 - } else { - mode & required == required - } -} - -#[cfg(not(any( - target_os = "aix", - target_os = "haiku", - target_os = "illumos", - target_os = "ios", - target_os = "macos", - target_os = "redox", - target_os = "solaris" -)))] -fn named_user_supplementary_groups(user_name: &str, primary_gid: Gid) -> Result> { - let user_name = CString::new(user_name).map_err(|_| miette::miette!("Invalid user name"))?; - nix::unistd::getgrouplist(user_name.as_c_str(), primary_gid).into_diagnostic() -} - -#[cfg(any( - target_os = "aix", - target_os = "haiku", - target_os = "illumos", - target_os = "ios", - target_os = "macos", - target_os = "redox", - target_os = "solaris" -))] -#[allow(clippy::unnecessary_wraps)] -fn named_user_supplementary_groups(_user_name: &str, _primary_gid: Gid) -> Result> { - // Privilege dropping does not call initgroups on these targets. - Ok(Vec::new()) -} - -#[cfg(unix)] -fn chown_children( - dir: &Path, - uid: Option, - gid: Option, - do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, -) -> Result<()> { - match std::fs::read_dir(dir) { - Ok(entries) => { - for entry in entries { - let entry = entry.into_diagnostic()?; - chown_recursive(&entry.path(), uid, gid, do_chown)?; - } - } - Err(error) => { - debug!( - path = %dir.display(), - %error, - "Cannot list directory during sandbox home chown" - ); - } - } - Ok(()) -} - -#[cfg(unix)] -fn chown_recursive( - path: &Path, - uid: Option, - gid: Option, - do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, -) -> Result<()> { - let meta = std::fs::symlink_metadata(path).into_diagnostic()?; - if meta.file_type().is_symlink() { - debug!(path = %path.display(), "Skipping symlink during sandbox home chown"); - return Ok(()); - } - - if let Err(error) = do_chown(path, uid, gid) { - if error == nix::errno::Errno::EROFS { - debug!(path = %path.display(), "Skipping read-only path during sandbox home chown"); - return Ok(()); - } - return Err(error).into_diagnostic(); - } - - if meta.is_dir() { - chown_children(path, uid, gid, do_chown)?; - } - - Ok(()) -} - -/// Prepare filesystem for the sandboxed process. -/// -/// Creates `read_write` directories if they don't exist and sets ownership -/// on newly-created paths to the configured sandbox user/group. This runs as -/// the supervisor (root) before forking the child process. -/// -/// Accepts both name-based identities (resolved via `/etc/passwd`) and numeric -/// UIDs/GIDs (passed directly to `chown` without a passwd lookup). -#[cfg(unix)] -pub fn prepare_filesystem(policy: &SandboxPolicy) -> Result<()> { - prepare_filesystem_with_identity(policy, ResolvedProcessIdentity::default(), None, false) -} - -#[cfg(unix)] -pub fn prepare_filesystem_with_identity( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - workdir: Option<&str>, - prepare_workspace: bool, -) -> Result<()> { - use nix::unistd::chown; - - // If no user/group configured, nothing to do - if policy - .process - .run_as_user - .as_deref() - .is_none_or(str::is_empty) - && policy - .process - .run_as_group - .as_deref() - .is_none_or(str::is_empty) - { - return Ok(()); - } - - let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; - - // Docker owns workspace resolution and must make the selected root usable - // by the final effective identity, including when both policy identity - // fields were explicit. Validate it before processing any user-authored - // read-write paths so an unsafe image path fails first. Other drivers - // retain their preparation. - if prepare_workspace { - let workspace = workdir.ok_or_else(|| { - miette::miette!("local container driver did not supply a workspace workdir") - })?; - let workspace = Path::new(workspace); - if workspace == Path::new(openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) { - info!(path = %workspace.display(), ?uid, ?gid, "Preparing managed workspace"); - prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; - } else { - info!(path = %workspace.display(), ?uid, ?gid, "Validating image workspace authority"); - #[cfg(target_os = "linux")] - validate_oci_workspace_in_subprocess(policy, resolved_identity, workspace)?; - #[cfg(not(target_os = "linux"))] - validate_oci_workspace(workspace, uid, gid, &supplementary_gids)?; - } - } - - // Create missing read_write paths and only chown the ones we created. - for path in &policy.filesystem.read_write { - if prepare_read_write_path(path)? { - debug!( - path = %path.display(), - ?uid, - ?gid, - "Setting ownership on newly created read_write path" - ); - chown(path, uid, gid).into_diagnostic()?; - } - } - - // Retain the existing Kubernetes/OpenShift behavior for driver-injected - // numeric identities. Docker clears this variable and does not receive - // identity-specific workspace preparation. - if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { - let sandbox_home = Path::new("/sandbox"); - if sandbox_home.exists() { - info!(?uid, ?gid, "Chowning /sandbox for driver-injected UID/GID"); - chown_sandbox_home(sandbox_home, uid, gid)?; - } - } - - Ok(()) -} - -#[cfg(unix)] -fn resolve_filesystem_identity( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, -) -> Result<(Option, Option, Vec)> { - let user_name = policy - .process - .run_as_user - .as_deref() - .filter(|name| !name.is_empty()); - let group_name = policy - .process - .run_as_group - .as_deref() - .filter(|name| !name.is_empty()); - - let uid = match resolved_identity.uid() { - Some(uid) => Some(Uid::from_raw(uid)), - None => match user_name { - Some(name) if name.parse::().is_ok() => { - Some(Uid::from_raw(name.parse().into_diagnostic()?)) - } - Some(name) => User::from_name(name).into_diagnostic()?.map(|u| u.uid), - _ => None, - }, - }; - - // Resolve GID: numeric values are passed directly; names resolve via group. - let gid = match resolved_identity.gid() { - Some(gid) => Some(Gid::from_raw(gid)), - None => match group_name { - Some(name) if name.parse::().is_ok() => { - Some(Gid::from_raw(name.parse().into_diagnostic()?)) - } - Some(name) => Group::from_name(name).into_diagnostic()?.map(|g| g.gid), - _ => None, - }, - }; - - let supplementary_gids = match user_name { - Some(name) if name.parse::().is_err() => { - let primary_gid = if let Some(gid) = gid { - gid - } else { - let uid = - uid.ok_or_else(|| miette::miette!("Failed to resolve sandbox user '{name}'"))?; - User::from_uid(uid) - .into_diagnostic()? - .ok_or_else(|| miette::miette!("Failed to resolve user from UID {uid}"))? - .gid - }; - if resolved_identity.uid().is_some() { - crate::identity::resolve_oci_supplementary_gids(name, primary_gid.as_raw())? - .into_iter() - .map(Gid::from_raw) - .collect() - } else { - named_user_supplementary_groups(name, primary_gid)? - } - } - _ => Vec::new(), - }; - - Ok((uid, gid, supplementary_gids)) -} - -#[cfg(not(unix))] -pub fn prepare_filesystem(_policy: &SandboxPolicy) -> Result<()> { - Ok(()) -} - -// `effective_gid`/`effective_uid` are intentionally parallel names (same role -// for different identifiers) and the noise from renaming would obscure intent. -#[cfg(unix)] -#[allow(clippy::similar_names)] -pub fn drop_privileges(policy: &SandboxPolicy) -> Result<()> { - drop_privileges_with_identity(policy, ResolvedProcessIdentity::default()) -} - -#[cfg(unix)] -#[allow(clippy::similar_names)] -pub fn drop_privileges_with_identity( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, -) -> Result<()> { - let user_name = match policy.process.run_as_user.as_deref() { - Some(name) if !name.is_empty() => Some(name), - _ => None, - }; - let group_name = match policy.process.run_as_group.as_deref() { - Some(name) if !name.is_empty() => Some(name), - _ => None, - }; - - // If no user/group is configured and we are running as root, fall back to - // "sandbox:sandbox" instead of silently keeping root. This covers the - // local/dev-mode path for drivers that provide no identity metadata. - // For non-root runtimes, the no-op is safe -- we are already unprivileged. - if user_name.is_none() && group_name.is_none() { - if nix::unistd::geteuid().is_root() { - let mut fallback = policy.clone(); - fallback.process.run_as_user = Some("sandbox".into()); - fallback.process.run_as_group = Some("sandbox".into()); - return drop_privileges_with_identity(&fallback, resolved_identity); - } - return Ok(()); - } - - // Resolve UID: numeric values are used directly; names resolve via passwd. - let target_uid = match resolved_identity.uid() { - Some(uid) => Uid::from_raw(uid), - None => match user_name { - Some(name) if name.parse::().is_ok() => { - Uid::from_raw(name.parse().into_diagnostic()?) - } - Some(name) => { - User::from_name(name) - .into_diagnostic()? - .ok_or_else(|| miette::miette!("Sandbox user not found: {name}"))? - .uid - } - None => nix::unistd::geteuid(), - }, - }; - - // Resolve group: if a numeric GID is configured use it directly. - // Otherwise try name resolution, then fall back to current user's primary group. - let target_gid = match resolved_identity.gid() { - Some(gid) => Gid::from_raw(gid), - None => match group_name { - Some(name) if name.parse::().is_ok() => { - Gid::from_raw(name.parse().into_diagnostic()?) - } - Some(name) => { - Group::from_name(name) - .into_diagnostic()? - .ok_or_else(|| miette::miette!("Sandbox group not found: {name}"))? - .gid - } - None => match target_uid.as_raw() { - 0 => nix::unistd::getegid(), - _ => Group::from_gid( - User::from_uid(target_uid) - .into_diagnostic()? - .ok_or_else(|| { - miette::miette!("Failed to resolve user from UID {target_uid}") - })? - .gid, - ) - .into_diagnostic()? - .map_or_else(nix::unistd::getegid, |g| g.gid), - }, - }, - }; - - // Resolve the name for initgroups only for the existing explicit-policy - // path. OCI-derived users carry a numeric UID from the bounded parser and - // must not be looked up again through NSS. - let user_name_is_numeric = user_name.is_some_and(|n| n.parse::().is_ok()); - let initgroups_name = - if user_name.is_some() && !user_name_is_numeric && resolved_identity.uid().is_none() { - Some( - User::from_uid(target_uid) - .into_diagnostic()? - .ok_or_else(|| { - miette::miette!("Failed to resolve user record for UID {target_uid}") - })? - .name, - ) - } else { - None - }; - - if target_uid != nix::unistd::geteuid() { - if resolved_identity.uses_oci_user_fallback() { - // OCI named users use the bounded /etc/group parser shared with - // workspace validation. Numeric OCI users resolve to an empty - // list. Never retain the root supervisor's inherited groups. - #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "haiku", - target_os = "redox" - )))] - { - let (_, _, supplementary_gids) = - resolve_filesystem_identity(policy, resolved_identity)?; - nix::unistd::setgroups(&supplementary_gids).into_diagnostic()?; - } - } else if let Some(ref user_name) = initgroups_name { - let user_cstr = CString::new(user_name.as_str()) - .map_err(|_| miette::miette!("Invalid user name"))?; - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "haiku", - target_os = "redox" - ))] - { - let _ = user_cstr; - } - #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "haiku", - target_os = "redox" - )))] - { - nix::unistd::initgroups(user_cstr.as_c_str(), target_gid).into_diagnostic()?; - } - } - } - - if target_gid != nix::unistd::getegid() { - nix::unistd::setgid(target_gid).into_diagnostic()?; - } - - // Verify effective GID actually changed (defense-in-depth, CWE-250 / CERT POS37-C) - let effective_gid = nix::unistd::getegid(); - if effective_gid != target_gid { - return Err(miette::miette!( - "Privilege drop verification failed: expected effective GID {}, got {}", - target_gid, - effective_gid - )); - } - - #[cfg(target_os = "linux")] - if nix::unistd::geteuid().is_root() { - drop_capability_bounding_set()?; - } - - if user_name.is_some() { - if target_uid != nix::unistd::geteuid() { - nix::unistd::setuid(target_uid).into_diagnostic()?; - } - - // Verify effective UID actually changed (defense-in-depth, CWE-250 / CERT POS37-C) - let effective_uid = nix::unistd::geteuid(); - if effective_uid != target_uid { - return Err(miette::miette!( - "Privilege drop verification failed: expected effective UID {}, got {}", - target_uid, - effective_uid - )); - } - - // Verify root cannot be re-acquired (CERT POS37-C hardening). - // If we dropped from root, setuid(0) must fail; success means privileges - // were not fully relinquished. - if nix::unistd::setuid(Uid::from_raw(0)).is_ok() && target_uid.as_raw() != 0 { - return Err(miette::miette!( - "Privilege drop verification failed: process can still re-acquire root (UID 0) \ - after switching to UID {}", - target_uid - )); - } - } - - Ok(()) -} - -/// Process exit status. -#[derive(Debug, Clone, Copy)] -pub struct ProcessStatus { - code: Option, - signal: Option, -} - -impl ProcessStatus { - /// Get the conventional exit code when the process exited normally. - #[must_use] - pub const fn exit_code(&self) -> Option { - self.code - } - - /// Get the exit code, or 128 + signal number if killed by signal. - #[must_use] - pub fn code(&self) -> i32 { - self.code - .or_else(|| self.signal.map(|s| 128 + s)) - .unwrap_or(-1) - } - - /// Check if the process exited successfully. - #[must_use] - pub fn success(&self) -> bool { - self.code == Some(0) - } - - /// Get the signal that killed the process, if any. - #[must_use] - pub const fn signal(&self) -> Option { - self.signal - } -} - -impl From for ProcessStatus { - fn from(status: std::process::ExitStatus) -> Self { - #[cfg(unix)] - { - use std::os::unix::process::ExitStatusExt; - Self { - code: status.code(), - signal: status.signal(), - } - } - - #[cfg(not(unix))] - { - Self { - code: status.code(), - signal: None, - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[cfg(unix)] - use nix::sys::wait::{WaitStatus, waitpid}; - #[cfg(unix)] - use nix::unistd::{ForkResult, fork}; - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, - }; - #[cfg(unix)] - use std::mem::size_of; - use std::process::Stdio as StdStdio; - - /// Helper to create a minimal `SandboxPolicy` with the given process policy. - fn policy_with_process(process: ProcessPolicy) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process, - } - } - - #[cfg(unix)] - #[tokio::test] - async fn canonical_tty_environment_replaces_supervisor_identity_defaults() { - let current_user = User::from_uid(nix::unistd::geteuid()) - .expect("look up current user") - .expect("current user entry"); - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some(current_user.name.clone()), - run_as_group: None, - }); - let workspace = ResolvedWorkspace::default(); - let mut cmd = Command::new("/usr/bin/env"); - cmd.env_clear() - .env("HOME", "/root") - .env("TERM", "dumb") - .stdout(StdStdio::piped()); - - apply_canonical_process_environment(&mut cmd, &policy, &workspace, true, &HashMap::new()); - - let output = cmd.output().await.expect("run environment probe"); - assert!(output.status.success()); - let environment = String::from_utf8(output.stdout).expect("environment is UTF-8"); - let variables: HashMap<_, _> = environment - .lines() - .filter_map(|line| line.split_once('=')) - .collect(); - - assert_eq!( - variables.get("HOME"), - Some(¤t_user.dir.to_string_lossy().as_ref()) - ); - assert_eq!(variables.get("USER"), Some(¤t_user.name.as_str())); - // SHELL is the shell detected in the current root filesystem, not a - // hardcoded path (bash-less images resolve to /bin/sh). - let expected_shell = openshell_core::shell::detect_login_shell(); - assert_eq!(variables.get("SHELL"), Some(&expected_shell.as_str())); - assert_eq!(variables.get("TERM"), Some(&"xterm-256color")); - } - - /// Unknown names may yield `Ok(None)` (`… not found …`) or `Err` when NSS fails first - /// (e.g. `ENOENT: No such file or directory`). - fn assert_unknown_identity_lookup_failed(msg: &str) { - assert!( - msg.contains("not found") - || msg.contains("ENOENT") - || msg.contains("No such file or directory"), - "expected unknown user/group lookup failure (…not found… or ENOENT): {msg}" - ); - } - - #[test] - #[cfg(unix)] - fn explicit_identity_accepts_non_root_system_ids() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some("101".into()), - run_as_group: Some("102".into()), - }); - - assert!(validate_sandbox_user(&policy).is_ok()); - assert!(validate_sandbox_group(&policy).is_ok()); - } - - #[test] - #[cfg(unix)] - fn resolved_oci_identity_accepts_non_root_system_ids() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some("app".into()), - run_as_group: Some("staff".into()), - }); - let resolved = ResolvedProcessIdentity::new(Some(101), Some(102)); - - assert!(validate_sandbox_user_with_identity(&policy, resolved).is_ok()); - assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); - } - - #[test] - #[cfg(unix)] - fn completed_runtime_identity_rejects_numeric_root() { - let root_user = policy_with_process(ProcessPolicy { - run_as_user: Some("0".into()), - run_as_group: Some("102".into()), - }); - let root_group = policy_with_process(ProcessPolicy { - run_as_user: Some("101".into()), - run_as_group: Some("0".into()), - }); - - assert!(validate_sandbox_user(&root_user).is_err()); - assert!(validate_sandbox_group(&root_group).is_err()); - } - - #[test] - #[cfg(unix)] - fn resolved_oci_components_do_not_repeat_nss_validation() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some("__oci_name_not_in_host_nss__".into()), - run_as_group: Some("__oci_group_not_in_host_nss__".into()), - }); - let resolved = ResolvedProcessIdentity::new(Some(1234), Some(1235)); - - assert!(validate_sandbox_user_with_identity(&policy, resolved).is_ok()); - assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); - } - - #[test] - #[cfg(unix)] - fn explicit_policy_components_keep_existing_validation_path() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some("__explicit_name_not_in_host_nss__".into()), - run_as_group: Some("__oci_group_not_in_host_nss__".into()), - }); - let resolved = ResolvedProcessIdentity::new(None, Some(1235)); - - assert!(validate_sandbox_user_with_identity(&policy, resolved).is_err()); - assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); - } - - #[test] - fn full_enforcement_uses_privileged_setup_and_child_sandbox() { - assert!(ProcessEnforcementMode::Full.uses_privileged_process_setup()); - assert!(ProcessEnforcementMode::Full.enforces_child_sandbox()); - } - - #[test] - fn network_only_enforcement_keeps_child_sandbox_without_privileged_setup() { - assert!(!ProcessEnforcementMode::NetworkOnly.uses_privileged_process_setup()); - assert!(ProcessEnforcementMode::NetworkOnly.enforces_child_sandbox()); - } - - #[cfg(target_os = "linux")] - fn capability_bounding_set_clear_available() -> bool { - capctl::caps::CapState::get_current() - .is_ok_and(|state| state.effective.has(capctl::caps::Cap::SETPCAP)) - || capctl::caps::bounding::probe().is_empty() - } - - #[test] - #[cfg(target_os = "linux")] - fn capability_bounding_set_clear_accepts_empty_eperm() { - let remaining = capctl::caps::CapSet::empty(); - - assert!( - validate_capability_bounding_set_clear( - Err(capctl::Error::from_code(libc::EPERM)), - remaining, - || Ok(()), - ) - .is_ok() - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn capability_bounding_set_clear_rejects_nonempty_eperm() { - let mut remaining = capctl::caps::CapSet::empty(); - remaining.add(capctl::caps::Cap::CHOWN); - - let result = validate_capability_bounding_set_clear( - Err(capctl::Error::from_code(libc::EPERM)), - remaining, - || panic!("unknown capabilities should not be checked when known caps remain"), - ); - - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("Failed to clear child capability bounding set") - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn capability_bounding_set_clear_rejects_nonempty_success() { - let mut remaining = capctl::caps::CapSet::empty(); - remaining.add(capctl::caps::Cap::CHOWN); - - let result = validate_capability_bounding_set_clear(Ok(()), remaining, || { - panic!("unknown capabilities should not be checked when known caps remain") - }); - - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("capabilities remain raised") - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn capability_bounding_set_clear_rejects_unknown_eperm() { - let remaining = capctl::caps::CapSet::empty(); - - let result = validate_capability_bounding_set_clear( - Err(capctl::Error::from_code(libc::EPERM)), - remaining, - || Err(capctl::Error::from_code(libc::EPERM)), - ); - - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("Failed to clear unknown child capability bounding set entries") - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn capability_probe_child() { - if std::env::var_os("OPENSHELL_TEST_PROBE_CHILD_CAPS").is_none() { - return; - } - - assert!( - capctl::caps::bounding::probe().is_empty(), - "child CapBnd should be empty after exec" - ); - } - - #[test] - fn drop_privileges_noop_when_no_user_or_group() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: None, - run_as_group: None, - }); - if nix::unistd::geteuid().is_root() { - // As root, drop_privileges falls back to "sandbox:sandbox". - // If that user exists, it succeeds; if not (e.g. CI), it - // must error rather than silently keep root. - let has_sandbox = User::from_name("sandbox").ok().flatten().is_some(); - assert_eq!(drop_privileges(&policy).is_ok(), has_sandbox); - } else { - assert!(drop_privileges(&policy).is_ok()); - } - } - - #[test] - fn drop_privileges_noop_when_empty_strings() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some(String::new()), - run_as_group: Some(String::new()), - }); - if nix::unistd::geteuid().is_root() { - let has_sandbox = User::from_name("sandbox").ok().flatten().is_some(); - assert_eq!(drop_privileges(&policy).is_ok(), has_sandbox); - } else { - assert!(drop_privileges(&policy).is_ok()); - } - } - - #[test] - fn drop_privileges_succeeds_for_current_group() { - // Set only run_as_group (no run_as_user) so that initgroups() is not - // called. initgroups(3) requires CAP_SETGID/root even when the target - // is the current user, so it cannot be exercised without elevated - // privileges. This test covers the setgid() + GID post-condition - // verification path without needing root. - let current_group = Group::from_gid(nix::unistd::getegid()) - .expect("getgrgid") - .expect("current group entry"); - - let policy = policy_with_process(ProcessPolicy { - run_as_user: None, - run_as_group: Some(current_group.name), - }); - - let result = drop_privileges(&policy); - #[cfg(target_os = "linux")] - { - if nix::unistd::geteuid().is_root() && !capability_bounding_set_clear_available() { - let msg = format!("{}", result.unwrap_err()); - assert!( - msg.contains("Failed to clear child capability bounding set"), - "unexpected failure: {msg}" - ); - return; - } - } - assert!(result.is_ok(), "drop_privileges failed: {result:?}"); - } - - #[test] - #[cfg(target_os = "linux")] - #[allow(unsafe_code)] - fn drop_privileges_clears_bounding_set_for_spawned_child_when_permitted() { - use std::os::unix::process::CommandExt; - - if !capability_bounding_set_clear_available() { - eprintln!( - "skipping: CAP_SETPCAP is not effective and the capability bounding set is nonempty" - ); - return; - } - - let current_group = Group::from_gid(nix::unistd::getegid()) - .expect("getgrgid") - .expect("current group entry"); - - let policy = policy_with_process(ProcessPolicy { - run_as_user: None, - run_as_group: Some(current_group.name), - }); - - let mut cmd = std::process::Command::new(std::env::current_exe().expect("current exe")); - cmd.arg("capability_probe_child") - .arg("--nocapture") - .env("OPENSHELL_TEST_PROBE_CHILD_CAPS", "1") - .stdin(StdStdio::null()) - .stdout(StdStdio::piped()) - .stderr(StdStdio::piped()); - - unsafe { - cmd.pre_exec(move || { - drop_privileges(&policy).map_err(|err| std::io::Error::other(err.to_string())) - }); - } - - let output = cmd.output().expect("spawn child status probe"); - assert!( - output.status.success(), - "status probe failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - #[test] - #[ignore = "initgroups(3) requires CAP_SETGID; run as root: sudo cargo test -- --ignored"] - fn drop_privileges_succeeds_for_current_user() { - // Exercises the full privilege-drop path including initgroups(), - // setgid(), setuid(), and the root-reacquisition check. Requires - // CAP_SETGID (root) because initgroups(3) calls setgroups(2) - // internally. Fixes: https://github.com/NVIDIA/OpenShell/issues/622 - let current_user = User::from_uid(nix::unistd::geteuid()) - .expect("getpwuid") - .expect("current user entry"); - let current_group = Group::from_gid(nix::unistd::getegid()) - .expect("getgrgid") - .expect("current group entry"); - - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some(current_user.name), - run_as_group: Some(current_group.name), - }); - - assert!(drop_privileges(&policy).is_ok()); - } - - #[test] - fn drop_privileges_fails_for_nonexistent_user() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some("__nonexistent_test_user_42__".to_string()), - run_as_group: None, - }); - - let result = drop_privileges(&policy); - assert!(result.is_err()); - let msg = format!("{}", result.unwrap_err()); - assert_unknown_identity_lookup_failed(&msg); - } - - #[test] - fn drop_privileges_fails_for_nonexistent_group() { - let policy = policy_with_process(ProcessPolicy { - run_as_user: None, - run_as_group: Some("__nonexistent_test_group_42__".to_string()), - }); - - let result = drop_privileges(&policy); - assert!(result.is_err()); - let msg = format!("{}", result.unwrap_err()); - assert_unknown_identity_lookup_failed(&msg); - } - - #[cfg(unix)] - #[allow(unsafe_code)] - fn probe_hardened_child(probe: unsafe fn() -> i64) -> i64 { - const HARDEN_FAILED: i64 = -2; - - let mut fds = [0; 2]; - let pipe_rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; - assert_eq!( - pipe_rc, - 0, - "pipe failed: {}", - std::io::Error::last_os_error() - ); - - match unsafe { fork() }.expect("fork should succeed") { - ForkResult::Child => { - unsafe { libc::close(fds[0]) }; - let value = match harden_child_process() { - Ok(()) => unsafe { probe() }, - Err(_) => HARDEN_FAILED, - }; - let bytes = value.to_ne_bytes(); - let written = unsafe { libc::write(fds[1], bytes.as_ptr().cast(), bytes.len()) }; - unsafe { - libc::close(fds[1]); - libc::_exit(i32::from(written != bytes.len().cast_signed())); - } - } - ForkResult::Parent { child } => { - unsafe { libc::close(fds[1]) }; - let mut bytes = [0u8; size_of::()]; - let read = unsafe { libc::read(fds[0], bytes.as_mut_ptr().cast(), bytes.len()) }; - unsafe { libc::close(fds[0]) }; - assert_eq!( - read.cast_unsigned(), - bytes.len(), - "expected {} probe bytes, got {}", - bytes.len(), - read - ); - - match waitpid(child, None).expect("waitpid should succeed") { - WaitStatus::Exited(_, 0) => {} - status => panic!("probe child exited unexpectedly: {status:?}"), - } - - i64::from_ne_bytes(bytes) - } - } - } - - #[cfg(unix)] - #[allow(unsafe_code)] - unsafe fn core_dump_limit_is_zero_probe() -> i64 { - let mut limit = std::mem::MaybeUninit::::uninit(); - let rc = unsafe { libc::getrlimit(libc::RLIMIT_CORE, limit.as_mut_ptr()) }; - if rc != 0 { - return -1; - } - let limit = unsafe { limit.assume_init() }; - i64::from(limit.rlim_cur == 0 && limit.rlim_max == 0) - } - - #[test] - #[cfg(unix)] - fn harden_child_process_disables_core_dumps() { - assert_eq!(probe_hardened_child(core_dump_limit_is_zero_probe), 1); - } - - #[cfg(target_os = "linux")] - #[allow(unsafe_code)] - unsafe fn dumpable_flag_probe() -> i64 { - unsafe { i64::from(libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0)) } - } - - #[test] - #[cfg(target_os = "linux")] - fn harden_child_process_marks_process_nondumpable() { - assert_eq!(probe_hardened_child(dumpable_flag_probe), 0); - } - - #[test] - #[cfg(target_os = "linux")] - fn parse_pids_max_detects_limited_runtime() { - assert_eq!( - parse_pids_max("2048\n"), - RuntimePidLimitStatus::Limited(2048) - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn parse_pids_max_detects_unlimited_runtime() { - assert_eq!(parse_pids_max("max\n"), RuntimePidLimitStatus::Unlimited); - } - - #[test] - #[cfg(target_os = "linux")] - fn parse_pids_max_reports_invalid_values() { - let status = parse_pids_max("not-a-number\n"); - assert!(matches!(status, RuntimePidLimitStatus::Unavailable(_))); - } - - #[test] - #[cfg(target_os = "linux")] - fn pid_limit_require_mode_rejects_missing_guardrail_statuses() { - for status in [ - RuntimePidLimitStatus::Unlimited, - RuntimePidLimitStatus::Unavailable("missing".to_string()), - ] { - let result = check_runtime_pid_limit_status(status, RuntimePidLimitMode::Require); - assert!(result.is_err()); - } - } - - #[test] - #[cfg(target_os = "linux")] - fn pid_limit_warn_mode_accepts_missing_guardrail_statuses() { - for status in [ - RuntimePidLimitStatus::Unlimited, - RuntimePidLimitStatus::Unavailable("missing".to_string()), - ] { - let result = check_runtime_pid_limit_status(status, RuntimePidLimitMode::Warn); - assert!(result.is_ok()); - } - } - - #[tokio::test] - async fn inject_provider_env_sets_placeholder_values() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.stdin(StdStdio::null()) - .stdout(StdStdio::piped()) - .stderr(StdStdio::null()); - - let provider_env = std::iter::once(( - "ANTHROPIC_API_KEY".to_string(), - "openshell:resolve:env:ANTHROPIC_API_KEY".to_string(), - )) - .collect(); - - inject_provider_env(&mut cmd, &provider_env); - - let output = cmd.output().await.expect("spawn env"); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - assert!(stdout.contains("ANTHROPIC_API_KEY=openshell:resolve:env:ANTHROPIC_API_KEY")); - } - - #[cfg(unix)] - fn sandbox_policy_with_read_write( - path: PathBuf, - run_as_user: Option, - run_as_group: Option, - ) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only: vec![], - read_write: vec![path], - include_workdir: false, - }, - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user, - run_as_group, - }, - } - } - - #[cfg(unix)] - #[test] - fn prepare_read_write_path_creates_missing_directory() { - let dir = tempfile::tempdir().unwrap(); - let missing = dir.path().join("missing").join("nested"); - - assert!(prepare_read_write_path(&missing).unwrap()); - assert!(missing.is_dir()); - } - - #[cfg(unix)] - #[test] - fn prepare_read_write_path_preserves_existing_directory() { - let dir = tempfile::tempdir().unwrap(); - let existing = dir.path().join("existing"); - std::fs::create_dir(&existing).unwrap(); - - assert!(!prepare_read_write_path(&existing).unwrap()); - assert!(existing.is_dir()); - } - - #[cfg(unix)] - #[test] - fn prepare_read_write_path_rejects_symlink() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let target = dir.path().join("target"); - let link = dir.path().join("link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let error = prepare_read_write_path(&link).unwrap_err(); - assert!( - error - .to_string() - .contains("is a symlink — refusing to chown"), - "unexpected error: {error}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_filesystem_skips_chown_for_existing_read_write_paths() { - use std::os::unix::fs::MetadataExt; - - if nix::unistd::geteuid().is_root() { - return; - } - - let Ok(Some(current_user)) = User::from_uid(nix::unistd::geteuid()) else { - eprintln!("skipping: current UID has no /etc/passwd entry"); - return; - }; - let restricted_group = Group::from_gid(Gid::from_raw(0)) - .unwrap() - .expect("gid 0 group entry"); - if restricted_group.gid == nix::unistd::getegid() { - return; - } - - let dir = tempfile::tempdir().unwrap(); - let existing = dir.path().join("existing"); - std::fs::create_dir(&existing).unwrap(); - let before = std::fs::metadata(&existing).unwrap(); - - let policy = sandbox_policy_with_read_write( - existing.clone(), - Some(current_user.name), - Some(restricted_group.name), - ); - - prepare_filesystem(&policy).expect("existing path should not be re-owned"); - - let after = std::fs::metadata(&existing).unwrap(); - assert_eq!(after.uid(), before.uid()); - assert_eq!(after.gid(), before.gid()); - } - - #[cfg(unix)] - #[test] - #[allow(clippy::similar_names)] - fn chown_sandbox_home_changes_ownership_recursively() { - use std::os::unix::fs::MetadataExt; - - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - std::fs::write(root.join("file.txt"), "hello").unwrap(); - std::fs::create_dir(root.join("subdir")).unwrap(); - std::fs::write(root.join("subdir").join("nested.txt"), "world").unwrap(); - - let expected_uid = nix::unistd::geteuid(); - let expected_gid = nix::unistd::getegid(); - chown_sandbox_home(&root, Some(expected_uid), Some(expected_gid)).unwrap(); - - for path in &[ - root.clone(), - root.join("file.txt"), - root.join("subdir"), - root.join("subdir").join("nested.txt"), - ] { - let meta = std::fs::metadata(path).unwrap(); - assert_eq!(meta.uid(), expected_uid.as_raw()); - assert_eq!(meta.gid(), expected_gid.as_raw()); - } - } - - #[cfg(unix)] - #[test] - fn chown_sandbox_home_rejects_symlink_root() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let target = dir.path().join("real"); - let link = dir.path().join("link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let err = chown_sandbox_home( - &link, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - ) - .unwrap_err(); - assert!( - err.to_string().contains("symlink"), - "expected symlink rejection: {err}" - ); - } - - #[cfg(unix)] - #[test] - fn chown_sandbox_home_skips_symlink_children() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - let target = dir.path().join("outside"); - std::fs::write(&target, "secret").unwrap(); - symlink(&target, root.join("link")).unwrap(); - - chown_sandbox_home( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - ) - .expect("symlink children should be skipped"); - } - - #[cfg(unix)] - #[test] - fn chown_recursive_skips_erofs_subtree_but_continues_siblings() { - use std::sync::{Arc, Mutex}; - - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - - let readonly_dir = root.join("ro-mount"); - std::fs::create_dir(&readonly_dir).unwrap(); - std::fs::write(readonly_dir.join("child-under-ro.txt"), "data").unwrap(); - std::fs::write(root.join("writable-sibling.txt"), "data").unwrap(); - - let chowned = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&chowned); - let readonly_dir_for_chown = readonly_dir.clone(); - let fake_chown = - move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - if path == readonly_dir_for_chown { - return Err(nix::errno::Errno::EROFS); - } - observed.lock().unwrap().push(path.to_path_buf()); - Ok(()) - }; - - chown_children( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &fake_chown, - ) - .expect("read-only subtree should be skipped"); - - let chowned = chowned.lock().unwrap(); - assert!( - !chowned.contains(&readonly_dir.join("child-under-ro.txt")), - "children under EROFS directory must not be traversed" - ); - assert!( - chowned.contains(&root.join("writable-sibling.txt")), - "writable sibling should still be chowned" - ); - } - - #[cfg(unix)] - #[test] - fn chown_recursive_propagates_non_erofs_errors() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - let fake_chown = |_path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - Err(nix::errno::Errno::EPERM) - }; - - let result = chown_recursive( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &fake_chown, - ); - assert!(result.is_err(), "non-EROFS errors should propagate"); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_chowns_only_root() { - use std::sync::{Arc, Mutex}; - - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - let child = root.join("image-content.txt"); - std::fs::write(&child, "image-owned").unwrap(); - - let chowned = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&chowned); - let fake_chown = - move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - observed.lock().unwrap().push(path.to_path_buf()); - Ok(()) - }; - - prepare_oci_workspace_with( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - &fake_chown, - ) - .expect("workspace root should be prepared"); - - assert_eq!(*chowned.lock().unwrap(), vec![root]); - assert!(child.exists(), "image-provided child should be untouched"); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_accepts_existing_owner_writable_directory() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - - validate_oci_workspace( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .expect("image owner already has write and traverse authority"); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_accepts_supplementary_group_write_authority() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o070)).unwrap(); - let metadata = std::fs::symlink_metadata(&root).unwrap(); - - validate_oci_workspace( - &root, - Some(Uid::from_raw(metadata.uid().wrapping_add(1))), - Some(Gid::from_raw(metadata.gid().wrapping_add(1))), - &[Gid::from_raw(metadata.gid())], - ) - .expect("supplementary group already has write and traverse authority"); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_unwritable_directory() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); - let metadata = std::fs::symlink_metadata(&root).unwrap(); - - let error = validate_oci_workspace( - &root, - Some(Uid::from_raw(metadata.uid().wrapping_add(1))), - Some(Gid::from_raw(metadata.gid().wrapping_add(1))), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("not writable and traversable")); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_missing_path() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("missing"); - - let error = validate_oci_workspace( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("does not exist")); - } - - #[cfg(target_os = "linux")] - #[test] - #[allow(unsafe_code)] - fn effective_identity_validation_honors_named_user_acl() { - const TEST_UID: u32 = 42_234; - const TEST_GID: u32 = 42_235; - const ACL_XATTR_VERSION: u32 = 2; - const ACL_USER_OBJ: u16 = 0x01; - const ACL_USER: u16 = 0x02; - const ACL_GROUP_OBJ: u16 = 0x04; - const ACL_MASK: u16 = 0x10; - const ACL_OTHER: u16 = 0x20; - const ACL_UNDEFINED_ID: u32 = u32::MAX; - - if !nix::unistd::geteuid().is_root() { - return; - } - - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - - let mut acl = ACL_XATTR_VERSION.to_ne_bytes().to_vec(); - for (tag, permissions, id) in [ - (ACL_USER_OBJ, 0o7_u16, ACL_UNDEFINED_ID), - (ACL_USER, 0o7_u16, TEST_UID), - (ACL_GROUP_OBJ, 0o0_u16, ACL_UNDEFINED_ID), - (ACL_MASK, 0o7_u16, ACL_UNDEFINED_ID), - (ACL_OTHER, 0o0_u16, ACL_UNDEFINED_ID), - ] { - acl.extend_from_slice(&tag.to_ne_bytes()); - acl.extend_from_slice(&permissions.to_ne_bytes()); - acl.extend_from_slice(&id.to_ne_bytes()); - } - let path = CString::new(root.as_os_str().as_encoded_bytes()).unwrap(); - let name = c"system.posix_acl_access"; - let result = unsafe { - libc::setxattr( - path.as_ptr(), - name.as_ptr(), - acl.as_ptr().cast(), - acl.len(), - 0, - ) - }; - assert_eq!( - result, - 0, - "setxattr failed: {}", - std::io::Error::last_os_error() - ); - - match unsafe { fork() }.expect("fork should succeed") { - ForkResult::Child => { - let credentials_dropped = unsafe { - libc::setgroups(0, std::ptr::null()) == 0 - && libc::setgid(TEST_GID) == 0 - && libc::setuid(TEST_UID) == 0 - }; - let valid = credentials_dropped - && validate_oci_workspace_as_effective_identity(&root).is_ok(); - unsafe { libc::_exit(i32::from(!valid)) }; - } - ForkResult::Parent { child } => { - assert_eq!( - waitpid(child, None).expect("waitpid should succeed"), - WaitStatus::Exited(child, 0), - "named ACL user should retain workspace authority" - ); - } - } - } - - #[cfg(target_os = "linux")] - #[test] - #[allow(unsafe_code)] - fn effective_identity_validation_honors_landlock_denial() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - - let mut policy = policy_with_process(ProcessPolicy::default()); - policy.filesystem = FilesystemPolicy { - read_only: vec![root.clone()], - read_write: Vec::new(), - include_workdir: false, - }; - policy.landlock = LandlockPolicy { - compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, - }; - let Ok(prepared) = sandbox::linux::prepare_current_user(&policy, None) else { - return; - }; - - match unsafe { fork() }.expect("fork should succeed") { - ForkResult::Child => { - let denied = sandbox::linux::enforce(prepared).is_ok() - && validate_oci_workspace_as_effective_identity(&root).is_err(); - unsafe { libc::_exit(i32::from(!denied)) }; - } - ForkResult::Parent { child } => { - assert_eq!( - waitpid(child, None).expect("waitpid should succeed"), - WaitStatus::Exited(child, 0), - "kernel-effective validation should honor an enforced LSM denial" - ); - } - } - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_restrictive_parent() { - let dir = tempfile::tempdir().unwrap(); - let parent = dir.path().canonicalize().unwrap().join("private"); - let root = parent.join("project"); - std::fs::create_dir_all(&root).unwrap(); - std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); - let metadata = std::fs::symlink_metadata(&parent).unwrap(); - - let error = validate_oci_workspace( - &root, - Some(Uid::from_raw(metadata.uid().wrapping_add(1))), - Some(Gid::from_raw(metadata.gid().wrapping_add(1))), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("not traversable")); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_symlink_component() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().canonicalize().unwrap(); - let target = base.join("target"); - let link = base.join("link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let error = validate_oci_workspace( - &link, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("symlink")); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_makes_existing_root_owner_writable() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o555)).unwrap(); - - prepare_oci_workspace_with(&root, None, None, &[], &|_, _, _| Ok(())) - .expect("read-only workspace root should be prepared"); - - let mode = std::fs::symlink_metadata(&root) - .unwrap() - .permissions() - .mode(); - assert_eq!(mode & 0o777, 0o755); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_symlink_root() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().canonicalize().unwrap(); - let target = base.join("real"); - let link = base.join("link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let err = prepare_oci_workspace( - &link, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - err.to_string().contains("symlink"), - "expected symlink rejection: {err}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_symlink_parent() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().canonicalize().unwrap(); - let target = base.join("real"); - let parent_link = base.join("parent-link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &parent_link).unwrap(); - - let err = prepare_oci_workspace( - &parent_link.join("workspace"), - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - err.to_string().contains("symlink"), - "expected parent symlink rejection: {err}" - ); - assert!( - !target.join("workspace").exists(), - "workspace must not be created through a symlink parent" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_parent_traversal() { - let err = prepare_oci_workspace( - Path::new("/tmp/workspace/../escape"), - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - err.to_string().contains("must be normalized"), - "expected traversal rejection: {err}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_inaccessible_existing_parent() { - let dir = tempfile::tempdir().unwrap(); - let parent = dir.path().canonicalize().unwrap().join("workspace"); - std::fs::create_dir(&parent).unwrap(); - std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); - let metadata = std::fs::symlink_metadata(&parent).unwrap(); - let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); - let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); - let root = parent.join("project"); - - let error = prepare_oci_workspace_with( - &root, - Some(different_user), - Some(different_group), - &[], - &|_, _, _| Ok(()), - ) - .unwrap_err(); - - assert!( - error.to_string().contains("is not traversable"), - "unexpected error: {error}" - ); - assert!( - !root.exists(), - "workspace must not be created below an inaccessible parent" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_accepts_supplementary_group_parent() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o710)).unwrap(); - let parent = dir.path().canonicalize().unwrap().join("workspace"); - std::fs::create_dir(&parent).unwrap(); - std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o710)).unwrap(); - let metadata = std::fs::symlink_metadata(&parent).unwrap(); - let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); - let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); - let supplementary_group = Gid::from_raw(metadata.gid()); - let root = parent.join("project"); - - prepare_oci_workspace_with( - &root, - Some(different_user), - Some(different_group), - &[supplementary_group], - &|_, _, _| Ok(()), - ) - .expect("supplementary group execute permission should allow traversal"); - - assert!(root.is_dir()); - } - - #[cfg(not(any( - target_os = "aix", - target_os = "haiku", - target_os = "illumos", - target_os = "ios", - target_os = "macos", - target_os = "redox", - target_os = "solaris" - )))] - #[test] - fn named_user_supplementary_groups_include_primary_group() { - let user = User::from_uid(nix::unistd::geteuid()) - .expect("resolve current UID") - .expect("current user exists"); - - let groups = named_user_supplementary_groups(&user.name, user.gid) - .expect("resolve named-user supplementary groups"); - - assert!(groups.contains(&user.gid)); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_non_directory_root() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::write(&root, "not a directory").unwrap(); - - let error = prepare_oci_workspace( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - error.to_string().contains("is not a directory"), - "unexpected error: {error}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_propagates_root_chown_error() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - let fake_chown = |_path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - Err(nix::errno::Errno::EROFS) - }; - - let error = prepare_oci_workspace_with( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - &fake_chown, - ) - .unwrap_err(); - - assert!( - error.to_string().contains("Read-only file system"), - "unexpected error: {error}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_creates_missing_root() { - use std::sync::{Arc, Mutex}; - - let dir = tempfile::tempdir().unwrap(); - let missing = dir - .path() - .canonicalize() - .unwrap() - .join("missing") - .join("sandbox"); - let chowned = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&chowned); - let fake_chown = - move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - observed.lock().unwrap().push(path.to_path_buf()); - Ok(()) - }; - - prepare_oci_workspace_with( - &missing, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - &fake_chown, - ) - .expect("missing OCI workspace should be created"); - - assert!(missing.is_dir()); - assert_eq!( - std::fs::symlink_metadata(missing.parent().unwrap()) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o755 - ); - assert_eq!(*chowned.lock().unwrap(), vec![missing]); - } - - #[cfg(unix)] - #[test] - fn rewrite_passwd_modifies_existing_sandbox_entry() { - let dir = tempfile::tempdir().unwrap(); - let passwd = dir.path().join("passwd"); - std::fs::write( - &passwd, - "root:x:0:0:root:/root:/bin/bash\nsandbox:x:1000:1000::/sandbox:/bin/bash\n", - ) - .unwrap(); - - rewrite_passwd_at(&passwd, "5000", "6000").unwrap(); - - let content = std::fs::read_to_string(&passwd).unwrap(); - assert!(content.contains("sandbox:x:5000:6000::/sandbox:/bin/bash")); - assert!(content.contains("root:x:0:0:root:/root:/bin/bash")); - } - - #[cfg(unix)] - #[test] - fn rewrite_passwd_appends_when_no_sandbox_entry() { - let dir = tempfile::tempdir().unwrap(); - let passwd = dir.path().join("passwd"); - std::fs::write(&passwd, "root:x:0:0:root:/root:/bin/bash\n").unwrap(); - - rewrite_passwd_at(&passwd, "5000", "6000").unwrap(); - - let content = std::fs::read_to_string(&passwd).unwrap(); - assert!(content.contains("root:x:0:0:root:/root:/bin/bash")); - assert!(content.contains("sandbox:x:5000:6000::/sandbox:/bin/sh")); - } - - #[cfg(unix)] - #[test] - fn rewrite_group_modifies_existing_sandbox_entry() { - let dir = tempfile::tempdir().unwrap(); - let group = dir.path().join("group"); - std::fs::write(&group, "root:x:0:\nsandbox:x:1000:\n").unwrap(); - - rewrite_group_at(&group, "6000").unwrap(); - - let content = std::fs::read_to_string(&group).unwrap(); - assert!(content.contains("sandbox:x:6000:")); - assert!(content.contains("root:x:0:")); - } - - #[cfg(unix)] - #[test] - fn rewrite_group_appends_when_no_sandbox_entry() { - let dir = tempfile::tempdir().unwrap(); - let group = dir.path().join("group"); - std::fs::write(&group, "root:x:0:\n").unwrap(); - - rewrite_group_at(&group, "6000").unwrap(); - - let content = std::fs::read_to_string(&group).unwrap(); - assert!(content.contains("root:x:0:")); - assert!(content.contains("sandbox:x:6000:")); - } - - #[cfg(unix)] - #[test] - fn rewrite_passwd_leaves_malformed_entry_unchanged() { - let dir = tempfile::tempdir().unwrap(); - let passwd = dir.path().join("passwd"); - // Only 3 fields — slice pattern should fall through instead of panic. - std::fs::write(&passwd, "sandbox:x:1000\n").unwrap(); - rewrite_passwd_at(&passwd, "5000", "6000").unwrap(); - let content = std::fs::read_to_string(&passwd).unwrap(); - assert!(content.contains("sandbox:x:1000")); - } - - #[cfg(unix)] - #[test] - fn rewrite_group_leaves_malformed_entry_unchanged() { - let dir = tempfile::tempdir().unwrap(); - let group = dir.path().join("group"); - // Only 2 fields — slice pattern should fall through instead of panic. - std::fs::write(&group, "sandbox:x\n").unwrap(); - rewrite_group_at(&group, "6000").unwrap(); - let content = std::fs::read_to_string(&group).unwrap(); - assert!(content.contains("sandbox:x")); - } - - #[cfg(unix)] - #[test] - fn rewrite_passwd_preserves_other_entries() { - let dir = tempfile::tempdir().unwrap(); - let passwd = dir.path().join("passwd"); - std::fs::write( - &passwd, - "root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534:nobody:/:/usr/sbin/nologin\nsandbox:x:1000:1000::/sandbox:/bin/bash\n", - ) - .unwrap(); - - rewrite_passwd_at(&passwd, "1234567", "1234567").unwrap(); - - let content = std::fs::read_to_string(&passwd).unwrap(); - assert!(content.contains("root:x:0:0:root:/root:/bin/bash")); - assert!(content.contains("nobody:x:65534:65534:nobody:/:/usr/sbin/nologin")); - assert!(content.contains("sandbox:x:1234567:1234567::/sandbox:/bin/bash")); - assert_eq!(content.lines().count(), 3); - } - - #[tokio::test] - async fn inject_provider_env_skips_supervisor_identity_material() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.env_clear() - .stdin(StdStdio::null()) - .stdout(StdStdio::piped()) - .stderr(StdStdio::null()); - - let provider_env = HashMap::from([ - ( - "ANTHROPIC_API_KEY".to_string(), - "openshell:resolve:env:ANTHROPIC_API_KEY".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), - "provider-token".to_string(), - ), - ( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), - "/spiffe-workload-api/spire-agent.sock".to_string(), - ), - ]); - - inject_provider_env(&mut cmd, &provider_env); - - let output = cmd.output().await.expect("spawn env"); - assert!(output.status.success()); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - assert!(stdout.contains("ANTHROPIC_API_KEY=openshell:resolve:env:ANTHROPIC_API_KEY")); - assert!(!stdout.contains(openshell_core::sandbox_env::SANDBOX_TOKEN)); - assert!(!stdout.contains(openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET)); - } - - #[tokio::test] - async fn strip_supervisor_only_env_removes_identity_material() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.stdin(StdStdio::null()) - .stdout(StdStdio::piped()) - .stderr(StdStdio::null()) - .env("OPENSHELL_ENDPOINT", "https://gateway.example.test"); - - for key in SUPERVISOR_ONLY_ENV_VARS { - cmd.env(key, format!("{key}-secret")); - } - - strip_supervisor_only_env(&mut cmd); - - let output = cmd.output().await.expect("spawn env"); - assert!(output.status.success()); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - - for key in SUPERVISOR_ONLY_ENV_VARS { - assert!( - !stdout - .lines() - .any(|line| line.starts_with(&format!("{key}="))), - "{key} must not be inherited by sandbox child processes" - ); - } - assert!(stdout.contains("OPENSHELL_ENDPOINT=https://gateway.example.test")); - } - - #[test] - fn supervisor_identity_mount_target_uses_socket_parent() { - assert_eq!( - supervisor_identity_mount_target("/spiffe-workload-api/spire-agent.sock") - .expect("plain path should parse"), - Some(PathBuf::from("/spiffe-workload-api")) - ); - assert_eq!( - supervisor_identity_mount_target("unix:/spiffe-workload-api/spire-agent.sock") - .expect("unix path should parse"), - Some(PathBuf::from("/spiffe-workload-api")) - ); - } - - #[test] - fn supervisor_identity_mount_target_ignores_empty_socket_path() { - assert_eq!( - supervisor_identity_mount_target(" ").expect("empty path should be ignored"), - None - ); - } - - #[test] - fn supervisor_identity_mount_target_rejects_unhideable_endpoints() { - assert_eq!( - supervisor_identity_mount_target("tcp:127.0.0.1:8081") - .expect("tcp endpoint should not require mount hiding"), - None - ); - assert!(supervisor_identity_mount_target("spiffe-workload-api/spire-agent.sock").is_err()); - assert!(supervisor_identity_mount_target("/spire-agent.sock").is_err()); - } - - #[test] - fn supervisor_identity_mount_target_rejects_shared_root_shadowing() { - for socket_path in [ - "/run/spire-agent.sock", - "/var/spire-agent.sock", - "/tmp/spire-agent.sock", - "/etc/spire-agent.sock", - ] { - let err = supervisor_identity_mount_target(socket_path) - .expect_err("shared root shadowing should be rejected"); - assert!(err.to_string().contains("dedicated subdirectory")); - } - - assert_eq!( - supervisor_identity_mount_target("/run/spire/spire-agent.sock") - .expect("dedicated subdirectory should be accepted"), - Some(PathBuf::from("/run/spire")) - ); - } - - // ---- Numeric UID tests (Phase 2) ---- - - #[test] - fn drop_privileges_accepts_numeric_uid() { - // When running as non-root, a numeric UID/GID that matches the - // current process should succeed without any passwd lookup. - if nix::unistd::geteuid().is_root() { - return; - } - - let uid_raw = nix::unistd::geteuid().as_raw(); - let gid_raw = nix::unistd::getegid().as_raw(); - - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some(uid_raw.to_string()), - run_as_group: Some(gid_raw.to_string()), - }); - - assert!( - drop_privileges(&policy).is_ok(), - "should accept current process UID/GID as numeric strings" - ); - } - - #[test] - fn drop_privileges_numeric_uid_skips_initgroups() { - // When running as non-root with a numeric user but group matches, - // initgroups should not be called (guard: target_uid != geteuid()). - if nix::unistd::geteuid().is_root() { - return; - } - - let current_uid = nix::unistd::geteuid().as_raw(); - - // Use a different group name that exists (the current one). - let current_group = Group::from_gid(nix::unistd::getegid()) - .expect("should resolve current group") - .expect("current group should exist"); - - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some(current_uid.to_string()), // numeric UID, no passwd entry needed - run_as_group: Some(current_group.name), // name-based group - }); - - assert!( - drop_privileges(&policy).is_ok(), - "should accept numeric UID with name-based group (initgroups guarded)" - ); - } - - #[test] - fn numeric_uid_privilege_drop_child() { - if std::env::var_os("OPENSHELL_TEST_NUMERIC_UID_CHILD").is_none() { - return; - } - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some("999999".into()), - run_as_group: Some("999999".into()), - }); - match drop_privileges(&policy) { - Ok(()) => {} - Err(e) => { - assert!( - !e.to_string().contains("Failed to resolve user record"), - "unexpected error for numeric UID without passwd entry: {e}" - ); - } - } - } - - #[test] - fn drop_privileges_numeric_uid_without_passwd_entry_skips_lookup() { - let mut cmd = std::process::Command::new(std::env::current_exe().expect("current exe")); - cmd.arg("numeric_uid_privilege_drop_child") - .arg("--nocapture") - .env("OPENSHELL_TEST_NUMERIC_UID_CHILD", "1") - .stdin(StdStdio::null()) - .stdout(StdStdio::piped()) - .stderr(StdStdio::piped()); - let output = cmd.output().expect("spawn child"); - assert!( - output.status.success(), - "numeric UID privilege drop child failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } -} diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs deleted file mode 100644 index bf42faede8..0000000000 --- a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs +++ /dev/null @@ -1,699 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Landlock filesystem sandboxing. - -use landlock::{ - ABI, Access, AccessFs, BitFlags, CompatLevel, Compatible, PathBeneath, PathFd, PathFdError, - Ruleset, RulesetAttr, RulesetCreatedAttr, -}; -use miette::{IntoDiagnostic, Result}; -use openshell_core::policy::{LandlockCompatibility, SandboxPolicy}; -use std::os::fd::AsFd; -use std::path::{Path, PathBuf}; -use tracing::debug; - -/// Result of probing the kernel for Landlock support. -#[derive(Debug)] -pub enum LandlockAvailability { - /// Landlock is available with the given ABI version. - Available { abi: i32 }, - /// Kernel does not implement Landlock (ENOSYS). - NotImplemented, - /// Landlock is compiled in but not enabled at boot (EOPNOTSUPP). - NotEnabled, - /// Landlock syscall is blocked, likely by a container seccomp profile (EPERM). - Blocked, - /// Unexpected error from the probe syscall. - Unknown(i32), -} - -impl std::fmt::Display for LandlockAvailability { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Available { abi } => write!(f, "available (ABI v{abi})"), - Self::NotImplemented => { - write!(f, "not implemented (kernel lacks CONFIG_SECURITY_LANDLOCK)") - } - Self::NotEnabled => write!( - f, - "not enabled (Landlock built into kernel but not in active LSM list)" - ), - Self::Blocked => write!( - f, - "blocked (container seccomp profile denies Landlock syscalls)" - ), - Self::Unknown(errno) => write!(f, "unexpected probe error (errno {errno})"), - } - } -} - -/// Probe the kernel for Landlock support by issuing the `landlock_create_ruleset` -/// syscall with the version-check flag. -/// -/// This is safe to call from the parent process and does not create any file -/// descriptors or modify process state. -pub fn probe_availability() -> LandlockAvailability { - // landlock_create_ruleset syscall number (same on x86_64 and aarch64). - const SYS_LANDLOCK_CREATE_RULESET: libc::c_long = 444; - // Flag: return the highest supported ABI version instead of creating a ruleset. - const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1 << 0; - - // SAFETY: landlock_create_ruleset(NULL, 0, LANDLOCK_CREATE_RULESET_VERSION) - // is a read-only probe that returns the ABI version or an error code. - // It does not allocate file descriptors or modify process state. - #[allow(unsafe_code)] - let ret = unsafe { - libc::syscall( - SYS_LANDLOCK_CREATE_RULESET, - std::ptr::null::(), - 0_usize, - LANDLOCK_CREATE_RULESET_VERSION, - ) - }; - - if ret >= 0 { - #[allow(clippy::cast_possible_truncation)] - LandlockAvailability::Available { abi: ret as i32 } - } else { - let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); - match errno { - libc::ENOSYS => LandlockAvailability::NotImplemented, - libc::EOPNOTSUPP => LandlockAvailability::NotEnabled, - libc::EPERM => LandlockAvailability::Blocked, - other => LandlockAvailability::Unknown(other), - } - } -} - -/// A prepared Landlock ruleset ready to be enforced via `restrict_self()`. -/// -/// Created by [`prepare`] while running as root (so `PathFd::new()` can open -/// any path regardless of DAC permissions). Enforced by [`enforce`] after -/// `drop_privileges()` — `restrict_self()` does not require elevated privileges. -pub struct PreparedRuleset { - ruleset: landlock::RulesetCreated, - compatibility: LandlockCompatibility, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PathOpenMode { - Privileged, - CurrentUser, -} - -/// Phase 1: Open `PathFds` and build the Landlock ruleset **as root**. -/// -/// This must run before `drop_privileges()` so that `PathFd::new()` can open -/// paths that are only accessible to root (e.g. mode 700 directories). -/// -/// Returns `None` if there are no filesystem paths to restrict (no-op). -/// Returns `Some(PreparedRuleset)` on success, or an error. -pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result> { - prepare_with_path_open_mode(policy, workdir, PathOpenMode::Privileged) -} - -/// Phase 1 for already-unprivileged workloads. -/// -/// Kubernetes sidecar mode starts the process supervisor as the sandbox UID, so -/// Landlock path FDs are opened as the same UID that will run the workload. -/// Paths this UID cannot open are already unavailable to the workload; omit -/// them from the allowlist and let the resulting ruleset deny everything else. -pub fn prepare_current_user( - policy: &SandboxPolicy, - workdir: Option<&str>, -) -> Result> { - prepare_with_path_open_mode(policy, workdir, PathOpenMode::CurrentUser) -} - -fn prepare_with_path_open_mode( - policy: &SandboxPolicy, - workdir: Option<&str>, - path_open_mode: PathOpenMode, -) -> Result> { - let read_only = policy.filesystem.read_only.clone(); - let mut read_write = policy.filesystem.read_write.clone(); - - if policy.filesystem.include_workdir - && let Some(dir) = workdir - { - let workdir_path = PathBuf::from(dir); - if !read_write.contains(&workdir_path) { - read_write.push(workdir_path); - } - } - - if read_only.is_empty() && read_write.is_empty() { - return Ok(None); - } - - let compatibility = &policy.landlock.compatibility; - - // Probe first: kernels without Landlock (e.g. gVisor's sentry returns - // ENOSYS) would otherwise log misleading "Applying"+"Built" events. - let availability = probe_availability(); - if !matches!(availability, LandlockAvailability::Available { .. }) { - match compatibility { - LandlockCompatibility::BestEffort => { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(openshell_ocsf::ActivityId::Open) - .severity(openshell_ocsf::SeverityId::High) - .confidence(openshell_ocsf::ConfidenceId::High) - .is_alert(true) - .finding_info( - openshell_ocsf::FindingInfo::new( - "landlock-unavailable", - "Landlock Filesystem Sandbox Unavailable", - ) - .with_desc(&format!( - "Running WITHOUT filesystem restrictions: Landlock is {availability}. \ - Set landlock.compatibility to 'hard_requirement' to make this fatal." - )), - ) - .message(format!( - "Landlock filesystem sandbox unavailable: {availability}" - )) - .build() - ); - return Ok(None); - } - LandlockCompatibility::HardRequirement => { - return Err(miette::miette!( - "Landlock unavailable in hard_requirement mode: {availability}" - )); - } - } - } - - let total_paths = read_only.len() + read_write.len(); - let abi = ABI::V2; - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "applying") - .message(format!( - "Applying Landlock filesystem sandbox [abi:{abi:?} compat:{:?} ro:{} rw:{}]", - policy.landlock.compatibility, - read_only.len(), - read_write.len(), - )) - .build() - ); - - let result: Result = (|| { - let access_all = AccessFs::from_all(abi); - let access_read = AccessFs::from_read(abi); - - let mut ruleset = Ruleset::default(); - ruleset = ruleset - .set_compatibility(compat_level(compatibility)) - .handle_access(access_all) - .into_diagnostic()?; - - let mut ruleset = ruleset.create().into_diagnostic()?; - let mut rules_applied: usize = 0; - - for path in &read_only { - if let Some(path_fd) = try_open_path(path, compatibility, path_open_mode)? { - let allowed_access = access_for_path_fd(&path_fd, access_read, abi)?; - debug!(path = %path.display(), "Landlock allow read-only"); - ruleset = ruleset - .add_rule(PathBeneath::new(path_fd, allowed_access)) - .into_diagnostic()?; - rules_applied += 1; - } - } - - for path in &read_write { - if let Some(path_fd) = try_open_path(path, compatibility, path_open_mode)? { - let allowed_access = access_for_path_fd(&path_fd, access_all, abi)?; - debug!(path = %path.display(), "Landlock allow read-write"); - ruleset = ruleset - .add_rule(PathBeneath::new(path_fd, allowed_access)) - .into_diagnostic()?; - rules_applied += 1; - } - } - - if rules_applied == 0 { - return Err(miette::miette!( - "Landlock ruleset has zero valid paths — all {} path(s) failed to open. \ - Refusing to apply an empty ruleset that would block all filesystem access.", - total_paths, - )); - } - - let skipped = total_paths - rules_applied; - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "built") - .message(format!( - "Landlock ruleset built [rules_applied:{rules_applied} skipped:{skipped}]" - )) - .build() - ); - - Ok(PreparedRuleset { - ruleset, - compatibility: compatibility.clone(), - }) - })(); - - match result { - Ok(prepared) => Ok(Some(prepared)), - Err(err) => { - if matches!(compatibility, LandlockCompatibility::BestEffort) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(openshell_ocsf::ActivityId::Open) - .severity(openshell_ocsf::SeverityId::High) - .confidence(openshell_ocsf::ConfidenceId::High) - .is_alert(true) - .finding_info( - openshell_ocsf::FindingInfo::new( - "landlock-unavailable", - "Landlock Filesystem Sandbox Unavailable", - ) - .with_desc(&format!( - "Running WITHOUT filesystem restrictions: {err}. \ - Set landlock.compatibility to 'hard_requirement' to make this fatal." - )), - ) - .message(format!("Landlock filesystem sandbox unavailable: {err}")) - .build() - ); - Ok(None) - } else { - Err(err) - } - } - } -} - -/// Phase 2: Enforce a prepared Landlock ruleset by calling `restrict_self()`. -/// -/// This runs **after** `drop_privileges()`. The `restrict_self()` syscall does -/// not require root — it only restricts the calling thread (and its future -/// children), which is always permitted. -/// -/// Respects the same `best_effort` / `hard_requirement` compatibility as -/// [`prepare`]: if `restrict_self()` fails and the policy is `best_effort`, -/// the error is logged and the sandbox continues without Landlock. -pub fn enforce(prepared: PreparedRuleset) -> Result<()> { - let result = prepared.ruleset.restrict_self().into_diagnostic(); - if let Err(err) = result { - if matches!(prepared.compatibility, LandlockCompatibility::BestEffort) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(openshell_ocsf::ActivityId::Open) - .severity(openshell_ocsf::SeverityId::High) - .confidence(openshell_ocsf::ConfidenceId::High) - .is_alert(true) - .finding_info( - openshell_ocsf::FindingInfo::new( - "landlock-enforce-failed", - "Landlock restrict_self Failed", - ) - .with_desc(&format!( - "Ruleset was prepared but restrict_self() failed: {err}. \ - Running WITHOUT filesystem restrictions. \ - Set landlock.compatibility to 'hard_requirement' to make this fatal." - )), - ) - .message(format!( - "Landlock restrict_self failed (best_effort): {err}" - )) - .build() - ); - return Ok(()); - } - return Err(err); - } - Ok(()) -} - -/// Legacy single-phase apply. Kept for non-Linux platforms and tests. -/// On Linux, callers should use [`prepare`] + [`enforce`] for correct -/// privilege ordering. -#[allow(dead_code)] // Retained for backward compat; live callers use prepare+enforce. -pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { - if let Some(prepared) = prepare(policy, workdir)? { - enforce(prepared)?; - } - Ok(()) -} - -/// Tailor a rule's access mask to the inode referenced by its already-open FD. -/// -/// Landlock directory-only rights such as `ReadDir` are invalid for regular -/// files and device nodes in hard-requirement mode. Classifying through the -/// same `PathFd` used by the rule avoids a pathname TOCTOU race. -fn access_for_path_fd( - path_fd: &PathFd, - requested_access: BitFlags, - abi: ABI, -) -> Result> { - let stat = rustix::fs::fstat(path_fd.as_fd()).into_diagnostic()?; - Ok(match rustix::fs::FileType::from_raw_mode(stat.st_mode) { - rustix::fs::FileType::Directory => requested_access, - _ => requested_access & AccessFs::from_file(abi), - }) -} - -/// Attempt to open a path for Landlock rule creation. -/// -/// In `BestEffort` mode, inaccessible paths (missing, permission denied, symlink -/// loops, etc.) are skipped with a warning and `Ok(None)` is returned so the -/// caller can continue building the ruleset from the remaining valid paths. -/// -/// In `HardRequirement` mode, any failure is fatal — the caller propagates the -/// error, which ultimately aborts sandbox startup. -fn try_open_path( - path: &Path, - compatibility: &LandlockCompatibility, - path_open_mode: PathOpenMode, -) -> Result> { - match PathFd::new(path) { - Ok(fd) => Ok(Some(fd)), - Err(err) => { - let reason = classify_path_fd_error(&err); - let is_not_found = matches!( - &err, - PathFdError::OpenCall { source, .. } - if source.kind() == std::io::ErrorKind::NotFound - ); - if matches!(path_open_mode, PathOpenMode::CurrentUser) { - if is_not_found { - debug!( - path = %path.display(), - reason, - "Skipping non-existent Landlock path for current user" - ); - } else { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Other, "already-denied") - .message(format!( - "Skipping inaccessible Landlock path for current user [path:{} error:{err}]", - path.display() - )) - .build() - ); - } - return Ok(None); - } - match compatibility { - LandlockCompatibility::BestEffort => { - // NotFound is expected for stale baseline paths (e.g. - // /app baked into the server-stored policy but absent - // in this container image). Log at debug! to avoid - // polluting SSH exec stdout — the pre_exec hook - // inherits the tracing subscriber whose writer targets - // fd 1 (the pipe/PTY). - // - // Other errors (permission denied, symlink loops, etc.) - // are genuinely unexpected and logged at warn!. - if is_not_found { - debug!( - path = %path.display(), - reason, - "Skipping non-existent Landlock path (best-effort mode)" - ); - } else { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Other, "degraded") - .message(format!( - "Skipping inaccessible Landlock path (best-effort) [path:{} error:{err}]", - path.display() - )) - .build() - ); - } - Ok(None) - } - LandlockCompatibility::HardRequirement => Err(miette::miette!( - "Landlock path unavailable in hard_requirement mode: {} ({}): {}", - path.display(), - reason, - err, - )), - } - } - } -} - -/// Classify a [`PathFdError`] into a human-readable reason. -/// -/// `PathFd::new()` wraps `open(path, O_PATH | O_CLOEXEC)` which can fail for -/// several reasons beyond simple non-existence. The `PathFdError::OpenCall` -/// variant wraps the underlying `std::io::Error`. -fn classify_path_fd_error(err: &PathFdError) -> &'static str { - match err { - PathFdError::OpenCall { source, .. } => classify_io_error(source), - // PathFdError is #[non_exhaustive], handle future variants gracefully. - _ => "unexpected error", - } -} - -/// Classify a `std::io::Error` into a human-readable reason string. -fn classify_io_error(err: &std::io::Error) -> &'static str { - match err.kind() { - std::io::ErrorKind::NotFound => "path does not exist", - std::io::ErrorKind::PermissionDenied => "permission denied", - _ => match err.raw_os_error() { - Some(40) => "too many symlink levels", // ELOOP - Some(36) => "path name too long", // ENAMETOOLONG - Some(20) => "path component is not a directory", // ENOTDIR - _ => "unexpected error", - }, - } -} - -fn compat_level(level: &LandlockCompatibility) -> CompatLevel { - match level { - LandlockCompatibility::BestEffort => CompatLevel::BestEffort, - LandlockCompatibility::HardRequirement => CompatLevel::HardRequirement, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy}; - - fn hard_requirement_policy(read_only: Vec, read_write: Vec) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only, - read_write, - include_workdir: false, - }, - network: NetworkPolicy::default(), - landlock: LandlockPolicy { - compatibility: LandlockCompatibility::HardRequirement, - }, - process: ProcessPolicy::default(), - } - } - - #[test] - fn prepare_hard_requirement_accepts_device_paths() { - if !matches!(probe_availability(), LandlockAvailability::Available { .. }) { - return; - } - - let policy = hard_requirement_policy( - vec![PathBuf::from("/tmp"), PathBuf::from("/dev/urandom")], - vec![PathBuf::from("/dev/null")], - ); - - let result = prepare(&policy, None); - if let Err(err) = result { - panic!("hard_requirement should accept mixed directory and device paths: {err}"); - } - } - fn tailored_access(path: &Path, requested_access: BitFlags) -> BitFlags { - let path_fd = PathFd::new(path).unwrap(); - access_for_path_fd(&path_fd, requested_access, ABI::V2).unwrap() - } - - #[test] - fn access_for_path_fd_preserves_directory_access() { - let dir = tempfile::tempdir().unwrap(); - let requested_access = AccessFs::from_all(ABI::V2); - - assert_eq!( - tailored_access(dir.path(), requested_access), - requested_access - ); - } - - #[test] - fn access_for_path_fd_limits_regular_file_access() { - let file = tempfile::NamedTempFile::new().unwrap(); - let requested_access = AccessFs::from_all(ABI::V2); - - assert_eq!( - tailored_access(file.path(), requested_access), - requested_access & AccessFs::from_file(ABI::V2) - ); - } - - #[test] - fn access_for_path_fd_limits_character_device_access() { - let requested_read = AccessFs::from_read(ABI::V2); - let requested_write = AccessFs::from_all(ABI::V2); - - assert_eq!( - tailored_access(Path::new("/dev/urandom"), requested_read), - requested_read & AccessFs::from_file(ABI::V2) - ); - assert_eq!( - tailored_access(Path::new("/dev/null"), requested_write), - requested_write & AccessFs::from_file(ABI::V2) - ); - } - - #[test] - fn access_for_path_fd_classifies_symlink_target() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let target = dir.path().join("target"); - let link = dir.path().join("link"); - std::fs::File::create(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let requested_access = AccessFs::from_all(ABI::V2); - assert_eq!( - tailored_access(&link, requested_access), - requested_access & AccessFs::from_file(ABI::V2) - ); - } - - #[test] - fn try_open_path_best_effort_returns_none_for_missing_path() { - let result = try_open_path( - &PathBuf::from("/nonexistent/openshell/test/path"), - &LandlockCompatibility::BestEffort, - PathOpenMode::Privileged, - ); - assert!(result.is_ok()); - assert!(result.unwrap().is_none()); - } - - #[test] - fn try_open_path_hard_requirement_errors_for_missing_path() { - let result = try_open_path( - &PathBuf::from("/nonexistent/openshell/test/path"), - &LandlockCompatibility::HardRequirement, - PathOpenMode::Privileged, - ); - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("hard_requirement"), - "error should mention hard_requirement mode: {err_msg}" - ); - assert!( - err_msg.contains("does not exist"), - "error should include the classified reason: {err_msg}" - ); - } - - #[test] - fn try_open_path_succeeds_for_existing_path() { - let dir = tempfile::tempdir().unwrap(); - let result = try_open_path( - dir.path(), - &LandlockCompatibility::BestEffort, - PathOpenMode::Privileged, - ); - assert!(result.is_ok()); - assert!(result.unwrap().is_some()); - } - - #[test] - fn try_open_path_current_user_skips_missing_path_in_hard_requirement() { - let result = try_open_path( - &PathBuf::from("/nonexistent/openshell/test/path"), - &LandlockCompatibility::HardRequirement, - PathOpenMode::CurrentUser, - ); - assert!(result.is_ok()); - assert!(result.unwrap().is_none()); - } - - #[test] - fn classify_not_found() { - let err = std::io::Error::from_raw_os_error(libc::ENOENT); - assert_eq!(classify_io_error(&err), "path does not exist"); - } - - #[test] - fn classify_permission_denied() { - let err = std::io::Error::from_raw_os_error(libc::EACCES); - assert_eq!(classify_io_error(&err), "permission denied"); - } - - #[test] - fn classify_symlink_loop() { - let err = std::io::Error::from_raw_os_error(libc::ELOOP); - assert_eq!(classify_io_error(&err), "too many symlink levels"); - } - - #[test] - fn classify_name_too_long() { - let err = std::io::Error::from_raw_os_error(libc::ENAMETOOLONG); - assert_eq!(classify_io_error(&err), "path name too long"); - } - - #[test] - fn classify_not_a_directory() { - let err = std::io::Error::from_raw_os_error(libc::ENOTDIR); - assert_eq!(classify_io_error(&err), "path component is not a directory"); - } - - #[test] - fn classify_unknown_error() { - let err = std::io::Error::from_raw_os_error(libc::EIO); - assert_eq!(classify_io_error(&err), "unexpected error"); - } - - #[test] - fn classify_path_fd_error_extracts_io_error() { - // Use PathFd::new on a non-existent path to get a real PathFdError - // (the OpenCall variant is #[non_exhaustive] and can't be constructed directly). - let err = PathFd::new("/nonexistent/openshell/classify/test").unwrap_err(); - assert_eq!(classify_path_fd_error(&err), "path does not exist"); - } - - #[test] - fn probe_availability_returns_a_result() { - // The probe should not panic regardless of whether Landlock is available. - // On Linux hosts with Landlock, this returns Available; on Docker Desktop - // linuxkit or older kernels, it returns NotImplemented/NotEnabled/Blocked. - let result = probe_availability(); - let display = format!("{result}"); - assert!( - !display.is_empty(), - "probe_availability Display should produce output" - ); - // Verify the Debug impl works too. - let debug = format!("{result:?}"); - assert!( - !debug.is_empty(), - "probe_availability Debug should produce output" - ); - } -} diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs b/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs deleted file mode 100644 index 107a50e370..0000000000 --- a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs +++ /dev/null @@ -1,181 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Linux sandbox implementation using Landlock and seccomp. - -mod landlock; -mod seccomp; - -use miette::Result; -use openshell_core::policy::SandboxPolicy; -use std::path::PathBuf; -use std::sync::Once; - -/// Opaque handle to a prepared-but-not-yet-enforced sandbox. -/// Holds the Landlock ruleset with `PathFds` opened before child exec. -pub struct PreparedSandbox { - landlock: Option, - policy: SandboxPolicy, -} - -/// Phase 1: Prepare sandbox restrictions **as root** (before `drop_privileges`). -/// -/// Opens Landlock `PathFds` while the process still has root privileges, -/// ensuring paths like mode-700 directories are accessible. -pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result { - let landlock = landlock::prepare(policy, workdir)?; - Ok(PreparedSandbox { - landlock, - policy: policy.clone(), - }) -} - -/// Phase 1 for already-unprivileged workloads. -/// -/// Opens Landlock `PathFds` as the current UID. This is used by Kubernetes -/// sidecar mode, where the agent container already runs as the sandbox user. -pub fn prepare_current_user( - policy: &SandboxPolicy, - workdir: Option<&str>, -) -> Result { - let landlock = landlock::prepare_current_user(policy, workdir)?; - Ok(PreparedSandbox { - landlock, - policy: policy.clone(), - }) -} - -/// Phase 2: Enforce prepared sandbox restrictions (after `drop_privileges`). -/// -/// Calls `restrict_self()` for Landlock and applies seccomp filters. -/// Neither operation requires root privileges. -pub fn enforce(prepared: PreparedSandbox) -> Result<()> { - if let Some(ruleset) = prepared.landlock { - landlock::enforce(ruleset)?; - } - seccomp::apply(&prepared.policy)?; - Ok(()) -} - -/// Apply the supervisor seccomp prelude after privileged bootstrap completes. -pub fn apply_supervisor_prelude() -> Result<()> { - seccomp::apply_supervisor_prelude() -} - -/// Legacy single-phase apply. Kept for backward compatibility. -/// New callers should use [`prepare`] + [`enforce`] for correct privilege ordering. -#[allow(dead_code)] // Retained for backward compat; live callers use prepare+enforce. -pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { - landlock::apply(policy, workdir)?; - seccomp::apply(policy)?; - Ok(()) -} - -/// Probe Landlock availability and emit OCSF logs from the parent process. -/// -/// This must be called **before** `pre_exec` / `fork()` so that the OCSF events -/// are emitted through the parent's tracing subscriber (the child process after -/// fork does not have a working tracing pipeline). -pub fn log_sandbox_readiness(policy: &SandboxPolicy, workdir: Option<&str>) { - static PROBED: Once = Once::new(); - let mut already_probed = true; - PROBED.call_once(|| already_probed = false); - if already_probed { - return; - } - - let mut read_write = policy.filesystem.read_write.clone(); - let read_only = &policy.filesystem.read_only; - - if policy.filesystem.include_workdir - && let Some(dir) = workdir - { - let workdir_path = PathBuf::from(dir); - if !read_write.contains(&workdir_path) { - read_write.push(workdir_path); - } - } - - let total_paths = read_only.len() + read_write.len(); - - if total_paths == 0 { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Other, "skipped") - .message("Landlock filesystem sandbox skipped: no paths configured".to_string()) - .build() - ); - return; - } - - let availability = landlock::probe_availability(); - if let landlock::LandlockAvailability::Available { abi } = &availability { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "probed") - .message(format!( - "Landlock filesystem sandbox available \ - [abi:v{abi} compat:{:?} ro:{} rw:{}]", - policy.landlock.compatibility, - read_only.len(), - read_write.len(), - )) - .build() - ); - } else { - // Landlock is NOT available — this is the critical log that was - // previously invisible because it only fired inside pre_exec. - let is_best_effort = matches!( - policy.landlock.compatibility, - openshell_core::policy::LandlockCompatibility::BestEffort - ); - let (desc, msg) = if is_best_effort { - ( - format!( - "Sandbox will run WITHOUT filesystem restrictions: {availability}. \ - Policy requests {total_paths} path rule(s) \ - (ro:{} rw:{}) but Landlock cannot enforce them. \ - Set landlock.compatibility to 'hard_requirement' to make this fatal.", - read_only.len(), - read_write.len(), - ), - format!( - "Landlock filesystem sandbox unavailable (best_effort, degraded): {availability}" - ), - ) - } else { - ( - format!( - "Landlock is unavailable: {availability}. \ - Policy requires {total_paths} path rule(s) \ - (ro:{} rw:{}) with hard_requirement — sandbox startup will fail.", - read_only.len(), - read_write.len(), - ), - format!( - "Landlock filesystem sandbox unavailable (hard_requirement, will fail): {availability}" - ), - ) - }; - openshell_ocsf::ocsf_emit!( - openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(openshell_ocsf::ActivityId::Open) - .severity(openshell_ocsf::SeverityId::High) - .confidence(openshell_ocsf::ConfidenceId::High) - .is_alert(true) - .finding_info( - openshell_ocsf::FindingInfo::new( - "landlock-unavailable", - "Landlock Filesystem Sandbox Unavailable", - ) - .with_desc(&desc), - ) - .message(msg) - .build() - ); - } -} diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs b/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs deleted file mode 100644 index a9c67af95a..0000000000 --- a/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs +++ /dev/null @@ -1,841 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Seccomp syscall filtering. -//! -//! The filter uses a default-allow policy with targeted blocks: -//! -//! 1. **Socket domain blocks** -- prevent raw/kernel sockets that bypass the proxy -//! 2. **Unconditional syscall blocks** -- block syscalls that enable sandbox escape -//! (fileless exec, ptrace, BPF, cross-process memory access, `io_uring`, mount) -//! 3. **Conditional syscall blocks** -- block dangerous flag combinations on otherwise -//! needed syscalls (`execveat+AT_EMPTY_PATH`, `unshare+CLONE_NEWUSER`, -//! `seccomp+SET_MODE_FILTER`) -//! -//! ## `AF_NETLINK` policy -//! -//! `AF_NETLINK` sockets are allowed **only** for the `NETLINK_ROUTE` protocol -//! (protocol value 0). All other netlink protocols are blocked with `EPERM`. -//! -//! `NETLINK_ROUTE` is required by `getifaddrs(3)` on Linux (used by Node.js, -//! Python, Go, and many HTTP/gRPC client libraries during startup). Without it -//! those runtimes fail to enumerate network interfaces even when they have no -//! intent to modify them. -//! -//! The risk is contained by existing sandbox layers: -//! - **Privilege drop**: `CAP_NET_ADMIN` is not granted, so all write operations -//! (add/delete routes, addresses, interfaces) fail with `EPERM` regardless. -//! - **Network namespace**: the sandboxed process sees only `lo` and one veth; -//! no host interfaces are visible. -//! - **nftables bypass rules**: all non-proxy traffic is rejected at the -//! netfilter level regardless of what the sandbox learns about its interfaces. -//! -//! Every other netlink protocol (`NETLINK_SOCK_DIAG`, `NETLINK_NETFILTER`, -//! `NETLINK_AUDIT`, `NETLINK_XFRM`, `NETLINK_GENERIC`, etc.) remains blocked. - -use miette::{IntoDiagnostic, Result}; -use openshell_core::policy::{NetworkMode, SandboxPolicy}; -use seccompiler::{ - SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, SeccompRule, - apply_filter, apply_filter_all_threads, -}; -use std::collections::BTreeMap; -use std::convert::TryInto; -use tracing::debug; - -/// Value of `SECCOMP_SET_MODE_FILTER` (linux/seccomp.h). -const SECCOMP_SET_MODE_FILTER: u64 = 1; - -// libc 0.2.185 omits `SYS_kexec_file_load` from the musl/aarch64 bindings even -// though the kernel exposes syscall 294. Fall back to the literal so the -// supervisor's seccomp filter still blocks fileless kernel-image loads when -// built statically against musl on aarch64. -#[cfg(all(target_arch = "aarch64", target_env = "musl"))] -#[allow(non_upper_case_globals)] -const SYS_kexec_file_load: libc::c_long = 294; -#[cfg(not(all(target_arch = "aarch64", target_env = "musl")))] -use libc::SYS_kexec_file_load; - -/// Apply the supervisor seccomp filter across the running process. -/// -/// This runs after privileged startup helpers complete and synchronizes the -/// filter across all supervisor threads via TSYNC. It intentionally blocks -/// only the privileged escape primitives that the long-lived supervisor no -/// longer needs once bootstrap is complete. -pub fn apply_supervisor_prelude() -> Result<()> { - let filter = build_supervisor_prelude_filter()?; - set_no_new_privs()?; - apply_filter_all_threads(&filter).into_diagnostic()?; - Ok(()) -} - -pub fn apply(policy: &SandboxPolicy) -> Result<()> { - let allow_inet = matches!(policy.network.mode, NetworkMode::Proxy | NetworkMode::Allow); - let main_filter = build_filter(allow_inet)?; - let clone3_filter = build_clone3_filter()?; - - set_no_new_privs()?; - apply_runtime_filters(&main_filter, &clone3_filter)?; - - Ok(()) -} - -fn build_filter(allow_inet: bool) -> Result { - let rules = build_filter_rules(allow_inet)?; - compile_filter(rules, SeccompAction::Errno(libc::EPERM as u32)) -} - -fn build_supervisor_prelude_filter() -> Result { - compile_filter( - build_supervisor_prelude_rules(), - SeccompAction::Errno(libc::EPERM as u32), - ) -} - -fn build_supervisor_prelude_rules() -> BTreeMap> { - let mut rules: BTreeMap> = BTreeMap::new(); - - for syscall in [ - libc::SYS_mount, - libc::SYS_fsopen, - libc::SYS_fsconfig, - libc::SYS_fsmount, - libc::SYS_fspick, - libc::SYS_move_mount, - libc::SYS_open_tree, - libc::SYS_pivot_root, - libc::SYS_umount2, - libc::SYS_bpf, - libc::SYS_perf_event_open, - libc::SYS_userfaultfd, - libc::SYS_init_module, - libc::SYS_finit_module, - libc::SYS_delete_module, - libc::SYS_kexec_load, - SYS_kexec_file_load, - ] { - rules.entry(syscall).or_default(); - } - - rules -} - -fn set_no_new_privs() -> Result<()> { - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - let rc = unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }; - if rc != 0 { - return Err(miette::miette!( - "Failed to set no_new_privs: {}", - std::io::Error::last_os_error() - )); - } - - Ok(()) -} - -fn compile_filter( - rules: BTreeMap>, - blocked_action: SeccompAction, -) -> Result { - let arch = std::env::consts::ARCH - .try_into() - .map_err(|_| miette::miette!("Unsupported architecture for seccomp"))?; - - let filter = - SeccompFilter::new(rules, SeccompAction::Allow, blocked_action, arch).into_diagnostic()?; - - filter.try_into().into_diagnostic() -} - -/// Build a minimal BPF filter that blocks clone3 with ENOSYS. -/// -/// This is a separate filter from the main one because seccomp BPF cannot -/// dereference the `struct clone_args *` pointer that clone3 takes as arg 0, -/// so we cannot selectively block `CLONE_NEWUSER`. We block clone3 -/// unconditionally with ENOSYS so glibc falls back to the older clone -/// syscall (where flags are a direct register argument and CAN be filtered). -/// -/// glibc's clone3 wrapper checks for ENOSYS specifically — EPERM would be -/// treated as a hard failure and propagated to the caller instead of -/// triggering the clone fallback. -fn build_clone3_filter() -> Result { - let mut rules: BTreeMap> = BTreeMap::new(); - rules.entry(libc::SYS_clone3).or_default(); - compile_filter(rules, SeccompAction::Errno(libc::ENOSYS as u32)) -} - -/// Install the sandbox seccomp filters in the required order. -/// -/// Order matters: -/// 1. Install the dedicated clone3 filter first so it can still call -/// `seccomp(SECCOMP_SET_MODE_FILTER)`. -/// 2. Install the main filter second. It blocks further seccomp filter -/// installation with `EPERM`, preserving the original hardening intent. -fn apply_runtime_filters( - main_filter: seccompiler::BpfProgramRef<'_>, - clone3_filter: seccompiler::BpfProgramRef<'_>, -) -> Result<()> { - apply_filter(clone3_filter).into_diagnostic()?; - apply_filter(main_filter).into_diagnostic()?; - Ok(()) -} - -fn build_filter_rules(allow_inet: bool) -> Result>> { - let mut rules: BTreeMap> = BTreeMap::new(); - - // --- Socket domain blocks --- - let mut blocked_domains = vec![ - libc::AF_PACKET, - libc::AF_BLUETOOTH, - libc::AF_VSOCK, - // AF_NETLINK is handled separately below: NETLINK_ROUTE (protocol 0) - // is allowed for getifaddrs(3); all other netlink protocols are blocked. - ]; - if !allow_inet { - blocked_domains.push(libc::AF_INET); - blocked_domains.push(libc::AF_INET6); - } - - for domain in blocked_domains { - debug!(domain, "Blocking socket domain via seccomp"); - add_socket_domain_rule(&mut rules, domain)?; - } - - // Allow AF_NETLINK only for NETLINK_ROUTE (protocol 0). - // - // NETLINK_ROUTE is needed by getifaddrs(3) which is called by Node.js, - // Python, Go, and many HTTP/gRPC client libraries during startup to - // enumerate local network interfaces. Blocking it causes runtime errors - // such as "getifaddrs returned an error" in tools like Claude Code. - // - // The rule blocks socket(AF_NETLINK, *, protocol) for any protocol != 0. - // Write operations via NETLINK_ROUTE still require CAP_NET_ADMIN, which - // the sandbox does not grant, so interface/route modification is not possible. - add_netlink_non_route_rule(&mut rules)?; - - // --- Unconditional syscall blocks --- - // These syscalls are blocked entirely (empty rule vec = unconditional EPERM). - - // Fileless binary execution via memfd bypasses Landlock filesystem restrictions. - rules.entry(libc::SYS_memfd_create).or_default(); - // Cross-process memory inspection and code injection. - rules.entry(libc::SYS_ptrace).or_default(); - // Kernel BPF program loading. - rules.entry(libc::SYS_bpf).or_default(); - // Cross-process memory read. - rules.entry(libc::SYS_process_vm_readv).or_default(); - // Cross-process memory write (symmetric with process_vm_readv). - rules.entry(libc::SYS_process_vm_writev).or_default(); - // Process handle acquisition, fd theft, and signalling via pidfd. - rules.entry(libc::SYS_pidfd_open).or_default(); - rules.entry(libc::SYS_pidfd_getfd).or_default(); - rules.entry(libc::SYS_pidfd_send_signal).or_default(); - // Async I/O subsystem with extensive CVE history. - rules.entry(libc::SYS_io_uring_setup).or_default(); - // Filesystem mount could subvert Landlock or overlay writable paths. - rules.entry(libc::SYS_mount).or_default(); - // New mount API syscalls (Linux 5.2+) bypass the SYS_mount block entirely. - rules.entry(libc::SYS_fsopen).or_default(); - rules.entry(libc::SYS_fsconfig).or_default(); - rules.entry(libc::SYS_fsmount).or_default(); - rules.entry(libc::SYS_fspick).or_default(); - rules.entry(libc::SYS_move_mount).or_default(); - rules.entry(libc::SYS_open_tree).or_default(); - // Namespace manipulation — setns enters existing namespaces, pivot_root/umount2 - // change the filesystem root. The supervisor calls setns before seccomp is applied, - // so blocking it here is safe. - rules.entry(libc::SYS_setns).or_default(); - rules.entry(libc::SYS_umount2).or_default(); - rules.entry(libc::SYS_pivot_root).or_default(); - // Kernel exploit primitives: userfaultfd enables race-condition exploitation (multiple - // CVEs), perf_event_open enables Spectre-class side channels. Both blocked by Docker's - // default seccomp profile. - rules.entry(libc::SYS_userfaultfd).or_default(); - rules.entry(libc::SYS_perf_event_open).or_default(); - - // --- Conditional syscall blocks --- - - // execveat with AT_EMPTY_PATH enables fileless execution from an anonymous fd. - add_masked_arg_rule( - &mut rules, - libc::SYS_execveat, - 4, // flags argument - libc::AT_EMPTY_PATH as u64, - )?; - - // unshare with CLONE_NEWUSER allows creating user namespaces to escalate privileges. - add_masked_arg_rule( - &mut rules, - libc::SYS_unshare, - 0, // flags argument - libc::CLONE_NEWUSER as u64, - )?; - - // clone with CLONE_NEWUSER achieves the same as unshare via a different syscall. - add_masked_arg_rule( - &mut rules, - libc::SYS_clone, - 0, // flags argument - libc::CLONE_NEWUSER as u64, - )?; - // clone3 is handled by a separate filter — see build_clone3_filter(). - - // seccomp(SECCOMP_SET_MODE_FILTER) would let sandboxed code replace the active filter. - let condition = SeccompCondition::new( - 0, // operation argument - SeccompCmpArgLen::Dword, - SeccompCmpOp::Eq, - SECCOMP_SET_MODE_FILTER, - ) - .into_diagnostic()?; - let rule = SeccompRule::new(vec![condition]).into_diagnostic()?; - rules.entry(libc::SYS_seccomp).or_default().push(rule); - - Ok(rules) -} - -#[allow(clippy::cast_sign_loss)] -fn add_socket_domain_rule(rules: &mut BTreeMap>, domain: i32) -> Result<()> { - let condition = - SeccompCondition::new(0, SeccompCmpArgLen::Dword, SeccompCmpOp::Eq, domain as u64) - .into_diagnostic()?; - - let rule = SeccompRule::new(vec![condition]).into_diagnostic()?; - rules.entry(libc::SYS_socket).or_default().push(rule); - Ok(()) -} - -/// Block `socket(AF_NETLINK, *, protocol)` for every protocol except -/// `NETLINK_ROUTE` (protocol 0). -/// -/// Two AND'd conditions are required: -/// - arg0 == `AF_NETLINK` (domain) -/// - arg2 != 0 (protocol is not `NETLINK_ROUTE`) -/// -/// A seccomp rule fires (and returns EPERM) only when **all** conditions -/// match, so this rule is triggered for any `socket(AF_NETLINK, *, non-zero)` -/// call while leaving `socket(AF_NETLINK, *, 0)` (`NETLINK_ROUTE`) through. -#[allow(clippy::cast_sign_loss)] -fn add_netlink_non_route_rule(rules: &mut BTreeMap>) -> Result<()> { - let domain_condition = SeccompCondition::new( - 0, // domain argument - SeccompCmpArgLen::Dword, - SeccompCmpOp::Eq, - libc::AF_NETLINK as u64, - ) - .into_diagnostic()?; - - let protocol_condition = SeccompCondition::new( - 2, // protocol argument - SeccompCmpArgLen::Dword, - SeccompCmpOp::Ne, - 0, // NETLINK_ROUTE = 0 - ) - .into_diagnostic()?; - - let rule = SeccompRule::new(vec![domain_condition, protocol_condition]).into_diagnostic()?; - rules.entry(libc::SYS_socket).or_default().push(rule); - Ok(()) -} - -/// Block a syscall when a specific bit pattern is set in an argument. -/// -/// Uses `MaskedEq` to check `(arg & flag_bit) == flag_bit`, which triggers -/// EPERM when the flag is present regardless of other bits in the argument. -fn add_masked_arg_rule( - rules: &mut BTreeMap>, - syscall: i64, - arg_index: u8, - flag_bit: u64, -) -> Result<()> { - let condition = SeccompCondition::new( - arg_index, - SeccompCmpArgLen::Dword, - SeccompCmpOp::MaskedEq(flag_bit), - flag_bit, - ) - .into_diagnostic()?; - let rule = SeccompRule::new(vec![condition]).into_diagnostic()?; - rules.entry(syscall).or_default().push(rule); - Ok(()) -} - -#[cfg(test)] -// libc/syscall FFI requires unsafe; these tests fork children and exercise -// blocked syscalls, so unsafe blocks/calls are pervasive. -#[allow( - unsafe_code, - unsafe_op_in_unsafe_fn, - unused_unsafe, - clippy::borrow_as_ptr, - trivial_numeric_casts -)] -mod tests { - use super::*; - - // These tests cover both filter construction (rule map shape and BPF - // compilation) and selected runtime behavior on Linux via forked children. - - #[test] - fn build_filter_proxy_mode_compiles() { - let filter = build_filter(true); - assert!(filter.is_ok(), "build_filter(true) should succeed"); - } - - #[test] - fn build_filter_block_mode_compiles() { - let filter = build_filter(false); - assert!(filter.is_ok(), "build_filter(false) should succeed"); - } - - #[test] - fn build_supervisor_prelude_filter_compiles() { - let filter = build_supervisor_prelude_filter(); - assert!( - filter.is_ok(), - "build_supervisor_prelude_filter() should succeed" - ); - } - - #[test] - fn add_masked_arg_rule_creates_entry() { - let mut rules: BTreeMap> = BTreeMap::new(); - let result = add_masked_arg_rule(&mut rules, libc::SYS_execveat, 4, 0x1000); - assert!(result.is_ok()); - assert!( - rules.contains_key(&libc::SYS_execveat), - "should have an entry for SYS_execveat" - ); - assert_eq!( - rules[&libc::SYS_execveat].len(), - 1, - "should have exactly one rule" - ); - } - - #[test] - fn unconditional_blocks_present_in_filter() { - // Build a real filter and verify all unconditional blocks are present. - let filter_rules = build_filter_rules(true).unwrap(); - - // Unconditional blocks have an empty Vec (no conditions = always match). - let expected = [ - libc::SYS_memfd_create, - libc::SYS_ptrace, - libc::SYS_bpf, - libc::SYS_process_vm_readv, - libc::SYS_process_vm_writev, - libc::SYS_pidfd_open, - libc::SYS_pidfd_getfd, - libc::SYS_pidfd_send_signal, - libc::SYS_io_uring_setup, - libc::SYS_mount, - libc::SYS_fsopen, - libc::SYS_fsconfig, - libc::SYS_fsmount, - libc::SYS_fspick, - libc::SYS_move_mount, - libc::SYS_open_tree, - libc::SYS_setns, - libc::SYS_umount2, - libc::SYS_pivot_root, - libc::SYS_userfaultfd, - libc::SYS_perf_event_open, - ]; - - for syscall in expected { - assert!( - filter_rules.contains_key(&syscall), - "syscall {syscall} should be in the rules map" - ); - assert!( - filter_rules[&syscall].is_empty(), - "syscall {syscall} should have empty rules (unconditional block)" - ); - } - } - - #[test] - fn conditional_blocks_have_rules() { - // Build a real filter and verify the conditional syscalls have rule entries - // (non-empty Vec means conditional match). - let filter_rules = build_filter_rules(true).unwrap(); - - for syscall in [ - libc::SYS_execveat, - libc::SYS_unshare, - libc::SYS_clone, - libc::SYS_seccomp, - ] { - assert!( - filter_rules.contains_key(&syscall), - "syscall {syscall} should be in the rules map" - ); - assert!( - !filter_rules[&syscall].is_empty(), - "syscall {syscall} should have conditional rules" - ); - } - } - - #[test] - fn netlink_socket_rules_are_conditional_not_unconditional() { - // SYS_socket must appear in the rules map (for domain blocks and the - // AF_NETLINK+non-ROUTE filter), but it must NOT be an unconditional block - // (empty Vec). An empty Vec would block ALL socket() calls, including - // socket(AF_NETLINK, *, NETLINK_ROUTE=0) which getifaddrs(3) needs. - let filter_rules = build_filter_rules(true).unwrap(); - - assert!( - filter_rules.contains_key(&libc::SYS_socket), - "SYS_socket should be in the rules map (domain blocks present)" - ); - - // The Vec for SYS_socket must be non-empty (rules are - // conditional), which is the opposite of an unconditional block. - assert!( - !filter_rules[&libc::SYS_socket].is_empty(), - "SYS_socket should have conditional rules, not an unconditional block" - ); - } - - #[test] - fn supervisor_prelude_blocks_expected_syscalls() { - let filter_rules = build_supervisor_prelude_rules(); - - for syscall in [ - libc::SYS_mount, - libc::SYS_fsopen, - libc::SYS_fsconfig, - libc::SYS_fsmount, - libc::SYS_fspick, - libc::SYS_move_mount, - libc::SYS_open_tree, - libc::SYS_pivot_root, - libc::SYS_umount2, - libc::SYS_bpf, - libc::SYS_perf_event_open, - libc::SYS_userfaultfd, - libc::SYS_init_module, - libc::SYS_finit_module, - libc::SYS_delete_module, - libc::SYS_kexec_load, - SYS_kexec_file_load, - ] { - assert!( - filter_rules.contains_key(&syscall), - "syscall {syscall} should be in the supervisor prelude rules" - ); - assert!( - filter_rules[&syscall].is_empty(), - "syscall {syscall} should be unconditionally blocked in the supervisor prelude" - ); - } - } - - #[test] - fn supervisor_prelude_keeps_required_setup_syscalls_available() { - let filter_rules = build_supervisor_prelude_rules(); - - for syscall in [ - libc::SYS_setns, - libc::SYS_clone, - libc::SYS_unshare, - libc::SYS_ptrace, - ] { - assert!( - !filter_rules.contains_key(&syscall), - "syscall {syscall} should remain available during supervisor startup" - ); - } - } - - #[test] - fn clone3_filter_compiles_and_blocks_clone3() { - let bpf = build_clone3_filter(); - assert!(bpf.is_ok(), "clone3 ENOSYS filter should compile"); - } - - #[test] - fn clone3_not_in_main_filter() { - // clone3 must NOT be in the main filter; it has its own ENOSYS filter. - let filter_rules = build_filter_rules(true).unwrap(); - assert!( - !filter_rules.contains_key(&libc::SYS_clone3), - "clone3 should not be in the main filter — it uses a separate ENOSYS filter" - ); - } - - // --- Behavioral tests --- - // - // These apply seccomp filters in a forked child and verify that blocked - // syscalls actually return the expected errno. They only compile and run - // on Linux (seccomp is a Linux kernel feature). - - /// Fork a child, apply the given filter, invoke `syscall_nr`, and return - /// the errno observed by the child. The child exits 0 if the syscall - /// returned the expected errno, 1 otherwise. - unsafe fn assert_blocked_in_child( - filter: &seccompiler::BpfProgram, - syscall_nr: i64, - expected_errno: i32, - ) { - let pid = libc::fork(); - assert!(pid >= 0, "fork failed"); - if pid == 0 { - // Child: apply filter and try the syscall. - libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - apply_filter(filter).expect("apply_filter"); - let ret = libc::syscall(syscall_nr, 0 as libc::c_ulong, 0 as libc::c_ulong); - let errno = *libc::__errno_location(); - if ret == -1 && errno == expected_errno { - libc::_exit(0); - } else { - // Write diagnostic before exiting so test failures are debuggable. - let msg = format!( - "syscall {syscall_nr}: expected errno={expected_errno}, got ret={ret} errno={errno}\n" - ); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - // Parent: wait for child. - let mut status: libc::c_int = 0; - libc::waitpid(pid, &mut status, 0); - assert!( - libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, - "child failed: syscall {syscall_nr} was not blocked with errno {expected_errno}" - ); - } - - unsafe fn install_runtime_filters_in_child( - main_filter: &seccompiler::BpfProgram, - clone3_filter: &seccompiler::BpfProgram, - ) { - libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - if let Err(err) = apply_runtime_filters(main_filter, clone3_filter) { - let msg = format!("failed to install runtime seccomp filters: {err}\n"); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - - #[test] - fn behavioral_memfd_create_blocked() { - let filter = build_filter(true).unwrap(); - unsafe { assert_blocked_in_child(&filter, libc::SYS_memfd_create, libc::EPERM) }; - } - - #[test] - fn behavioral_ptrace_blocked() { - let filter = build_filter(true).unwrap(); - unsafe { assert_blocked_in_child(&filter, libc::SYS_ptrace, libc::EPERM) }; - } - - #[test] - fn behavioral_process_vm_writev_blocked() { - let filter = build_filter(true).unwrap(); - unsafe { assert_blocked_in_child(&filter, libc::SYS_process_vm_writev, libc::EPERM) }; - } - - #[test] - fn behavioral_userfaultfd_blocked() { - let filter = build_filter(true).unwrap(); - unsafe { assert_blocked_in_child(&filter, libc::SYS_userfaultfd, libc::EPERM) }; - } - - #[test] - fn behavioral_perf_event_open_blocked() { - let filter = build_filter(true).unwrap(); - unsafe { assert_blocked_in_child(&filter, libc::SYS_perf_event_open, libc::EPERM) }; - } - - #[test] - fn behavioral_setns_blocked() { - let filter = build_filter(true).unwrap(); - unsafe { assert_blocked_in_child(&filter, libc::SYS_setns, libc::EPERM) }; - } - - #[test] - fn behavioral_supervisor_prelude_mount_blocked() { - let pid = unsafe { libc::fork() }; - assert!(pid >= 0, "fork failed"); - if pid == 0 { - unsafe { - if let Err(err) = apply_supervisor_prelude() { - let msg = format!("failed to install supervisor prelude: {err}\n"); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - let ret = libc::syscall( - libc::SYS_mount, - std::ptr::null::(), - std::ptr::null::(), - std::ptr::null::(), - 0 as libc::c_ulong, - std::ptr::null::(), - ); - let errno = *libc::__errno_location(); - if ret == -1 && errno == libc::EPERM { - libc::_exit(0); - } else { - let msg = format!( - "mount: expected EPERM after supervisor prelude, got ret={ret} errno={errno}\n" - ); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - } - - let mut status: libc::c_int = 0; - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!( - unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, - "mount should be blocked by the supervisor prelude filter" - ); - } - - #[test] - fn behavioral_clone3_returns_enosys() { - // clone3 uses a separate filter that returns ENOSYS (not EPERM) so - // glibc falls back to clone. - let main_filter = build_filter(true).unwrap(); - let clone3_filter = build_clone3_filter().unwrap(); - // Apply in the same order as apply(): clone3 filter first, main filter second. - let pid = unsafe { libc::fork() }; - assert!(pid >= 0, "fork failed"); - if pid == 0 { - unsafe { - install_runtime_filters_in_child(&main_filter, &clone3_filter); - let ret = libc::syscall(libc::SYS_clone3, 0 as libc::c_ulong, 0 as libc::c_ulong); - let errno = *libc::__errno_location(); - if ret == -1 && errno == libc::ENOSYS { - libc::_exit(0); - } else { - let msg = format!("clone3: expected ENOSYS, got ret={ret} errno={errno}\n"); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - } - let mut status: libc::c_int = 0; - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!( - unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, - "clone3 should be blocked with ENOSYS, not EPERM" - ); - } - - #[test] - fn behavioral_third_filter_install_blocked_after_startup() { - let main_filter = build_filter(true).unwrap(); - let clone3_filter = build_clone3_filter().unwrap(); - let third_filter = build_clone3_filter().unwrap(); - - let pid = unsafe { libc::fork() }; - assert!(pid >= 0, "fork failed"); - if pid == 0 { - unsafe { - install_runtime_filters_in_child(&main_filter, &clone3_filter); - match apply_filter(&third_filter) { - Err(seccompiler::Error::Seccomp(e)) - if e.raw_os_error() == Some(libc::EPERM) => - { - libc::_exit(0); - } - Err(err) => { - let msg = - format!("third filter install failed with unexpected error: {err}\n"); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - Ok(()) => { - let msg = "third filter unexpectedly installed\n"; - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - } - } - - let mut status: libc::c_int = 0; - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!( - unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, - "additional seccomp filter installation should be blocked after startup" - ); - } - - #[test] - fn behavioral_netlink_route_allowed() { - // socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE=0) must succeed (not blocked). - // This is the call getifaddrs(3) makes on Linux to enumerate interfaces. - let filter = build_filter(true).unwrap(); - let pid = unsafe { libc::fork() }; - assert!(pid >= 0, "fork failed"); - if pid == 0 { - unsafe { - libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - apply_filter(&filter).expect("apply_filter"); - // NETLINK_ROUTE = 0 - let fd = libc::socket(libc::AF_NETLINK, libc::SOCK_RAW, 0); - if fd >= 0 { - libc::close(fd); - libc::_exit(0); - } else { - let errno = *libc::__errno_location(); - let msg = format!( - "socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE): expected success, got errno={errno}\n" - ); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - } - let mut status: libc::c_int = 0; - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!( - unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, - "socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE) should be allowed for getifaddrs(3)" - ); - } - - #[test] - fn behavioral_netlink_non_route_blocked() { - // socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG=4) must be blocked. - // NETLINK_SOCK_DIAG is representative of non-ROUTE netlink protocols - // that have no legitimate use inside the sandbox. - let filter = build_filter(true).unwrap(); - let pid = unsafe { libc::fork() }; - assert!(pid >= 0, "fork failed"); - if pid == 0 { - unsafe { - libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - apply_filter(&filter).expect("apply_filter"); - // NETLINK_SOCK_DIAG = 4 - let fd = libc::socket(libc::AF_NETLINK, libc::SOCK_RAW, 4); - let errno = *libc::__errno_location(); - if fd == -1 && errno == libc::EPERM { - libc::_exit(0); - } else { - if fd >= 0 { - libc::close(fd); - } - let msg = format!( - "socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG): expected EPERM, got fd={fd} errno={errno}\n" - ); - libc::write(2, msg.as_ptr().cast(), msg.len()); - libc::_exit(1); - } - } - } - let mut status: libc::c_int = 0; - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!( - unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, - "socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG) should be blocked with EPERM" - ); - } -} diff --git a/crates/openshell-supervisor-process/src/sandbox/mod.rs b/crates/openshell-supervisor-process/src/sandbox/mod.rs deleted file mode 100644 index ff44f8ba10..0000000000 --- a/crates/openshell-supervisor-process/src/sandbox/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Platform sandboxing implementation. - -use miette::Result; -use openshell_core::policy::SandboxPolicy; - -#[cfg(target_os = "linux")] -pub mod linux; - -/// Apply sandboxing rules for the current platform. -/// -/// # Errors -/// -/// Returns an error if the sandbox cannot be applied. -// On Linux the spawn path uses `prepare`+`enforce` directly; this single-phase -// apply is only invoked from the non-Linux spawn_impl. -#[cfg_attr(target_os = "linux", allow(dead_code))] -#[cfg_attr(not(target_os = "linux"), allow(clippy::unnecessary_wraps))] -pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { - #[cfg(target_os = "linux")] - { - linux::apply(policy, workdir) - } - - #[cfg(not(target_os = "linux"))] - { - let _ = (policy, workdir); - openshell_ocsf::ocsf_emit!( - openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(openshell_ocsf::ActivityId::Open) - .severity(openshell_ocsf::SeverityId::Medium) - .finding_info(openshell_ocsf::FindingInfo::new( - "platform-sandbox-unavailable", - "Platform Sandboxing Not Implemented", - ).with_desc("Sandbox policy provided but platform sandboxing is not yet implemented on this OS")) - .message("Platform sandboxing not yet implemented") - .build() - ); - Ok(()) - } -} - -/// Apply seccomp hardening for the long-lived supervisor process itself. -#[cfg_attr(not(target_os = "linux"), allow(clippy::unnecessary_wraps))] -pub fn apply_supervisor_startup_hardening() -> Result<()> { - #[cfg(target_os = "linux")] - { - linux::apply_supervisor_prelude() - } - - #[cfg(not(target_os = "linux"))] - { - Ok(()) - } -} From d51a3337c0e165cb0e3a7eecb0a38d6066a97ddd Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 22:14:02 -0700 Subject: [PATCH 10/10] ci(e2e): select migrated drivers per stack layer Signed-off-by: Drew Newberry --- .github/actions/check-job-results/action.yml | 14 ++++- .github/workflows/branch-e2e.yml | 60 ++++++++++++++++---- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/.github/actions/check-job-results/action.yml b/.github/actions/check-job-results/action.yml index bd6beb9456..442e074b29 100644 --- a/.github/actions/check-job-results/action.yml +++ b/.github/actions/check-job-results/action.yml @@ -8,6 +8,10 @@ inputs: results: description: JSON-encoded GitHub Actions needs context required: true + allowed-skipped-jobs: + description: Comma-separated job IDs that may be skipped but must not fail + required: false + default: "" runs: using: composite @@ -16,12 +20,20 @@ runs: shell: bash env: JOB_RESULTS: ${{ inputs.results }} + ALLOWED_SKIPPED_JOBS: ${{ inputs.allowed-skipped-jobs }} run: | set -euo pipefail failures="$( - jq -r ' + jq -r --arg allowed_skipped "$ALLOWED_SKIPPED_JOBS" ' + ($allowed_skipped | split(",") | map(select(length > 0))) as $allowed_skipped_jobs + | to_entries[] + | . as $job | select(.value.result != "success") + | select( + .value.result != "skipped" + or ($allowed_skipped_jobs | index($job.key)) == null + ) | "\(.key) concluded \(.value.result)" ' <<< "$JOB_RESULTS" )" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 4c894e3ea3..dcdc3905f6 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -28,6 +28,11 @@ jobs: run_kubernetes_ha_e2e: ${{ steps.labels.outputs.run_kubernetes_ha_e2e }} run_kubernetes_credential_drivers_e2e: ${{ steps.labels.outputs.run_kubernetes_credential_drivers_e2e }} run_any_e2e: ${{ steps.labels.outputs.run_any_e2e }} + run_docker_e2e: ${{ steps.labels.outputs.run_docker_e2e }} + run_podman_e2e: ${{ steps.labels.outputs.run_podman_e2e }} + run_vm_e2e: ${{ steps.labels.outputs.run_vm_e2e }} + run_kubernetes_e2e: ${{ steps.labels.outputs.run_kubernetes_e2e }} + allowed_skipped_core_jobs: ${{ steps.labels.outputs.allowed_skipped_core_jobs }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - id: gate @@ -68,12 +73,46 @@ jobs: else run_any_e2e=false fi + + run_docker_e2e="$run_core_e2e" + run_podman_e2e="$run_core_e2e" + run_vm_e2e="$run_core_e2e" + run_kubernetes_e2e="$run_core_e2e" + allowed_skipped_core_jobs="" + + # The RFC 0012 stack deliberately introduces the split runtime before + # migrating each driver. At those intermediate layers, run only the + # drivers whose isolation adapter is present. Before and after the + # stack this resolves to the complete legacy or migrated driver set. + if [ "$run_core_e2e" = "true" ] && [ -f crates/openshell-supervisor/Cargo.toml ]; then + if [ ! -f crates/openshell-driver-docker/src/isolation.rs ]; then + run_docker_e2e=false + allowed_skipped_core_jobs="docker-e2e,docker-external-driver-e2e" + fi + if [ ! -f crates/openshell-driver-podman/src/isolation.rs ]; then + run_podman_e2e=false + allowed_skipped_core_jobs="${allowed_skipped_core_jobs:+$allowed_skipped_core_jobs,}podman-e2e,podman-external-driver-e2e" + fi + if [ ! -f crates/openshell-driver-vm/src/isolation/mod.rs ]; then + run_vm_e2e=false + allowed_skipped_core_jobs="${allowed_skipped_core_jobs:+$allowed_skipped_core_jobs,}vm-e2e,vm-external-driver-e2e" + fi + if [ ! -f crates/openshell-driver-kubernetes/src/isolation.rs ]; then + run_kubernetes_e2e=false + allowed_skipped_core_jobs="${allowed_skipped_core_jobs:+$allowed_skipped_core_jobs,}kubernetes-e2e,kubernetes-external-driver-e2e,kubernetes-workspace-managed-e2e,kubernetes-workspace-operator-e2e" + fi + fi { echo "run_core_e2e=$run_core_e2e" echo "run_gpu_e2e=$run_gpu_e2e" echo "run_kubernetes_ha_e2e=$run_kubernetes_ha_e2e" echo "run_kubernetes_credential_drivers_e2e=$run_kubernetes_credential_drivers_e2e" echo "run_any_e2e=$run_any_e2e" + echo "run_docker_e2e=$run_docker_e2e" + echo "run_podman_e2e=$run_podman_e2e" + echo "run_vm_e2e=$run_vm_e2e" + echo "run_kubernetes_e2e=$run_kubernetes_e2e" + echo "allowed_skipped_core_jobs=$allowed_skipped_core_jobs" } >> "$GITHUB_OUTPUT" version: @@ -268,7 +307,7 @@ jobs: docker-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_docker_e2e == 'true' permissions: actions: read contents: read @@ -281,7 +320,7 @@ jobs: podman-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_podman_e2e == 'true' permissions: actions: read contents: read @@ -293,7 +332,7 @@ jobs: vm-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-vm-driver] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_vm_e2e == 'true' permissions: actions: read contents: read @@ -304,7 +343,7 @@ jobs: docker-external-driver-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-driver-docker, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_docker_e2e == 'true' permissions: actions: read contents: read @@ -321,7 +360,7 @@ jobs: podman-external-driver-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-driver-podman, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_podman_e2e == 'true' permissions: actions: read contents: read @@ -337,7 +376,7 @@ jobs: vm-external-driver-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-vm-driver] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_vm_e2e == 'true' permissions: actions: read contents: read @@ -363,7 +402,7 @@ jobs: kubernetes-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_e2e == 'true' strategy: fail-fast: false matrix: @@ -394,7 +433,7 @@ jobs: kubernetes-workspace-managed-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_e2e == 'true' permissions: actions: read contents: read @@ -408,7 +447,7 @@ jobs: kubernetes-external-driver-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-driver-kubernetes, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_e2e == 'true' permissions: actions: read contents: read @@ -425,7 +464,7 @@ jobs: kubernetes-workspace-operator-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_e2e == 'true' permissions: actions: read contents: read @@ -478,6 +517,7 @@ jobs: - uses: ./.github/actions/check-job-results with: results: ${{ toJSON(needs) }} + allowed-skipped-jobs: ${{ needs.pr_metadata.outputs.allowed_skipped_core_jobs }} gpu-e2e-result: name: GPU E2E result