diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcdc..948ea01b1 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -57,6 +57,10 @@ public final class LinuxContainer: Container, Sendable { public var cpus: Int = 4 /// The memory in bytes to give to the container. public var memoryInBytes: UInt64 = 1024.mib() + /// The optional maximum number of processes for the container. + /// + /// OCI semantics are preserved: `-1` means unlimited and `0` is a valid limit. + public var pidsLimit: Int64? /// The hostname for the container. public var hostname: String? /// The system control options for the container. @@ -66,6 +70,9 @@ public final class LinuxContainer: Container, Sendable { /// The Unix domain socket relays to setup for the container. public var sockets: [UnixSocketConfiguration] = [] /// The mounts for the container. + /// + /// When ``pidsLimit`` is finite, cgroup2 mounts and mounts at or below + /// `/sys/fs/cgroup` are made read-only before the container is created. public var mounts: [Mount] = LinuxContainer.defaultMounts() /// Paths inside the container that vmexec hides from the workload. /// Defaults to the OCI standard set (``LinuxContainer/defaultMaskedPaths()``), @@ -105,6 +112,7 @@ public final class LinuxContainer: Container, Sendable { process: LinuxProcessConfiguration, cpus: Int = 4, memoryInBytes: UInt64 = 1024.mib(), + pidsLimit: Int64? = nil, hostname: String? = nil, sysctl: [String: String] = [:], interfaces: [any Interface] = [], @@ -124,6 +132,7 @@ public final class LinuxContainer: Container, Sendable { self.process = process self.cpus = cpus self.memoryInBytes = memoryInBytes + self.pidsLimit = pidsLimit self.hostname = hostname self.sysctl = sysctl self.interfaces = interfaces @@ -349,12 +358,23 @@ public final class LinuxContainer: Container, Sendable { configuration: LinuxContainer.Configuration, logger: Logger? = nil ) throws { + var configuration = configuration guard id.count <= Self.maxIDLength else { throw ContainerizationError( .invalidArgument, message: "container id length \(id.count) exceeds maximum of \(Self.maxIDLength) characters" ) } + if let pidsLimit = configuration.pidsLimit, pidsLimit < -1 { + throw ContainerizationError( + .invalidArgument, + message: "pidsLimit must be greater than or equal to -1" + ) + } + configuration.mounts = Self.mountsEnforcingPidsLimit( + configuration.mounts, + pidsLimit: configuration.pidsLimit + ) if let writableLayer { guard writableLayer.isBlock else { throw ContainerizationError( @@ -426,7 +446,8 @@ public final class LinuxContainer: Container, Sendable { cpu: LinuxCPU( quota: Int64(config.cpus * 100_000), period: 100_000 - ) + ), + pids: config.pidsLimit.map { LinuxPids(limit: $0) } ) spec.linux?.namespaces = [ @@ -506,6 +527,36 @@ public final class LinuxContainer: Container, Sendable { ] } + /// Prevent a workload with a finite PID ceiling from raising or removing + /// that ceiling through a guest-visible cgroup mount. Unlimited and omitted + /// limits retain the existing mount behavior. + static func mountsEnforcingPidsLimit(_ mounts: [Mount], pidsLimit: Int64?) -> [Mount] { + guard let pidsLimit, pidsLimit >= 0 else { + return mounts + } + + return mounts.map { mount in + let destination = FilePath(mount.destination).lexicallyNormalized().string + let source = FilePath(mount.source).lexicallyNormalized().string + let exposesCgroup = + mount.type == "cgroup2" || Self.isCgroupPath(destination) || Self.isCgroupPath(source) + guard exposesCgroup else { + return mount + } + + var mount = mount + mount.options.removeAll { $0 == "rw" } + if !mount.options.contains("ro") { + mount.options.append("ro") + } + return mount + } + } + + private static func isCgroupPath(_ path: String) -> Bool { + path == "/sys/fs/cgroup" || path.hasPrefix("/sys/fs/cgroup/") + } + private static func guestRootfsPath(_ id: String) -> String { "/run/container/\(id)/rootfs" } diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index 9c8edd880..a4a418640 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -905,6 +905,14 @@ extension IntegrationSuite { config.process.arguments = ["sleep", "infinity"] config.cpus = 2 config.memoryInBytes = 512.mib() + config.pidsLimit = 64 + config.mounts.append( + .any( + type: "none", + source: "/sys/fs/cgroup", + destination: "/alternate-cgroup", + options: ["bind", "rw"] + )) config.bootLog = bs.bootLog } @@ -981,6 +989,71 @@ extension IntegrationSuite { throw IntegrationError.assert(msg: "cpu.max '\(cpuLimit)' != expected '\(expectedCpu)'") } + // Verify PID limit + let pidsBuffer = BufferWriter() + let pidsExec = try await container.exec("check-pids") { config in + config.arguments = ["cat", "/sys/fs/cgroup/pids.max"] + config.stdout = pidsBuffer + } + try await pidsExec.start() + status = try await pidsExec.wait() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "check-pids status \(status) != 0") + } + try await pidsExec.delete() + + guard let pidsLimit = String(data: pidsBuffer.data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) else { + throw IntegrationError.assert(msg: "failed to parse pids.max") + } + let expectedPids = "64" + guard pidsLimit == expectedPids else { + throw IntegrationError.assert(msg: "pids.max '\(pidsLimit)' != expected '\(expectedPids)'") + } + + // A workload must not be able to weaken its own PID ceiling. The + // management agent applies resources from a separate mount + // namespace, so the container-visible cgroup filesystem can and + // should be read-only. + for (processID, attemptedLimit) in [("raise-pids", "65"), ("unlimit-pids", "max")] { + for (pathID, path) in [ + ("canonical", "/sys/fs/cgroup/pids.max"), + ("bind-alias", "/alternate-cgroup/pids.max"), + ] { + let mutationError = BufferWriter() + let mutationExec = try await container.exec("\(processID)-\(pathID)") { config in + config.arguments = ["sh", "-c", "echo \(attemptedLimit) > \(path)"] + config.stderr = mutationError + } + try await mutationExec.start() + status = try await mutationExec.wait() + try await mutationExec.delete() + guard status.exitCode != 0 else { + throw IntegrationError.assert( + msg: "workload unexpectedly changed \(path) to \(attemptedLimit)" + ) + } + } + + let verifyBuffer = BufferWriter() + let verifyExec = try await container.exec("verify-\(processID)") { config in + config.arguments = ["cat", "/sys/fs/cgroup/pids.max"] + config.stdout = verifyBuffer + } + try await verifyExec.start() + status = try await verifyExec.wait() + try await verifyExec.delete() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "verify-\(processID) status \(status) != 0") + } + let observedLimit = String(data: verifyBuffer.data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) + guard observedLimit == expectedPids else { + throw IntegrationError.assert( + msg: "pids.max '\(observedLimit ?? "")' changed after rejected write" + ) + } + } + try await sleepExec.delete() try await container.kill(.kill) @@ -992,12 +1065,341 @@ extension IntegrationSuite { } } + func testPidsLimitExhaustion() async throws { + let id = "test-pids-limit-exhaustion" + let limit: UInt64 = 8 + let extraMarker = "/tmp/pids-extra-marker" + let forkMarker = "/tmp/pids-fork-marker" + + let bs = try await bootstrap(id) + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["sleep", "infinity"] + config.pidsLimit = Int64(limit) + config.bootLog = bs.bootLog + } + + var sleepers: [LinuxProcess] = [] + + func terminate(_ process: LinuxProcess) async { + try? await process.kill(.kill) + _ = try? await process.wait(timeoutInSeconds: 5) + try? await process.delete() + } + + do { + try await container.create() + try await container.start() + + // The init process plus these seven direct sleep execs hold the + // cgroup exactly at pids.max without an unbounded fork workload. + for index in 0..<7 { + let process = try await container.exec("pids-sleeper-\(index)") { config in + config.arguments = ["sleep", "infinity"] + } + try await process.start() + sleepers.append(process) + } + + let cappedStats = try await Timeout.run(for: .seconds(5)) { + try await container.statistics(categories: .process) + } + guard let cappedPids = cappedStats.process else { + throw IntegrationError.assert(msg: "missing process statistics at pids.max") + } + guard cappedPids.current == limit, cappedPids.limit == limit else { + throw IntegrationError.assert( + msg: "expected process statistics \(limit)/\(limit), got \(cappedPids.current)/\(cappedPids.limit)" + ) + } + + // Repeated admission attempts must fail promptly and must never + // execute the requested workload marker. + for index in 0..<3 { + let denied = try await container.exec("pids-denied-\(index)") { config in + config.arguments = ["sh", "-c", "touch \(extraMarker)"] + } + + var startError: Error? + do { + try await Timeout.run(for: .seconds(5)) { + try await denied.start() + } + } catch { + startError = error + } + + guard let startError else { + _ = try? await denied.wait(timeoutInSeconds: 5) + try? await denied.delete() + throw IntegrationError.assert(msg: "process admission unexpectedly succeeded at pids.max") + } + guard !(startError is CancellationError) else { + try? await denied.delete() + throw IntegrationError.assert(msg: "process admission did not fail within five seconds") + } + try await denied.delete() + } + + // Signal, wait, and delete are management-plane RPCs. They must + // remain responsive while the workload cgroup is full. + let released = sleepers.removeLast() + try await Timeout.run(for: .seconds(5)) { + try await released.kill(.kill) + } + _ = try await Timeout.run(for: .seconds(5)) { + try await released.wait(timeoutInSeconds: 5) + } + try await Timeout.run(for: .seconds(5)) { + try await released.delete() + } + + let extraMarkerCheck = try await container.exec("check-extra-marker") { config in + config.arguments = ["test", "!", "-e", extraMarker] + } + try await extraMarkerCheck.start() + var status = try await extraMarkerCheck.wait(timeoutInSeconds: 5) + try await extraMarkerCheck.delete() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "a denied process executed its marker") + } + + // With seven resident processes, the shell itself occupies the + // eighth slot. Its single bounded background fork must fail. + let forkError = BufferWriter() + let forkProbe = try await container.exec("pids-fork-probe") { config in + config.arguments = [ + "sh", + "-c", + "(touch \(forkMarker)) & child=$!; wait \"$child\"", + ] + config.stderr = forkError + } + try await forkProbe.start() + status = try await forkProbe.wait(timeoutInSeconds: 5) + try await forkProbe.delete() + guard status.exitCode != 0 else { + throw IntegrationError.assert(msg: "fork unexpectedly succeeded at pids.max") + } + + let forkMarkerCheck = try await container.exec("check-fork-marker") { config in + config.arguments = ["test", "!", "-e", forkMarker] + } + try await forkMarkerCheck.start() + status = try await forkMarkerCheck.wait(timeoutInSeconds: 5) + try await forkMarkerCheck.delete() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "the exhausted fork executed its marker") + } + + let eventsBuffer = BufferWriter() + let eventsProbe = try await container.exec("check-pids-events") { config in + config.arguments = ["cat", "/sys/fs/cgroup/pids.events"] + config.stdout = eventsBuffer + } + try await eventsProbe.start() + status = try await eventsProbe.wait(timeoutInSeconds: 5) + try await eventsProbe.delete() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "failed to read pids.events") + } + let events = String(data: eventsBuffer.data, encoding: .utf8) ?? "" + let maxEvents = + events + .split(separator: "\n") + .first { $0.hasPrefix("max ") }? + .split(separator: " ") + .last + .flatMap { UInt64($0) } + guard let maxEvents, maxEvents > 0 else { + throw IntegrationError.assert(msg: "pids.events did not record exhaustion: \(events)") + } + + // Successful admission after one slot is released proves the + // failed attempts did not strand guest-side process records. + let recoveryProbe = try await container.exec("pids-recovery-probe") { config in + config.arguments = ["/bin/true"] + } + try await recoveryProbe.start() + status = try await recoveryProbe.wait(timeoutInSeconds: 5) + try await recoveryProbe.delete() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "management recovery probe failed: \(status)") + } + + let recoveredStats = try await Timeout.run(for: .seconds(5)) { + try await container.statistics(categories: .process) + } + guard let recoveredPids = recoveredStats.process, recoveredPids.current <= limit, + recoveredPids.limit == limit + else { + throw IntegrationError.assert(msg: "invalid process statistics after recovery") + } + + for process in sleepers.reversed() { + await terminate(process) + } + sleepers.removeAll() + + try await container.kill(.kill) + try await container.wait(timeoutInSeconds: 5) + try await container.stop() + } catch { + for process in sleepers.reversed() { + await terminate(process) + } + try? await container.kill(.kill) + _ = try? await container.wait(timeoutInSeconds: 5) + try? await container.stop() + throw error + } + } + + func testZeroPidsLimitFailsClosed() async throws { + let id = "test-zero-pids-limit" + let bs = try await bootstrap(id) + let markerBuffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["echo", "unexpected-zero-pids-start"] + config.process.stdout = markerBuffer + config.pidsLimit = 0 + config.bootLog = bs.bootLog + } + + try await container.create() + var startError: Error? + do { + try await Timeout.run(for: .seconds(5)) { + try await container.start() + } + } catch { + startError = error + } + + guard let startError else { + try? await container.kill(.kill) + _ = try? await container.wait(timeoutInSeconds: 5) + try? await container.stop() + throw IntegrationError.assert(msg: "container started with pidsLimit 0") + } + guard !(startError is CancellationError) else { + throw IntegrationError.assert(msg: "pidsLimit 0 startup did not fail within five seconds") + } + let startFailure = String(describing: startError) + guard + startFailure.contains("clone3(CLONE_INTO_CGROUP)") + && (startFailure.contains("Code=11") || startFailure.contains("errno 11") + || startFailure.contains("Resource temporarily unavailable")) + else { + throw IntegrationError.assert( + msg: "pidsLimit 0 failed for an unexpected reason: \(startFailure)" + ) + } + guard markerBuffer.data.isEmpty else { + throw IntegrationError.assert(msg: "pidsLimit 0 workload executed before rejection") + } + } + + func testPidsControllerUnavailableFailsClosed() async throws { + let kernelArguments = ["cgroup_disable=pids"] + + // Establish that this exact kernel boot really removed the controller; + // otherwise a passing startup failure would be ambiguous fixture noise. + let controlID = "test-pids-controller-control" + let controlBootstrap = try await bootstrap(controlID, kernelArguments: kernelArguments) + let controllersBuffer = BufferWriter() + let control = try LinuxContainer( + controlID, + rootfs: controlBootstrap.rootfs, + vmm: controlBootstrap.vmm + ) { config in + config.process.arguments = ["sleep", "infinity"] + config.bootLog = controlBootstrap.bootLog + } + + do { + try await control.create() + try await control.start() + + // No PID limit was requested, so disabling the controller must not + // break ordinary exec compatibility. + let controllersProbe = try await control.exec("read-controllers") { config in + config.arguments = ["cat", "/sys/fs/cgroup/cgroup.controllers"] + config.stdout = controllersBuffer + } + try await controllersProbe.start() + let status = try await controllersProbe.wait(timeoutInSeconds: 5) + try await controllersProbe.delete() + try await control.kill(.kill) + try await control.wait(timeoutInSeconds: 5) + try await control.stop() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "controller control exited with \(status)") + } + } catch { + try? await control.stop() + throw error + } + + let controllers = String(data: controllersBuffer.data, encoding: .utf8) ?? "" + guard !controllers.split(whereSeparator: { $0.isWhitespace }).contains("pids") else { + throw IntegrationError.assert(msg: "test kernel still exposes the pids controller: \(controllers)") + } + + let blockedID = "test-pids-controller-blocked" + let blockedBootstrap = try await bootstrap(blockedID, kernelArguments: kernelArguments) + let markerBuffer = BufferWriter() + let blocked = try LinuxContainer( + blockedID, + rootfs: blockedBootstrap.rootfs, + vmm: blockedBootstrap.vmm + ) { config in + config.process.arguments = ["echo", "unexpected-pids-start"] + config.process.stdout = markerBuffer + config.pidsLimit = 8 + config.bootLog = blockedBootstrap.bootLog + } + + try await blocked.create() + var startError: Error? + do { + try await Timeout.run(for: .seconds(5)) { + try await blocked.start() + } + } catch { + startError = error + } + + guard let startError else { + try? await blocked.kill(.kill) + _ = try? await blocked.wait(timeoutInSeconds: 5) + try? await blocked.stop() + throw IntegrationError.assert(msg: "container started without the requested pids controller") + } + guard !(startError is CancellationError) else { + throw IntegrationError.assert(msg: "controller-unavailable startup did not fail within five seconds") + } + let startFailure = String(describing: startError) + guard + startFailure.contains("pids.max") + && (startFailure.contains("errno 2") || startFailure.contains("Code=2") + || startFailure.contains("No such file or directory")) + else { + throw IntegrationError.assert( + msg: "controller-unavailable startup failed for an unexpected reason: \(startFailure)" + ) + } + guard markerBuffer.data.isEmpty else { + throw IntegrationError.assert(msg: "workload executed before PID resource application failed") + } + } + func testMemoryEventsOOMKill() async throws { let id = "test-memory-events-oom-kill" let bs = try await bootstrap(id) let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in config.process.arguments = ["sleep", "infinity"] + config.memoryInBytes = 64.mib() config.bootLog = bs.bootLog } @@ -1007,11 +1409,11 @@ extension IntegrationSuite { // Run a process that will exceed the memory limit and get OOM-killed let exec = try await container.exec("oom-trigger") { config in - // First set a 2MB memory limit on the container's cgroup, then allocate more config.arguments = [ - "sh", - "-c", - "echo 2097152 > /sys/fs/cgroup/memory.max && dd if=/dev/zero of=/dev/null bs=100M", + "dd", + "if=/dev/zero", + "of=/dev/null", + "bs=100M", ] } diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec86..546ee87de 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -446,6 +446,7 @@ extension IntegrationSuite { try await pod.addContainer("container1", rootfs: bs.rootfs) { config in config.process.arguments = ["/bin/sleep", "infinity"] + config.memoryInBytes = 64.mib() } do { @@ -454,9 +455,10 @@ extension IntegrationSuite { let exec = try await pod.execInContainer("container1", processID: "oom-trigger") { config in config.arguments = [ - "sh", - "-c", - "echo 2097152 > /sys/fs/cgroup/memory.max && dd if=/dev/zero of=/dev/null bs=100M", + "dd", + "if=/dev/zero", + "of=/dev/null", + "bs=100M", ] } diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index bf11e1cef..6d79d8e4c 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -195,7 +195,10 @@ struct IntegrationSuite: AsyncParsableCommand { static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) - func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { + func bootstrap( + _ testID: String, + kernelArguments: [String] = [] + ) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { let reference = "ghcr.io/linuxcontainers/alpine:3.20" let store = Self.imageStore @@ -217,7 +220,13 @@ struct IntegrationSuite: AsyncParsableCommand { } }() - let testKernel = Kernel(path: .init(filePath: kernel), platform: .linuxArm) + var commandLine = Kernel.CommandLine(debug: false, panic: 0) + commandLine.kernelArgs.append(contentsOf: kernelArguments) + let testKernel = Kernel( + path: .init(filePath: kernel), + platform: .linuxArm, + commandline: commandLine + ) // Intentionally NOT adding `debug` or `earlycon=pl011,...` here. // Both look free, but each costs real wall-clock per VM boot: // * `debug` floods printk through hvc0 (which CH writes to the @@ -233,7 +242,9 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + // OCI digests include a `sha256:` prefix. Keep it out of the host + // filesystem name used for the disposable ext4 image. + let fsPath = Self.testDir.appending(component: image.digest.trimmingDigestPrefix) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -259,12 +270,12 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + let preserve = fsPath.lastPathComponent if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries where url.lastPathComponent != preserve { try? FileManager.default.removeItem(at: url) } } @@ -438,6 +449,9 @@ struct IntegrationSuite: AsyncParsableCommand { // Statistics / cgroups / memory Test("container statistics", testContainerStatistics), Test("container cgroup limits", testCgroupLimits), + Test("container zero pids limit", testZeroPidsLimitFailsClosed), + Test("container pids exhaustion", testPidsLimitExhaustion), + Test("container pids controller unavailable", testPidsControllerUnavailableFailsClosed), Test("container memory events OOM kill", testMemoryEventsOOMKill), // Console / boot / lifecycle diff --git a/Tests/ContainerizationTests/LinuxContainerTests.swift b/Tests/ContainerizationTests/LinuxContainerTests.swift index 713e3982d..8e8b98609 100644 --- a/Tests/ContainerizationTests/LinuxContainerTests.swift +++ b/Tests/ContainerizationTests/LinuxContainerTests.swift @@ -14,6 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerizationError import ContainerizationOCI import ContainerizationOS import Foundation @@ -21,6 +22,12 @@ import Testing @testable import Containerization +private struct StubVirtualMachineManager: VirtualMachineManager { + func create(config: some VMCreationConfig) throws -> any VirtualMachineInstance { + fatalError("not used") + } +} + struct LinuxContainerTests { @Test func processInitFromImageConfigWithAllFields() { @@ -119,4 +126,78 @@ struct LinuxContainerTests { #expect(pod.maskedPaths == expectedMasked) #expect(pod.readonlyPaths == expectedReadonly) } + + @Test func finitePidsLimitMakesCgroupMountsReadOnly() throws { + for mounts in [LinuxContainer.defaultMounts(), LinuxContainer.defaultOCIMounts()] { + let cgroupMount = mounts.first { $0.destination == "/sys/fs/cgroup" } + #expect(cgroupMount?.type == "cgroup2") + #expect(cgroupMount?.options.contains("ro") == false) + + let finite = LinuxContainer.mountsEnforcingPidsLimit(mounts, pidsLimit: 64) + let finiteCgroupMount = finite.first { $0.destination == "/sys/fs/cgroup" } + #expect(finiteCgroupMount?.options.contains("ro") == true) + #expect(finiteCgroupMount?.options.contains("rw") == false) + + let omitted = LinuxContainer.mountsEnforcingPidsLimit(mounts, pidsLimit: nil) + let unlimited = LinuxContainer.mountsEnforcingPidsLimit(mounts, pidsLimit: -1) + #expect(omitted.first { $0.destination == "/sys/fs/cgroup" }?.options.contains("ro") == false) + #expect(unlimited.first { $0.destination == "/sys/fs/cgroup" }?.options.contains("ro") == false) + } + + let custom = Mount.any( + type: "cgroup2", + source: "none", + destination: "/alternate-cgroup", + options: ["rw"] + ) + let hardenedCustom = LinuxContainer.mountsEnforcingPidsLimit([custom], pidsLimit: 1) + #expect(hardenedCustom[0].options == ["ro"]) + + let bindAlias = Mount.any( + type: "none", + source: "/sys/fs/cgroup", + destination: "/alternate-cgroup", + options: ["bind", "rw"] + ) + let hardenedAlias = LinuxContainer.mountsEnforcingPidsLimit([bindAlias], pidsLimit: 1) + #expect(hardenedAlias[0].options == ["bind", "ro"]) + } + + @Test func pidsLimitPreservesOmissionAndOCIValues() { + let omitted = LinuxContainer.Configuration() + let omittedFromInitializer = LinuxContainer.Configuration( + process: LinuxProcessConfiguration(arguments: ["/bin/sh"]) + ) + let zero = LinuxContainer.Configuration( + process: LinuxProcessConfiguration(arguments: ["/bin/sh"]), + pidsLimit: 0 + ) + let unlimited = LinuxContainer.Configuration( + process: LinuxProcessConfiguration(arguments: ["/bin/sh"]), + pidsLimit: -1 + ) + + #expect(omitted.pidsLimit == nil) + #expect(omittedFromInitializer.pidsLimit == nil) + #expect(zero.pidsLimit == 0) + #expect(unlimited.pidsLimit == -1) + } + + @Test func pidsLimitRejectsValuesBelowUnlimitedSentinel() { + let rootfs = Mount.any(type: "none", source: "none", destination: "/") + + for pidsLimit in [-2, Int64.min] { + var configuration = LinuxContainer.Configuration() + configuration.pidsLimit = pidsLimit + + #expect(throws: ContainerizationError.self) { + _ = try LinuxContainer( + "invalid-pids-limit", + rootfs: rootfs, + vmm: StubVirtualMachineManager(), + configuration: configuration + ) + } + } + } } diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 62fa78c7a..d1af3740f 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -122,6 +122,14 @@ public struct Cgroup2Manager: Sendable { ) } + package func openForCloning() throws -> Int32 { + let fd = open(self.path.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC) + guard fd >= 0 else { + throw Error.errno(errno: errno, message: "failed to open cgroup \(self.path.path)") + } + return fd + } + private static func writeValue(path: URL, value: String, fileName: String) throws { let file = path.appending(path: fileName) let fd = open(file.path, O_WRONLY, 0) diff --git a/vminitd/Sources/LCShim/include/syscall.h b/vminitd/Sources/LCShim/include/syscall.h index 815dd4768..0420e4fc4 100644 --- a/vminitd/Sources/LCShim/include/syscall.h +++ b/vminitd/Sources/LCShim/include/syscall.h @@ -99,6 +99,11 @@ int CZ_pidfd_open(pid_t pid, unsigned int flags); #endif int CZ_pidfd_getfd(int pidfd, int targetfd, unsigned int flags); +// Fork a child directly into the supplied cgroup v2 directory. This keeps +// pids.max admission atomic; moving an already-forked process through +// cgroup.procs is explicitly allowed to exceed the limit. +pid_t CZ_clone_into_cgroup(int cgroup_fd); + int CZ_prctl_set_no_new_privs(); #endif diff --git a/vminitd/Sources/LCShim/syscall.c b/vminitd/Sources/LCShim/syscall.c index 094f6c61b..f5523feed 100644 --- a/vminitd/Sources/LCShim/syscall.c +++ b/vminitd/Sources/LCShim/syscall.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include "syscall.h" @@ -38,6 +40,38 @@ int CZ_pidfd_getfd(int pidfd, int targetfd, unsigned int flags) { return syscall(SYS_pidfd_getfd, pidfd, targetfd, flags); } +#ifndef SYS_clone3 +#define SYS_clone3 435 +#endif + +#ifndef CLONE_INTO_CGROUP +#define CLONE_INTO_CGROUP 0x200000000ULL +#endif + +// Keep this layout in sync with Linux's struct clone_args. Defining the +// syscall ABI locally avoids relying on a particular userspace header age. +struct cz_clone_args { + uint64_t flags; + uint64_t pidfd; + uint64_t child_tid; + uint64_t parent_tid; + uint64_t exit_signal; + uint64_t stack; + uint64_t stack_size; + uint64_t tls; + uint64_t set_tid; + uint64_t set_tid_size; + uint64_t cgroup; +}; + +pid_t CZ_clone_into_cgroup(int cgroup_fd) { + struct cz_clone_args args = {0}; + args.flags = CLONE_INTO_CGROUP; + args.exit_signal = SIGCHLD; + args.cgroup = (uint64_t)cgroup_fd; + return (pid_t)syscall(SYS_clone3, &args, sizeof(args)); +} + int CZ_prctl_set_no_new_privs() { return prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); } diff --git a/vminitd/Sources/VminitdCore/CgroupProcessAdmission.swift b/vminitd/Sources/VminitdCore/CgroupProcessAdmission.swift new file mode 100644 index 000000000..5e415f617 --- /dev/null +++ b/vminitd/Sources/VminitdCore/CgroupProcessAdmission.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(Linux) + +import Cgroup + +/// Narrow cross-package access to cgroup descriptors used by vmexec's +/// clone3(CLONE_INTO_CGROUP) process-admission path. +public enum CgroupProcessAdmission { + public static func openForCloning(_ manager: Cgroup2Manager) throws -> Int32 { + try manager.openForCloning() + } + + public static func openForCloning(parentPid: Int32) throws -> Int32 { + let manager = try Cgroup2Manager.loadFromPid(pid: parentPid) + return try manager.openForCloning() + } +} + +#endif diff --git a/vminitd/Sources/VminitdCore/ManagedContainer.swift b/vminitd/Sources/VminitdCore/ManagedContainer.swift index 545046a8f..0c31395e8 100644 --- a/vminitd/Sources/VminitdCore/ManagedContainer.swift +++ b/vminitd/Sources/VminitdCore/ManagedContainer.swift @@ -31,6 +31,7 @@ public actor ManagedContainer { private let log: Logger private let bundle: ContainerizationOCI.Bundle private let needsCgroupCleanup: Bool + private let enforcePidsLimit: Bool private var execs: [String: any ContainerProcess] = [:] public var pid: Int32? { @@ -56,6 +57,7 @@ public actor ManagedContainer { spec: spec ) log.debug("created bundle with spec \(spec)") + let enforcePidsLimit = spec.linux?.resources?.pids.map { $0.limit >= 0 } ?? false let cgManager = Cgroup2Manager( group: URL(filePath: cgroupsPath), @@ -103,6 +105,7 @@ public actor ManagedContainer { self.id = id self.bundle = bundle self.log = log + self.enforcePidsLimit = enforcePidsLimit } catch { try? cgManager.delete() throw error @@ -177,6 +180,7 @@ extension ManagedContainer { stdio: stdio, bundle: self.bundle, owningPid: self.initProcess.pid, + enforcePidsLimit: self.enforcePidsLimit, log: self.log ) self.execs[id] = process diff --git a/vminitd/Sources/VminitdCore/ManagedProcess.swift b/vminitd/Sources/VminitdCore/ManagedProcess.swift index ba4cd2d1e..82818e7de 100644 --- a/vminitd/Sources/VminitdCore/ManagedProcess.swift +++ b/vminitd/Sources/VminitdCore/ManagedProcess.swift @@ -74,6 +74,7 @@ final class ManagedProcess: ContainerProcess, Sendable { stdio: HostStdio, bundle: ContainerizationOCI.Bundle, owningPid: Int32? = nil, + enforcePidsLimit: Bool = false, log: Logger ) throws { self.id = id @@ -94,7 +95,7 @@ final class ManagedProcess: ContainerProcess, Sendable { try errorPipe.setCloexec() self.errorPipe = errorPipe - let args: [String] + var args: [String] if let owningPid { args = [ "exec", @@ -103,6 +104,9 @@ final class ManagedProcess: ContainerProcess, Sendable { "--process-path", bundle.getExecSpecPath(id: id).path, ] + if enforcePidsLimit { + args.append("--enforce-pids-limit") + } } else { args = ["run", "--bundle-path", bundle.path.path] } diff --git a/vminitd/Sources/vmexec/ExecCommand.swift b/vminitd/Sources/vmexec/ExecCommand.swift index b5a87f8e6..94027bc5f 100644 --- a/vminitd/Sources/vmexec/ExecCommand.swift +++ b/vminitd/Sources/vmexec/ExecCommand.swift @@ -21,6 +21,7 @@ import FoundationEssentials import LCShim import Logging import SystemPackage +import VminitdCore #if canImport(Musl) import Musl @@ -40,6 +41,9 @@ struct ExecCommand: ParsableCommand { @Option(name: .long, help: "pid of the init process for the container") var parentPid: Int + @Flag(name: .long, help: "enforce finite pids.max admission with clone3") + var enforcePidsLimit: Bool = false + func run() throws { do { let src = URL(fileURLWithPath: processPath) @@ -65,6 +69,21 @@ struct ExecCommand: ParsableCommand { let syncPipe = FileDescriptor(rawValue: 3) let ackPipe = FileDescriptor(rawValue: 4) + // The pids controller does not reject migration through cgroup.procs. + // When this container has a finite ceiling, fork the exec child into + // the cgroup atomically so kernel admission enforces pids.max. + let cgroupFD: Int32? + if enforcePidsLimit { + cgroupFD = try CgroupProcessAdmission.openForCloning(parentPid: Int32(parentPid)) + } else { + cgroupFD = nil + } + defer { + if let cgroupFD { + close(cgroupFD) + } + } + let pidFd = CZ_pidfd_open(Int32(parentPid), 0) guard pidFd > 0 else { throw App.Errno(stage: "pidfd_open(\(parentPid))") @@ -74,13 +93,14 @@ struct ExecCommand: ParsableCommand { nsType: CLONE_NEWCGROUP | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWNS ) - let processID = fork() + let processID = cgroupFD.map { CZ_clone_into_cgroup($0) } ?? fork() guard processID != -1 else { try? syncPipe.close() try? ackPipe.close() - throw App.Errno(stage: "fork") + let stage = cgroupFD == nil ? "fork" : "clone3(CLONE_INTO_CGROUP)" + throw App.Errno(stage: stage, info: "\(stage):") } if processID == 0 { // child diff --git a/vminitd/Sources/vmexec/RunCommand.swift b/vminitd/Sources/vmexec/RunCommand.swift index e20b86ad5..4440d8fde 100644 --- a/vminitd/Sources/vmexec/RunCommand.swift +++ b/vminitd/Sources/vmexec/RunCommand.swift @@ -21,6 +21,7 @@ import ContainerizationOS import FoundationEssentials import LCShim import SystemPackage +import VminitdCore #if canImport(Musl) import Musl @@ -328,34 +329,50 @@ struct RunCommand: ParsableCommand { let syncPipe = FileDescriptor(rawValue: 3) let ackPipe = FileDescriptor(rawValue: 4) + var cgroupManager: Cgroup2Manager? + var cgroupFD: Int32? + var cloneIntoCgroup = false + if let linux = spec.linux, !linux.cgroupsPath.isEmpty { + let manager = try Cgroup2Manager.load(group: URL(filePath: linux.cgroupsPath)) + if let resources = linux.resources { + // Apply limits before the workload child exists. In particular, + // controller-unavailable PID requests must fail before fork. + try manager.applyResources(resources: resources) + cloneIntoCgroup = resources.pids.map { $0.limit >= 0 } ?? false + } + if cloneIntoCgroup { + cgroupFD = try CgroupProcessAdmission.openForCloning(manager) + } + cgroupManager = manager + } + defer { + if let cgroupFD { + close(cgroupFD) + } + } + let unshareFlags = try setupNamespaces(namespaces: spec.linux?.namespaces) guard unshare(unshareFlags) == 0 else { throw App.Errno(stage: "unshare(\(unshareFlags))") } - let processID = fork() + let processID = cgroupFD.map { CZ_clone_into_cgroup($0) } ?? fork() guard processID != -1 else { try? syncPipe.close() try? ackPipe.close() - throw App.Errno(stage: "fork") + let stage = cgroupFD == nil ? "fork" : "clone3(CLONE_INTO_CGROUP)" + throw App.Errno(stage: stage, info: "\(stage):") } if processID == 0 { // child try childSetup(spec: spec, ackPipe: ackPipe, syncPipe: syncPipe) } else { // parent process - // Setup cgroup before child enters cgroup namespace - if let linux = spec.linux { - let cgroupPath = linux.cgroupsPath - if !cgroupPath.isEmpty { - let cgroupManager = try Cgroup2Manager.load(group: URL(filePath: cgroupPath)) - - if let resources = linux.resources { - try cgroupManager.applyResources(resources: resources) - } - - try cgroupManager.addProcess(pid: processID) - } + // Without a finite PID limit, retain the legacy migration path. + // Finite limits use clone3(CLONE_INTO_CGROUP) above because Linux + // permits cgroup.procs migration to exceed pids.max. + if !cloneIntoCgroup, let cgroupManager { + try cgroupManager.addProcess(pid: processID) } // Send our child's pid before we exit.