From cab0484510cab951ea19065f1be5dfd268864ffe Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 29 Jul 2026 15:40:56 +0200 Subject: [PATCH 01/11] refactor: move all monitor resources in InitialSetup Gather everything related to the guest rootfs (along with the preStart) command in once during InitialSetup and store them in monitorResources. Exec simply loads the struct and uses them. No rootfs decision is made in Exec. Signed-off-by: Charalampos Mainas --- pkg/unikontainers/block.go | 2 +- pkg/unikontainers/initrd_rootfs.go | 2 +- pkg/unikontainers/rootfs.go | 8 +- pkg/unikontainers/shared_fs.go | 17 +- pkg/unikontainers/types/types.go | 11 ++ pkg/unikontainers/unikontainers.go | 262 +++++++++++++---------------- pkg/unikontainers/utils.go | 26 ++- 7 files changed, 153 insertions(+), 175 deletions(-) diff --git a/pkg/unikontainers/block.go b/pkg/unikontainers/block.go index 69db4629a..aaac2446b 100644 --- a/pkg/unikontainers/block.go +++ b/pkg/unikontainers/block.go @@ -426,7 +426,7 @@ func (b blockRootfs) getSharedDirs() (types.SharedfsParams, error) { return types.SharedfsParams{}, nil } -func (b blockRootfs) preStart() error { +func (b blockRootfs) preStartCmd() []string { return nil } diff --git a/pkg/unikontainers/initrd_rootfs.go b/pkg/unikontainers/initrd_rootfs.go index ee70d9da2..136b77365 100644 --- a/pkg/unikontainers/initrd_rootfs.go +++ b/pkg/unikontainers/initrd_rootfs.go @@ -57,6 +57,6 @@ func (i initrdRootfs) getSharedDirs() (types.SharedfsParams, error) { return types.SharedfsParams{}, nil } -func (i initrdRootfs) preStart() error { +func (i initrdRootfs) preStartCmd() []string { return nil } diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index bd1c800f0..40dd5b963 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -30,17 +30,13 @@ import ( // TODO: Find and set the correct size for the tmpfs in the host const tmpfsSizeForNoRootfs = "65536k" -// annotRootfsParams holds JSON RootfsParams after shim chooseGuestRootfs. -// When present in bundle config.json, Exec reuses it; otherwise Exec runs ChooseRootfs. -const annotRootfsParams = "com.urunc.internal.rootfs.params" - type rootfsBuilder interface { preSetup() error postSetup() error getMounts() ([]specs.Mount, error) getBlockDevs() ([]types.BlockDevParams, error) getSharedDirs() (types.SharedfsParams, error) - preStart() error + preStartCmd() []string } // tmpfsMount creates a mount for a tmpfs in the form of "/tmp" at target @@ -162,7 +158,7 @@ func (n noRootfs) getSharedDirs() (types.SharedfsParams, error) { return types.SharedfsParams{}, nil } -func (n noRootfs) preStart() error { +func (n noRootfs) preStartCmd() []string { return nil } diff --git a/pkg/unikontainers/shared_fs.go b/pkg/unikontainers/shared_fs.go index 9441f39a7..875d9dc6f 100644 --- a/pkg/unikontainers/shared_fs.go +++ b/pkg/unikontainers/shared_fs.go @@ -15,7 +15,6 @@ package unikontainers import ( - "fmt" "path/filepath" "strings" @@ -73,26 +72,24 @@ func (s sharedfsRootfs) getSharedDirs() (types.SharedfsParams, error) { }, nil } -func (s sharedfsRootfs) preStart() error { +func (s sharedfsRootfs) preStartCmd() []string { if s.sfsType == "9pfs" { return nil } - // Start the virtiofsd process - args := []string{ + // The virtiofsd argv, with the binary itself as the first element so it can be + // both stored in the monitor spec and spawned later by spawnProcess. + argv := []string{ + s.vfsdConfig.Path, "--socket-path=/tmp/vhostqemu", "--shared-dir", s.sharedPath, } if s.vfsdConfig.Options != "" { - args = append(args, strings.Fields(s.vfsdConfig.Options)...) + argv = append(argv, strings.Fields(s.vfsdConfig.Options)...) } - err := spawnProcess(s.vfsdConfig.Path, args) - if err != nil { - err = fmt.Errorf("failed to start virtiofsd: %w", err) - } - return err + return argv } func chooseTmpfsSize(sfsType string, mem uint64) string { diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index f5d668b0d..8950d9388 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -142,3 +142,14 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization } + +// MonitorSpec is everything the post-pivot urunc process needs in order to +// finalize the monitor's process execution environment and exec the monitor. +type MonitorSpec struct { + ContainerID string `json:"containerID"` + UnikernelType string `json:"unikernelType"` + MonitorType string `json:"monitorType"` + MonitorCfg MonitorConfig `json:"monitorCfg"` + ExecArgs ExecArgs `json:"execArgs"` + GuestParams UnikernelParams `json:"guestParams"` +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index ad0bf2897..dcdd570a8 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -169,26 +169,10 @@ func (u *Unikontainer) InitialSetup() error { // if the respective annotation is set then, depending on the guest // (supports block or 9pfs), it will use the supported option. In case // both ae supported, then the block option will be used by default. - var rootfsParams types.RootfsParams - - // Read the rootfs choice written by the shim. - if rootfsParamsJSON := u.Spec.Annotations[annotRootfsParams]; rootfsParamsJSON != "" { - if err := json.Unmarshal([]byte(rootfsParamsJSON), &rootfsParams); err != nil { - return fmt.Errorf("could not decode guest rootfs params: %w", err) - } - } - - if rootfsParams.MonRootfs == "" { - rootfsParams, err = ChooseRootfs(bundleDir, rootfsDir, u.State.Annotations, u.UruncCfg) - if err != nil { - uniklog.Errorf("could not choose guest rootfs: %v", err) - return err - } - encoded, err := json.Marshal(rootfsParams) - if err != nil { - return err - } - u.State.Annotations[annotRootfsParams] = string(encoded) + rootfsParams, err := ChooseRootfs(bundleDir, rootfsDir, u.State.Annotations, u.UruncCfg) + if err != nil { + uniklog.Errorf("could not choose guest rootfs: %v", err) + return err } uniklog.WithFields(logrus.Fields{ "rootfs_type": rootfsParams.Type, @@ -216,6 +200,12 @@ func (u *Unikontainer) InitialSetup() error { if err != nil { return err } + monRes.Rootfs = rootfsParams + + err = rfsBuilder.postSetup() + if err != nil { + return fmt.Errorf("post setup step for rootfs failed: %w", err) + } u.State.Status = specs.StateCreating // FIXME: should we really create this base dir @@ -388,6 +378,13 @@ func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm } res.Devices = append(res.Devices, blockDevs...) + res.Sharedfs, err = rfs.getSharedDirs() + if err != nil { + return res, fmt.Errorf("failed to get directories to share with sandbox: %w", err) + } + + res.PreStartCmd = rfs.preStartCmd() + return res, nil } @@ -446,45 +443,19 @@ func monitorMemoryBytes(defaultMem uint, resources *specs.LinuxResources) uint64 return mem } -// nolint:gocyclo -func (u *Unikontainer) Exec(metrics m.Writer) error { - metrics.Capture(m.TS15) - - // container Paths - // Make sure paths are clean - bundleDir := filepath.Clean(u.State.Bundle) - rootfsDir := filepath.Clean(u.Spec.Root.Path) - rootfsDir, err := resolveAgainstBase(bundleDir, rootfsDir) - if err != nil { - uniklog.Errorf("could not resolve rootfs directory %s: %v", rootfsDir, err) - return err - } +// buildMonitorSpec assembles the base MonitorSpec: everything the monitor needs +// that can be derived from the OCI spec, the container's annotations and the +// monitor resources gathered during InitialSetup. +func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) types.MonitorSpec { + var mSpec types.MonitorSpec - // unikernel unikernelType := u.State.Annotations[annotType] - unikernel, err := unikernels.New(unikernelType) - if err != nil { - return err - } - - // Vmm vmmType := u.State.Annotations[annotHypervisor] - vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors) - if err != nil { - return err - } - - // unikernelParams unikernelVersion := u.State.Annotations[annotVersion] - - // ExecArgs unikernelPath := u.State.Annotations[annotBinary] initrdPath := u.State.Annotations[annotInitrd] - // debug uniklog.WithFields(logrus.Fields{ - "bundle directory": bundleDir, - "rootfs directory": rootfsDir, "vmm type": vmmType, "unikernel type": unikernelType, "unikernel version": unikernelVersion, @@ -492,14 +463,12 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { "initrd Path": initrdPath, }).Debug("Initialization values") - // ExecArgs defaultVCPUs := u.UruncCfg.Monitors[vmmType].DefaultVCPUs if defaultVCPUs < 1 { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB - // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, UnikernelPath: unikernelPath, @@ -510,127 +479,139 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { Environment: os.Environ(), } - // ExecArgs // Check if container is set to unconfined -- disable seccomp if u.Spec.Linux.Seccomp == nil { uniklog.Warn("Seccomp is disabled") vmmArgs.Seccomp = false } - procAttrs := types.ProcessConfig{ - UID: u.Spec.Process.User.UID, - GID: u.Spec.Process.User.GID, - WorkDir: u.Spec.Process.Cwd, - } - // UnikernelParams - // populate unikernel params - unikernelParams := types.UnikernelParams{ - CmdLine: u.Spec.Process.Args, - EnvVars: u.Spec.Process.Env, - Monitor: vmmType, - Version: unikernelVersion, - ProcConf: procAttrs, + guest := types.UnikernelParams{ + CmdLine: u.Spec.Process.Args, + EnvVars: u.Spec.Process.Env, + Monitor: vmmType, + Version: unikernelVersion, + ProcConf: types.ProcessConfig{ + UID: u.Spec.Process.User.UID, + GID: u.Spec.Process.User.GID, + WorkDir: u.Spec.Process.Cwd, + }, NetDevName: u.State.Annotations[annotNetDev], BlkDevName: u.State.Annotations[annotBlkDev], + Rootfs: rootfsParams, + Block: monRes.BlockArgs, } - if len(unikernelParams.CmdLine) == 0 { - unikernelParams.CmdLine = strings.Fields(u.State.Annotations[annotCmdLine]) + if len(guest.CmdLine) == 0 { + guest.CmdLine = strings.Fields(u.State.Annotations[annotCmdLine]) } - // handle network - netArgs, err := u.SetupNet() - if err != nil { - uniklog.Errorf("failed to setup network: %v", err) - return err + if rootfsParams.Type == "virtiofs" || rootfsParams.Type == "9pfs" { + // Update the paths of the files we need to pass in the monitor process. + vmmArgs.UnikernelPath = adjustPathsForSharedfs(vmmArgs.UnikernelPath) + vmmArgs.InitrdPath = adjustPathsForSharedfs(vmmArgs.InitrdPath) } - metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" + vmmArgs.Sharedfs = monRes.Sharedfs - // UnikernelParams - unikernelParams.Net = netArgs - - // ExecArgs - vmmArgs.Net = netArgs + mSpec.ContainerID = u.State.ID + mSpec.UnikernelType = unikernelType + mSpec.MonitorType = vmmType + mSpec.MonitorCfg = u.UruncCfg.Monitors[vmmType] + mSpec.ExecArgs = vmmArgs + mSpec.GuestParams = guest - // guest rootfs - // block - // handle guest's rootfs. - // There are three options: - // 1. No rootfs for guest - // 2. Use the devmapper snapshot as a block device for the guest's rootfs - // 3. Use 9pfs to share the container's rootfs as the guest's rootfs - // By default, urunc will not set any rootfs for the guest. However, - // if the respective annotation is set then, depending on the guest - // (supports block or 9pfs), it will use the supported option. In case - // both ae supported, then the block option will be used by default. - var rootfsParams types.RootfsParams + return mSpec +} - // Read the rootfs choice written by the shim. - if rootfsParamsJSON := u.State.Annotations[annotRootfsParams]; rootfsParamsJSON != "" { - if err := json.Unmarshal([]byte(rootfsParamsJSON), &rootfsParams); err != nil { - return fmt.Errorf("could not decode guest rootfs params: %w", err) - } +// setupMonitorRootfs prepares the monitor rootfs: it makes sure the directory +// exists and is mounted with a propagation flag that allows a later pivot, then +// replicates the gathered mounts and devices inside it and gives the monitor a +// console. +func (u *Unikontainer) setupMonitorRootfs(monRootfs string, monRes monitorResources, withTUNTAP bool) error { + err := os.MkdirAll(monRootfs, 0o755) + if err != nil { + return fmt.Errorf("failed to create monitor rootfs directory %s: %w", monRootfs, err) } - if rootfsParams.MonRootfs == "" { - uniklog.Errorf("missing annotations from selected rootfs") - return fmt.Errorf("missing metadata for rootfs preparation") + // Make sure that rootfs is mounted with the correct propagation + // flags so we can later pivot if needed. + err = prepareRoot(monRootfs, u.Spec.Linux.RootfsPropagation) + if err != nil { + return err } - uniklog.WithFields(logrus.Fields{ - "rootfs_type": rootfsParams.Type, - "rootfs_path": rootfsParams.Path, - "mon_rootfs": rootfsParams.MonRootfs, - }).Debug("guest rootfs params") - rfsBuilder := u.newRootfsBuilder(rootfsParams, unikernel, unikernelPath, initrdPath, vmmArgs.MemSizeB) - if rootfsParams.Type == "virtiofs" || rootfsParams.Type == "9pfs" { - // Update the paths of the files we need to pass in the monitor process. - vmmArgs.UnikernelPath = adjustPathsForSharedfs(vmmArgs.UnikernelPath) - vmmArgs.InitrdPath = adjustPathsForSharedfs(vmmArgs.InitrdPath) + err = applyMounts(monRootfs, monRes.Mounts) + if err != nil { + return fmt.Errorf("failed to apply rootfs mounts: %w", err) } - if err = os.MkdirAll(rootfsParams.MonRootfs, 0o755); err != nil { - return fmt.Errorf("failed to create monitor rootfs directory %s: %w", rootfsParams.MonRootfs, err) + // setupDevices decides whether to create the TUN/TAP device based on the + // container's network configuration. + err = setupDevices(monRootfs, monRes.Devices, withTUNTAP) + if err != nil { + return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } - // Prepare Monitor rootfs - // Make sure that rootfs is mounted with the correct propagation - // flags so we can later pivot if needed. - err = prepareRoot(rootfsParams.MonRootfs, u.Spec.Linux.RootfsPropagation) + err = setupConsole(monRootfs) if err != nil { - return err + return fmt.Errorf("failed to setup console: %w", err) } - // The monitor mounts and devices were gathered and stored in a file during - // InitialSetup; here we just apply them. postSetup applies the container's own - // bind mounts on top of the shared rootfs, and setupDevices decides whether to - // create the TUN/TAP device based on the container's network configuration. + return nil +} + +// nolint:gocyclo +func (u *Unikontainer) Exec(metrics m.Writer) error { + metrics.Capture(m.TS15) + + // The chosen guest rootfs params, together with the monitor mounts, devices + // and block args, were gathered and stored in monitor.json during + // InitialSetup. Load them back here. monRes, err := loadMonitorResources(u.BaseDir) if err != nil { return fmt.Errorf("failed to load monitor resources: %w", err) } + rootfsParams := monRes.Rootfs + if rootfsParams.MonRootfs == "" { + uniklog.Errorf("missing metadata for selected rootfs") + return fmt.Errorf("missing metadata for rootfs preparation") + } + uniklog.WithFields(logrus.Fields{ + "rootfs_type": rootfsParams.Type, + "rootfs_path": rootfsParams.Path, + "mon_rootfs": rootfsParams.MonRootfs, + }).Debug("guest rootfs params") + + ms := u.buildMonitorSpec(rootfsParams, monRes) + vmmArgs := ms.ExecArgs + unikernelParams := ms.GuestParams - err = applyMounts(rootfsParams.MonRootfs, monRes.Mounts) + // The spec carries the monitor and unikernel by type; rebuild the behaviour + // objects it refers to, exactly as the monitor process does. + unikernel, err := unikernels.New(ms.UnikernelType) if err != nil { - return fmt.Errorf("failed to apply rootfs mounts: %w", err) + return err } - - err = rfsBuilder.postSetup() + vmm, err := hypervisors.NewVMM(hypervisors.VmmType(ms.MonitorType), u.UruncCfg.Monitors) if err != nil { - return fmt.Errorf("post setup step for rootfs failed: %w", err) + return err } - err = setupDevices(rootfsParams.MonRootfs, monRes.Devices, withTUNTAP) + // handle network + netArgs, err := u.SetupNet() if err != nil { - return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) + uniklog.Errorf("failed to setup network: %v", err) + return err } + metrics.Capture(m.TS16) + withTUNTAP := netArgs.IP != "" + unikernelParams.Net = netArgs + vmmArgs.Net = netArgs - err = setupConsole(rootfsParams.MonRootfs) + err = u.setupMonitorRootfs(rootfsParams.MonRootfs, monRes, withTUNTAP) if err != nil { return err } - metrics.Capture(m.TS17) + // vAccel setup vAccelType, vsockSocketPath, rpcAddress, err := resolveVAccelConfig(u.State.Annotations[annotHypervisor], u.Spec.Annotations) if err != nil { @@ -664,20 +645,6 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } - unikernelParams.Rootfs = rootfsParams - - // unikernelParams - // The block parameters were gathered in InitialSetup and stored in the - // monitor resources file. - unikernelParams.Block = monRes.BlockArgs - - // ExecArgs - sharedfsArgs, err := rfsBuilder.getSharedDirs() - if err != nil { - return fmt.Errorf("failed to get directories to share with sandbox: %w", err) - } - vmmArgs.Sharedfs = sharedfsArgs - // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || @@ -687,14 +654,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - // unikernel // build the unikernel command unikernelCmd, err := unikernel.CommandString() if err != nil { return err } - - // ExecArgs vmmArgs.Command = unikernelCmd // pivot @@ -728,7 +692,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - err = rfsBuilder.preStart() + err = spawnProcess(monRes.PreStartCmd) if err != nil { return err } diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index 90b9500cf..68977dd0d 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -44,12 +44,16 @@ const ( rootfsDirName = "rootfs" ) -// monitorResources holds the mounts and devices that must be replicated inside -// the monitor's rootfs, along with the block parameters the guest needs. +// monitorResources holds the chosen guest rootfs params, the mounts and devices +// that must be replicated inside the monitor's rootfs, and the block parameters +// the guest needs. type monitorResources struct { - Mounts []specs.Mount `json:"mounts"` - Devices []specs.LinuxDevice `json:"devices"` - BlockArgs []types.BlockDevParams `json:"blockArgs"` + Rootfs types.RootfsParams `json:"rootfs"` + Mounts []specs.Mount `json:"mounts"` + Devices []specs.LinuxDevice `json:"devices"` + BlockArgs []types.BlockDevParams `json:"blockArgs"` + Sharedfs types.SharedfsParams `json:"sharedfs"` + PreStartCmd []string `json:"preStartCmd,omitempty"` } // saveMonitorResources stores the monitorResources passed as an argument in a JSON @@ -243,14 +247,20 @@ func convertUint32ToIntSlice(valSlice []uint32, size int) []int { // return data.Bytes(), nil // } -func spawnProcess(binaryPath string, args []string) error { - cmd := exec.Command(binaryPath, args...) +// spawnProcess starts the process described by argv, whose first element is the +// binary. An empty argv is a no-op. +func spawnProcess(argv []string) error { + if len(argv) == 0 { + return nil + } + // argv is built by urunc, based on its config, not untrusted input + cmd := exec.Command(argv[0], argv[1:]...) //nolint:gosec cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { - return err + return fmt.Errorf("failed to start %s: %w", argv[0], err) } return nil From 27fc414a0fcf2b0d07a48dfa9e2693efca8874af Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 29 Jul 2026 16:41:10 +0200 Subject: [PATCH 02/11] refactor: Break Exec and create in smaller reusable functions Break the large Exec function and the create function into smaller functions that cna be reused: - split createUnikontainer into newUnikontainer (bundle parse and InitialSetup) and the reexec handshake - make SetupNet a standalone function - pull buildUnikernelCommand and execMonitor out of Exec The rationale is to let the later port of libcontainer to use some of this functionality directly instead of duplicating logic. Signed-off-by: Charalampos Mainas --- cmd/urunc/create.go | 43 ++++++++++++------- pkg/unikontainers/unikontainers.go | 68 +++++++++++++++++------------- 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/cmd/urunc/create.go b/cmd/urunc/create.go index 5a9e2f94d..75118af98 100644 --- a/cmd/urunc/create.go +++ b/cmd/urunc/create.go @@ -82,16 +82,15 @@ var createCommand = &cli.Command{ }, } -// createUnikontainer creates a Unikernel struct from bundle data, -// initializes it's base dir and state.json, -// setups terminal if required and spawns reexec process, -// waits for reexec process to notify, executes CreateRuntime hooks, -// sends ACK to reexec process -func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (err error) { - err = nil +// newUnikontainer parses the bundle and performs the host-side preparation for +// the monitor execution environment (Unikontainer, base directory, state and +// monitor resources). It never returns for a container that is not a urunc +// container: those are handed over to the real runc with an execve. +func newUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (*unikontainers.Unikontainer, error) { containerID := cmd.Args().First() - if err = validateID(containerID); err != nil { - return err + err := validateID(containerID) + if err != nil { + return nil, err } metrics.SetLoggerContainerID(containerID) metrics.Capture(m.TS00) @@ -105,7 +104,7 @@ func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( if bundlePath == "" { bundlePath, err = os.Getwd() if err != nil { - return err + return nil, err } } @@ -114,21 +113,33 @@ func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( if err != nil { if errors.Is(err, unikontainers.ErrQueueProxy) || errors.Is(err, unikontainers.ErrNotUnikernel) { - // Exec runc to handle non unikernel containers - err = runcExec() - return err + // Exec runc to handle non urunc containers. + // It should never return. + return nil, runcExec() } - return err + return nil, err } metrics.Capture(m.TS01) err = unikontainer.InitialSetup() if err != nil { - return err + return nil, err } - metrics.Capture(m.TS02) + return unikontainer, nil +} + +// createUnikontainer creates a Unikernel struct from bundle data, initializes +// it's base dir and state.json, setups terminal if required and spawns reexec +// process, waits for reexec process to notify, executes CreateRuntime hooks, +// sends ACK to reexec process +func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (err error) { + unikontainer, err := newUnikontainer(cmd, uruncCfg) + if err != nil { + return err + } + // Create socket for nsenter initSockParent, initSockChild, err := newSockPair("init") if err != nil { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index dcdd570a8..5b3db8f24 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -245,8 +245,9 @@ func (u *Unikontainer) SetRunningState() error { return u.saveContainerState() } -func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { - networkType := u.getNetworkType() +// SetupNet creates the sandbox's network device (tap) in the current network +// namespace and returns its parameters; uid and gid own the tap device. +func SetupNet(networkType string, uid, gid uint32) (types.NetDevParams, error) { uniklog.WithField("network type", networkType).Debug("Retrieved network type") netArgs := types.NetDevParams{} netManager, err := network.NewNetworkManager(networkType) @@ -254,7 +255,7 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } - networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID) + networkInfo, err := netManager.NetworkSetup(uid, gid) if err != nil { // TODO: Handle this case better. We do not need to show an error // since there was no network in the container. Therefore, we @@ -443,9 +444,6 @@ func monitorMemoryBytes(defaultMem uint, resources *specs.LinuxResources) uint64 return mem } -// buildMonitorSpec assembles the base MonitorSpec: everything the monitor needs -// that can be derived from the OCI spec, the container's annotations and the -// monitor resources gathered during InitialSetup. func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) types.MonitorSpec { var mSpec types.MonitorSpec @@ -596,7 +594,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // handle network - netArgs, err := u.SetupNet() + netArgs, err := SetupNet(u.getNetworkType(), u.Spec.Process.User.UID, u.Spec.Process.User.GID) if err != nil { uniklog.Errorf("failed to setup network: %v", err) return err @@ -646,20 +644,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // unikernel - err = unikernel.Init(unikernelParams) - if errors.Is(err, unikernels.ErrUndefinedVersion) || - errors.Is(err, unikernels.ErrVersionParsing) { - uniklog.WithError(err).Error("an error occurred while initializing the unikernel") - } else if err != nil { - return err - } - // build the unikernel command - unikernelCmd, err := unikernel.CommandString() + vmmArgs.Command, err = buildUnikernelCommand(unikernel, unikernelParams) if err != nil { return err } - vmmArgs.Command = unikernelCmd // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) @@ -697,30 +686,49 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - uniklog.Debug("calling vmm execve") - metrics.Capture(m.TS18) - - // Build the VMM command once and verify it can be constructed successfully. - // This ensures we don't report the container as started if command building fails. + // Build the VMM command once and verify it can be constructed successfully, so + // we do not report the container as started if command building fails. execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) if err != nil { uniklog.WithError(err).Error("failed to build VMM command") return err } - // Notify urunc start that the monitor is ready to execute. - // We send this after BuildExecCmd succeeds to avoid reporting a container - // as started when the VMM command cannot be built. - // TODO: The container can still be reported as running if the PreExec step - // (e.g., BPF/seccomp filter setup) fails after this point. We should find - // a way to handle that case as well. + // Notify urunc start that the monitor is ready to execute, only after the + // command builds so a container is never reported started when it cannot be. err = u.SendMessage(StartSuccess) if err != nil { return err } + return execMonitor(metrics, vmm, vmmArgs, execCmd) +} + +// buildUnikernelCommand initializes the unikernel with the collected parameters +// and returns its command line. +func buildUnikernelCommand(unikernel types.Unikernel, params types.UnikernelParams) (string, error) { + err := unikernel.Init(params) + if errors.Is(err, unikernels.ErrUndefinedVersion) || + errors.Is(err, unikernels.ErrVersionParsing) { + uniklog.WithError(err).Error("an error occurred while initializing the unikernel") + } else if err != nil { + return "", err + } + + return unikernel.CommandString() +} + +// execMonitor runs the monitor's pre-exec setup and finally execve's the monitor. +// It does not return on success: +// +// TODO: The container can still be reported as running if the PreExec step +// (e.g., BPF/seccomp filter setup) fails after the caller reported success. We +// should find a way to handle that case as well. +func execMonitor(metrics m.Writer, vmm types.VMM, execArgs types.ExecArgs, execCmd []string) error { + uniklog.Debug("calling vmm execve") + metrics.Capture(m.TS18) // Perform any monitor-specific pre-exec setup (e.g., seccomp filters for HVT). - err = vmm.PreExec(vmmArgs) + err := vmm.PreExec(execArgs) if err != nil { uniklog.WithError(err).Error("failed to perform pre-exec setup") return err @@ -728,7 +736,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") - return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec + return syscall.Exec(vmm.Path(), execCmd, execArgs.Environment) //nolint: gosec } func setupUser(user specs.User) error { From 5e3a17dc4dbc1a319d5f9760b982d7709fe27d0e Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Thu, 9 Jul 2026 19:12:00 +0200 Subject: [PATCH 03/11] feat(libcontainer): Add configuration option for libcontainer Add a configuration option in urunc's configuration to let users choose between libcontianer's and urunc's own implementation for the setup of the monitor's execution environment. THe option is under the [runtime] section of the configuration which should hold generic runtime options. For the time being it is off by default. Signed-off-by: Charalampos Mainas --- docs/configuration.md | 26 +++++++++ pkg/unikontainers/unikontainers.go | 3 ++ pkg/unikontainers/urunc_config.go | 38 +++++++++++++ pkg/unikontainers/urunc_config_test.go | 75 ++++++++++++++++++++++++-- 4 files changed, 138 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..6e3d65a7a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,9 @@ The configuration file uses the [TOML](https://toml.io/) format and is organized into several sections: ```toml +[runtime] +libcontainer = false + [log] level = "info" syslog = false @@ -42,6 +45,29 @@ options = "--sandbox none" ## Configuration Sections +### Runtime + +The `[runtime]` section controls runtime-wide behavior. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `libcontainer` | boolean | `false` | Use libcontainer to set up the monitor's execution environment | + +> **⚠️ Experimental:** the use of `libcontainer` to prepare the monitor execution +> environment is under active development. It is off by default. + +The value is resolved when the container is created and recorded in the +container's `state.json`, so `start`, `kill` and `delete` keep using the mode +the container was created with even if the configuration file changes in +between. + +**Example:** + +```toml +[runtime] +libcontainer = false +``` + ### Log Configuration The `[log]` section controls logging behavior for `urunc`: diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 5b3db8f24..54962b852 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -90,6 +90,8 @@ func New(bundlePath string, containerID string, rootDir string, cfg *UruncConfig return nil, ErrNotUnikernel } + uniklog.Debugf("libcontainer runtime enabled: %t", cfg.Runtime.Libcontainer) + confMap := config.Map() maps.Copy(confMap, cfg.Map()) @@ -136,6 +138,7 @@ func Get(containerID string, rootDir string) (*Unikontainer, error) { u.RootDir = rootDir u.Spec = spec u.UruncCfg = UruncConfigFromMap(state.Annotations) + uniklog.Debugf("libcontainer runtime enabled: %t", u.UruncCfg.Runtime.Libcontainer) return u, nil } diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..b4f1a027b 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -34,13 +34,42 @@ type UruncTimestamps struct { Destination string `toml:"destination"` // Used to specify a file for timestamps } +// UruncRuntime holds runtime-wide behavior options. +type UruncRuntime struct { + // Libcontainer selects whether the monitor's execution environment + // is set up through runc's libcontainer instead of urunc's own implementation. + Libcontainer bool `toml:"libcontainer"` +} + +// libcontainerKey is the state.json annotation with the runtime setting +// for libcontainer. +const libcontainerKey = "urunc_config.runtime.libcontainer" + type UruncConfig struct { Log UruncLog `toml:"log"` Timestamps UruncTimestamps `toml:"timestamps"` + Runtime UruncRuntime `toml:"runtime"` Monitors map[string]types.MonitorConfig `toml:"monitors"` ExtraBins map[string]types.ExtraBinConfig `toml:"extra_binaries"` } +// runtimeFromMap rebuilds the runtime options from the state.json annotations. +// A missing or malformed value falls back to the default (libcontainer off). +func runtimeFromMap(cfgMap map[string]string) UruncRuntime { + rt := defaultRuntimeConfig() + val, ok := cfgMap[libcontainerKey] + if !ok { + return rt + } + choice, err := strconv.ParseBool(val) + if err != nil { + uniklog.Warnf("Invalid libcontainer value %q. Using default (false).", val) + return rt + } + rt.Libcontainer = choice + return rt +} + // this struct is used to parse only the log and timestamp section of the urunc config file type LogMetricsUruncConfig struct { Log UruncLog `toml:"log"` @@ -78,6 +107,12 @@ func defaultTimestampsConfig() UruncTimestamps { } } +func defaultRuntimeConfig() UruncRuntime { + return UruncRuntime{ + Libcontainer: false, + } +} + const ( defaultMonitorMemoryMB uint = 256 defaultMonitorVCPUs uint = 1 @@ -103,6 +138,7 @@ func defaultUruncConfig() *UruncConfig { return &UruncConfig{ Log: defaultLogConfig(), Timestamps: defaultTimestampsConfig(), + Runtime: defaultRuntimeConfig(), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } @@ -138,6 +174,7 @@ func (p *UruncConfig) Map() map[string]string { // them to this map. this map will be used to save the rest of the urunc config to state.json cfgMap := make(map[string]string) + cfgMap[libcontainerKey] = strconv.FormatBool(p.Runtime.Libcontainer) for hv, hvCfg := range p.Monitors { prefix := "urunc_config.monitors." + hv + "." cfgMap[prefix+"default_memory_mb"] = strconv.FormatUint(uint64(hvCfg.DefaultMemoryMB), 10) @@ -158,6 +195,7 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { // since log and timestamps are loaded at the start of urunc, we will not be reading // them from this map. this map will be used to parse the rest of the urunc config from state.json cfg := &UruncConfig{ + Runtime: runtimeFromMap(cfgMap), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } diff --git a/pkg/unikontainers/urunc_config_test.go b/pkg/unikontainers/urunc_config_test.go index a51a3eb96..a1ba139bf 100644 --- a/pkg/unikontainers/urunc_config_test.go +++ b/pkg/unikontainers/urunc_config_test.go @@ -25,6 +25,7 @@ import ( // Constants for test configuration keys and values const ( + testLibcontainerKey = "urunc_config.runtime.libcontainer" testQemuMemoryKey = "urunc_config.monitors.qemu.default_memory_mb" testQemuVCPUsKey = "urunc_config.monitors.qemu.default_vcpus" testQemuBinaryKey = "urunc_config.monitors.qemu.binary_path" @@ -425,7 +426,7 @@ func TestUruncConfigMap(t *testing.T) { assert.Equal(t, config.ExtraBins["custom"].Options, cfgMap["urunc_config.extra_binaries.custom.options"]) }) - t.Run("empty monitors map produces empty result", func(t *testing.T) { + t.Run("empty monitors map produces no monitor keys", func(t *testing.T) { t.Parallel() config := &UruncConfig{ Monitors: map[string]types.MonitorConfig{}, @@ -434,10 +435,10 @@ func TestUruncConfigMap(t *testing.T) { cfgMap := config.Map() assert.NotNil(t, cfgMap) - assert.Empty(t, cfgMap) + assert.Equal(t, map[string]string{testLibcontainerKey: "false"}, cfgMap) }) - t.Run("empty extra binaries map produces empty result", func(t *testing.T) { + t.Run("empty extra binaries map produces no extra binary keys", func(t *testing.T) { t.Parallel() config := &UruncConfig{ ExtraBins: map[string]types.ExtraBinConfig{}, @@ -446,7 +447,7 @@ func TestUruncConfigMap(t *testing.T) { cfgMap := config.Map() assert.NotNil(t, cfgMap) - assert.Empty(t, cfgMap) + assert.Equal(t, map[string]string{testLibcontainerKey: "false"}, cfgMap) }) t.Run("vhost true is serialized correctly", func(t *testing.T) { @@ -525,6 +526,7 @@ func TestDefaultConfigs(t *testing.T) { assert.False(t, config.Log.Syslog) assert.False(t, config.Timestamps.Enabled) assert.Equal(t, testTimestampsPath, config.Timestamps.Destination) + assert.False(t, config.Runtime.Libcontainer) assert.Len(t, config.Monitors, 5) assert.Len(t, config.ExtraBins, 1) }) @@ -618,3 +620,68 @@ path = "/usr/bin/mon" assert.Equal(t, defaultMonitorsConfig(), config.Monitors) }) } + +func TestRuntimeLibcontainer(t *testing.T) { + t.Run("absent from the config file defaults to false", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[monitors.qemu] +default_memory_mb = 512 +`) + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + assert.False(t, config.Runtime.Libcontainer) + }) + + t.Run("libcontainer true is honored", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[runtime] +libcontainer = true +`) + + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + assert.True(t, config.Runtime.Libcontainer) + }) + + t.Run("libcontainer false is honored", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[runtime] +libcontainer = false +`) + + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + assert.False(t, config.Runtime.Libcontainer) + }) + + t.Run("survives the round trip through state.json annotations", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[runtime] +libcontainer = true +`) + + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + + cfgMap := config.Map() + assert.Equal(t, "true", cfgMap[testLibcontainerKey]) + + assert.True(t, UruncConfigFromMap(cfgMap).Runtime.Libcontainer) + }) + + t.Run("absent annotation defaults to false", func(t *testing.T) { + t.Parallel() + assert.False(t, UruncConfigFromMap(map[string]string{}).Runtime.Libcontainer) + }) + + t.Run("malformed annotation falls back to false", func(t *testing.T) { + t.Parallel() + cfgMap := map[string]string{testLibcontainerKey: "notBool"} + + assert.False(t, UruncConfigFromMap(cfgMap).Runtime.Libcontainer) + }) +} From 8a87a2cfed74f00b5536826944e9b86d3f4e0a5b Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Mon, 13 Jul 2026 17:13:10 +0200 Subject: [PATCH 04/11] feat(libcontainer): Use runc's init Since, we are going to use libcontainers, we need the init of runc which intercepts the "normal flow" of starting a GO program from main and continues with the setup of the container process ( after the namespaces have been created). Signed-off-by: Charalampos Mainas --- .github/linters/urunc-dict.txt | 1 + cmd/urunc/init.go | 33 +++++++++++++++++++++++++++++++++ cmd/urunc/main.go | 1 - go.mod | 7 +++++++ go.sum | 11 +++++++++++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 cmd/urunc/init.go diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 933cca0b1..5469f41cd 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -433,3 +433,4 @@ hyperlight Hyperlight Odysseas Kalaitsidis +nsexec diff --git a/cmd/urunc/init.go b/cmd/urunc/init.go new file mode 100644 index 000000000..a32e5e24c --- /dev/null +++ b/cmd/urunc/init.go @@ -0,0 +1,33 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// File taken from https://github.com/opencontainers/runc v1.5.1 Please +// check the license there too. +package main + +import ( + "os" + + "github.com/opencontainers/runc/libcontainer" + _ "github.com/opencontainers/runc/libcontainer/nsenter" +) + +// Taken from https://github.com/opencontainers/runc/blob/v1.5.1/init.go +func init() { + if len(os.Args) > 1 && os.Args[1] == "init" { + // This is the golang entry point for runc init, executed + // before main() but after libcontainer/nsenter's nsexec(). + libcontainer.Init() + } +} diff --git a/cmd/urunc/main.go b/cmd/urunc/main.go index 7a103aacf..7431e1f16 100644 --- a/cmd/urunc/main.go +++ b/cmd/urunc/main.go @@ -30,7 +30,6 @@ import ( m "github.com/urunc-dev/urunc/internal/metrics" "github.com/urunc-dev/urunc/pkg/unikontainers" - _ "github.com/opencontainers/runc/libcontainer/nsenter" "github.com/urfave/cli/v3" ) diff --git a/go.mod b/go.mod index 526ab367d..0b3601ef2 100644 --- a/go.mod +++ b/go.mod @@ -36,9 +36,11 @@ require ( ) require ( + cyphar.com/go-pathrs v0.2.5 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.13.0 // indirect + github.com/checkpoint-restore/go-criu/v6 v6.3.0 // indirect github.com/cilium/ebpf v0.21.0 // indirect github.com/containerd/cgroups/v3 v3.1.0 // indirect github.com/containerd/console v1.0.5 // indirect @@ -63,11 +65,16 @@ require ( github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect + github.com/moby/sys/capability v0.4.0 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect + github.com/mrunalp/fileutils v0.5.1 // indirect + github.com/opencontainers/cgroups v0.0.4 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/seccomp/libseccomp-golang v0.10.0 // indirect go.opencensus.io v0.24.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.36.0 // indirect diff --git a/go.sum b/go.sum index bee2d7796..029ddfa1b 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,7 @@ github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwl github.com/containerd/typeurl/v2 v2.3.0/go.mod h1:Qk+PAdUYArVj41TnGi6rJ+48RF0PkcTc4i/taoBcK0w= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= @@ -102,6 +103,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -110,6 +112,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= @@ -119,6 +122,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackpal/gateway v1.2.0 h1:euPRe4t7JfTaqC5Lr78HXl2wSHo54XndTtiAcIxkb5g= github.com/jackpal/gateway v1.2.0/go.mod h1:/jchvRi4HukAqV24da70iaBMFcSrX3rNWdR5K9VHd0A= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= @@ -190,11 +194,14 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -266,6 +273,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -310,11 +318,14 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 974e96989261ccf197baa8fa39a06f15d631231c Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Tue, 21 Jul 2026 13:58:13 +0200 Subject: [PATCH 05/11] feat(libcontainer): write the monitor spec on create The monitor process runs in its own rootfs and cannot reach the state dir, so InitialSetup marshals everything it needs into .monitor_spec.json inside that rootfs. - Move MonitorSpec out of types into monitor_spec.go and extend it with everything the urunc monitor process needs to finalize the setup. - Guest rootfs path rewritten to "/": the urunc monitor process has pivoted inside the new rootfs. - Environment is nulled before writing because the monitor inherits it from libcontainer anyway, and persisting host env into the rootfs is exposure with no upside. Signed-off-by: Charalampos Mainas --- pkg/unikontainers/monitor_spec.go | 79 ++++++++++++ pkg/unikontainers/monitor_spec_test.go | 161 +++++++++++++++++++++++++ pkg/unikontainers/types/types.go | 11 -- pkg/unikontainers/unikontainers.go | 16 ++- 4 files changed, 254 insertions(+), 13 deletions(-) create mode 100644 pkg/unikontainers/monitor_spec.go create mode 100644 pkg/unikontainers/monitor_spec_test.go diff --git a/pkg/unikontainers/monitor_spec.go b/pkg/unikontainers/monitor_spec.go new file mode 100644 index 000000000..31c0033cb --- /dev/null +++ b/pkg/unikontainers/monitor_spec.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "encoding/json" + "fmt" + "os" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// monitorSpecFilename is the file that contains the information to finalize the +// setup of the monitor's execution environment. It is written into the monitor rootfs +// in order to let the process that libcontainer spawns to read it. +const monitorSpecFilename = ".monitor_spec.json" + +// MonitorSpec is everything the urunc monitor process needs in order to +// finalize the execution environment for the monitor process and then execve it. +type MonitorSpec struct { + ContainerID string `json:"containerID"` + UnikernelType string `json:"unikernelType"` + MonitorType string `json:"monitorType"` + MonitorCfg types.MonitorConfig `json:"monitorCfg"` + ExecArgs types.ExecArgs `json:"execArgs"` + GuestParams types.UnikernelParams `json:"guestParams"` + NetworkType string `json:"networkType"` + User specs.User `json:"user"` + PreStartCmd []string `json:"preStartCmd,omitempty"` +} + +// writeMonitorSpec builds the monitor spec and writes it into the monitor rootfs. +func (u *Unikontainer) writeMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) error { + mSpec := u.buildMonitorSpec(rootfsParams, monRes) + + mSpec.NetworkType = u.getNetworkType() + mSpec.User = u.Spec.Process.User + + // The post-pivot process sees the monitor rootfs as "/", so the guest rootfs + // path it is handed has to be relative to it. + mSpec.GuestParams.Rootfs.MonRootfs = "/" + + // The monitor's environment is not persisted: the urunc monitor process + // inherits it from libcontainer anyway, and writing the host's environment + // into a file inside the monitor rootfs is exposure with no upside. + mSpec.ExecArgs.Environment = nil + + data, err := json.Marshal(mSpec) + if err != nil { + return fmt.Errorf("could not encode the monitor spec: %w", err) + } + + path, err := securejoin.SecureJoin(rootfsParams.MonRootfs, monitorSpecFilename) + if err != nil { + return fmt.Errorf("could not resolve path for monitor spec: %w", err) + } + + err = os.WriteFile(path, data, 0o600) + if err != nil { + return fmt.Errorf("could not write the monitor spec: %w", err) + } + + return nil +} diff --git a/pkg/unikontainers/monitor_spec_test.go b/pkg/unikontainers/monitor_spec_test.go new file mode 100644 index 000000000..86eda8571 --- /dev/null +++ b/pkg/unikontainers/monitor_spec_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// newSpecUnikontainer builds the minimum Unikontainer that writeMonitorSpec +// needs: a spec, a state with the unikernel annotations, and the default urunc +// config. The returned RootfsParams points MonRootfs at monRootfs so the spec +// file lands in a real, writable directory. +func newSpecUnikontainer(t *testing.T, monRootfs string) (*Unikontainer, types.RootfsParams) { + t.Helper() + + u := &Unikontainer{ + State: &specs.State{ + ID: "test-container", + Annotations: map[string]string{ + annotType: "unikraft", + annotHypervisor: "qemu", + annotVersion: "1.0", + annotBinary: "/unikernel", + }, + }, + Spec: &specs.Spec{ + Version: specs.Version, + Root: &specs.Root{Path: "rootfs", Readonly: true}, + Process: &specs.Process{ + Args: []string{"/unikernel", "--flag"}, + Env: []string{"HOME=/root"}, + Cwd: "/guest/workdir", + User: specs.User{UID: 1000, GID: 1000}, + }, + Linux: &specs.Linux{ + Seccomp: &specs.LinuxSeccomp{DefaultAction: specs.ActErrno}, + }, + Annotations: map[string]string{}, + }, + UruncCfg: defaultUruncConfig(), + } + + rootfsParams := types.RootfsParams{Type: "initrd", Path: "initrd.cpio", MonRootfs: monRootfs} + + return u, rootfsParams +} + +// readMonitorSpecFile decodes the spec file written into dir directly, so the +// write-side test does not depend on LoadMonitorSpec (the read side). +func readMonitorSpecFile(t *testing.T, dir string) MonitorSpec { + t.Helper() + + data, err := os.ReadFile(filepath.Join(dir, monitorSpecFilename)) + require.NoError(t, err) + + var ms MonitorSpec + err = json.Unmarshal(data, &ms) + require.NoError(t, err) + + return ms +} + +func TestWriteMonitorSpec(t *testing.T) { + t.Run("writes a spec that decodes back unchanged", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + + assert.Equal(t, "test-container", got.ContainerID) + assert.Equal(t, "unikraft", got.UnikernelType) + assert.Equal(t, "qemu", got.MonitorType) + assert.Equal(t, u.UruncCfg.Monitors["qemu"], got.MonitorCfg) + assert.Equal(t, specs.User{UID: 1000, GID: 1000}, got.User) + // No knative annotation, so the network type is dynamic. + assert.Equal(t, "dynamic", got.NetworkType) + // The post-pivot process sees the monitor rootfs as "/". + assert.Equal(t, "/", got.GuestParams.Rootfs.MonRootfs) + }) + + t.Run("does not persist the monitor environment", func(t *testing.T) { + // t.Setenv forbids t.Parallel. + t.Setenv("URUNC_TEST_SECRET", "do-not-write-me") + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + assert.Nil(t, got.ExecArgs.Environment) + + // Not just absent from the decoded struct: absent from the file. + data, err := os.ReadFile(filepath.Join(monRootfs, monitorSpecFilename)) + require.NoError(t, err) + assert.NotContains(t, string(data), "do-not-write-me") + }) + + t.Run("keeps seccomp enabled when the spec confines it", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + assert.True(t, got.ExecArgs.Seccomp) + }) + + t.Run("disables seccomp for an unconfined spec", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + u.Spec.Linux.Seccomp = nil + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + assert.False(t, got.ExecArgs.Seccomp) + }) + + t.Run("writes the file owner-only inside the monitor rootfs", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + info, err := os.Stat(filepath.Join(monRootfs, monitorSpecFilename)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + }) +} diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index 8950d9388..f5d668b0d 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -142,14 +142,3 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization } - -// MonitorSpec is everything the post-pivot urunc process needs in order to -// finalize the monitor's process execution environment and exec the monitor. -type MonitorSpec struct { - ContainerID string `json:"containerID"` - UnikernelType string `json:"unikernelType"` - MonitorType string `json:"monitorType"` - MonitorCfg MonitorConfig `json:"monitorCfg"` - ExecArgs ExecArgs `json:"execArgs"` - GuestParams UnikernelParams `json:"guestParams"` -} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 54962b852..9a7f6c8ff 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -222,6 +222,17 @@ func (u *Unikontainer) InitialSetup() error { return fmt.Errorf("failed to store monitor resources: %w", err) } + // In libcontainer mode the monitor process runs inside a dedicated rootfs and + // cannot reach the state directory, so everything it needs is written into that + // rootfs now. + // TODO: Switch to fifo + if u.UruncCfg.Runtime.Libcontainer { + err = u.writeMonitorSpec(rootfsParams, monRes) + if err != nil { + return fmt.Errorf("failed to store the monitor spec: %w", err) + } + } + return u.saveContainerState() } @@ -447,8 +458,8 @@ func monitorMemoryBytes(defaultMem uint, resources *specs.LinuxResources) uint64 return mem } -func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) types.MonitorSpec { - var mSpec types.MonitorSpec +func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) MonitorSpec { + var mSpec MonitorSpec unikernelType := u.State.Annotations[annotType] vmmType := u.State.Annotations[annotHypervisor] @@ -518,6 +529,7 @@ func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes mSpec.MonitorCfg = u.UruncCfg.Monitors[vmmType] mSpec.ExecArgs = vmmArgs mSpec.GuestParams = guest + mSpec.PreStartCmd = monRes.PreStartCmd return mSpec } From 2bcb01968d0e095ea209ea15daa55778cef016c0 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Tue, 21 Jul 2026 15:01:45 +0200 Subject: [PATCH 06/11] feat(libcontainer): create the monitor container with libcontainer Build the monitor's libcontainer config using as a base the contianer's OCI spec. SPecify the root based on the monitor rootfs, append the monitor mounts/devices and create+start the init that "urunc monitor" becomes. - Process argv is "/proc/self/exe monitor ": /proc is always mounted post-pivot, so the urunc binary re-execs without being copied into the rootfs. - Seccomp stripped, since urunc applies monitor-specific filters - Allow-all cgroup device rule, no resource limits, just placement for the time being. - Caps = container's set + CAP_NET_ADMIN (tap setup) in all five sets. - Drop Poststart/Poststop from the config, keep them on urunc's execution. - libcontainer state nested under /libcontainer to avoid the state.json collision. - On failure after Create/Start, destroy and reap the init before the cgroup teardown, matching runc's ordering. Signed-off-by: Charalampos Mainas --- .github/linters/urunc-dict.txt | 3 + cmd/urunc/create.go | 12 +- cmd/urunc/create_libcontainer.go | 152 +++++++++++++ go.mod | 4 +- go.sum | 12 +- pkg/unikontainers/libcontainer.go | 99 ++++++++ pkg/unikontainers/libcontainer_cgo.go | 85 +++++++ pkg/unikontainers/libcontainer_test.go | 301 +++++++++++++++++++++++++ pkg/unikontainers/unikontainers.go | 2 + 9 files changed, 658 insertions(+), 12 deletions(-) create mode 100644 cmd/urunc/create_libcontainer.go create mode 100644 pkg/unikontainers/libcontainer.go create mode 100644 pkg/unikontainers/libcontainer_cgo.go create mode 100644 pkg/unikontainers/libcontainer_test.go diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 5469f41cd..9e6a6289c 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -434,3 +434,6 @@ Hyperlight Odysseas Kalaitsidis nsexec +specconv +Readonlyfs +capab diff --git a/cmd/urunc/create.go b/cmd/urunc/create.go index 75118af98..14bbffea1 100644 --- a/cmd/urunc/create.go +++ b/cmd/urunc/create.go @@ -73,12 +73,16 @@ var createCommand = &cli.Command{ if err := checkArgs(cmd, 1, exactArgs); err != nil { return err } - if !cmd.Bool("reexec") { - uruncCfg, _ := unikontainers.LoadUruncConfig(unikontainers.UruncConfigPath) // ignore the error and use default config - return createUnikontainer(cmd, uruncCfg) + if cmd.Bool("reexec") { + return reexecUnikontainer(cmd) } - return reexecUnikontainer(cmd) + uruncCfg, _ := unikontainers.LoadUruncConfig(unikontainers.UruncConfigPath) // ignore the error and use default config + if uruncCfg.Runtime.Libcontainer { + return libcontainerCreate(cmd, uruncCfg) + } + + return createUnikontainer(cmd, uruncCfg) }, } diff --git a/cmd/urunc/create_libcontainer.go b/cmd/urunc/create_libcontainer.go new file mode 100644 index 000000000..c229543b4 --- /dev/null +++ b/cmd/urunc/create_libcontainer.go @@ -0,0 +1,152 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + "os" + "strconv" + + "github.com/opencontainers/runc/libcontainer" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + m "github.com/urunc-dev/urunc/internal/metrics" + "github.com/urunc-dev/urunc/pkg/unikontainers" +) + +// monitorArgv is the internal argv of the urunc monitor process. +const monitorArgv = "monitor" + +// libcontainerCreate is "urunc create" when the libcontainer runtime is +// enabled. It configures libcontainer to spawn a specific urunc process +// ("urunc monitor") which will finalize the execution environment for a +// monitor and then execve it. +func libcontainerCreate(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (err error) { + unikontainer, err := newUnikontainer(cmd, uruncCfg) + if err != nil { + return err + } + + config, err := unikontainer.BuildContainerConfig(cmd.Bool("systemd-cgroup")) + if err != nil { + return err + } + + container, err := libcontainer.Create(unikontainer.LibcontainerRoot(), unikontainer.State.ID, config) + if err != nil { + return fmt.Errorf("failed to create the monitor's container: %w", err) + } + defer func() { + // Best effort cleanup in case of failure. Any cgroups or libcontainer + // metadata residuals can cause issues with later container creates. + if err != nil { + tmpErr := container.Destroy() + if tmpErr != nil { + logrus.WithError(tmpErr).Error("failed to destroy the monitor's container") + } + } + }() + + process, err := monitorProcess(cmd, unikontainer) + if err != nil { + return err + } + metrics.Capture(m.TS03) + + err = container.Start(process) + if err != nil { + return fmt.Errorf("failed to start the monitor's init process: %w", err) + } + // The init process is now waiting the exec fifo. If anything below fails, + // terminate and reap it before the Destroy above tears down the cgroup, so the + // reap does not race the cgroup removal (same order as runc). + defer func() { + if err != nil { + _ = process.Signal(os.Kill) + _, _ = process.Wait() + } + }() + + state, err := container.State() + if err != nil { + return fmt.Errorf("failed to read the monitor's container state: %w", err) + } + metrics.Capture(m.TS06) + + err = unikontainer.Create(state.InitProcessPid, cmd.String("pid-file")) + if err != nil { + return err + } + metrics.Capture(m.TS08) + + return nil +} + +// monitorProcess describes the urunc process that libcontainer starts. +// Args[0] is "/proc/self/exe" because libcontainer resolves it after the +// pivot: /proc is always mounted in the monitor rootfs, so this re-execs the +// sealed urunc binary without urunc having to be copied into that rootfs. +func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer) (*libcontainer.Process, error) { + process := &libcontainer.Process{ + Args: []string{"/proc/self/exe", monitorArgv, u.State.ID}, + Env: os.Environ(), + // "/" rather than the spec's Cwd: that one is the guest's working + // directory + Cwd: "/", + // TODO set uid, gid and additional groups here + Init: true, + Capabilities: unikontainers.MonitorCapabilities(u.Spec), + LogLevel: strconv.Itoa(int(logrus.GetLevel())), + } + + if !u.Spec.Process.Terminal { + process.Stdin = os.Stdin + process.Stdout = os.Stdout + process.Stderr = os.Stderr + + return process, nil + } + + // With a terminal, libcontainer's init allocates the pty and passes + // the master end over the console socket itself The console socket is + // an AF_UNIX socket the caller (e.g. containerd) is listening on, so + // it must be dialed, not opened as a file: open(2) on a socket inode + // fails with ENXIO. + consoleSocket := cmd.String("console-socket") + if consoleSocket == "" { + return nil, fmt.Errorf("the container requests a terminal but no console socket was given") + } + conn, err := net.Dial("unix", consoleSocket) + if err != nil { + return nil, fmt.Errorf("failed to dial the console socket %s: %w", consoleSocket, err) + } + defer conn.Close() + + unixConn, ok := conn.(*net.UnixConn) + if !ok { + return nil, fmt.Errorf("console socket connection is not a unix socket") + } + // File returns a dup of the socket fd, independent of conn, so closing conn + // above does not affect it. libcontainer passes it to its init as an extra + // file and sends the pty master over it. + socket, err := unixConn.File() + if err != nil { + return nil, fmt.Errorf("failed to get the console socket file: %w", err) + } + process.ConsoleSocket = socket + + return process, nil +} diff --git a/go.mod b/go.mod index 0b3601ef2..161c7b140 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/nubificus/hedge_cli v0.0.3 github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 + github.com/opencontainers/cgroups v0.0.4 github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runc v1.3.6 github.com/opencontainers/runtime-spec v1.2.1 @@ -41,7 +42,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.13.0 // indirect github.com/checkpoint-restore/go-criu/v6 v6.3.0 // indirect - github.com/cilium/ebpf v0.21.0 // indirect + github.com/cilium/ebpf v0.17.3 // indirect github.com/containerd/cgroups/v3 v3.1.0 // indirect github.com/containerd/console v1.0.5 // indirect github.com/containerd/continuity v0.5.0 // indirect @@ -69,7 +70,6 @@ require ( github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/mrunalp/fileutils v0.5.1 // indirect - github.com/opencontainers/cgroups v0.0.4 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/selinux v1.13.1 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 029ddfa1b..027790197 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA= github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI= -github.com/cilium/ebpf v0.21.0 h1:4dpx1J/B/1apeTmWBH5BkVLayHTkFrMovVPnHEk+l3k= -github.com/cilium/ebpf v0.21.0/go.mod h1:1kHKv6Kvh5a6TePP5vvvoMa1bclRyzUXELSs272fmIQ= +github.com/cilium/ebpf v0.17.3 h1:FnP4r16PWYSE4ux6zN+//jMcW4nMVRvuTLVTvCjyyjg= +github.com/cilium/ebpf v0.17.3/go.mod h1:G5EDHij8yiLzaqn0WjyfJHvRa+3aDlReIaLVRMvOyJk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/cgroups/v3 v3.1.0 h1:azxYVj+91ZgSnIBp2eI3k9y2iYQSR/ZQIgh9vKO+HSY= @@ -81,8 +81,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= -github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= @@ -145,8 +145,8 @@ github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= -github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= -github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= diff --git a/pkg/unikontainers/libcontainer.go b/pkg/unikontainers/libcontainer.go new file mode 100644 index 000000000..93d0cc8a0 --- /dev/null +++ b/pkg/unikontainers/libcontainer.go @@ -0,0 +1,99 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "slices" + + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// monCaps all the necessary capabilities for the "urunc init" process +// to finalize the setup of the monitor's execution environment. +// - CAP_NET_ADMIN: required for the creation and setup of th the tap device +// inside the container's network namespace. +var monCaps = []string{"CAP_NET_ADMIN"} + +// monitorOCISpec builds the OCI spec that describes the monitor's +// execution environment. It consists of the container's spec, +// but altered accordingly for the respective monitor and rootfs. +func monitorOCISpec(spec *specs.Spec, monRes monitorResources, rootfsParams types.RootfsParams) *specs.Spec { + monSpec := *spec + monSpec.Root = &specs.Root{Path: rootfsParams.MonRootfs, Readonly: false} + monSpec.Mounts = monRes.Mounts + + process := *spec.Process + process.Cwd = "/" + monSpec.Process = &process + + linux := *spec.Linux + linux.Devices = monRes.Devices + linux.Namespaces = spec.Linux.Namespaces + monSpec.Linux = &linux + + return &monSpec +} + +// MonitorCapabilities returns the capability set the monitor process requires +// to finalize the execution environment for the monitor. Currently, it consists +// of the container's own set plus CAP_NET_ADMIN in all five sets. +// TODO: Narrow down to the extremely necessary capabilities. +func MonitorCapabilities(spec *specs.Spec) *configs.Capabilities { + caps := specCapabilities(spec) + if caps == nil { + uniklog.Warn("The container declares no capabilities. The monitor inherits urunc's own set.") + return nil + } + + for _, capab := range monCaps { + caps.Bounding = withCap(caps.Bounding, capab) + caps.Effective = withCap(caps.Effective, capab) + caps.Permitted = withCap(caps.Permitted, capab) + caps.Inheritable = withCap(caps.Inheritable, capab) + caps.Ambient = withCap(caps.Ambient, capab) + } + + return caps +} + +// specCapabilities copies the container's capability set out of the OCI spec, +// returning nil when the spec declares none. +func specCapabilities(spec *specs.Spec) *configs.Capabilities { + if spec.Process == nil || spec.Process.Capabilities == nil { + return nil + } + + caps := spec.Process.Capabilities + + return &configs.Capabilities{ + Bounding: slices.Clone(caps.Bounding), + Effective: slices.Clone(caps.Effective), + Permitted: slices.Clone(caps.Permitted), + Inheritable: slices.Clone(caps.Inheritable), + Ambient: slices.Clone(caps.Ambient), + } +} + +// withCap returns caps with add appended, unless it is already there. +func withCap(caps []string, add string) []string { + if slices.Contains(caps, add) { + return caps + } + + return append(caps, add) +} diff --git a/pkg/unikontainers/libcontainer_cgo.go b/pkg/unikontainers/libcontainer_cgo.go new file mode 100644 index 000000000..5549cf090 --- /dev/null +++ b/pkg/unikontainers/libcontainer_cgo.go @@ -0,0 +1,85 @@ +//go:build cgo + +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file holds the parts of the libcontainer monitor path that import runc's +// specconv and libcontainer packages, both of which require cgo. urunc is always +// built with cgo, so it always has them. The containerd shim is built without +// cgo and never uses the libcontainer runtime path, so gating these behind the +// cgo build tag keeps the shim free of the cgo-only dependency. See +// libcontainer_nocgo.go for the non-cgo stub that lets Delete still link. + +package unikontainers + +import ( + "fmt" + "path/filepath" + + "github.com/opencontainers/cgroups" + // Registers the cgroup device-rule setter (DevicesSetV1/V2). Without this + // blank import the setter stays nil and starting a container with device + // rules fails with "cgroup manager is not configured to set device rules". + _ "github.com/opencontainers/cgroups/devices" + devices "github.com/opencontainers/cgroups/devices/config" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/specconv" +) + +// BuildContainerConfig modifies the container's OCI spec, according to the monitor +// mounts and devices gathered during InitialSetup +func (u *Unikontainer) BuildContainerConfig(systemdCgroup bool) (*configs.Config, error) { + monRes, err := loadMonitorResources(u.BaseDir) + if err != nil { + return nil, fmt.Errorf("failed to load monitor resources: %w", err) + } + + monSpec := monitorOCISpec(u.Spec, monRes, monRes.Rootfs) + + config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{ + CgroupName: u.State.ID, + UseSystemdCgroup: systemdCgroup, + Spec: monSpec, + }) + if err != nil { + return nil, fmt.Errorf("failed to build the monitor's libcontainer config: %w", err) + } + + // urunc applies monitor-specific seccomp filters, therefore libcontainer + // should not apply any. + config.Seccomp = nil + + // Allow all devices through cgroups to avoid issues with device access from + // "urunc init" or the monitor process. + // TODO: Revisit this in the future and apply a stricter model. + config.Cgroups.Resources = &cgroups.Resources{ + Devices: []*devices.Rule{{Type: devices.WildcardDevice, Allow: true}}, + } + + // The Prestart, CreateRuntime, CreateContainer and StartContainer + // hooks are part of the configuration and are libcontainer's to run, + // Poststart and Poststop stay with urunc, in start and delete + // respectively, due to the extra steps urunc performs (sandbox and + // network cleanup). + delete(config.Hooks, configs.Poststart) + delete(config.Hooks, configs.Poststop) + + return config, nil +} + +// LibcontainerRoot returns the state directory libcontainer uses for the +// monitor's process execution environment. It is placed under the root directory +func (u *Unikontainer) LibcontainerRoot() string { + return filepath.Join(u.RootDir, libcontainerDirName) +} diff --git a/pkg/unikontainers/libcontainer_test.go b/pkg/unikontainers/libcontainer_test.go new file mode 100644 index 000000000..1187e7304 --- /dev/null +++ b/pkg/unikontainers/libcontainer_test.go @@ -0,0 +1,301 @@ +//go:build cgo + +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "path/filepath" + "testing" + + devices "github.com/opencontainers/cgroups/devices/config" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// newTestUnikontainer builds the minimum Unikontainer that BuildContainerConfig +// needs: a spec and a monitor resources file. It returns the chosen rootfs +// params so the caller can pass them to BuildContainerConfig. The monitor rootfs +// is a real directory, because specconv resolves it and libcontainer's +// validation insists on an existing absolute path. +func newTestUnikontainer(t *testing.T, spec *specs.Spec, monRes monitorResources) (*Unikontainer, types.RootfsParams) { + t.Helper() + + baseDir := t.TempDir() + monRootfs := t.TempDir() + + rootfsParams := newRootfsResult("initrd", "initrd.cpio", "", monRootfs) + monRes.Rootfs = rootfsParams + + err := saveMonitorResources(baseDir, monRes) + require.NoError(t, err) + + u := &Unikontainer{ + BaseDir: baseDir, + RootDir: filepath.Dir(baseDir), + Spec: spec, + State: &specs.State{ + ID: "test-container", + }, + UruncCfg: defaultUruncConfig(), + } + + return u, rootfsParams +} + +// testSpec returns a spec close to what a real container carries: a couple of +// namespaces, a capability set and a cgroup path. +func testSpec() *specs.Spec { + return &specs.Spec{ + Version: specs.Version, + Root: &specs.Root{Path: "rootfs", Readonly: true}, + Process: &specs.Process{ + Cwd: "/guest/workdir", + Args: []string{"/unikernel"}, + Capabilities: &specs.LinuxCapabilities{ + Bounding: []string{"CAP_CHOWN", "CAP_KILL"}, + Effective: []string{"CAP_CHOWN"}, + Permitted: []string{"CAP_CHOWN", "CAP_KILL"}, + Inheritable: []string{"CAP_CHOWN"}, + Ambient: []string{"CAP_CHOWN"}, + }, + }, + Linux: &specs.Linux{ + CgroupsPath: "/pods/test-container", + Namespaces: []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: "/proc/1234/ns/net"}, + {Type: specs.MountNamespace}, + {Type: specs.UTSNamespace}, + }, + }, + } +} + +// countString returns how many times want appears in values. +func countString(values []string, want string) int { + count := 0 + for _, v := range values { + if v == want { + count++ + } + } + + return count +} + +func TestBuildContainerConfig(t *testing.T) { + t.Run("roots the monitor at the monitor rootfs", func(t *testing.T) { + t.Parallel() + spec := testSpec() + u, rootfsParams := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.Equal(t, rootfsParams.MonRootfs, config.Rootfs) + // The container's rootfs is read-only, the monitor's is not. + assert.False(t, config.Readonlyfs) + assert.False(t, config.NoPivotRoot) + // The container's spec must not have been modified. + assert.Equal(t, "rootfs", spec.Root.Path) + assert.True(t, spec.Root.Readonly) + assert.Equal(t, "/guest/workdir", spec.Process.Cwd) + }) + + t.Run("maps the namespaces from the spec", func(t *testing.T) { + t.Parallel() + u, _ := newTestUnikontainer(t, testSpec(), monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.True(t, config.Namespaces.Contains(configs.NEWNET)) + assert.True(t, config.Namespaces.Contains(configs.NEWNS)) + assert.True(t, config.Namespaces.Contains(configs.NEWUTS)) + assert.False(t, config.Namespaces.Contains(configs.NEWPID)) + assert.False(t, config.Namespaces.Contains(configs.NEWUSER)) + assert.Equal(t, "/proc/1234/ns/net", config.Namespaces.PathOf(configs.NEWNET)) + }) + + t.Run("carries the monitor mounts and devices", func(t *testing.T) { + t.Parallel() + monRes := monitorResources{ + Mounts: []specs.Mount{ + tmpfsMount("/tmp", "65536k"), + bindMount("/usr/bin/qemu-system-x86_64", "/usr/bin/qemu-system-x86_64", true), + }, + Devices: []specs.LinuxDevice{ + {Path: "/dev/kvm", Type: "c", Major: 10, Minor: 232}, + {Path: "/dev/null", Type: "c", Major: 1, Minor: 3}, + }, + } + u, _ := newTestUnikontainer(t, testSpec(), monRes) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + require.Len(t, config.Mounts, 2) + assert.Equal(t, "/tmp", config.Mounts[0].Destination) + assert.Equal(t, "/usr/bin/qemu-system-x86_64", config.Mounts[1].Destination) + + // libcontainer adds the standard device nodes on top of the ones urunc + // gathered, and drops its own entry for a device the spec already + // declares, so /dev/null must appear exactly once. + paths := make([]string, 0, len(config.Devices)) + for _, dev := range config.Devices { + paths = append(paths, dev.Path) + } + assert.Contains(t, paths, "/dev/kvm") + assert.Contains(t, paths, "/dev/zero") + assert.Equal(t, 1, countString(paths, "/dev/null")) + }) + + t.Run("places the monitor in the container cgroup without limits", func(t *testing.T) { + t.Parallel() + spec := testSpec() + limit := int64(64 * 1024 * 1024) + spec.Linux.Resources = &specs.LinuxResources{ + Memory: &specs.LinuxMemory{Limit: &limit}, + } + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.Equal(t, "/pods/test-container", config.Cgroups.Path) + assert.False(t, config.Cgroups.Systemd) + + // Only the allow-all device rule, and in particular no memory limit: + // the guest's RAM is sized from the same limit, so applying it to the + // monitor's cgroup would OOM-kill the VMM. + require.NotNil(t, config.Cgroups.Resources) + assert.Zero(t, config.Cgroups.Resources.Memory) + assert.Zero(t, config.Cgroups.Resources.CpuQuota) + require.Len(t, config.Cgroups.Resources.Devices, 1) + assert.Equal(t, devices.WildcardDevice, config.Cgroups.Resources.Devices[0].Type) + assert.True(t, config.Cgroups.Resources.Devices[0].Allow) + }) + + t.Run("honors the systemd cgroup driver", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Linux.CgroupsPath = "system.slice:urunc:test-container" + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(true) + require.NoError(t, err) + + assert.True(t, config.Cgroups.Systemd) + assert.Equal(t, "system.slice", config.Cgroups.Parent) + assert.Equal(t, "urunc", config.Cgroups.ScopePrefix) + assert.Equal(t, "test-container", config.Cgroups.Name) + }) + + t.Run("does not register the poststart and poststop hooks", func(t *testing.T) { + t.Parallel() + spec := testSpec() + hook := specs.Hook{Path: "/bin/true"} + spec.Hooks = &specs.Hooks{ + Prestart: []specs.Hook{hook}, + CreateRuntime: []specs.Hook{hook}, + CreateContainer: []specs.Hook{hook}, + StartContainer: []specs.Hook{hook}, + Poststart: []specs.Hook{hook}, + Poststop: []specs.Hook{hook}, + } + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + // urunc keeps its own timing for these two, since libcontainer would run + // Poststart from container.Start(), i.e. during create. + assert.False(t, config.HasHook(configs.Poststart)) + assert.False(t, config.HasHook(configs.Poststop)) + // The rest are libcontainer's to run. + assert.True(t, config.HasHook(configs.Prestart)) + assert.True(t, config.HasHook(configs.CreateRuntime)) + assert.True(t, config.HasHook(configs.CreateContainer)) + assert.True(t, config.HasHook(configs.StartContainer)) + }) + + // urunc delegates syscall filtering to the monitor and is never built with + // the seccomp build tag, so a non-nil Seccomp here would fail libcontainer's + // init. This guards against a specconv bump re-introducing it. + t.Run("never carries a seccomp profile", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Linux.Seccomp = &specs.LinuxSeccomp{ + DefaultAction: specs.ActErrno, + Architectures: []specs.Arch{specs.ArchX86_64}, + Syscalls: []specs.LinuxSyscall{ + {Names: []string{"read", "write"}, Action: specs.ActAllow}, + }, + } + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.Nil(t, config.Seccomp) + }) +} + +func TestMonitorCapabilities(t *testing.T) { + t.Run("adds CAP_NET_ADMIN to all five sets", func(t *testing.T) { + t.Parallel() + spec := testSpec() + + caps := MonitorCapabilities(spec) + + require.NotNil(t, caps) + for name, set := range map[string][]string{ + "bounding": caps.Bounding, + "effective": caps.Effective, + "permitted": caps.Permitted, + "inheritable": caps.Inheritable, + "ambient": caps.Ambient, + } { + assert.Contains(t, set, monCaps[0], "%s must carry CAP_NET_ADMIN", name) + assert.Contains(t, set, "CAP_CHOWN", "%s must keep the container's own capabilities", name) + } + + // The container's spec must not have been touched. + assert.NotContains(t, spec.Process.Capabilities.Bounding, monCaps[0]) + }) + + t.Run("does not duplicate a CAP_NET_ADMIN already present", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Process.Capabilities.Bounding = []string{monCaps[0]} + + caps := MonitorCapabilities(spec) + + require.NotNil(t, caps) + assert.Equal(t, []string{monCaps[0]}, caps.Bounding) + }) + + t.Run("a spec without capabilities gives nil", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Process.Capabilities = nil + + assert.Nil(t, MonitorCapabilities(spec)) + }) +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 9a7f6c8ff..d61fd0890 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -44,6 +44,8 @@ import ( const ( monitorRootfsDirName string = "monRootfs" containerRootfsMountPath string = "/cntrRootfs" + // libcontainerDirName is the directory under urunc's root used from libcontainer + libcontainerDirName string = "libcontainer" ) var uniklog = logrus.WithField("subsystem", "unikontainers") From 59ed961fc14266a50981fbf6fbc73e5ab3c3c13a Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 29 Jul 2026 07:15:22 +0200 Subject: [PATCH 07/11] feat(libcontainer): Create a FIFO for urunc IPC in libcontainer The "urunc monitor init" process can't reach the urunc-mode socket, so signal readiness over a FIFO in the state dir. Create wires the write end to the monitor as its single ExtraFile (fd 3 / ReadyPipeFD). Start reads the message. - Create opens O_RDWR so a writer is always present: start blocks instead of seeing a premature EOF, yet still gets EOF if the monitor dies without writing. - Read side opens O_RDONLY|O_NONBLOCK (open never waits on a writer); the read itself blocks via the runtime poller. - One byte: message readyOK(0)=success; any other byte, read error, or EOF=fail. Signed-off-by: Charalampos Mainas --- .github/linters/urunc-dict.txt | 2 + cmd/urunc/create_libcontainer.go | 17 ++++- pkg/unikontainers/ipc.go | 114 +++++++++++++++++++++++++++++ pkg/unikontainers/unikontainers.go | 17 +++++ 4 files changed, 148 insertions(+), 2 deletions(-) diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 9e6a6289c..0c867b64c 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -437,3 +437,5 @@ nsexec specconv Readonlyfs capab +werr +cerr diff --git a/cmd/urunc/create_libcontainer.go b/cmd/urunc/create_libcontainer.go index c229543b4..981955c4c 100644 --- a/cmd/urunc/create_libcontainer.go +++ b/cmd/urunc/create_libcontainer.go @@ -60,7 +60,17 @@ func libcontainerCreate(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( } }() - process, err := monitorProcess(cmd, unikontainer) + // The write end of the ready pipe. The monitor process inherits the + // write end of the ready pipe and reports the outcome of its setup + // back to "urunc start" over it. + + readyFile, err := unikontainers.CreateReadyPipe(unikontainer.BaseDir) + if err != nil { + return err + } + defer readyFile.Close() + + process, err := monitorProcess(cmd, unikontainer, readyFile) if err != nil { return err } @@ -99,7 +109,7 @@ func libcontainerCreate(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( // Args[0] is "/proc/self/exe" because libcontainer resolves it after the // pivot: /proc is always mounted in the monitor rootfs, so this re-execs the // sealed urunc binary without urunc having to be copied into that rootfs. -func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer) (*libcontainer.Process, error) { +func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile *os.File) (*libcontainer.Process, error) { process := &libcontainer.Process{ Args: []string{"/proc/self/exe", monitorArgv, u.State.ID}, Env: os.Environ(), @@ -110,6 +120,9 @@ func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer) (*libcontai Init: true, Capabilities: unikontainers.MonitorCapabilities(u.Spec), LogLevel: strconv.Itoa(int(logrus.GetLevel())), + // The ready pipe's write end. It must be the only/first ExtraFile so the + // monitor inherits it at unikontainers.ReadyPipeFD: + ExtraFiles: []*os.File{readyFile}, } if !u.Spec.Process.Terminal { diff --git a/pkg/unikontainers/ipc.go b/pkg/unikontainers/ipc.go index 86d54e4dc..2fc7327d1 100644 --- a/pkg/unikontainers/ipc.go +++ b/pkg/unikontainers/ipc.go @@ -17,10 +17,12 @@ package unikontainers import ( "errors" "fmt" + "io" "io/fs" "net" "os" "path/filepath" + "syscall" "time" "github.com/sirupsen/logrus" @@ -29,6 +31,14 @@ import ( type IPCMessage string const ( + // readyPipeName is the FIFO in the container's state dir (BaseDir). + readyPipeName = "urunc-ready.fifo" + // readyOK is the byte the urunc monitor process writes on a successful setup. + readyOK byte = 0 + // ReadyPipeFD is the descriptor the FIFO's write end lands on inside + // the urunc monitor process.The ready pipe is passed as the single, + // first ExtraFile so it is always fd 3. All other files are placed after it. + ReadyPipeFD = 3 // Socket for messages towards reexec. The reexec process listens in this socket reexecSock = "reexec.sock" // Socket for messages from reexec. The reexec process writes in this socket @@ -55,6 +65,10 @@ func getReexecSockAddr(baseDir string) string { return getSockAddr(baseDir, reexecSock) } +func getReadyPipePath(baseDir string) string { + return filepath.Join(baseDir, readyPipeName) +} + func ensureValidSockAddr(sockAddr string) error { if sockAddr == "" { return fmt.Errorf("socket address is empty") @@ -169,3 +183,103 @@ func AwaitMessage(listener *net.UnixListener, expectedMessage IPCMessage) error } return nil } + +// CreateReadyPipe creates the ready FIFO in the container's state dir and +// opens its write end. The returned file is meant to be passed as the urunc's +// monitor process' single Process.ExtraFiles entry, so the monitor inherits it +// at ReadyPipeFD. +func CreateReadyPipe(baseDir string) (*os.File, error) { + path := getReadyPipePath(baseDir) + + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("failed to remove stale ready pipe %s: %w", path, err) + } + + err = syscall.Mkfifo(path, 0o600) + if err != nil { + return nil, fmt.Errorf("failed to create ready pipe %s: %w", path, err) + } + + // Open O_RDWR, which never blocks on a FIFO and, more importantly, + // keeps a write end open from the moment the monitor is born. That way + // the FIFO always "has a writer" while the monitor runs, so "urunc + // start" blocks reading it rather than seeing a premature EOF, yet it + // still reports EOF if the monitor dies without writing. + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("failed to open ready pipe %s: %w", path, err) + } + + return file, nil +} + +// openReadReadyPipe opens the read end of the ready FIFO that "urunc create" left in +// the state dir. +func (u *Unikontainer) openReadReadyPipe() error { + path := getReadyPipePath(u.BaseDir) + + // Open as non-blocking so the open never waits on a writer; + // the actual read in awaitReady blocks through the Go runtime poller. + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return fmt.Errorf("failed to open ready pipe %s: %w", path, err) + } + u.readyPipe = os.NewFile(uintptr(fd), path) + + return nil +} + +// closeReadReadyPipe closes the read end of the ready FIFO and removes it. +func (u *Unikontainer) closeReadReadyPipe() error { + if u.readyPipe != nil { + err := u.readyPipe.Close() + if err != nil { + uniklog.WithError(err).Error("failed to close the ready pipe") + } + } + + path := getReadyPipePath(u.BaseDir) + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove ready pipe %s: %w", path, err) + } + + return nil +} + +// awaitReadyPipe blocks until the urunc monitor process reports the outcome of its setup +// over the ready FIFO. A single readyOK byte means success; any other byte, a read +// error, or EOF (the monitor exited before reporting) means failure. +func (u *Unikontainer) awaitReadyPipe() error { + buf := make([]byte, 1) + + n, err := u.readyPipe.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("the monitor exited before reporting a successful start") + } + return fmt.Errorf("failed to read from the ready pipe: %w", err) + } + // buf is make([]byte, 1); index 0 is a valid value + if n != 1 || buf[0] != readyOK { //nolint:gosec + return fmt.Errorf("the monitor reported a failed start") + } + + return nil +} + +// signalReady reports the outcome of the monitor setup to "urunc start" over the +// ready FIFO inherited, then closes it so it is not left open in the +// monitor. +func signalReady(ok bool) error { + status := byte(1) + if ok { + status = readyOK + } + + _, werr := syscall.Write(ReadyPipeFD, []byte{status}) + cerr := syscall.Close(ReadyPipeFD) + + return errors.Join(werr, cerr) +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index d61fd0890..7c6daba65 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -63,6 +63,9 @@ type Unikontainer struct { UruncCfg *UruncConfig Listener *net.UnixListener Conn *net.UnixConn + // readyPipe is the read end of the libcontainer-mode ready FIFO, held by + // "urunc start" between opening it and reading the monitor's outcome. + readyPipe *os.File } // New parses the bundle and creates a new Unikontainer object @@ -1303,6 +1306,12 @@ func (u *Unikontainer) FormatNsenterInfo() (rdr io.Reader, retErr error) { // If it is not the reexec process then the listener will refer to the // uruncSock, the socket that holds messages from reexec to urunc instances func (u *Unikontainer) CreateListener(isReexec bool) error { + // In libcontainer mode the start side reads the monitor's outcome from a FIFO + // in the state dir, whose write end the monitor inherited from create. + if !isReexec && u.UruncCfg.Runtime.Libcontainer { + return u.openReadReadyPipe() + } + sockAddr := getUruncSockAddr(u.BaseDir) if isReexec { sockAddr = getReexecSockAddr(u.BaseDir) @@ -1321,6 +1330,10 @@ func (u *Unikontainer) CreateListener(isReexec bool) error { // DestroyListener destroys an existing listener over a socket func (u *Unikontainer) DestroyListener(isReexec bool) error { + if !isReexec && u.UruncCfg.Runtime.Libcontainer { + return u.closeReadReadyPipe() + } + sockAddr := getUruncSockAddr(u.BaseDir) if isReexec { sockAddr = getReexecSockAddr(u.BaseDir) @@ -1378,6 +1391,10 @@ func (u *Unikontainer) DestroyConn(isReexec bool) error { // AwaitMessage waits for a specific message in the listener of unikontainer instance func (u *Unikontainer) AwaitMsg(msg IPCMessage) error { + // In libcontainer mode the monitor reports over the ready FIFO. + if u.UruncCfg.Runtime.Libcontainer { + return u.awaitReadyPipe() + } return AwaitMessage(u.Listener, msg) } From 411c8af044bee5d289232e9c377b670e526f88c5 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 29 Jul 2026 07:55:43 +0200 Subject: [PATCH 08/11] feat(libcontainer): branch start between monitor init and reexec Split startContainer: in libcontainer mode Load the container and Exec() it (opens the exec fifo, releasing the init blocked since create); otherwise keep the reexec socket handshake. Both paths then wait on AwaitMsg, FIFO- or socket-backed by mode Signed-off-by: Charalampos Mainas --- cmd/urunc/start.go | 71 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/cmd/urunc/start.go b/cmd/urunc/start.go index a020b5471..51e15dd45 100644 --- a/cmd/urunc/start.go +++ b/cmd/urunc/start.go @@ -20,6 +20,7 @@ import ( "fmt" "os" + "github.com/opencontainers/runc/libcontainer" "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" m "github.com/urunc-dev/urunc/internal/metrics" @@ -70,38 +71,27 @@ func startUnikontainer(cmd *cli.Command) error { defer func() { tmpErr := unikontainer.DestroyListener(!unikontainers.FromReexec) if tmpErr != nil { - logrus.WithError(tmpErr).Error("failed to destroy listener on reexec socket") + logrus.WithError(tmpErr).Error("failed to destroy the start listener") } }() - // Send message to reexec to start the monitor - err = unikontainer.CreateConn(!unikontainers.FromReexec) - if err != nil { - err = fmt.Errorf("failed to create connection with reexec socket: %w", err) - return err - } - sendErr := unikontainer.SendMessage(unikontainers.StartExecve) - if sendErr != nil { - logrus.WithError(sendErr).Error("failed to send START message to reexec") - sendErr = fmt.Errorf("error sending START message: %w", sendErr) - } - // Regardless of the SendMessage status, make sure to clean up the socket, - // since it is not required anymore - cleanErr := unikontainer.DestroyConn(!unikontainers.FromReexec) - if cleanErr != nil { - logrus.WithError(cleanErr).Error("failed to destroy connection to reexec socket") - cleanErr = fmt.Errorf("error destroying connection to reexec socket: %w", cleanErr) + // Start the monitor environment setup process. For libcontainer the heavy lifting + // has be done from libcontainer and we just need to finalize the setup. + // For urunc native way, we need to setup the monitor rootfs too. + if unikontainer.UruncCfg.Runtime.Libcontainer { + err = startMonitorInit(unikontainer) + } else { + err = startReexec(unikontainer) } - err = errors.Join(sendErr, cleanErr) if err != nil { return err } metrics.Capture(m.TS13) - // wait ContainerStarted message on start.sock from reexec process + // Wait for the monitor process to report a successful start. err = unikontainer.AwaitMsg(unikontainers.StartSuccess) if err != nil { - err = fmt.Errorf("failed to get message from successful start from reexec: %w", err) + err = fmt.Errorf("failed to get the success message from the monitor: %w", err) return err } @@ -113,3 +103,42 @@ func startUnikontainer(cmd *cli.Command) error { return unikontainer.ExecuteHooks("Poststart") } + +// startMonitorInit starts the urunc monitor process, blocked since create from +// libcontainers, by opening the exec fifo. +func startMonitorInit(unikontainer *unikontainers.Unikontainer) error { + container, err := libcontainer.Load(unikontainer.LibcontainerRoot(), unikontainer.State.ID) + if err != nil { + return fmt.Errorf("failed to load the monitor's container: %w", err) + } + + err = container.Exec() + if err != nil { + return fmt.Errorf("failed to release the monitor process: %w", err) + } + + return nil +} + +// startReexec tells urunc's reexec process to set up the monitor's +// environment and launch it. +func startReexec(unikontainer *unikontainers.Unikontainer) error { + err := unikontainer.CreateConn(!unikontainers.FromReexec) + if err != nil { + return fmt.Errorf("failed to create connection with reexec socket: %w", err) + } + sendErr := unikontainer.SendMessage(unikontainers.StartExecve) + if sendErr != nil { + logrus.WithError(sendErr).Error("failed to send START message to reexec") + sendErr = fmt.Errorf("error sending START message: %w", sendErr) + } + // Regardless of the SendMessage status, make sure to clean up the socket, + // since it is not required anymore + cleanErr := unikontainer.DestroyConn(!unikontainers.FromReexec) + if cleanErr != nil { + logrus.WithError(cleanErr).Error("failed to destroy connection to reexec socket") + cleanErr = fmt.Errorf("error destroying connection to reexec socket: %w", cleanErr) + } + + return errors.Join(sendErr, cleanErr) +} From 90bcbf253012998b8b4c8522767908bb5dced686 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 5 Aug 2026 14:33:05 +0200 Subject: [PATCH 09/11] feat(libcontainer): add the internal monitor subcommand This is the command that libcontainer's init will execve to inside the monitor's container. Then ExecMonitor loads the spec, sets up the tap, drops to the container user, signals ready, and execve's the monitor (no return on success). - Skip root/XDG_RUNTIME_DIR resolution for this invocation: it never touches urunc's state dir, and the inherited XDG_RUNTIME_DIR names a path absent inside the rootfs, so preparing it would fail the monitor. - Container ID is only for log/ps labelling; everything real comes from the spec file. Env is the one libcontainer set (spec carries none). - signalReady(true) fires only after the exec command is built and closes the pipe so it isn't left open across the execve; failures signal false. Signed-off-by: Charalampos Mainas --- cmd/urunc/create_libcontainer.go | 25 ++++- cmd/urunc/main.go | 32 ++++--- cmd/urunc/monitor.go | 57 +++++++++++ pkg/unikontainers/monitor.go | 125 +++++++++++++++++++++++++ pkg/unikontainers/monitor_spec.go | 27 ++++++ pkg/unikontainers/monitor_spec_test.go | 37 ++++++++ 6 files changed, 290 insertions(+), 13 deletions(-) create mode 100644 cmd/urunc/monitor.go create mode 100644 pkg/unikontainers/monitor.go diff --git a/cmd/urunc/create_libcontainer.go b/cmd/urunc/create_libcontainer.go index 981955c4c..8778387a4 100644 --- a/cmd/urunc/create_libcontainer.go +++ b/cmd/urunc/create_libcontainer.go @@ -105,13 +105,36 @@ func libcontainerCreate(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( return nil } +// buildMonitorArgs assembles the "urunc monitor" argv setting the same log format +// and enabling debug logs based on the current urunc process. +// TODO: THe logs will be printed in stdout. We need to pass another fd with the logs +// in order to store them in the correct file. +func buildMonitorArgs(id string, logFormat string) []string { + args := []string{"/proc/self/exe"} + logLevel := logrus.GetLevel() + if logLevel >= logrus.DebugLevel { + // TODO: We need to pass the log level not just debug. + // However, this needs to change the cli args of urunc. + // So let's do it in future iteration. + args = append(args, "--debug") + } + + if logFormat != "" { + args = append(args, "--log-format", logFormat) + } + + args = append(args, monitorArgv, id) + + return args +} + // monitorProcess describes the urunc process that libcontainer starts. // Args[0] is "/proc/self/exe" because libcontainer resolves it after the // pivot: /proc is always mounted in the monitor rootfs, so this re-execs the // sealed urunc binary without urunc having to be copied into that rootfs. func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile *os.File) (*libcontainer.Process, error) { process := &libcontainer.Process{ - Args: []string{"/proc/self/exe", monitorArgv, u.State.ID}, + Args: buildMonitorArgs(u.State.ID, cmd.String("log-format")), Env: os.Environ(), // "/" rather than the spec's Cwd: that one is the guest's working // directory diff --git a/cmd/urunc/main.go b/cmd/urunc/main.go index 7431e1f16..fdbcda979 100644 --- a/cmd/urunc/main.go +++ b/cmd/urunc/main.go @@ -111,6 +111,7 @@ func main() { createCommand, deleteCommand, killCommand, + monitorCommand, runCommand, psCommand, // specCommand, @@ -118,20 +119,27 @@ func main() { // stateCommand, }, Before: func(_ context.Context, cmd *cli.Command) (context.Context, error) { - if !cmd.IsSet("root") { - xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") - if xdgRuntimeDir != "" && ShouldHonorXDGRuntimeDir() { - root := xdgRuntimeDir + "/urunc" - if err := prepareXDGRuntimeDir(root); err != nil { - return nil, err - } - if err := cmd.Set("root", root); err != nil { - return nil, err + // "urunc monitor" runs inside the monitor rootfs and + // never touches urunc's state directory, so it has no + // root to resolve. Also the inherited XDG_RUNTIME_DIR + // names a directory that does not exist in there, so + // preparing it would fail the monitor. + if !isMonitorInvocation(cmd) { + if !cmd.IsSet("root") { + xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") + if xdgRuntimeDir != "" && ShouldHonorXDGRuntimeDir() { + root := xdgRuntimeDir + "/urunc" + if err := prepareXDGRuntimeDir(root); err != nil { + return nil, err + } + if err := cmd.Set("root", root); err != nil { + return nil, err + } } } - } - if err := reviseRootDir(cmd); err != nil { - return nil, err + if err := reviseRootDir(cmd); err != nil { + return nil, err + } } // ignore error since ParseLogMetricsConfig will print a warning and return default values cfg, _ := unikontainers.ParseLogMetricsConfig(unikontainers.UruncConfigPath) diff --git a/cmd/urunc/monitor.go b/cmd/urunc/monitor.go new file mode 100644 index 000000000..23204ace9 --- /dev/null +++ b/cmd/urunc/monitor.go @@ -0,0 +1,57 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + "github.com/urunc-dev/urunc/pkg/unikontainers" +) + +// monitorCommand is not part of the OCI runtime interface and no user or shim +// ever calls it: it is the argv libcontainer's init execve's inside the +// monitor rootfs. By that point the vast majority of the execution environment +// (namespaces, cgroups, mounts, etc.) are all in place and all +// that is left is to create the tap device and become the monitor. +// It takes the container ID purely so that the logs and the process listing name +// the container; everything it actually needs comes from the monitor spec file that +// create wrote inside the monitor's rootfs. +var monitorCommand = &cli.Command{ + Name: monitorArgv, + Usage: "internal: launch the monitor from inside its own rootfs", + ArgsUsage: ``, + Hidden: true, + Action: func(_ context.Context, cmd *cli.Command) error { + logrus.WithField("command", "MONITOR").WithField("args", os.Args).Debug("urunc INVOKED") + err := checkArgs(cmd, 1, exactArgs) + if err != nil { + return err + } + metrics.SetLoggerContainerID(cmd.Args().First()) + + // ExecMonitor does not return on success: the monitor replaces this + // process and becomes the container process. + return unikontainers.ExecMonitor(metrics) + }, +} + +// isMonitorInvocation reports whether this process is the "urunc monitor" that +// libcontainer starts inside the monitor container. +func isMonitorInvocation(cmd *cli.Command) bool { + return cmd.Args().First() == monitorArgv +} diff --git a/pkg/unikontainers/monitor.go b/pkg/unikontainers/monitor.go new file mode 100644 index 000000000..d282d5af5 --- /dev/null +++ b/pkg/unikontainers/monitor.go @@ -0,0 +1,125 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "errors" + "fmt" + "os" + "syscall" + + m "github.com/urunc-dev/urunc/internal/metrics" + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" + "github.com/urunc-dev/urunc/pkg/unikontainers/unikernels" +) + +// ExecMonitor is the entry point of the urunc process libcontainer starts +// inside the monitor's container. It reads the monitor spec, finalizes the +// environment for the monitor execution and execs it. +func ExecMonitor(metrics m.Writer) error { + // The ReadyPipeFD is passed as ExtraFle with libcontainer, but in order to do + // so, it gets CLOEXEC cleared. Therefore, restore it before spawning any child + // (e.g. virtiofsd from PreStartCmd): so the child does not inherit it. + syscall.CloseOnExec(ReadyPipeFD) + // this function is supposed to be called after the libcontainer has created the + // monitor container and is therefore in "/". + const pivotedRoot = "/" + + ms, err := LoadMonitorSpec(pivotedRoot) + if err != nil { + sigErr := signalReady(false) + if sigErr != nil { + uniklog.WithError(sigErr).Error("failed to signal the failed start") + } + return fmt.Errorf("failed to read the monitor spec: %w", err) + } + metrics.Capture(m.TS14) + + err = runMonitor(metrics, ms) + if err != nil { + uniklog.WithError(err).Error("setting up execution environment for monitor") + sigErr := signalReady(false) + if sigErr != nil { + uniklog.WithError(sigErr).Error("failed to signal the failed start") + } + + return errors.Join(err, sigErr) + } + + return nil +} + +// runMonitor sets up the monitor's network and guest command, drops to the +// container user and execve's the monitor. Every failure returns to caller +// On success it does not return. +func runMonitor(metrics m.Writer, ms MonitorSpec) error { + // The monitor's environment is not carried in the spec: libcontainer already + // gave this process the environment it configured for the monitor. + ms.ExecArgs.Environment = os.Environ() + + unikernel, err := unikernels.New(ms.UnikernelType) + if err != nil { + return err + } + + // TODO simplify this + monitorsConfig := map[string]types.MonitorConfig{ms.MonitorType: ms.MonitorCfg} + vmm, err := hypervisors.NewVMM(hypervisors.VmmType(ms.MonitorType), monitorsConfig) + if err != nil { + return err + } + + netArgs, err := SetupNet(ms.NetworkType, ms.User.UID, ms.User.GID) + if err != nil { + return fmt.Errorf("failed to setup network: %w", err) + } + metrics.Capture(m.TS16) + ms.ExecArgs.Net = netArgs + ms.GuestParams.Net = netArgs + + ms.ExecArgs.Command, err = buildUnikernelCommand(unikernel, ms.GuestParams) + if err != nil { + return err + } + + // Drop to the container user, which also clears the capabilities the tap + // device needed. From here on the monitor runs unprivileged. + err = setupUser(ms.User) + if err != nil { + return err + } + metrics.Capture(m.TS17) + + err = spawnProcess(ms.PreStartCmd) + if err != nil { + return err + } + + execCmd, err := vmm.BuildExecCmd(ms.ExecArgs, unikernel) + if err != nil { + return err + } + + // Report a successful setup to "urunc start" over the ready pipe, only after + // the command has been built. signalReady closes the pipe, so it is not left + // open in the monitor after the execve. + err = signalReady(true) + if err != nil { + return err + } + + return execMonitor(metrics, vmm, ms.ExecArgs, execCmd) +} diff --git a/pkg/unikontainers/monitor_spec.go b/pkg/unikontainers/monitor_spec.go index 31c0033cb..85907fe0f 100644 --- a/pkg/unikontainers/monitor_spec.go +++ b/pkg/unikontainers/monitor_spec.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" securejoin "github.com/cyphar/filepath-securejoin" "github.com/opencontainers/runtime-spec/specs-go" @@ -77,3 +78,29 @@ func (u *Unikontainer) writeMonitorSpec(rootfsParams types.RootfsParams, monRes return nil } + +// LoadMonitorSpec reads the monitor spec-file from dir +func LoadMonitorSpec(dir string) (MonitorSpec, error) { + var ms MonitorSpec + + data, err := os.ReadFile(filepath.Join(dir, monitorSpecFilename)) + if err != nil { + return ms, err + } + + err = json.Unmarshal(data, &ms) + if err != nil { + return ms, fmt.Errorf("could not decode the monitor spec: %w", err) + } + + return ms, nil +} + +// RemoveMonitorSpec deletes the monitor spec file from dir. +func RemoveMonitorSpec(dir string) error { + path, err := securejoin.SecureJoin(dir, monitorSpecFilename) + if err != nil { + return fmt.Errorf("could not resolve path for monitor spec: %w", err) + } + return os.Remove(path) +} diff --git a/pkg/unikontainers/monitor_spec_test.go b/pkg/unikontainers/monitor_spec_test.go index 86eda8571..bb5113824 100644 --- a/pkg/unikontainers/monitor_spec_test.go +++ b/pkg/unikontainers/monitor_spec_test.go @@ -159,3 +159,40 @@ func TestWriteMonitorSpec(t *testing.T) { assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) }) } + +func TestMonitorSpecFile(t *testing.T) { + t.Run("can be loaded back after being written", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got, err := LoadMonitorSpec(monRootfs) + require.NoError(t, err) + assert.Equal(t, "test-container", got.ContainerID) + }) + + t.Run("can be removed once it has been read", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + err = RemoveMonitorSpec(monRootfs) + require.NoError(t, err) + + _, err = LoadMonitorSpec(monRootfs) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("reports a missing file", func(t *testing.T) { + t.Parallel() + + _, err := LoadMonitorSpec(t.TempDir()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} From a7082425a2b4638d1797cc9e5f07ed1994f0ca90 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Fri, 7 Aug 2026 11:00:42 +0200 Subject: [PATCH 10/11] feat(libcontainer): tear down the monitor container on delete In libcontainer mode Delete() destroys the monitor's libcontainer state + cgroup (Load then Destroy, runc-style) and removes the .monitor_spec.json left in the container rootfs. - destroyLibcontainer split cgo/nocgo: the real implementation imports runc's libcontainer (cgo-only); the shim is built without cgo and never takes this path, so a no-op stub keeps Delete linking. - Failed/absent Load = "nothing to destroy" (create that failed before libcontainer.Create, or an already-completed teardown). - Missing spec file on removal is ignored (os.ErrNotExist). Signed-off-by: Charalampos Mainas --- pkg/unikontainers/libcontainer_cgo.go | 20 +++++++++++++++++ pkg/unikontainers/libcontainer_nocgo.go | 29 +++++++++++++++++++++++++ pkg/unikontainers/unikontainers.go | 19 ++++++++++++++-- 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 pkg/unikontainers/libcontainer_nocgo.go diff --git a/pkg/unikontainers/libcontainer_cgo.go b/pkg/unikontainers/libcontainer_cgo.go index 5549cf090..65bcb0b8e 100644 --- a/pkg/unikontainers/libcontainer_cgo.go +++ b/pkg/unikontainers/libcontainer_cgo.go @@ -33,6 +33,7 @@ import ( // rules fails with "cgroup manager is not configured to set device rules". _ "github.com/opencontainers/cgroups/devices" devices "github.com/opencontainers/cgroups/devices/config" + "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/specconv" ) @@ -83,3 +84,22 @@ func (u *Unikontainer) BuildContainerConfig(systemdCgroup bool) (*configs.Config func (u *Unikontainer) LibcontainerRoot() string { return filepath.Join(u.RootDir, libcontainerDirName) } + +// destroyLibcontainer removes the monitor's libcontainer state directory and its +// cgroup. Following runc's delete. +func (u *Unikontainer) destroyLibcontainer() error { + container, err := libcontainer.Load(u.LibcontainerRoot(), u.State.ID) + if err != nil { + // Nothing to destroy: e.g. a create that failed before + // libcontainer.Create, or an already-completed teardown. + uniklog.Debugf("no monitor libcontainer state to destroy: %v", err) + return nil + } + + err = container.Destroy() + if err != nil { + return fmt.Errorf("failed to destroy the monitor's libcontainer state: %w", err) + } + + return nil +} diff --git a/pkg/unikontainers/libcontainer_nocgo.go b/pkg/unikontainers/libcontainer_nocgo.go new file mode 100644 index 000000000..daba9b773 --- /dev/null +++ b/pkg/unikontainers/libcontainer_nocgo.go @@ -0,0 +1,29 @@ +//go:build !cgo + +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The libcontainer monitor path requires cgo (runc's specconv and libcontainer +// packages). In a non-cgo build -- the containerd shim -- that path is never +// taken, so this stub stands in for destroyLibcontainer, which Delete references +// unconditionally. BuildContainerConfig and LibcontainerRoot need no stub: +// nothing compiled in a non-cgo build references them. + +package unikontainers + +// destroyLibcontainer is a no-op in a non-cgo build; see libcontainer_cgo.go for +// the real implementation. +func (u *Unikontainer) destroyLibcontainer() error { + return nil +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 7c6daba65..5b0a367b2 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -840,6 +840,15 @@ func (u *Unikontainer) Delete() error { return fmt.Errorf("cannot delete running container: %s", u.State.ID) } + // In libcontainer mode, tear down the monitor's libcontainer state and cgroup. + // Like runc. + if u.UruncCfg.Runtime.Libcontainer { + err := u.destroyLibcontainer() + if err != nil { + return err + } + } + // Restore the block volume mounts that were unmounted during create, // so their sources become discoverable by future containers. Do it in // a best-effort way, since a failure to restore a mount should not @@ -883,11 +892,17 @@ func (u *Unikontainer) Delete() error { dirs = append(dirs, monitorRootfsDirName) prefPath = bundleDir } else { - // Otherwise remove the enw directories we created inside the - // container's rootfs. + // Otherwise remove the enw directories we created and the monitor spec + // file inside the container's rootfs. // We do not need to unmount anything here, since we rely on Linux // to do the cleanup for us. This will happen automatically, // when the mount namespace gets destroyed + err = RemoveMonitorSpec(rootfsDir) + // Ignore the case where the file does not exist. + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove the monitor spec: %w", err) + } + dirs = []string{ "/lib", "/lib64", From 3d72bea4c903b29223d5b99dbefe91167c67b9ab Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 19 Aug 2026 11:12:07 +0200 Subject: [PATCH 11/11] feat(libcontainer): Add tests for libcontainer in CI Since we currently have 2 versions of urunc, we need to test both urunc native way and the libcontainer one for the setup of the monitor's execution environment setup To do that, repeat the vm_tests and kind test by simply editing the urunc configuration enabling libcontainer. We might want to check how to improve this in the future. Signed-off-by: Charalampos Mainas --- .github/workflows/kind_test.yml | 42 +++++++++++++++++++++++++++++++++ .github/workflows/vm_test.yml | 18 ++++++++++++++ Makefile | 20 +++++++++------- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kind_test.yml b/.github/workflows/kind_test.yml index 43719e70c..c77f757de 100644 --- a/.github/workflows/kind_test.yml +++ b/.github/workflows/kind_test.yml @@ -335,6 +335,48 @@ jobs: grep "Hello world" /tmp/logs.txt kubectl describe pod hello-spt-rumprun-block || true + # Re-run the hello-world deployment with urunc's libcontainer runtime + # enabled, reusing the same kind cluster and urunc install. + - name: Deploy and verify with libcontainer + run: | + docker exec urunc-test-control-plane bash -c ' + cat >> /etc/urunc/config.toml < /dev/null <<'EOF' + + [runtime] + libcontainer = true + EOF + export GOROOT=$(go env GOROOT) + export PATH="$GOROOT/bin:$PATH" + if [ "${{ matrix.arch }}" = "arm64" ]; then + sudo -E env "PATH=$PATH" "GOROOT=$GOROOT" make ${{ matrix.test }}_Spt + else + sudo -E env "PATH=$PATH" "GOROOT=$GOROOT" make ${{ matrix.test }} SKIP="Firecracker-unikraft-httpreply-static-net" + fi + - name: Dump urunc logs on failure if: failure() run: | diff --git a/Makefile b/Makefile index b0329d2dc..42a0bbb7d 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,10 @@ CGO := CGO_ENABLED=1 NOCGO := CGO_ENABLED=0 TEST_FLAGS := "-count=1" TEST_OPTS += -timeout 20m +# Skip the execution of a test +# make test_crictl SKIP="Firecracker-unikraft-httpreply-static-net" +# make test_crictl SKIP="Crictl.*(CaseA|CaseB)" +GINKGO_SKIP := $(if $(SKIP),--ginkgo.skip="$(SKIP)") BUILD_TAGS ?= netgo osusergo # Linking variables @@ -273,56 +277,56 @@ test_unikernels: .PHONY: test_nerdctl test_nerdctl: @echo "Testing nerdctl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Nerdctl" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Nerdctl" $(GINKGO_SKIP) @echo " " ## test_ctr Run all end-to-end tests with ctr .PHONY: test_ctr test_ctr: @echo "Testing ctr" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Ctr" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Ctr" $(GINKGO_SKIP) @echo " " ## test_crictl Run all end-to-end tests with crictl .PHONY: test_crictl test_crictl: @echo "Testing crictl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Crictl" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Crictl" $(GINKGO_SKIP) @echo " " ## test_docker Run all end-to-end tests with docker .PHONY: test_docker test_docker: @echo "Testing docker" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Docker" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Docker" $(GINKGO_SKIP) @echo " " ## test_nerdctl_[pattern] Run all end-to-end tests with nerdctl that match pattern .PHONY: test_nerdctl_% test_nerdctl_%: @echo "Testing nerdctl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Nerdctl.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Nerdctl.*$*" $(GINKGO_SKIP) @echo " " ## test_ctr_[pattern] Run all end-to-end tests with ctr that match pattern .PHONY: test_ctr_% test_ctr_%: @echo "Testing ctr" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Ctr.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Ctr.*$*" $(GINKGO_SKIP) @echo " " ## test_crictl_[pattern] Run all end-to-end tests with crictl that match pattern .PHONY: test_crictl_% test_crictl_%: @echo "Testing crictl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Crictl.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Crictl.*$*" $(GINKGO_SKIP) @echo " " ## test_docker_[pattern] Run all end-to-end tests with docker that match pattern .PHONY: test_docker_% test_docker_%: @echo "Testing docker" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Docker.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Docker.*$*" $(GINKGO_SKIP) @echo " " ## help Show this help message