vm: per-VM cgroup v2 CPU scope with Guaranteed-at-N defaults (#182)

cgroup/cgroup.go

@@ -0,0 +1,239 @@
+// Package cgroup places VMM processes into per-VM cgroup v2 CPU scopes.
+package cgroup
+
+import (
+	"cmp"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/cocoonstack/cocoon/types"
+	"github.com/cocoonstack/cocoon/utils"
+)
+
+const (
+	// Root is the cgroup v2 unified hierarchy mount point.
+	Root = "/sys/fs/cgroup"
+	// DefaultParent holds every per-VM scope unless cgroup_parent overrides it.
+	DefaultParent = "cocoon.slice"
+	// DefaultPeriodUs is the kernel's default cpu.max period.
+	DefaultPeriodUs = 100000
+
+	// MinWeight/MaxWeight are the kernel's cpu.weight bounds.
+	MinWeight = 1
+	MaxWeight = 10000
+	// MinPeriodUs/MaxPeriodUs are the kernel's cpu.max period bounds.
+	MinPeriodUs = 1000
+	MaxPeriodUs = 1000000
+	// MinQuotaUs is the kernel's minimum cpu.max quota.
+	MinQuotaUs = 1000
+
+	scopePrefix = "vm-"
+	scopeSuffix = ".scope"
+
+	subtreeControlName = "cgroup.subtree_control"
+	killName           = "cgroup.kill"
+	weightName         = "cpu.weight"
+	maxName            = "cpu.max"
+	burstName          = "cpu.max.burst"
+	statName           = "cpu.stat"
+
+	removeWait         = time.Second
+	removePollInterval = 10 * time.Millisecond
+)
+
+// Knobs is the resolved cgroup CPU configuration for one VM scope.
+type Knobs struct {
+	Weight   int
+	QuotaUs  int64
+	PeriodUs int64
+	BurstUs  int64
+}
+
+// ResolveKnobs applies the Guaranteed-at-N defaults: weight = vCPU count, quota = vCPU count x period, burst = 0.
+func ResolveKnobs(cfg *types.Config) Knobs {
+	period := cmp.Or(cfg.CPUPeriodUs, int64(DefaultPeriodUs))
+	return Knobs{
+		Weight:   cmp.Or(cfg.CPUWeight, cfg.CPU),
+		QuotaUs:  cmp.Or(cfg.CPUQuotaUs, int64(cfg.CPU)*period),
+		PeriodUs: period,
+		BurstUs:  cfg.CPUBurstUs,
+	}
+}
+
+// Validate checks resolved knob values against the kernel's accepted ranges.
+func (k Knobs) Validate() error {
+	if k.Weight < MinWeight || k.Weight > MaxWeight {
+		return fmt.Errorf("--cpu-weight must be %d..%d, got %d", MinWeight, MaxWeight, k.Weight)
+	}
+	if k.PeriodUs < MinPeriodUs || k.PeriodUs > MaxPeriodUs {
+		return fmt.Errorf("--cpu-period-us must be %d..%d, got %d", MinPeriodUs, MaxPeriodUs, k.PeriodUs)
+	}
+	if k.QuotaUs < MinQuotaUs {
+		return fmt.Errorf("--cpu-quota-us must be at least %d, got %d", MinQuotaUs, k.QuotaUs)
+	}
+	if k.BurstUs < 0 || k.BurstUs > k.QuotaUs {
+		return fmt.Errorf("--cpu-burst-us must be 0..quota (%d), got %d", k.QuotaUs, k.BurstUs)
+	}
+	return nil
+}
+
+// ScopeDir returns vmID's scope directory under parentDir.
+func ScopeDir(parentDir, vmID string) string {
+	return filepath.Join(parentDir, scopePrefix+vmID+scopeSuffix)
+}
+
+// Prepare creates or reconfigures vmID's scope and returns its opened directory for CLONE_INTO_CGROUP; idempotent, so a relaunch reuses a scope its dying predecessor still occupies.
+func Prepare(parentDir, vmID string, k Knobs) (*os.File, error) {
+	if vmID == "" {
+		return nil, errors.New("cgroup scope: empty vm id")
+	}
+	if err := ensureParent(parentDir); err != nil {
+		return nil, err
+	}
+	dir := ScopeDir(parentDir, vmID)
+	mkErr := os.Mkdir(dir, 0o750)
+	if mkErr != nil && !errors.Is(mkErr, fs.ErrExist) {
+		return nil, fmt.Errorf("create scope: %w", mkErr)
+	}
+	if err := writeControl(dir, weightName, strconv.Itoa(k.Weight)); err != nil {
+		return nil, err
+	}
+	// A reused scope may hold a leftover burst > target quota, which blocks the cpu.max write (kernel requires burst <= quota): zero it first. ENOENT tolerated — pre-5.14 kernels lack the file.
+	if errors.Is(mkErr, fs.ErrExist) {
+		if err := writeControl(dir, burstName, "0"); err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return nil, err
+		}
+	}
+	if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil {
+		return nil, err
+	}
+	if k.BurstUs > 0 {
+		if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil {
+			return nil, err
+		}
+	}
+	scope, err := os.Open(dir) //nolint:gosec // path derives from config parent + generated VM ID
+	if err != nil {
+		return nil, fmt.Errorf("open scope: %w", err)
+	}
+	return scope, nil
+}
+
+// Remove kills everything left in an owned scope and removes it; the VMM must already be confirmed dead. ENOENT counts as success.
+func Remove(ctx context.Context, parentDir, vmID string) error {
+	dir := ScopeDir(parentDir, vmID)
+	if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+	// Best-effort: catches the CH pty child and stray forks; an empty scope has nothing to kill.
+	_ = writeControl(dir, killName, "1")
+	// EBUSY polls: killed members need a moment to exit before rmdir succeeds.
+	if err := utils.WaitFor(ctx, removeWait, removePollInterval, func() (bool, error) {
+		switch rmErr := os.Remove(dir); {
+		case rmErr == nil || errors.Is(rmErr, fs.ErrNotExist):
+			return true, nil
+		case errors.Is(rmErr, syscall.EBUSY):
+			return false, nil
+		default:
+			return false, rmErr
+		}
+	}); err != nil {
+		return fmt.Errorf("remove scope %s: %w", dir, err)
+	}
+	return nil
+}
+
+// RemoveEmpty removes vmID's scope only if empty; ENOENT counts as success. GC's variant for unowned scopes — it never kills.
+func RemoveEmpty(parentDir, vmID string) error {
+	err := os.Remove(ScopeDir(parentDir, vmID))
+	if err == nil || errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+	return err
+}
+
+// ListScopeVMIDs returns the VM IDs of all scopes under parentDir; a missing parent is empty.
+func ListScopeVMIDs(parentDir string) ([]string, error) {
+	names, err := utils.ScanSubdirs(parentDir)
+	if err != nil {
+		return nil, err
+	}
+	var ids []string
+	for _, name := range names {
+		if strings.HasPrefix(name, scopePrefix) && strings.HasSuffix(name, scopeSuffix) {
+			ids = append(ids, strings.TrimSuffix(strings.TrimPrefix(name, scopePrefix), scopeSuffix))
+		}
+	}
+	return ids, nil
+}
+
+// ReadStat parses vmID's cpu.stat into key/value pairs.
+func ReadStat(parentDir, vmID string) (map[string]int64, error) {
+	data, err := os.ReadFile(filepath.Join(ScopeDir(parentDir, vmID), statName))
+	if err != nil {
+		return nil, err
+	}
+	return parseStat(string(data)), nil
+}
+
+func parseStat(data string) map[string]int64 {
+	stat := make(map[string]int64)
+	for line := range strings.Lines(data) {
+		key, val, ok := strings.Cut(strings.TrimSpace(line), " ")
+		if !ok {
+			continue
+		}
+		if n, err := strconv.ParseInt(val, 10, 64); err == nil {
+			stat[key] = n
+		}
+	}
+	return stat
+}
+
+// ensureParent enables cpu at every ancestor, not just the leaf — cgroup v2 subtree delegation is hierarchical.
+func ensureParent(parentDir string) error {
+	rel, err := filepath.Rel(Root, parentDir)
+	if err != nil || rel == "." || strings.HasPrefix(rel, "..") {
+		return fmt.Errorf("cgroup parent %q must be under %s", parentDir, Root)
+	}
+	if err := utils.EnsureDirs(parentDir); err != nil {
+		return err
+	}
+	if err := enableCPU(Root); err != nil {
+		return err
+	}
+	dir := Root
+	for part := range strings.SplitSeq(rel, string(filepath.Separator)) {
+		dir = filepath.Join(dir, part)
+		if err := enableCPU(dir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// enableCPU reads before writing: subtree_control writes take the kernel's hierarchy-wide cgroup_mutex, so steady-state launches must not contend on a no-op write.
+func enableCPU(dir string) error {
+	path := filepath.Join(dir, subtreeControlName)
+	if data, err := os.ReadFile(path); err == nil && slices.Contains(strings.Fields(string(data)), "cpu") { //nolint:gosec // fixed name under the config-derived parent
+		return nil
+	}
+	return writeControl(dir, subtreeControlName, "+cpu")
+}
+
+func writeControl(dir, name, value string) error {
+	path := filepath.Join(dir, name)
+	if err := os.WriteFile(path, []byte(value), 0); err != nil {
+		return fmt.Errorf("write %s: %w", path, err)
+	}
+	return nil
+}

cgroup/cgroup_test.go

@@ -0,0 +1,111 @@
+package cgroup
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/cocoonstack/cocoon/types"
+)
+
+func TestResolveKnobsDefaults(t *testing.T) {
+	k := ResolveKnobs(&types.Config{CPU: 2})
+	want := Knobs{Weight: 2, QuotaUs: 200000, PeriodUs: 100000, BurstUs: 0}
+	if k != want {
+		t.Errorf("got %+v, want %+v", k, want)
+	}
+}
+
+func TestResolveKnobsOverrides(t *testing.T) {
+	k := ResolveKnobs(&types.Config{CPU: 4, CPUWeight: 100, CPUQuotaUs: 50000, CPUPeriodUs: 20000, CPUBurstUs: 10000})
+	want := Knobs{Weight: 100, QuotaUs: 50000, PeriodUs: 20000, BurstUs: 10000}
+	if k != want {
+		t.Errorf("got %+v, want %+v", k, want)
+	}
+}
+
+func TestResolveKnobsDefaultQuotaUsesExplicitPeriod(t *testing.T) {
+	k := ResolveKnobs(&types.Config{CPU: 2, CPUPeriodUs: 50000})
+	if k.QuotaUs != 100000 {
+		t.Errorf("quota %d, want CPU x period = 100000", k.QuotaUs)
+	}
+}
+
+func TestKnobsValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		k       Knobs
+		wantErr bool
+	}{
+		{"defaults for 1 cpu", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000}, false},
+		{"burst at quota", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000, BurstUs: 100000}, false},
+		{"weight too high", Knobs{Weight: 10001, QuotaUs: 100000, PeriodUs: 100000}, true},
+		{"weight zero", Knobs{Weight: 0, QuotaUs: 100000, PeriodUs: 100000}, true},
+		{"period below kernel min", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 999}, true},
+		{"period above kernel max", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 1000001}, true},
+		{"quota below kernel min", Knobs{Weight: 1, QuotaUs: 999, PeriodUs: 100000}, true},
+		{"burst above quota", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000, BurstUs: 100001}, true},
+		{"negative burst", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000, BurstUs: -1}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.k.Validate(); (err != nil) != tt.wantErr {
+				t.Errorf("Validate() = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestListScopeVMIDs(t *testing.T) {
+	parent := t.TempDir()
+	for _, dir := range []string{"vm-A.scope", "vm-B.scope", "other-dir"} {
+		if err := os.Mkdir(filepath.Join(parent, dir), 0o755); err != nil {
+			t.Fatalf("setup: %v", err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(parent, "vm-C.scope"), nil, 0o600); err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+
+	ids, err := ListScopeVMIDs(parent)
+	if err != nil {
+		t.Fatalf("ListScopeVMIDs: %v", err)
+	}
+	slices.Sort(ids)
+	if want := []string{"A", "B"}; !slices.Equal(ids, want) {
+		t.Errorf("got %v, want %v", ids, want)
+	}
+
+	ids, err = ListScopeVMIDs(filepath.Join(parent, "missing"))
+	if err != nil || ids != nil {
+		t.Errorf("missing parent: got %v, %v; want nil, nil", ids, err)
+	}
+}
+
+func TestRemoveEmpty(t *testing.T) {
+	parent := t.TempDir()
+	if err := os.Mkdir(ScopeDir(parent, "X"), 0o755); err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	if err := RemoveEmpty(parent, "X"); err != nil {
+		t.Errorf("empty scope: %v", err)
+	}
+	if err := RemoveEmpty(parent, "X"); err != nil {
+		t.Errorf("missing scope: %v", err)
+	}
+
+	if err := os.MkdirAll(filepath.Join(ScopeDir(parent, "Y"), "child"), 0o755); err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	if err := RemoveEmpty(parent, "Y"); err == nil {
+		t.Error("populated scope: want error, got nil")
+	}
+}
+
+func TestParseStat(t *testing.T) {
+	stat := parseStat("usage_usec 1000\nnr_throttled 3\nthrottled_usec 250\nbad line here\n")
+	if stat["nr_throttled"] != 3 || stat["throttled_usec"] != 250 {
+		t.Errorf("got %v", stat)
+	}
+}

cmd/core/gc.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/cocoonstack/cocoon/config"
 	"github.com/cocoonstack/cocoon/gc"
+	"github.com/cocoonstack/cocoon/hypervisor"
 	"github.com/cocoonstack/cocoon/lock/vmlock"
 	"github.com/cocoonstack/cocoon/network/bridge"
 	"github.com/cocoonstack/cocoon/snapshot/localfile"
@@ -36,6 +37,7 @@ func NewGCOrchestrator(ctx context.Context, conf *config.Config, snapOpts ...loc
 	for _, hyper := range hypers {
 		hyper.RegisterGC(o)
 	}
+	gc.Register(o, hypervisor.CgroupGCModule(conf.CgroupParentDir()))
 	netProvider.RegisterGC(o)
 	gc.Register(o, bridge.GCModule())
 	gc.Register(o, vmlock.GCModule(conf.RootDir))

cmd/core/metering.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"cmp"
 	"context"
 	"os"
 	"path/filepath"
@@ -62,10 +63,7 @@ func buildRecorder(ctx context.Context, conf *config.Config) metering.Recorder {
 
 func buildFileRecorder(ctx context.Context, conf *config.Config) metering.Recorder {
 	logger := log.WithFunc("core.buildFileRecorder")
-	path := conf.Metering.File.Path
-	if path == "" {
-		path = filepath.Join(conf.RootDir, meteringSubdir, meteringFile)
-	}
+	path := cmp.Or(conf.Metering.File.Path, filepath.Join(conf.RootDir, meteringSubdir, meteringFile))
 	if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
 		logger.Warnf(ctx, "mkdir %s: %v; metering disabled", filepath.Dir(path), err)
 		return metering.NopRecorder{}