diff --git a/cgroup/cgroup.go b/cgroup/cgroup.go new file mode 100644 index 00000000..bd05d406 --- /dev/null +++ b/cgroup/cgroup.go @@ -0,0 +1,239 @@ +// Package cgroup places VMM processes into per-VM cgroup v2 CPU scopes. +package cgroup + +import ( + "cmp" + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "time" + + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +const ( + // Root is the cgroup v2 unified hierarchy mount point. + Root = "/sys/fs/cgroup" + // DefaultParent holds every per-VM scope unless cgroup_parent overrides it. + DefaultParent = "cocoon.slice" + // DefaultPeriodUs is the kernel's default cpu.max period. + DefaultPeriodUs = 100000 + + // MinWeight/MaxWeight are the kernel's cpu.weight bounds. + MinWeight = 1 + MaxWeight = 10000 + // MinPeriodUs/MaxPeriodUs are the kernel's cpu.max period bounds. + MinPeriodUs = 1000 + MaxPeriodUs = 1000000 + // MinQuotaUs is the kernel's minimum cpu.max quota. + MinQuotaUs = 1000 + + scopePrefix = "vm-" + scopeSuffix = ".scope" + + subtreeControlName = "cgroup.subtree_control" + killName = "cgroup.kill" + weightName = "cpu.weight" + maxName = "cpu.max" + burstName = "cpu.max.burst" + statName = "cpu.stat" + + removeWait = time.Second + removePollInterval = 10 * time.Millisecond +) + +// Knobs is the resolved cgroup CPU configuration for one VM scope. +type Knobs struct { + Weight int + QuotaUs int64 + PeriodUs int64 + BurstUs int64 +} + +// ResolveKnobs applies the Guaranteed-at-N defaults: weight = vCPU count, quota = vCPU count x period, burst = 0. +func ResolveKnobs(cfg *types.Config) Knobs { + period := cmp.Or(cfg.CPUPeriodUs, int64(DefaultPeriodUs)) + return Knobs{ + Weight: cmp.Or(cfg.CPUWeight, cfg.CPU), + QuotaUs: cmp.Or(cfg.CPUQuotaUs, int64(cfg.CPU)*period), + PeriodUs: period, + BurstUs: cfg.CPUBurstUs, + } +} + +// Validate checks resolved knob values against the kernel's accepted ranges. +func (k Knobs) Validate() error { + if k.Weight < MinWeight || k.Weight > MaxWeight { + return fmt.Errorf("--cpu-weight must be %d..%d, got %d", MinWeight, MaxWeight, k.Weight) + } + if k.PeriodUs < MinPeriodUs || k.PeriodUs > MaxPeriodUs { + return fmt.Errorf("--cpu-period-us must be %d..%d, got %d", MinPeriodUs, MaxPeriodUs, k.PeriodUs) + } + if k.QuotaUs < MinQuotaUs { + return fmt.Errorf("--cpu-quota-us must be at least %d, got %d", MinQuotaUs, k.QuotaUs) + } + if k.BurstUs < 0 || k.BurstUs > k.QuotaUs { + return fmt.Errorf("--cpu-burst-us must be 0..quota (%d), got %d", k.QuotaUs, k.BurstUs) + } + return nil +} + +// ScopeDir returns vmID's scope directory under parentDir. +func ScopeDir(parentDir, vmID string) string { + return filepath.Join(parentDir, scopePrefix+vmID+scopeSuffix) +} + +// Prepare creates or reconfigures vmID's scope and returns its opened directory for CLONE_INTO_CGROUP; idempotent, so a relaunch reuses a scope its dying predecessor still occupies. +func Prepare(parentDir, vmID string, k Knobs) (*os.File, error) { + if vmID == "" { + return nil, errors.New("cgroup scope: empty vm id") + } + if err := ensureParent(parentDir); err != nil { + return nil, err + } + dir := ScopeDir(parentDir, vmID) + mkErr := os.Mkdir(dir, 0o750) + if mkErr != nil && !errors.Is(mkErr, fs.ErrExist) { + return nil, fmt.Errorf("create scope: %w", mkErr) + } + if err := writeControl(dir, weightName, strconv.Itoa(k.Weight)); err != nil { + return nil, err + } + // A reused scope may hold a leftover burst > target quota, which blocks the cpu.max write (kernel requires burst <= quota): zero it first. ENOENT tolerated — pre-5.14 kernels lack the file. + if errors.Is(mkErr, fs.ErrExist) { + if err := writeControl(dir, burstName, "0"); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil { + return nil, err + } + if k.BurstUs > 0 { + if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil { + return nil, err + } + } + scope, err := os.Open(dir) //nolint:gosec // path derives from config parent + generated VM ID + if err != nil { + return nil, fmt.Errorf("open scope: %w", err) + } + return scope, nil +} + +// Remove kills everything left in an owned scope and removes it; the VMM must already be confirmed dead. ENOENT counts as success. +func Remove(ctx context.Context, parentDir, vmID string) error { + dir := ScopeDir(parentDir, vmID) + if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) { + return nil + } + // Best-effort: catches the CH pty child and stray forks; an empty scope has nothing to kill. + _ = writeControl(dir, killName, "1") + // EBUSY polls: killed members need a moment to exit before rmdir succeeds. + if err := utils.WaitFor(ctx, removeWait, removePollInterval, func() (bool, error) { + switch rmErr := os.Remove(dir); { + case rmErr == nil || errors.Is(rmErr, fs.ErrNotExist): + return true, nil + case errors.Is(rmErr, syscall.EBUSY): + return false, nil + default: + return false, rmErr + } + }); err != nil { + return fmt.Errorf("remove scope %s: %w", dir, err) + } + return nil +} + +// RemoveEmpty removes vmID's scope only if empty; ENOENT counts as success. GC's variant for unowned scopes — it never kills. +func RemoveEmpty(parentDir, vmID string) error { + err := os.Remove(ScopeDir(parentDir, vmID)) + if err == nil || errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +// ListScopeVMIDs returns the VM IDs of all scopes under parentDir; a missing parent is empty. +func ListScopeVMIDs(parentDir string) ([]string, error) { + names, err := utils.ScanSubdirs(parentDir) + if err != nil { + return nil, err + } + var ids []string + for _, name := range names { + if strings.HasPrefix(name, scopePrefix) && strings.HasSuffix(name, scopeSuffix) { + ids = append(ids, strings.TrimSuffix(strings.TrimPrefix(name, scopePrefix), scopeSuffix)) + } + } + return ids, nil +} + +// ReadStat parses vmID's cpu.stat into key/value pairs. +func ReadStat(parentDir, vmID string) (map[string]int64, error) { + data, err := os.ReadFile(filepath.Join(ScopeDir(parentDir, vmID), statName)) + if err != nil { + return nil, err + } + return parseStat(string(data)), nil +} + +func parseStat(data string) map[string]int64 { + stat := make(map[string]int64) + for line := range strings.Lines(data) { + key, val, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok { + continue + } + if n, err := strconv.ParseInt(val, 10, 64); err == nil { + stat[key] = n + } + } + return stat +} + +// ensureParent enables cpu at every ancestor, not just the leaf — cgroup v2 subtree delegation is hierarchical. +func ensureParent(parentDir string) error { + rel, err := filepath.Rel(Root, parentDir) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return fmt.Errorf("cgroup parent %q must be under %s", parentDir, Root) + } + if err := utils.EnsureDirs(parentDir); err != nil { + return err + } + if err := enableCPU(Root); err != nil { + return err + } + dir := Root + for part := range strings.SplitSeq(rel, string(filepath.Separator)) { + dir = filepath.Join(dir, part) + if err := enableCPU(dir); err != nil { + return err + } + } + return nil +} + +// enableCPU reads before writing: subtree_control writes take the kernel's hierarchy-wide cgroup_mutex, so steady-state launches must not contend on a no-op write. +func enableCPU(dir string) error { + path := filepath.Join(dir, subtreeControlName) + if data, err := os.ReadFile(path); err == nil && slices.Contains(strings.Fields(string(data)), "cpu") { //nolint:gosec // fixed name under the config-derived parent + return nil + } + return writeControl(dir, subtreeControlName, "+cpu") +} + +func writeControl(dir, name, value string) error { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(value), 0); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + return nil +} diff --git a/cgroup/cgroup_test.go b/cgroup/cgroup_test.go new file mode 100644 index 00000000..0eb2af69 --- /dev/null +++ b/cgroup/cgroup_test.go @@ -0,0 +1,111 @@ +package cgroup + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/cocoon/types" +) + +func TestResolveKnobsDefaults(t *testing.T) { + k := ResolveKnobs(&types.Config{CPU: 2}) + want := Knobs{Weight: 2, QuotaUs: 200000, PeriodUs: 100000, BurstUs: 0} + if k != want { + t.Errorf("got %+v, want %+v", k, want) + } +} + +func TestResolveKnobsOverrides(t *testing.T) { + k := ResolveKnobs(&types.Config{CPU: 4, CPUWeight: 100, CPUQuotaUs: 50000, CPUPeriodUs: 20000, CPUBurstUs: 10000}) + want := Knobs{Weight: 100, QuotaUs: 50000, PeriodUs: 20000, BurstUs: 10000} + if k != want { + t.Errorf("got %+v, want %+v", k, want) + } +} + +func TestResolveKnobsDefaultQuotaUsesExplicitPeriod(t *testing.T) { + k := ResolveKnobs(&types.Config{CPU: 2, CPUPeriodUs: 50000}) + if k.QuotaUs != 100000 { + t.Errorf("quota %d, want CPU x period = 100000", k.QuotaUs) + } +} + +func TestKnobsValidate(t *testing.T) { + tests := []struct { + name string + k Knobs + wantErr bool + }{ + {"defaults for 1 cpu", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000}, false}, + {"burst at quota", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000, BurstUs: 100000}, false}, + {"weight too high", Knobs{Weight: 10001, QuotaUs: 100000, PeriodUs: 100000}, true}, + {"weight zero", Knobs{Weight: 0, QuotaUs: 100000, PeriodUs: 100000}, true}, + {"period below kernel min", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 999}, true}, + {"period above kernel max", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 1000001}, true}, + {"quota below kernel min", Knobs{Weight: 1, QuotaUs: 999, PeriodUs: 100000}, true}, + {"burst above quota", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000, BurstUs: 100001}, true}, + {"negative burst", Knobs{Weight: 1, QuotaUs: 100000, PeriodUs: 100000, BurstUs: -1}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.k.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestListScopeVMIDs(t *testing.T) { + parent := t.TempDir() + for _, dir := range []string{"vm-A.scope", "vm-B.scope", "other-dir"} { + if err := os.Mkdir(filepath.Join(parent, dir), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + } + if err := os.WriteFile(filepath.Join(parent, "vm-C.scope"), nil, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + + ids, err := ListScopeVMIDs(parent) + if err != nil { + t.Fatalf("ListScopeVMIDs: %v", err) + } + slices.Sort(ids) + if want := []string{"A", "B"}; !slices.Equal(ids, want) { + t.Errorf("got %v, want %v", ids, want) + } + + ids, err = ListScopeVMIDs(filepath.Join(parent, "missing")) + if err != nil || ids != nil { + t.Errorf("missing parent: got %v, %v; want nil, nil", ids, err) + } +} + +func TestRemoveEmpty(t *testing.T) { + parent := t.TempDir() + if err := os.Mkdir(ScopeDir(parent, "X"), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + if err := RemoveEmpty(parent, "X"); err != nil { + t.Errorf("empty scope: %v", err) + } + if err := RemoveEmpty(parent, "X"); err != nil { + t.Errorf("missing scope: %v", err) + } + + if err := os.MkdirAll(filepath.Join(ScopeDir(parent, "Y"), "child"), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + if err := RemoveEmpty(parent, "Y"); err == nil { + t.Error("populated scope: want error, got nil") + } +} + +func TestParseStat(t *testing.T) { + stat := parseStat("usage_usec 1000\nnr_throttled 3\nthrottled_usec 250\nbad line here\n") + if stat["nr_throttled"] != 3 || stat["throttled_usec"] != 250 { + t.Errorf("got %v", stat) + } +} diff --git a/cmd/core/gc.go b/cmd/core/gc.go index 69df4f37..8610e389 100644 --- a/cmd/core/gc.go +++ b/cmd/core/gc.go @@ -5,6 +5,7 @@ import ( "github.com/cocoonstack/cocoon/config" "github.com/cocoonstack/cocoon/gc" + "github.com/cocoonstack/cocoon/hypervisor" "github.com/cocoonstack/cocoon/lock/vmlock" "github.com/cocoonstack/cocoon/network/bridge" "github.com/cocoonstack/cocoon/snapshot/localfile" @@ -36,6 +37,7 @@ func NewGCOrchestrator(ctx context.Context, conf *config.Config, snapOpts ...loc for _, hyper := range hypers { hyper.RegisterGC(o) } + gc.Register(o, hypervisor.CgroupGCModule(conf.CgroupParentDir())) netProvider.RegisterGC(o) gc.Register(o, bridge.GCModule()) gc.Register(o, vmlock.GCModule(conf.RootDir)) diff --git a/cmd/core/metering.go b/cmd/core/metering.go index 8d4a3655..7c33b27f 100644 --- a/cmd/core/metering.go +++ b/cmd/core/metering.go @@ -1,6 +1,7 @@ package core import ( + "cmp" "context" "os" "path/filepath" @@ -62,10 +63,7 @@ func buildRecorder(ctx context.Context, conf *config.Config) metering.Recorder { func buildFileRecorder(ctx context.Context, conf *config.Config) metering.Recorder { logger := log.WithFunc("core.buildFileRecorder") - path := conf.Metering.File.Path - if path == "" { - path = filepath.Join(conf.RootDir, meteringSubdir, meteringFile) - } + path := cmp.Or(conf.Metering.File.Path, filepath.Join(conf.RootDir, meteringSubdir, meteringFile)) if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { logger.Warnf(ctx, "mkdir %s: %v; metering disabled", filepath.Dir(path), err) return metering.NopRecorder{} diff --git a/cmd/core/vmconfig.go b/cmd/core/vmconfig.go index 8d45d361..3913dc37 100644 --- a/cmd/core/vmconfig.go +++ b/cmd/core/vmconfig.go @@ -9,6 +9,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/cobra" + "github.com/cocoonstack/cocoon/cgroup" "github.com/cocoonstack/cocoon/config" "github.com/cocoonstack/cocoon/hypervisor" "github.com/cocoonstack/cocoon/images" @@ -22,6 +23,10 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error storStr, _ := cmd.Flags().GetString("storage") queueSize, _ := cmd.Flags().GetInt("queue-size") diskQueueSize, _ := cmd.Flags().GetInt("disk-queue-size") + cpuWeight, _ := cmd.Flags().GetInt("cpu-weight") + cpuQuotaUs, _ := cmd.Flags().GetInt64("cpu-quota-us") + cpuPeriodUs, _ := cmd.Flags().GetInt64("cpu-period-us") + cpuBurstUs, _ := cmd.Flags().GetInt64("cpu-burst-us") network, _ := cmd.Flags().GetString("network") user, _ := cmd.Flags().GetString("user") password, _ := cmd.Flags().GetString("password") @@ -31,9 +36,7 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error hugePages, _ := cmd.Flags().GetBool("hugepages") dataDiskRaw, _ := cmd.Flags().GetStringArray("data-disk") - if vmName == "" { - vmName = sanitizeVMName(image) - } + vmName = cmp.Or(vmName, sanitizeVMName(image)) memBytes, err := units.RAMInBytes(memStr) if err != nil { @@ -63,6 +66,10 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error Windows: windows, SharedMemory: sharedMemory, HugePages: hugePages, + CPUWeight: cpuWeight, + CPUQuotaUs: cpuQuotaUs, + CPUPeriodUs: cpuPeriodUs, + CPUBurstUs: cpuBurstUs, }, User: user, Password: password, @@ -71,9 +78,13 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error if err := cfg.Validate(); err != nil { return nil, err } + if err := cgroup.ResolveKnobs(&cfg.Config).Validate(); err != nil { + return nil, err + } return cfg, nil } +// CloneVMConfigFromFlags builds VMConfig for a clone. The snapshot's cgroup knobs record its source VM's policy and are never applied; the clone's policy comes from flags alone. func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (*types.VMConfig, error) { vmName, _ := cmd.Flags().GetString("name") flagNetwork, _ := cmd.Flags().GetString("network") @@ -82,6 +93,10 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (* queueSize := cmp.Or(flagQueueSize, snapCfg.QueueSize) flagDiskQueueSize, _ := cmd.Flags().GetInt("disk-queue-size") diskQueueSize := cmp.Or(flagDiskQueueSize, snapCfg.DiskQueueSize) + flagCPUWeight, _ := cmd.Flags().GetInt("cpu-weight") + flagCPUQuotaUs, _ := cmd.Flags().GetInt64("cpu-quota-us") + flagCPUPeriodUs, _ := cmd.Flags().GetInt64("cpu-period-us") + flagCPUBurstUs, _ := cmd.Flags().GetInt64("cpu-burst-us") noDirectIO := snapCfg.NoDirectIO if cmd.Flags().Changed("no-direct-io") { noDirectIO, _ = cmd.Flags().GetBool("no-direct-io") @@ -97,7 +112,7 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (* return nil, err } - return &types.VMConfig{ + cfg := &types.VMConfig{ Name: vmName, Config: types.Config{ CPU: snapCfg.CPU, @@ -113,13 +128,21 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (* Windows: snapCfg.Windows, SharedMemory: snapCfg.SharedMemory, HugePages: snapCfg.HugePages, + CPUWeight: flagCPUWeight, + CPUQuotaUs: flagCPUQuotaUs, + CPUPeriodUs: flagCPUPeriodUs, + CPUBurstUs: flagCPUBurstUs, }, DataDisks: dataDisks, RestoreMode: restoreMode, - }, nil + } + if err := cgroup.ResolveKnobs(&cfg.Config).Validate(); err != nil { + return nil, err + } + return cfg, nil } -// RestoreVMConfigFromFlags builds VMConfig for restore: resources from the snapshot, Name/Network from the VM (CNI namespace survives restore). +// RestoreVMConfigFromFlags builds VMConfig for restore: guest resources from the snapshot; Name, Network, and cgroup knobs from the VM (host-side state survives restore). func RestoreVMConfigFromFlags(cmd *cobra.Command, vm *types.VM, snapCfg types.SnapshotConfig) (*types.VMConfig, error) { if snapCfg.NICs != len(vm.NetworkConfigs) { return nil, fmt.Errorf("nic count mismatch: vm has %d, snapshot has %d", @@ -127,6 +150,11 @@ func RestoreVMConfigFromFlags(cmd *cobra.Command, vm *types.VM, snapCfg types.Sn } cfg := snapCfg.Config cfg.Network = vm.Config.Network + // Host-side policy stays with the VM, like Network; the snapshot's knobs describe its source VM. + cfg.CPUWeight = vm.Config.CPUWeight + cfg.CPUQuotaUs = vm.Config.CPUQuotaUs + cfg.CPUPeriodUs = vm.Config.CPUPeriodUs + cfg.CPUBurstUs = vm.Config.CPUBurstUs restoreMode, err := restoreModeFromFlags(cmd) if err != nil { return nil, err @@ -139,6 +167,10 @@ func RestoreVMConfigFromFlags(cmd *cobra.Command, vm *types.VM, snapCfg types.Sn if err := result.Validate(); err != nil { return nil, fmt.Errorf("snapshot config: %w", err) } + // The kept knobs must fit the snapshot's vCPU count before the destructive phase — a bad combination retries into the same failure. + if err := cgroup.ResolveKnobs(&result.Config).Validate(); err != nil { + return nil, err + } return result, nil } diff --git a/cmd/core/vmconfig_test.go b/cmd/core/vmconfig_test.go index e69f5470..190aca1b 100644 --- a/cmd/core/vmconfig_test.go +++ b/cmd/core/vmconfig_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/spf13/cobra" + + "github.com/cocoonstack/cocoon/types" ) func TestRestoreModeFromFlags(t *testing.T) { @@ -38,3 +40,101 @@ func TestRestoreModeFromFlags(t *testing.T) { }) } } + +func TestRestoreVMConfigKeepsHostCPUPolicy(t *testing.T) { + vm := &types.VM{Config: types.VMConfig{ + Name: "v", + Config: types.Config{CPU: 1, CPUWeight: 25, CPUQuotaUs: 150000, CPUPeriodUs: 50000, CPUBurstUs: 10000, Network: "keepnet"}, + }} + snapCfg := types.SnapshotConfig{Config: types.Config{ + CPU: 2, Memory: 1 << 30, Storage: 10 << 30, + CPUWeight: 9999, CPUQuotaUs: 999999, CPUPeriodUs: 100000, CPUBurstUs: 999999, + }} + + cmd := &cobra.Command{} + cmd.Flags().String("restore-mode", "", "") + got, err := RestoreVMConfigFromFlags(cmd, vm, snapCfg) + if err != nil { + t.Fatalf("RestoreVMConfigFromFlags: %v", err) + } + if got.CPU != 2 { + t.Errorf("CPU = %d, want snapshot's 2", got.CPU) + } + want := vm.Config + if got.CPUWeight != want.CPUWeight || got.CPUQuotaUs != want.CPUQuotaUs || + got.CPUPeriodUs != want.CPUPeriodUs || got.CPUBurstUs != want.CPUBurstUs { + t.Errorf("knobs = %d/%d/%d/%d, want the VM's %d/%d/%d/%d", + got.CPUWeight, got.CPUQuotaUs, got.CPUPeriodUs, got.CPUBurstUs, + want.CPUWeight, want.CPUQuotaUs, want.CPUPeriodUs, want.CPUBurstUs) + } + if got.Network != "keepnet" { + t.Errorf("Network = %q, want the VM's", got.Network) + } +} + +func TestRestoreVMConfigRejectsKnobsUnfitForSnapshotCPU(t *testing.T) { + vm := &types.VM{Config: types.VMConfig{ + Name: "v", + Config: types.Config{CPU: 2, CPUBurstUs: 150000}, + }} + snapCfg := types.SnapshotConfig{Config: types.Config{CPU: 1, Memory: 1 << 30, Storage: 10 << 30}} + + cmd := &cobra.Command{} + cmd.Flags().String("restore-mode", "", "") + if _, err := RestoreVMConfigFromFlags(cmd, vm, snapCfg); err == nil { + t.Fatal("want error: kept burst 150000 exceeds the 1-CPU derived quota 100000") + } +} + +func TestCloneVMConfigKnobFlagsOverrideSnapshot(t *testing.T) { + snapCfg := types.SnapshotConfig{Config: types.Config{ + CPU: 2, Memory: 1 << 30, Storage: 10 << 30, + CPUWeight: 40, CPUQuotaUs: 200000, CPUBurstUs: 50000, + }} + + tests := []struct { + name string + set map[string]string + wantWeight int + wantQuota int64 + wantBurst int64 + wantErr bool + }{ + {name: "no flags ignore snapshot knobs", wantWeight: 0, wantQuota: 0, wantBurst: 0}, + {name: "flags set the clone's policy", set: map[string]string{"cpu-weight": "10", "cpu-burst-us": "100000", "cpu-quota-us": "150000"}, wantWeight: 10, wantQuota: 150000, wantBurst: 100000}, + {name: "invalid flag rejected", set: map[string]string{"cpu-weight": "20000"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().String("name", "c", "") + cmd.Flags().Int("nics", 0, "") + cmd.Flags().Int("queue-size", 0, "") + cmd.Flags().Int("disk-queue-size", 0, "") + cmd.Flags().Int("cpu-weight", 0, "") + cmd.Flags().Int64("cpu-quota-us", 0, "") + cmd.Flags().Int64("cpu-period-us", 0, "") + cmd.Flags().Int64("cpu-burst-us", 0, "") + cmd.Flags().String("network", "", "") + cmd.Flags().Bool("no-direct-io", false, "") + cmd.Flags().String("restore-mode", "", "") + cmd.Flags().StringArray("data-disk", nil, "") + for k, v := range tt.set { + if err := cmd.Flags().Set(k, v); err != nil { + t.Fatalf("set %s: %v", k, err) + } + } + got, err := CloneVMConfigFromFlags(cmd, snapCfg) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if got.CPUWeight != tt.wantWeight || got.CPUQuotaUs != tt.wantQuota || got.CPUBurstUs != tt.wantBurst { + t.Errorf("knobs = %d/%d/%d, want %d/%d/%d", + got.CPUWeight, got.CPUQuotaUs, got.CPUBurstUs, tt.wantWeight, tt.wantQuota, tt.wantBurst) + } + }) + } +} diff --git a/cmd/root.go b/cmd/root.go index 287a71f9..c1e63f7b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/cocoonstack/cocoon/cgroup" "github.com/cocoonstack/cocoon/cmd/cliutil" cmdcore "github.com/cocoonstack/cocoon/cmd/core" cmddaemon "github.com/cocoonstack/cocoon/cmd/daemon" @@ -79,6 +80,7 @@ func newRootCmd() *cobra.Command { viper.SetDefault("pull_conns", 8) // Empty default keeps the key registered — AutomaticEnv only binds registered keys. viper.SetDefault("meta_backend", "") + viper.SetDefault("cgroup_parent", cgroup.DefaultParent) viper.SetDefault("log.level", "info") viper.SetDefault("log.max_size", 500) viper.SetDefault("log.max_age", 28) diff --git a/cmd/storebench/main.go b/cmd/storebench/main.go index 360dda31..226caf08 100644 --- a/cmd/storebench/main.go +++ b/cmd/storebench/main.go @@ -36,6 +36,7 @@ func (c benchConfig) RunDir() string { return c.dir } func (c benchConfig) LogDir() string { return c.dir } func (c benchConfig) VMRunDir(id string) string { return filepath.Join(c.dir, id) } func (c benchConfig) VMLogDir(id string) string { return filepath.Join(c.dir, id) } +func (c benchConfig) CgroupParentDir() string { return filepath.Join(c.dir, "cgroup") } func main() { if len(os.Args) < 3 { diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index d921adf0..31f9c32b 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -326,6 +326,10 @@ func addVMFlags(cmd *cobra.Command) { cmd.Flags().Int("nics", 1, "number of network interfaces (0 = no network); multiple NICs with auto IP config only works for cloudimg; OCI images auto-configure only the last NIC, others require manual setup inside the guest") cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = default 512; tradeoff: larger improves download throughput, smaller improves RPC latency)") //nolint:mnd cmd.Flags().Int("disk-queue-size", 0, "virtio-blk ring depth per device (0 = default 512; CH only, ignored by FC)") //nolint:mnd + cmd.Flags().Int("cpu-weight", 0, "cgroup cpu.weight, 1..10000 (0 = vCPU count)") + cmd.Flags().Int64("cpu-quota-us", 0, "cgroup cpu.max quota in us per period (0 = vCPU count x period)") + cmd.Flags().Int64("cpu-period-us", 0, "cgroup cpu.max period in us (0 = 100000)") + cmd.Flags().Int64("cpu-burst-us", 0, "cgroup cpu.max.burst credit in us (0 = none)") cmd.Flags().String("network", "", "CNI conflist name (empty = default); mutually exclusive with --bridge") cmd.Flags().String("bridge", "", "use TAP-on-bridge instead of CNI (value is bridge device, e.g. cni0); VM gets IP via DHCP from the bridge") cmd.Flags().String("user", "root", "guest username for cloud-init (cloudimg only)") @@ -342,6 +346,10 @@ func addCloneFlags(cmd *cobra.Command) { cmd.Flags().Int("nics", 0, "override NIC count (omit to inherit from snapshot)") cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)") //nolint:mnd cmd.Flags().Int("disk-queue-size", 0, "virtio-blk ring depth per device (0 = inherit from snapshot)") //nolint:mnd + cmd.Flags().Int("cpu-weight", 0, "cgroup cpu.weight, 1..10000 (0 = vCPU count; snapshot knobs are never inherited)") + cmd.Flags().Int64("cpu-quota-us", 0, "cgroup cpu.max quota in us per period (0 = vCPU count x period)") + cmd.Flags().Int64("cpu-period-us", 0, "cgroup cpu.max period in us (0 = 100000)") + cmd.Flags().Int64("cpu-burst-us", 0, "cgroup cpu.max.burst credit in us (0 = none)") cmd.Flags().String("network", "", "CNI conflist name (empty = inherit from source VM)") cmd.Flags().String("bridge", "", "use TAP-on-bridge instead of CNI (value is bridge device, e.g. cni0)") cmd.Flags().Bool("no-direct-io", false, "disable O_DIRECT on writable disks (inherit from snapshot if not set)") diff --git a/cmd/vm/status.go b/cmd/vm/status.go index 1e5f72fa..13c93ec5 100644 --- a/cmd/vm/status.go +++ b/cmd/vm/status.go @@ -14,6 +14,7 @@ import ( "github.com/projecteru2/core/log" "github.com/spf13/cobra" + "github.com/cocoonstack/cocoon/cgroup" "github.com/cocoonstack/cocoon/cmd/cliutil" cmdcore "github.com/cocoonstack/cocoon/cmd/core" "github.com/cocoonstack/cocoon/config" @@ -46,7 +47,7 @@ func (h Handler) List(cmd *cobra.Command, _ []string) error { return err } format, _ := cmd.Flags().GetString("format") - return statusOnce(ctx, hypers, nil, format) + return statusOnce(ctx, hypers, nil, format, conf.CgroupParentDir()) } func (h Handler) Status(cmd *cobra.Command, args []string) error { @@ -69,7 +70,7 @@ func (h Handler) Status(cmd *cobra.Command, args []string) error { } if !eventMode && !watchMode { - return statusOnce(ctx, hypers, args, format) + return statusOnce(ctx, hypers, args, format, conf.CgroupParentDir()) } watchCh := metaEvents(ctx, conf) @@ -84,23 +85,23 @@ func (h Handler) Status(cmd *cobra.Command, args []string) error { } } else { isTTY := term.IsTerminal(os.Stdout.Fd()) - statusRefreshLoop(ctx, hypers, args, watchCh, ticker.C, isTTY) + statusRefreshLoop(ctx, hypers, args, watchCh, ticker.C, isTTY, conf.CgroupParentDir()) } return nil } // statusOnce prints one snapshot; propagates ListAllVMs error (loop callers swallow). -func statusOnce(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string, format string) error { +func statusOnce(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string, format, scopeDir string) error { vms, err := cmdcore.ListAllVMs(ctx, hypers) if err != nil { return err } vms = applyFilters(vms, filters) sortVMs(vms) - return renderVMList(vms, format) + return renderVMList(vms, format, scopeDir) } -func renderVMList(vms []*types.VM, format string) error { +func renderVMList(vms []*types.VM, format, scopeDir string) error { if format == "json" { if vms == nil { vms = []*types.VM{} @@ -112,7 +113,7 @@ func renderVMList(vms []*types.VM, format string) error { return nil } return cliutil.OutputFormattedStr(format, vms, func(w *tabwriter.Writer) { - printVMTable(w, vms) + printVMTable(w, vms, scopeDir) }) } @@ -147,7 +148,7 @@ func runLoop(ctx context.Context, watchCh <-chan struct{}, tick <-chan time.Time } } -func statusRefreshLoop(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string, watchCh <-chan struct{}, tick <-chan time.Time, isTTY bool) { +func statusRefreshLoop(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string, watchCh <-chan struct{}, tick <-chan time.Time, isTTY bool, scopeDir string) { var prev []vmSnapshot runLoop(ctx, watchCh, tick, func() { vms := listAndFilter(ctx, hypers, filters) @@ -167,7 +168,7 @@ func statusRefreshLoop(ctx context.Context, hypers []hypervisor.Hypervisor, filt return } w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - printVMTable(w, vms) + printVMTable(w, vms, scopeDir) _ = w.Flush() }) } @@ -296,18 +297,31 @@ func sortVMs(vms []*types.VM) { slices.SortFunc(vms, func(a, b *types.VM) int { return a.CreatedAt.Compare(b.CreatedAt) }) } -func printVMTable(w *tabwriter.Writer, vms []*types.VM) { - fmt.Fprintln(w, "ID\tNAME\tSTATE\tCPU\tMEMORY\tSTORAGE\tIP\tIMAGE\tCREATED") //nolint:errcheck +func printVMTable(w *tabwriter.Writer, vms []*types.VM, scopeDir string) { + fmt.Fprintln(w, "ID\tNAME\tSTATE\tCPU\tMEMORY\tSTORAGE\tTHROTTLED\tIP\tIMAGE\tCREATED") //nolint:errcheck for _, vm := range vms { - fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n", //nolint:errcheck + fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\n", //nolint:errcheck vm.ID, vm.Config.Name, cmdcore.ReconcileState(vm), vm.Config.CPU, cliutil.FormatSize(vm.Config.Memory), cliutil.FormatSize(vm.Config.Storage), + vmThrottled(scopeDir, vm), vmIPs(vm), vm.Config.Image, vm.CreatedAt.Local().Format(time.DateTime)) } } +// vmThrottled renders cpu.stat throttling as "count/duration"; "-" when the VM is down or the scope is absent (non-Linux). +func vmThrottled(scopeDir string, vm *types.VM) string { + if vm.State != types.VMStateRunning { + return "-" + } + stat, err := cgroup.ReadStat(scopeDir, vm.ID) + if err != nil || stat["nr_throttled"] == 0 { + return "-" + } + return fmt.Sprintf("%d/%s", stat["nr_throttled"], time.Duration(stat["throttled_usec"])*time.Microsecond) +} + func vmIPs(vm *types.VM) string { var ips []string for _, nc := range vm.NetworkConfigs { diff --git a/cmd/vm/status_test.go b/cmd/vm/status_test.go index 60016701..e7f230da 100644 --- a/cmd/vm/status_test.go +++ b/cmd/vm/status_test.go @@ -114,7 +114,7 @@ func TestRenderVMList(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { out := captureStdout(t, func() { - if err := renderVMList(tt.vms, tt.format); err != nil { + if err := renderVMList(tt.vms, tt.format, t.TempDir()); err != nil { t.Fatalf("renderVMList: %v", err) } }) diff --git a/config/config.go b/config/config.go index 6fc2ae60..5a39b09c 100644 --- a/config/config.go +++ b/config/config.go @@ -1,12 +1,15 @@ package config import ( + "cmp" "fmt" "net" + "path/filepath" "strings" coretypes "github.com/projecteru2/core/types" + "github.com/cocoonstack/cocoon/cgroup" "github.com/cocoonstack/cocoon/utils" ) @@ -58,8 +61,10 @@ type Config struct { // SocketWaitTimeoutSeconds: wait for the CH API socket after start. Default: 5; increase for slow storage. SocketWaitTimeoutSeconds int `json:"socket_wait_timeout_seconds" mapstructure:"socket_wait_timeout_seconds"` // TerminateGracePeriodSeconds: SIGTERM→SIGKILL window when force-killing CH. Default: 5. - TerminateGracePeriodSeconds int `json:"terminate_grace_period_seconds" mapstructure:"terminate_grace_period_seconds"` - Log *coretypes.ServerLogConfig `json:"log" mapstructure:"log"` + TerminateGracePeriodSeconds int `json:"terminate_grace_period_seconds" mapstructure:"terminate_grace_period_seconds"` + // CgroupParent: cgroup v2 slice under /sys/fs/cgroup holding per-VM CPU scopes. Default: cocoon.slice. + CgroupParent string `json:"cgroup_parent" mapstructure:"cgroup_parent"` + Log *coretypes.ServerLogConfig `json:"log" mapstructure:"log"` // Metering selects the lifecycle-event recorder backend. Metering MeteringConfig `json:"metering,omitzero" mapstructure:"metering"` } @@ -82,6 +87,11 @@ func (c *Config) EffectivePullConns() int { return min(utils.OrDefault(c.PullConns, defaultPullConns), maxPullConns) } +// CgroupParentDir returns the absolute cgroup v2 directory holding per-VM CPU scopes. +func (c *Config) CgroupParentDir() string { + return filepath.Join(cgroup.Root, cmp.Or(c.CgroupParent, cgroup.DefaultParent)) +} + // Validate checks that all config fields are within acceptable ranges. // Should be called once at startup after unmarshalling. func (c *Config) Validate() error { diff --git a/daemon/api.go b/daemon/api.go index 087be4be..a08a6d35 100644 --- a/daemon/api.go +++ b/daemon/api.go @@ -1,6 +1,7 @@ package daemon import ( + "cmp" "context" "encoding/json" "errors" @@ -74,10 +75,7 @@ func (d *Daemon) listen() (net.Listener, error) { if err != nil { return nil, err } - mode := fs.FileMode(d.conf.APISockMode) - if mode == 0 { - mode = DefaultAPISockMode - } + mode := cmp.Or(fs.FileMode(d.conf.APISockMode), DefaultAPISockMode) if err := os.Chmod(d.conf.APIAddr, mode); err != nil { _ = ln.Close() return nil, err diff --git a/hypervisor/backend.go b/hypervisor/backend.go index 25b7f411..b26d0999 100644 --- a/hypervisor/backend.go +++ b/hypervisor/backend.go @@ -60,6 +60,7 @@ type BackendConfig interface { LogDir() string VMRunDir(id string) string VMLogDir(id string) string + CgroupParentDir() string } var _ Supervisable = (*Backend)(nil) @@ -96,13 +97,14 @@ func NewBackend(typ string, conf BackendConfig, rec metering.Recorder, store met func (b *Backend) Type() string { return b.Typ } -// LaunchSpec is the per-call input to Backend.LaunchVMProcess. +// LaunchSpec is the per-call input to Backend.LaunchVMProcess; PID-file and socket paths derive from Rec.RunDir. type LaunchSpec struct { Cmd *exec.Cmd - PIDPath string - SockPath string NetnsPath string OnFail func() + + // Rec names the VM whose CPU scope the process enters at spawn (ID + cgroup knobs); every VM enters a scope. + Rec *VMRecord } // PreflightHook validates rec against the snapshot source dir before anything is applied. diff --git a/hypervisor/cgroup_linux.go b/hypervisor/cgroup_linux.go new file mode 100644 index 00000000..a03e40d7 --- /dev/null +++ b/hypervisor/cgroup_linux.go @@ -0,0 +1,16 @@ +package hypervisor + +import ( + "os" + "os/exec" + "syscall" +) + +// setCmdCgroupFD lands cmd's clone3 directly in the scope (CLONE_INTO_CGROUP), so there is no post-fork attach race. +func setCmdCgroupFD(cmd *exec.Cmd, scope *os.File) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.UseCgroupFD = true + cmd.SysProcAttr.CgroupFD = int(scope.Fd()) +} diff --git a/hypervisor/cgroup_other.go b/hypervisor/cgroup_other.go new file mode 100644 index 00000000..b8d92be4 --- /dev/null +++ b/hypervisor/cgroup_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package hypervisor + +import ( + "os" + "os/exec" +) + +// setCmdCgroupFD is Linux-only (CLONE_INTO_CGROUP); other platforms never launch VMMs. +func setCmdCgroupFD(*exec.Cmd, *os.File) {} diff --git a/hypervisor/cgroupgc.go b/hypervisor/cgroupgc.go new file mode 100644 index 00000000..df26589d --- /dev/null +++ b/hypervisor/cgroupgc.go @@ -0,0 +1,44 @@ +package hypervisor + +import ( + "context" + "errors" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/cocoon/cgroup" + "github.com/cocoonstack/cocoon/gc" + "github.com/cocoonstack/cocoon/utils" +) + +// CgroupGCModule removes empty scopes owned by no VM in any backend snapshot; it never kills scope members. +func CgroupGCModule(parentDir string) gc.Module[[]string] { + return gc.Module[[]string]{ + Name: "cgroup", + ReadDB: func(context.Context) ([]string, error) { + return cgroup.ListScopeVMIDs(parentDir) + }, + Resolve: func(_ context.Context, scopes []string, others map[string]any) []string { + return utils.FilterUnreferenced(scopes, gc.Collect(others, gc.VMIDs)) + }, + Collect: func(ctx context.Context, ids, _ []string) error { + logger := log.WithFunc("gc.cgroup") + var errs []error + for _, id := range ids { + if err := cgroup.RemoveEmpty(parentDir, id); err != nil { + errs = append(errs, err) + continue + } + logger.Infof(ctx, "collected scope vm-%s reason=orphan-scope", id) + } + return errors.Join(errs...) + }, + } +} + +// removeCgroupScope reclaims id's scope after its VMM is confirmed dead; failure is left for the next converge/GC pass. +func (b *Backend) removeCgroupScope(ctx context.Context, id string) { + if err := cgroup.Remove(ctx, b.Conf.CgroupParentDir(), id); err != nil { + log.WithFunc(b.Typ+".removeCgroupScope").Warnf(ctx, "remove scope for %s: %v (left for gc)", id, err) + } +} diff --git a/hypervisor/cgroupgc_test.go b/hypervisor/cgroupgc_test.go new file mode 100644 index 00000000..fb049fdc --- /dev/null +++ b/hypervisor/cgroupgc_test.go @@ -0,0 +1,66 @@ +package hypervisor + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/cocoon/cgroup" +) + +func TestCgroupGCModuleSweepsUnownedScopes(t *testing.T) { + parent := t.TempDir() + for _, id := range []string{"OWNED-CH", "OWNED-FC", "ORPHAN"} { + if err := os.Mkdir(cgroup.ScopeDir(parent, id), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + } + + m := CgroupGCModule(parent) + scopes, err := m.ReadDB(t.Context()) + if err != nil { + t.Fatalf("ReadDB: %v", err) + } + slices.Sort(scopes) + if want := []string{"ORPHAN", "OWNED-CH", "OWNED-FC"}; !slices.Equal(scopes, want) { + t.Fatalf("scopes = %v, want %v", scopes, want) + } + + others := map[string]any{ + "cloud-hypervisor": VMGCSnapshot{vmIDs: map[string]struct{}{"OWNED-CH": {}}}, + "firecracker": VMGCSnapshot{vmIDs: map[string]struct{}{"OWNED-FC": {}}}, + } + ids := m.Resolve(t.Context(), scopes, others) + if want := []string{"ORPHAN"}; !slices.Equal(ids, want) { + t.Fatalf("Resolve = %v, want %v", ids, want) + } + + if err := m.Collect(t.Context(), ids, nil); err != nil { + t.Fatalf("Collect: %v", err) + } + if _, err := os.Stat(cgroup.ScopeDir(parent, "ORPHAN")); !os.IsNotExist(err) { + t.Error("orphan scope not removed") + } + for _, id := range []string{"OWNED-CH", "OWNED-FC"} { + if _, err := os.Stat(cgroup.ScopeDir(parent, id)); err != nil { + t.Errorf("owned scope %s removed: %v", id, err) + } + } +} + +func TestCgroupGCModuleLeavesPopulatedScope(t *testing.T) { + parent := t.TempDir() + dir := cgroup.ScopeDir(parent, "BUSY") + if err := os.MkdirAll(filepath.Join(dir, "child"), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + + m := CgroupGCModule(parent) + if err := m.Collect(t.Context(), []string{"BUSY"}, nil); err == nil { + t.Error("want error for populated scope, got nil") + } + if _, err := os.Stat(dir); err != nil { + t.Errorf("populated scope removed: %v", err) + } +} diff --git a/hypervisor/cloudhypervisor/clone.go b/hypervisor/cloudhypervisor/clone.go index 66df5884..408b8615 100644 --- a/hypervisor/cloudhypervisor/clone.go +++ b/hypervisor/cloudhypervisor/clone.go @@ -122,9 +122,10 @@ func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, vmID str ch.saveCmdline(ctx, &hypervisor.VMRecord{RunDir: runDir}, args) pid, err := ch.launchProcess(ctx, &hypervisor.VMRecord{ + VM: types.VM{ID: vmID, Config: *vmCfg}, RunDir: runDir, LogDir: logDir, - }, sockPath, args, net.NetnsPath) + }, args, net.NetnsPath) if err != nil { ch.MarkError(ctx, vmID) return nil, fmt.Errorf("launch CH: %w", err) diff --git a/hypervisor/cloudhypervisor/patch.go b/hypervisor/cloudhypervisor/patch.go index d45f7918..83f0bd51 100644 --- a/hypervisor/cloudhypervisor/patch.go +++ b/hypervisor/cloudhypervisor/patch.go @@ -76,10 +76,7 @@ func patchCHConfig(path string, opts *patchOptions) error { } func patchDisks(diskRaw json.RawMessage, opts *patchOptions) (json.RawMessage, error) { - diskQueueSize := opts.diskQueueSize - if diskQueueSize <= 0 { - diskQueueSize = defaultDiskQueueSize - } + diskQueueSize := utils.OrDefault(opts.diskQueueSize, defaultDiskQueueSize) return patchRawArray(diskRaw, len(opts.storageConfigs), func(i int, elem map[string]json.RawMessage) error { sc := opts.storageConfigs[i] if e := setField(elem, "path", sc.Path); e != nil { diff --git a/hypervisor/cloudhypervisor/restore.go b/hypervisor/cloudhypervisor/restore.go index 760d4a21..9ff3abe1 100644 --- a/hypervisor/cloudhypervisor/restore.go +++ b/hypervisor/cloudhypervisor/restore.go @@ -91,7 +91,9 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string, args := []string{apiSocketFlag, sockPath} ch.saveCmdline(ctx, rec, args) - pid, launchErr := ch.launchProcess(ctx, rec, sockPath, args, rec.ResolvedNetnsPath()) + // Launch under the target config: a --force cross-config restore changes the vCPU count the scope derives from. + rec.Config = *vmCfg + pid, launchErr := ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath()) if launchErr != nil { return nil, fmt.Errorf("launch CH: %w", launchErr) } diff --git a/hypervisor/cloudhypervisor/start.go b/hypervisor/cloudhypervisor/start.go index ffa9dcf4..d97b6e5e 100644 --- a/hypervisor/cloudhypervisor/start.go +++ b/hypervisor/cloudhypervisor/start.go @@ -22,12 +22,12 @@ func (ch *CloudHypervisor) startOne(ctx context.Context, id string) error { vmCfg := buildVMConfig(rec, hypervisor.ConsoleSockPath(rec.RunDir)) args := buildCLIArgs(vmCfg, sockPath) ch.saveCmdline(ctx, rec, args) - return ch.launchProcess(ctx, rec, sockPath, args, rec.ResolvedNetnsPath()) + return ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath()) }, }) } -func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, socketPath string, args []string, netnsPath string) (int, error) { +func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, args []string, netnsPath string) (int, error) { processLog := ch.LogFilePath(rec.LogDir) logFile, err := os.Create(processLog) //nolint:gosec if err != nil { @@ -47,9 +47,8 @@ func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VM pid, err := ch.LaunchVMProcess(ctx, hypervisor.LaunchSpec{ Cmd: cmd, - PIDPath: ch.PIDFilePath(rec.RunDir), - SockPath: socketPath, NetnsPath: netnsPath, + Rec: rec, }) if err != nil { return 0, err diff --git a/hypervisor/firecracker/clone.go b/hypervisor/firecracker/clone.go index 516c6fdc..cc1d7a91 100644 --- a/hypervisor/firecracker/clone.go +++ b/hypervisor/firecracker/clone.go @@ -98,7 +98,7 @@ func (fc *Firecracker) cloneAfterExtract(ctx context.Context, vmID string, vmCfg sockPath := hypervisor.SocketPath(runDir) launch := func(leaseFiles []*os.File) (int, *cloneLeaseControl, error) { return fc.launchProcessWithLeases(ctx, &hypervisor.VMRecord{ - VM: types.VM{ID: vmID}, + VM: types.VM{ID: vmID, Config: *vmCfg}, RunDir: runDir, LogDir: logDir, }, sockPath, net.NetnsPath, leaseFiles) diff --git a/hypervisor/firecracker/restore.go b/hypervisor/firecracker/restore.go index e88382cd..1444f27a 100644 --- a/hypervisor/firecracker/restore.go +++ b/hypervisor/firecracker/restore.go @@ -61,6 +61,8 @@ func (fc *Firecracker) restoreAfterExtract(ctx context.Context, vmID string, vmC sockPath := hypervisor.SocketPath(rec.RunDir) + // Launch under the target config: a --force cross-config restore changes the vCPU count the scope derives from. + rec.Config = *vmCfg pid, launchErr := fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath()) if launchErr != nil { return nil, fmt.Errorf("launch FC: %w", launchErr) diff --git a/hypervisor/firecracker/start.go b/hypervisor/firecracker/start.go index 7db635b6..770ef8d9 100644 --- a/hypervisor/firecracker/start.go +++ b/hypervisor/firecracker/start.go @@ -184,10 +184,9 @@ func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervi fcCmd.Stdout = slave pid, err := fc.LaunchVMProcess(ctx, hypervisor.LaunchSpec{ Cmd: fcCmd, - PIDPath: fc.PIDFilePath(rec.RunDir), - SockPath: sockPath, NetnsPath: netnsPath, OnFail: func() { _ = master.Close() }, + Rec: rec, }) if err != nil { return 0, nil, err diff --git a/hypervisor/snapshot.go b/hypervisor/snapshot.go index 7dcac713..c2cf25e2 100644 --- a/hypervisor/snapshot.go +++ b/hypervisor/snapshot.go @@ -143,6 +143,7 @@ func (b *Backend) HibernateSequence(ctx context.Context, ref string, spec Hibern } CleanupRuntimeFiles(ctx, rec.RunDir, spec.RuntimeFiles) + b.removeCgroupScope(ctx, vmID) // Warn-and-continue like StopAll: the VMM is dead and the flip self-heals; the snapshot is already durable. uErr := b.UpdateStates(ctx, []string{vmID}, types.VMStateStopped) if uErr != nil { diff --git a/hypervisor/start.go b/hypervisor/start.go index eadb042c..5ed4d66d 100644 --- a/hypervisor/start.go +++ b/hypervisor/start.go @@ -11,6 +11,7 @@ import ( "github.com/projecteru2/core/log" "golang.org/x/sys/unix" + "github.com/cocoonstack/cocoon/cgroup" "github.com/cocoonstack/cocoon/types" "github.com/cocoonstack/cocoon/utils" ) @@ -115,6 +116,8 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int started := false pidWritten := false binaryName := b.Conf.BinaryName() + pidPath := b.PIDFilePath(spec.Rec.RunDir) + sockPath := SocketPath(spec.Rec.RunDir) defer func() { if err == nil { return @@ -124,13 +127,20 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int _ = spec.Cmd.Wait() } if pidWritten { - _ = os.Remove(spec.PIDPath) + _ = os.Remove(pidPath) } if spec.OnFail != nil { spec.OnFail() } }() + scope, err := cgroup.Prepare(b.Conf.CgroupParentDir(), spec.Rec.ID, cgroup.ResolveKnobs(&spec.Rec.Config.Config)) + if err != nil { + return 0, fmt.Errorf("prepare cgroup scope: %w", err) + } + defer scope.Close() //nolint:errcheck + setCmdCgroupFD(spec.Cmd, scope) + if spec.NetnsPath != "" { restore, nsErr := EnterNetns(spec.NetnsPath) if nsErr != nil { @@ -146,12 +156,12 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int started = true pid = spec.Cmd.Process.Pid - if err = utils.WritePIDFile(spec.PIDPath, pid); err != nil { + if err = utils.WritePIDFile(pidPath, pid); err != nil { return 0, fmt.Errorf("write PID file: %w", err) } pidWritten = true - if err = WaitForSocket(ctx, spec.SockPath, pid, b.Conf.SocketWaitTimeout(), binaryName); err != nil { + if err = WaitForSocket(ctx, sockPath, pid, b.Conf.SocketWaitTimeout(), binaryName); err != nil { return 0, err } return pid, nil diff --git a/hypervisor/state_test.go b/hypervisor/state_test.go index e52d1904..4a3b7b53 100644 --- a/hypervisor/state_test.go +++ b/hypervisor/state_test.go @@ -644,6 +644,8 @@ func (stubBackendConfig) VMRunDir(string) string { panic("VMRunDir: not implemen func (stubBackendConfig) VMLogDir(string) string { panic("VMLogDir: not implemented in stub") } +func (c stubBackendConfig) CgroupParentDir() string { return filepath.Join(c.rootDir, "cgroup") } + // meteringStubConfig gives the metering stub a real VMRunDir so sequences // can take the per-VM ops lock and MkdirTemp under it. type meteringStubConfig struct { diff --git a/hypervisor/stop.go b/hypervisor/stop.go index f261d2cb..696345b1 100644 --- a/hypervisor/stop.go +++ b/hypervisor/stop.go @@ -111,6 +111,7 @@ func (b *Backend) HandleStopResult(ctx context.Context, id, runDir string, runti return shutdownErr } CleanupRuntimeFiles(ctx, runDir, runtimeFiles) + b.removeCgroupScope(ctx, id) return nil } diff --git a/hypervisor/supervisor.go b/hypervisor/supervisor.go index 6c866027..23e58e51 100644 --- a/hypervisor/supervisor.go +++ b/hypervisor/supervisor.go @@ -104,6 +104,7 @@ func (b *Backend) ConvergeDead(ctx context.Context, id string, gen uint64, obser if err := b.convergeDeadRecord(ctx, id, gen, observedAt); err != nil { return err } + b.removeCgroupScope(ctx, id) return b.QuiesceIfPending(ctx, id) } diff --git a/hypervisor/teardown.go b/hypervisor/teardown.go index 822c8962..460ec386 100644 --- a/hypervisor/teardown.go +++ b/hypervisor/teardown.go @@ -80,6 +80,7 @@ func (b *Backend) finishVMTeardown(ctx context.Context, id, leaseID string, cl v if err := RemoveVMDirs(cl.RunDir, cl.LogDir); err != nil { return fmt.Errorf("cleanup VM dirs (tombstone kept, retry or gc resumes): %w", err) } + b.removeCgroupScope(ctx, id) ts := b.tombstones() err := b.update(ctx, func(t *vmTx) error { if err := t.Del(id); err != nil { diff --git a/images/gc.go b/images/gc.go index b475960b..52f9c056 100644 --- a/images/gc.go +++ b/images/gc.go @@ -72,8 +72,7 @@ func BuildGCModule[E any](cfg GCModuleConfig[E]) gc.Module[ImageGCSnapshot] { }, Resolve: func(_ context.Context, snap ImageGCSnapshot, others map[string]any) []string { used := gc.Collect(others, gc.BlobIDs) - allRefs := utils.MergeSets(snap.refs, used) - candidates := utils.FilterUnreferenced(snap.diskIDs, allRefs) + candidates := utils.FilterUnreferenced(snap.diskIDs, snap.refs, used) slices.Sort(candidates) return slices.Compact(candidates) }, diff --git a/images/index.go b/images/index.go index 081cd36a..5951103f 100644 --- a/images/index.go +++ b/images/index.go @@ -9,7 +9,6 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/cocoon/types" - "github.com/cocoonstack/cocoon/utils" ) const minHexLen = 12 @@ -145,7 +144,15 @@ func entryToImage[E Entry](entry *E, typ string, sizer func(*E) int64) *types.Im } func listImages[E Entry](images map[string]*E, typ string, sizer func(*E) int64) []*types.Image { - return utils.MapValues(images, func(ep *E) *types.Image { - return entryToImage(ep, typ, sizer) - }) + if len(images) == 0 { + return nil + } + out := make([]*types.Image, 0, len(images)) + for _, ep := range images { + if ep == nil { + continue + } + out = append(out, entryToImage(ep, typ, sizer)) + } + return out } diff --git a/network/cni/teardown.go b/network/cni/teardown.go index 27034702..70736a39 100644 --- a/network/cni/teardown.go +++ b/network/cni/teardown.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/fs" + "slices" "github.com/projecteru2/core/log" @@ -169,11 +170,5 @@ func filterRecords(records []networkRecord, ids []string) []networkRecord { for _, id := range ids { want[id] = true } - out := make([]networkRecord, 0, len(ids)) - for _, r := range records { - if want[r.ID] { - out = append(out, r) - } - } - return out + return slices.DeleteFunc(records, func(r networkRecord) bool { return !want[r.ID] }) } diff --git a/types/config.go b/types/config.go index d2495423..ef41d39b 100644 --- a/types/config.go +++ b/types/config.go @@ -23,4 +23,10 @@ type Config struct { SharedMemory bool `json:"shared_memory,omitempty"` // HugePages backs CH guest memory with hugetlbfs (costs snapshots the mmap fast path); fixed at create, persists through clone/restore. HugePages bool `json:"hugepages,omitempty"` + + // Raw cgroup v2 CPU knobs; zero derives the Guaranteed-at-N defaults from CPU. + CPUWeight int `json:"cpu_weight,omitempty"` + CPUQuotaUs int64 `json:"cpu_quota_us,omitempty"` + CPUPeriodUs int64 `json:"cpu_period_us,omitempty"` + CPUBurstUs int64 `json:"cpu_burst_us,omitempty"` } diff --git a/utils/map.go b/utils/map.go deleted file mode 100644 index 10c110c8..00000000 --- a/utils/map.go +++ /dev/null @@ -1,33 +0,0 @@ -package utils - -import ( - "maps" -) - -// MergeSets unions any number of set maps into a new set. -func MergeSets[K comparable](sets ...map[K]struct{}) map[K]struct{} { - total := 0 - for _, s := range sets { - total += len(s) - } - out := make(map[K]struct{}, total) - for _, s := range sets { - maps.Copy(out, s) - } - return out -} - -// MapValues projects every non-nil value in m through fn into a slice. -func MapValues[K comparable, V, R any](m map[K]*V, fn func(*V) R) []R { - if len(m) == 0 { - return nil - } - out := make([]R, 0, len(m)) - for _, v := range m { - if v == nil { - continue - } - out = append(out, fn(v)) - } - return out -} diff --git a/utils/map_test.go b/utils/map_test.go deleted file mode 100644 index 110965b6..00000000 --- a/utils/map_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package utils - -import ( - "testing" -) - -func TestMergeSets_Basic(t *testing.T) { - s1 := map[string]struct{}{"a": {}, "b": {}} - s2 := map[string]struct{}{"b": {}, "c": {}} - - got := MergeSets(s1, s2) - if len(got) != 3 { - t.Errorf("got %d items, want 3", len(got)) - } - for _, k := range []string{"a", "b", "c"} { - if _, ok := got[k]; !ok { - t.Errorf("missing key %q", k) - } - } -} - -func TestMergeSets_Empty(t *testing.T) { - got := MergeSets[string]() - if len(got) != 0 { - t.Errorf("expected empty, got %d", len(got)) - } -} - -func TestMergeSets_NilSets(t *testing.T) { - var s1 map[string]struct{} - got := MergeSets(s1, nil) - if len(got) != 0 { - t.Errorf("expected empty, got %d", len(got)) - } -} - -func TestMergeSets_Single(t *testing.T) { - s := map[int]struct{}{1: {}, 2: {}} - got := MergeSets(s) - if len(got) != 2 { - t.Errorf("got %d, want 2", len(got)) - } -} - -func TestMergeSets_DoesNotModifyInput(t *testing.T) { - s1 := map[string]struct{}{"a": {}} - s2 := map[string]struct{}{"b": {}} - - _ = MergeSets(s1, s2) - - if len(s1) != 1 { - t.Error("s1 was modified") - } - if len(s2) != 1 { - t.Error("s2 was modified") - } -} - -func TestMapValues_Basic(t *testing.T) { - type rec struct{ N int } - m := map[string]*rec{"a": {N: 1}, "b": {N: 2}, "c": {N: 3}} - - got := MapValues(m, func(r *rec) int { return r.N }) - if len(got) != 3 { - t.Fatalf("got %d items, want 3", len(got)) - } - sum := 0 - for _, v := range got { - sum += v - } - if sum != 6 { - t.Errorf("sum %d, want 6", sum) - } -} - -func TestMapValues_SkipsNil(t *testing.T) { - type rec struct{ N int } - m := map[string]*rec{"a": {N: 1}, "b": nil, "c": {N: 3}} - - got := MapValues(m, func(r *rec) int { return r.N }) - if len(got) != 2 { - t.Errorf("got %d items, want 2 (nil should be skipped)", len(got)) - } -} - -func TestMapValues_Empty(t *testing.T) { - got := MapValues(map[string]*int{}, func(*int) int { return 0 }) - if got != nil { - t.Errorf("expected nil for empty map, got %v", got) - } -} - -func TestMapValues_Nil(t *testing.T) { - var m map[string]*int - got := MapValues(m, func(*int) int { return 0 }) - if got != nil { - t.Errorf("expected nil for nil map, got %v", got) - } -}