Reclaim creating-state stale VMs safely; document dense-node host tuning

docs/deploy.md

@@ -187,6 +187,45 @@ The last setting also removes the VMM's network-namespace quarantine. On a
 measurement. Pause `cocoon daemon` or lengthen its reconcile interval during a
 mass fill.
 
+### Host tuning for dense egress-lane nodes
+
+The no-network lane is CPU-bound and needs none of this. The egress lane
+serializes on the kernel's global rtnl lock — tap create, bridge attach, and
+the VMM's own tap open all take it — so at hundreds of VMs host configuration
+dominates fill throughput:
+
+- **Keep the kernel console quiet** (`loglevel=4` on the kernel cmdline, or
+  drop `console=ttyS0`; dmesg and the journal keep everything either way).
+  Bridge port transitions printk synchronously to every registered console
+  *while holding rtnl*: measured on a serial+fbcon host, one bridge attach is
+  44 ms noisy vs 3 ms quiet, and a 1000-VM fill roughly triples. Fix the
+  console before tuning anything else — netlink-level optimizations are
+  unmeasurable, or negative, on a noisy console.
+- **Keep udev off the sandbox taps.** Every tap add/remove fires a udev
+  event; rule evaluation plus hooks like `ifupdown-hotplug` (a fork+exec per
+  interface) contend on the same rtnl lock — stopping the udev exec queue
+  during a fill measured ~4x throughput, and a backed-up event queue can
+  collapse a fill outright. Shadow the net-setup rules for these taps, or
+  pause the exec queue around planned mass fills.
+- **Set `refill_concurrency` explicitly (~64) on egress-heavy nodes.** The
+  auto default scales with cores (a 384-core node gets 256), which suits the
+  no-network lane but overshoots the bridge lane: rtnl does not plateau under
+  pressure, it collapses — on a quiet host RC 256 halves the fill rate RC 64
+  achieves.
+
+### Reserving CPU for the control plane
+
+A saturated node can leave clone/wake execution and sandboxd itself nothing
+to run on — under full-node load CH clone p95 has measured in the tens of
+seconds. cocoon newer than v0.5.8 runs every VMM in its own cgroup v2 CPU
+scope (Guaranteed-at-N: a VM's CPU count is also its hard host-CPU cap) and
+adds `cgroup_cpus`, a one-time cpuset fence keeping the whole VM population —
+including the virtio and io-wq worker threads vCPU affinity cannot reach —
+off reserved host cores. On dense nodes set the fence in cocoon's config
+(e.g. `cgroup_cpus: "0-379"` on a 384-core host) so the reserved cores stay
+free for sandboxd, its cocoon invocations, and the OS; cocoon's CPU-isolation
+docs cover validation semantics and the per-VM knobs.
+
 ### Auth model
 
 Three token kinds. The root `api_token` has full access — operators and
@@ -210,7 +249,11 @@ sandboxd -config /etc/sandboxd/config.json
 ```
 
 On start the node reconciles: persisted claims whose VMs still run are
-re-adopted, everything else `sbx-`-prefixed is removed. Then the refill loop
+re-adopted, everything else `sbx-`-prefixed is removed. A record still in
+cocoon's `creating` state is reclaimed through `vm reconcile-stale-create`
+(cocoon ≥ v0.5.8), which refuses while a clone is in flight instead of
+deleting the VM out from under it; older cocoons fall back to forced
+removal. Then the refill loop
 builds one golden snapshot per pool (a one-time cold boot + snapshot export,
 tens of seconds) and keeps each pool topped up with claim-ready clones.
 `GET /v1/info` shows `"golden": true` and `warm` at target when the node is

e2e/fakeengine_test.go

@@ -61,6 +61,10 @@ func (f *fakeEngine) Remove(_ context.Context, name string) error {
 	return nil
 }
 
+func (f *fakeEngine) ReconcileStaleCreate(context.Context, string) (engine.StaleCreateOutcome, error) {
+	return engine.StaleCreateNotCreating, nil
+}
+
 func (f *fakeEngine) SnapshotSave(_ context.Context, _, _ string) error { return nil }
 
 func (f *fakeEngine) SnapshotExport(_ context.Context, _, toDir string) error {

protocol/wire/frame.go

@@ -713,8 +713,7 @@ func fastBulk(tag string, slow func([]byte) (Response, error), mk func([]byte) R
 
 // AppendBulkRequest renders a data-carrying request frame —
 // {"v":1,"op":<op>,"data":"<base64>"} plus newline — into buf, reused across
-// calls: the zero-alloc twin of EncodeRequest for the bulk send paths
-// (base64's alphabet needs no JSON escaping).
+// calls on the bulk send paths (base64's alphabet needs no JSON escaping).
 func AppendBulkRequest(buf []byte, op string, data []byte) []byte {
 	buf = append(buf[:0], requestHead...)
 	buf = append(buf, op...)

sandboxd/engine/engine.go

@@ -32,6 +32,12 @@ const (
 	// RequiredCocoon carries the snapshot/store performance baseline.
 	RequiredCocoon = "v0.5.2"
 
+	// Stale-create verdicts, mirroring cocoon's reconcile-stale-create verb.
+	StaleCreateCollected   StaleCreateOutcome = "collected"
+	StaleCreateBusy        StaleCreateOutcome = "busy"
+	StaleCreateNotCreating StaleCreateOutcome = "not-creating"
+	StaleCreateNotFound    StaleCreateOutcome = "not-found"
+
 	argName       = "--name"
 	argOutput     = "--output"
 	argNetwork    = "--network"
@@ -59,6 +65,9 @@ var capacitySignatures = []string{
 	"no space left on device",
 }
 
+// StaleCreateOutcome reports what reconcile-stale-create did with a record.
+type StaleCreateOutcome string
+
 // Engine runs cocoon commands on the local node.
 type Engine struct {
 	bin         string
@@ -141,6 +150,22 @@ func (e *Engine) Remove(ctx context.Context, name string) error {
 	return err
 }
 
+// ReconcileStaleCreate reclaims a creating-state record via cocoon's
+// free-ops-lock predicate — safe against an in-flight clone where rm --force is not.
+func (e *Engine) ReconcileStaleCreate(ctx context.Context, name string) (StaleCreateOutcome, error) {
+	out, err := e.run(ctx, "vm", "reconcile-stale-create", name, argOutput, formatJSON)
+	if err != nil {
+		return "", err
+	}
+	var res struct {
+		Outcome StaleCreateOutcome `json:"outcome"`
+	}
+	if err := json.Unmarshal(out, &res); err != nil {
+		return "", fmt.Errorf("parse reconcile-stale-create: %w", err)
+	}
+	return res.Outcome, nil
+}
+
 // SnapshotSave snapshots a running VM under snapName.
 func (e *Engine) SnapshotSave(ctx context.Context, vmName, snapName string) error {
 	_, err := e.run(ctx, "snapshot", "save", argName, snapName, vmName)

sandboxd/pool/archive.go

@@ -67,12 +67,7 @@ func (m *Manager) archiveOnce(ctx context.Context) {
 		logger := log.WithFunc("pool.archiveOnce")
 		m.runBounded(ctx, len(victims), func(ctx context.Context, i int) {
 			sb := victims[i]
-			switch err := m.archive(ctx, sb); {
-			case err == nil:
-				logger.Infof(ctx, "archived %s", sb.ID)
-			case !benignSweepErr(err):
-				logger.Errorf(ctx, err, "archive %s", sb.ID)
-			}
+			logSweepResult(ctx, logger, m.archive(ctx, sb), "archived "+sb.ID, "archive "+sb.ID)
 		}).Wait()
 	}()
 }

sandboxd/pool/claim.go

@@ -363,12 +363,7 @@ func (m *Manager) reapOnce(ctx context.Context) {
 			m.purgeArchiveCk(ctx, v.id, v.ck, v.tenant)
 			logger.Infof(ctx, "purged archived sandbox %s", v.id)
 		case reapArchive:
-			switch err := m.archive(ctx, v.sb); {
-			case err == nil:
-				logger.Infof(ctx, "archived expired sandbox %s", v.id)
-			case !benignSweepErr(err):
-				logger.Errorf(ctx, err, "archive expired %s", v.id)
-			}
+			logSweepResult(ctx, logger, m.archive(ctx, v.sb), "archived expired sandbox "+v.id, "archive expired sandbox "+v.id)
 		default:
 			m.disarmEgress(v.id, m.removeOrRetry(ctx, v.vmName, v.id, ""))
 			m.dropSnap(ctx, v.snap)

sandboxd/pool/hibernate.go

@@ -220,12 +220,7 @@ func (m *Manager) idleOnce(ctx context.Context) {
 		logger := log.WithFunc("pool.idleOnce")
 		m.runBounded(ctx, len(victims), func(ctx context.Context, i int) {
 			v := victims[i]
-			switch err := m.idleHibernate(ctx, v.id, v.token, now); {
-			case err == nil:
-				logger.Infof(ctx, "idle-hibernated %s", v.id)
-			case !benignSweepErr(err):
-				logger.Errorf(ctx, err, "idle-hibernate %s", v.id)
-			}
+			logSweepResult(ctx, logger, m.idleHibernate(ctx, v.id, v.token, now), "idle-hibernated "+v.id, "idle-hibernate "+v.id)
 		}).Wait()
 	}()
 }

sandboxd/pool/pool.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/cocoonstack/sandbox/sandboxd/config"
 	"github.com/cocoonstack/sandbox/sandboxd/egress"
+	"github.com/cocoonstack/sandbox/sandboxd/engine"
 	"github.com/cocoonstack/sandbox/sandboxd/netfilter"
 	"github.com/cocoonstack/sandbox/sandboxd/store"
 	"github.com/cocoonstack/sandbox/sandboxd/store/dir"
@@ -73,6 +74,7 @@ const (
 	hibernatePrefix = "sbx-hib-"
 	forkPrefix      = "sbx-fork-"
 	vmStateRunning  = "running"
+	vmStateCreating = "creating"
 
 	caSidecarSuffix = ".cafp"
 )
@@ -100,6 +102,7 @@ type Engine interface {
 	CloneSnap(ctx context.Context, snap, name string, key types.PoolKey) (types.VMRecord, error)
 	RunCold(ctx context.Context, name string, key types.PoolKey) (types.VMRecord, error)
 	Remove(ctx context.Context, name string) error
+	ReconcileStaleCreate(ctx context.Context, name string) (engine.StaleCreateOutcome, error)
 	SnapshotSave(ctx context.Context, vmName, snapName string) error
 	SnapshotExport(ctx context.Context, snapName, toDir string) error
 	SnapshotRemove(ctx context.Context, snapName string) error
@@ -149,8 +152,9 @@ type Gauges struct {
 }
 
 type pendingRemoval struct {
-	sandboxID string
-	tap       string
+	sandboxID   string
+	tap         string
+	staleCreate bool
 }
 
 type pool struct {
@@ -685,6 +689,16 @@ func tenantOwns(tenant, owner string) bool {
 
 // benignSweepErr reports whether err is the expected outcome of a housekeeping
 // sweep (victim released, woke mid-sweep, or a lane that never hibernates).
+// logSweepResult reports one background-sweep outcome; benign races stay silent.
+func logSweepResult(ctx context.Context, logger *log.Fields, err error, okMsg, failMsg string) {
+	switch {
+	case err == nil:
+		logger.Info(ctx, okMsg)
+	case !benignSweepErr(err):
+		logger.Error(ctx, err, failMsg)
+	}
+}
+
 func benignSweepErr(err error) bool {
 	return errors.Is(err, ErrUnknownSandbox) || errors.Is(err, errWokeMeanwhile) ||
 		errors.Is(err, ErrNoEgressHibernate)

sandboxd/pool/pool_test.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/cocoonstack/sandbox/sandboxd/config"
 	"github.com/cocoonstack/sandbox/sandboxd/egress"
+	"github.com/cocoonstack/sandbox/sandboxd/engine"
 	"github.com/cocoonstack/sandbox/sandboxd/types"
 )
 
@@ -765,8 +766,12 @@ type fakeEngine struct {
 	hibernates, restores, snapRemoves []string
 	snapSaves, exports, snapshots     []string
 	caInstalls                        []string // vsock sockets InstallCACert was called on
+	staleReconciles                   []string // VM names ReconcileStaleCreate was called on
 	installCAErr                      error
 	stopped                           map[string]bool
+	creating                          map[string]bool // VMs List reports in the creating state
+	staleOutcome                      engine.StaleCreateOutcome
+	staleErr                          error
 	pids                              map[string]int // VM name → PID, for Stats' resident-set lookup
 	vsockLateN                        int            // List calls that report no socket yet
 
@@ -787,7 +792,7 @@ type fakeEngine struct {
 }
 
 func newFakeEngine() *fakeEngine {
-	return &fakeEngine{vms: map[string]string{}, stopped: map[string]bool{}, pids: map[string]int{}}
+	return &fakeEngine{vms: map[string]string{}, stopped: map[string]bool{}, creating: map[string]bool{}, pids: map[string]int{}}
 }
 
 func (f *fakeEngine) Clone(_ context.Context, fromDir, name string, _ types.PoolKey) (types.VMRecord, error) {
@@ -830,6 +835,24 @@ func (f *fakeEngine) Remove(ctx context.Context, name string) error {
 	return nil
 }
 
+func (f *fakeEngine) ReconcileStaleCreate(ctx context.Context, name string) (engine.StaleCreateOutcome, error) {
+	// Models exec.CommandContext: a canceled ctx never runs cocoon at all.
+	if err := ctx.Err(); err != nil {
+		return "", err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.staleReconciles = append(f.staleReconciles, name)
+	if f.staleErr != nil {
+		return "", f.staleErr
+	}
+	if f.staleOutcome == engine.StaleCreateCollected || f.staleOutcome == engine.StaleCreateNotFound {
+		delete(f.vms, name)
+		delete(f.creating, name)
+	}
+	return f.staleOutcome, nil
+}
+
 func (f *fakeEngine) SnapshotSave(_ context.Context, _, snapName string) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -918,7 +941,10 @@ func (f *fakeEngine) List(_ context.Context, filters ...string) ([]types.VMRecor
 		}
 		sock = f.lateVsock(sock)
 		state := vmStateRunning
-		if f.stopped[name] {
+		switch {
+		case f.creating[name]:
+			state = vmStateCreating
+		case f.stopped[name]:
 			state = "stopped"
 		}
 		rec := types.VMRecord{State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}}

sandboxd/pool/reconcile.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/projecteru2/core/log"
 
+	"github.com/cocoonstack/sandbox/sandboxd/engine"
 	"github.com/cocoonstack/sandbox/sandboxd/netfilter"
 	"github.com/cocoonstack/sandbox/sandboxd/types"
 )
@@ -131,7 +132,7 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec
 	}
 	gone := make([]bool, len(stale)) // distinct indices: no lock under the Wait barrier
 	m.runBounded(ctx, len(stale), func(ctx context.Context, i int) {
-		if m.removeOrRetry(ctx, stale[i], "", live[stale[i]].TapDevice()) {
+		if m.removeStaleVM(ctx, stale[i], live[stale[i]]) {
 			gone[i] = true
 			logger.Infof(ctx, "removed stale VM %s", stale[i])
 		}
@@ -145,6 +146,31 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec
 	return removed
 }
 
+// removeStaleVM reclaims one unowned VM, reporting whether it is gone; a
+// creating-state record goes through reconcile-stale-create first, since
+// rm --force would queue on the ops lock and then delete the VM an
+// in-flight clone just produced.
+func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool {
+	logger := log.WithFunc("pool.removeStaleVM")
+	if rec.State == vmStateCreating {
+		// Cancellation-immune like removeVM: a canceled ctx must not skip
+		// the busy check and fall through to the forced remove it guards.
+		switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); {
+		case err != nil:
+			// Verb missing (cocoon < v0.5.8) or failed: keep the old sweep.
+			logger.Warnf(ctx, "reconcile stale create %s: %v; removing", name, err)
+		case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound:
+			return true
+		case outcome == engine.StaleCreateBusy:
+			logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name)
+			m.queueStaleCreate(name, rec.TapDevice())
+			return false
+		}
+		// not-creating: the record moved on under the lock; remove normally.
+	}
+	return m.removeOrRetry(ctx, name, "", rec.TapDevice())
+}
+
 // resyncEgress re-locks adopted egress claims after a restart, quarantines any
 // it cannot lock, and sweeps tables orphaned by VMs confirmed gone (in removed).
 func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) {