feat(microvm): virtio-fs mount-ns isolation + volume-quota verification (RIG-2497)

go/cmd/compass-runner/backend_flags_test.go

@@ -0,0 +1,235 @@
+//go:build unix
+
+package main
+
+// The operator-knob wiring for the microVM backend. Every knob here is dead
+// code unless it reaches MicroVMConfig, and a knob that silently does not is the
+// worst kind of gap: QuotaRequired was declared on the config, consumed by the
+// D7 preflight, and unsettable by any operator — so the multi-tenant gate could
+// never fire, on any profile. These tests assert BOTH resolution paths (the flag
+// and the environment fallback) for the quota knobs, so that failure mode
+// cannot recur silently.
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/RigelBuild/compass/go/internal/runtime"
+)
+
+// flagsFor builds a backendFlags whose pointers address local values, so a test
+// sets a "flag" without touching the global flag set (which flag.Parse would
+// own and which cannot be re-registered across tests).
+func flagsFor() (backendFlags, *bool, *string) {
+	var (
+		empty                                                     string
+		zero                                                      int
+		backend, vmm, virtiofsd, kernel, rootfs, initrd, manifest = empty, empty, empty, empty, empty, empty, empty
+		runRoot, volumeRoot                                       = empty, empty
+		cpus, memoryMB                                            = zero, zero
+		quotaRequired                                             bool
+	)
+	return backendFlags{
+		backend:       &backend,
+		vmm:           &vmm,
+		virtiofsd:     &virtiofsd,
+		kernel:        &kernel,
+		rootfs:        &rootfs,
+		initrd:        &initrd,
+		imageManifest: &manifest,
+		runRoot:       &runRoot,
+		volumeRoot:    &volumeRoot,
+		cpus:          &cpus,
+		memoryMB:      &memoryMB,
+		quotaRequired: &quotaRequired,
+	}, &quotaRequired, &volumeRoot
+}
+
+// TestQuotaRequiredReachesConfigFromFlag: --microvm-quota-required must land on
+// MicroVMConfig.QuotaRequired. Without this the D7 preflight gate is
+// unreachable dead code and every Runner runs fail-open.
+func TestQuotaRequiredReachesConfigFromFlag(t *testing.T) {
+	f, quotaRequired, _ := flagsFor()
+	*quotaRequired = true
+
+	cfg, err := f.backendConfig()
+	if err != nil {
+		t.Fatalf("backendConfig() = %v, want nil", err)
+	}
+	if !cfg.MicroVM.QuotaRequired {
+		t.Fatal("--microvm-quota-required did not reach MicroVMConfig.QuotaRequired; " +
+			"the D7 multi-tenant quota gate would be unsettable and could never fire")
+	}
+}
+
+// TestQuotaRequiredReachesConfigFromEnv: the documented
+// $COMPASS_MICROVM_QUOTA_REQUIRED fallback must resolve too — it is how the
+// managed profile sets the knob, since it ships env, not argv.
+func TestQuotaRequiredReachesConfigFromEnv(t *testing.T) {
+	for _, raw := range []string{"true", "1", "TRUE"} {
+		t.Run(raw, func(t *testing.T) {
+			t.Setenv("COMPASS_MICROVM_QUOTA_REQUIRED", raw)
+			f, _, _ := flagsFor()
+
+			cfg, err := f.backendConfig()
+			if err != nil {
+				t.Fatalf("backendConfig() = %v, want nil", err)
+			}
+			if !cfg.MicroVM.QuotaRequired {
+				t.Fatalf("$COMPASS_MICROVM_QUOTA_REQUIRED=%q did not reach MicroVMConfig.QuotaRequired", raw)
+			}
+		})
+	}
+}
+
+// TestQuotaRequiredDefaultsOff pins the transitional default: neither knob set
+// means the Dogfood single-tenant posture, where an absent quota is documented
+// and logged rather than fatal.
+func TestQuotaRequiredDefaultsOff(t *testing.T) {
+	t.Setenv("COMPASS_MICROVM_QUOTA_REQUIRED", "")
+	f, _, _ := flagsFor()
+
+	cfg, err := f.backendConfig()
+	if err != nil {
+		t.Fatalf("backendConfig() = %v, want nil", err)
+	}
+	if cfg.MicroVM.QuotaRequired {
+		t.Fatal("QuotaRequired defaulted to true with no knob set; the transitional default is the single-tenant posture")
+	}
+}
+
+// TestQuotaRequiredEnvRefusesGarbage: an unparseable value must REFUSE at
+// startup naming the variable, never read as false. Silently defaulting a
+// misspelled `=yes` to off is exactly the fail-open this knob exists to close.
+func TestQuotaRequiredEnvRefusesGarbage(t *testing.T) {
+	t.Setenv("COMPASS_MICROVM_QUOTA_REQUIRED", "yes-please")
+	f, _, _ := flagsFor()
+
+	_, err := f.backendConfig()
+	if err == nil {
+		t.Fatal("an unparseable $COMPASS_MICROVM_QUOTA_REQUIRED = nil error; a typo must refuse at startup, not read as false")
+	}
+	for _, part := range []string{"COMPASS_MICROVM_QUOTA_REQUIRED", "yes-please"} {
+		if !strings.Contains(err.Error(), part) {
+			t.Errorf("error %q does not name %q", err.Error(), part)
+		}
+	}
+}
+
+// TestQuotaRequiredFlagWinsOverEnv pins the precedence every other knob has
+// (orEnv/intOrEnv): an explicitly-passed flag beats the environment fallback.
+func TestQuotaRequiredFlagWinsOverEnv(t *testing.T) {
+	t.Setenv("COMPASS_MICROVM_QUOTA_REQUIRED", "false")
+	f, quotaRequired, _ := flagsFor()
+	*quotaRequired = true
+
+	cfg, err := f.backendConfig()
+	if err != nil {
+		t.Fatalf("backendConfig() = %v, want nil", err)
+	}
+	if !cfg.MicroVM.QuotaRequired {
+		t.Fatal("--microvm-quota-required=true lost to $COMPASS_MICROVM_QUOTA_REQUIRED=false; the flag must win")
+	}
+}
+
+// TestQuotaRequiredEnvTrueBeatsExplicitFlagFalse pins the OTHER direction of
+// boolOrEnv's precedence, which its sibling above does not cover: an env true
+// beats an explicitly-passed `--microvm-quota-required=false`.
+//
+// This is a DOCUMENTED CONTRACT, not a latent bug (boolOrEnv's doc). A false
+// flag is indistinguishable from an unset one under the zero-value convention
+// every knob here shares, and the asymmetry resolves in the SAFE direction: the
+// input that loses is the one that would turn the multi-tenant quota gate OFF,
+// so a conflicting pair leaves the gate ON. Pinning it means a future switch to
+// flag.Visit — which WOULD honor the explicit false and thereby move the
+// conflict resolution to fail-open — is a deliberate, reviewed change rather
+// than an unnoticed regression.
+func TestQuotaRequiredEnvTrueBeatsExplicitFlagFalse(t *testing.T) {
+	t.Setenv("COMPASS_MICROVM_QUOTA_REQUIRED", "true")
+	f, quotaRequired, _ := flagsFor()
+	// Explicitly false — which, under the zero-value convention, is
+	// indistinguishable from "not passed".
+	*quotaRequired = false
+
+	cfg, err := f.backendConfig()
+	if err != nil {
+		t.Fatalf("backendConfig() = %v, want nil", err)
+	}
+	if !cfg.MicroVM.QuotaRequired {
+		t.Fatal("$COMPASS_MICROVM_QUOTA_REQUIRED=true lost to an explicit --microvm-quota-required=false; " +
+			"the documented contract is that env-true wins (a false flag is indistinguishable from an unset " +
+			"one, and the conflict must resolve with the multi-tenant gate ON, not off). To disable the gate " +
+			"on such a box, unset the environment variable")
+	}
+}
+
+// TestVolumeRootReachesConfig: --microvm-volume-root / its env fallback must
+// land on MicroVMConfig.VolumeRoot. It is what makes the quota probe target the
+// session-volume filesystem instead of the RunRoot socket dir, so an unwired
+// knob would leave the preflight either fail-closed forever or (worse) reporting
+// a verdict about the wrong mount.
+func TestVolumeRootReachesConfig(t *testing.T) {
+	t.Run("flag", func(t *testing.T) {
+		f, _, volumeRoot := flagsFor()
+		*volumeRoot = "/srv/compass/volumes"
+
+		cfg, err := f.backendConfig()
+		if err != nil {
+			t.Fatalf("backendConfig() = %v, want nil", err)
+		}
+		if cfg.MicroVM.VolumeRoot != "/srv/compass/volumes" {
+			t.Fatalf("VolumeRoot = %q, want the flag value", cfg.MicroVM.VolumeRoot)
+		}
+	})
+	t.Run("env", func(t *testing.T) {
+		t.Setenv("COMPASS_MICROVM_VOLUME_ROOT", "/mnt/volumes")
+		f, _, _ := flagsFor()
+
+		cfg, err := f.backendConfig()
+		if err != nil {
+			t.Fatalf("backendConfig() = %v, want nil", err)
+		}
+		if cfg.MicroVM.VolumeRoot != "/mnt/volumes" {
+			t.Fatalf("VolumeRoot = %q, want the env value", cfg.MicroVM.VolumeRoot)
+		}
+	})
+}
+
+// TestVolumeRootIsNotTheRunRoot: the two knobs must resolve INDEPENDENTLY. A
+// wiring that aliased VolumeRoot to RunRoot would reintroduce the exact bug the
+// field exists to fix — a quota verdict read on the short /tmp socket dir rather
+// than the durable session-volume filesystem.
+func TestVolumeRootIsNotTheRunRoot(t *testing.T) {
+	t.Setenv("COMPASS_MICROVM_RUNROOT", "/tmp/cvm")
+	t.Setenv("COMPASS_MICROVM_VOLUME_ROOT", "/srv/compass/volumes")
+	f, _, _ := flagsFor()
+
+	cfg, err := f.backendConfig()
+	if err != nil {
+		t.Fatalf("backendConfig() = %v, want nil", err)
+	}
+	if cfg.MicroVM.RunRoot != "/tmp/cvm" {
+		t.Errorf("RunRoot = %q, want /tmp/cvm", cfg.MicroVM.RunRoot)
+	}
+	if cfg.MicroVM.VolumeRoot != "/srv/compass/volumes" {
+		t.Errorf("VolumeRoot = %q, want /srv/compass/volumes", cfg.MicroVM.VolumeRoot)
+	}
+}
+
+// TestBackendConfigSelectsMicroVM is the end-to-end shape: the resolved config
+// still drives backend selection, so the backendConfig split did not detach the
+// knobs from the engine they configure.
+func TestBackendConfigSelectsMicroVM(t *testing.T) {
+	f, quotaRequired, volumeRoot := flagsFor()
+	*f.backend = "microvm"
+	*quotaRequired = true
+	*volumeRoot = "/srv/compass/volumes"
+
+	engine, err := f.selectEngine()
+	if err != nil {
+		t.Fatalf("selectEngine() = %v, want the microVM backend", err)
+	}
+	if _, ok := engine.(*runtime.MicroVMRuntime); !ok {
+		t.Fatalf("selectEngine() = %T, want *runtime.MicroVMRuntime", engine)
+	}
+}

go/cmd/compass-runner/main.go

@@ -283,7 +283,9 @@ func setupOtel(ctx context.Context) (shutdown func(), err error) {
 // default flag set before flag.Parse and resolved into a runtime after it.
 type backendFlags struct {
 	backend, vmm, virtiofsd, kernel, rootfs, initrd, imageManifest, runRoot *string
+	volumeRoot                                                              *string
 	cpus, memoryMB                                                          *int
+	quotaRequired                                                           *bool
 }
 
 // registerBackendFlags declares the backend-selection flags. Call before
@@ -315,21 +317,46 @@ func registerBackendFlags() backendFlags {
 		memoryMB: flag.Int("microvm-memory-mb", 0,
 			"Default guest RAM in MiB per session (microvm backend); 0 leaves the VMM default. "+
 				"Defaults to $COMPASS_MICROVM_MEMORY_MB."),
+		volumeRoot: flag.String("microvm-volume-root", "",
+			"Parent dir the per-session workspace volumes are minted under (microvm backend); "+
+				"the filesystem the D7 session-volume quota is verified on. "+
+				"Defaults to $COMPASS_MICROVM_VOLUME_ROOT."),
+		quotaRequired: flag.Bool("microvm-quota-required", false,
+			"Require an operator-provisioned project quota on the session-volume filesystem "+
+				"(multi-tenant profile); startup fails when none is active (microvm backend). "+
+				"Defaults to $COMPASS_MICROVM_QUOTA_REQUIRED."),
 	}
 }
 
 // selectEngine resolves the configured runtime backend from the parsed flags
 // and their environment fallbacks.
 func (f backendFlags) selectEngine() (runtime.ContainerRuntime, error) {
-	cpus, err := intOrEnv(*f.cpus, "COMPASS_MICROVM_CPUS")
+	cfg, err := f.backendConfig()
 	if err != nil {
 		return nil, err
 	}
+	return runtime.SelectBackend(cfg)
+}
+
+// backendConfig resolves every backend knob from its flag with an environment
+// fallback. Split from selectEngine so the resolution is assertable without
+// constructing a real backend: a knob that never reaches the config is dead
+// code an operator cannot set (the QuotaRequired fail-open), and only a test
+// over THIS function catches that.
+func (f backendFlags) backendConfig() (runtime.BackendConfig, error) {
+	cpus, err := intOrEnv(*f.cpus, "COMPASS_MICROVM_CPUS")
+	if err != nil {
+		return runtime.BackendConfig{}, err
+	}
 	memoryMB, err := intOrEnv(*f.memoryMB, "COMPASS_MICROVM_MEMORY_MB")
 	if err != nil {
-		return nil, err
+		return runtime.BackendConfig{}, err
 	}
-	return runtime.SelectBackend(runtime.BackendConfig{
+	quotaRequired, err := boolOrEnv(*f.quotaRequired, "COMPASS_MICROVM_QUOTA_REQUIRED")
+	if err != nil {
+		return runtime.BackendConfig{}, err
+	}
+	return runtime.BackendConfig{
 		Backend: orEnv(*f.backend, "COMPASS_RUNTIME_BACKEND"),
 		MicroVM: runtime.MicroVMConfig{
 			VMMPath:         orEnv(*f.vmm, "COMPASS_MICROVM_VMM"),
@@ -339,10 +366,12 @@ func (f backendFlags) selectEngine() (runtime.ContainerRuntime, error) {
 			InitrdImage:     orEnv(*f.initrd, "COMPASS_MICROVM_INITRD"),
 			ImageManifest:   orEnv(*f.imageManifest, "COMPASS_MICROVM_IMAGE_MANIFEST"),
 			RunRoot:         orEnv(*f.runRoot, "COMPASS_MICROVM_RUNROOT"),
+			VolumeRoot:      orEnv(*f.volumeRoot, "COMPASS_MICROVM_VOLUME_ROOT"),
 			DefaultCPUs:     cpus,
 			DefaultMemoryMB: memoryMB,
+			QuotaRequired:   quotaRequired,
 		},
-	})
+	}, nil
 }
 
 // orEnv returns flagVal when non-empty, else the named environment variable.
@@ -373,6 +402,45 @@ func intOrEnv(flagVal int, envKey string) (int, error) {
 	return parsed, nil
 }
 
+// boolOrEnv returns flagVal when it is true, else the named environment
+// variable parsed as a bool. The flag defaults to false, so a set flag always
+// wins and an unset one falls through to the env — the same precedence orEnv
+// and intOrEnv give their zero values. An empty env var is false (unset); a
+// present-but-unparseable one is an error naming the offending variable and
+// value, so `COMPASS_MICROVM_QUOTA_REQUIRED=yes` refuses at startup instead of
+// silently reading as false and fail-opening the D7 gate.
+//
+// PRECEDENCE IS ASYMMETRIC, deliberately, and it is a CONTRACT, not an
+// accident: a false flag is indistinguishable from an unset one here (there is
+// no flag.Visit and no *bool nil sentinel), so `--microvm-quota-required=false`
+// does NOT override `$COMPASS_MICROVM_QUOTA_REQUIRED=true` — the env's true
+// wins. Two reasons to keep it that way rather than honoring an explicit false:
+//
+//   - It matches the zero-value convention of every sibling knob (orEnv,
+//     intOrEnv), so one knob does not read its flags differently from the rest.
+//   - It fails in the SAFE direction for THIS knob: the losing input is the one
+//     that would turn the multi-tenant quota gate OFF, so a conflict leaves the
+//     gate ON.
+//
+// An operator who genuinely needs the gate off on a box whose environment sets
+// it must unset the environment variable; passing the flag is not enough.
+// TestQuotaRequiredEnvTrueBeatsExplicitFlagFalse pins this direction so the
+// asymmetry cannot be silently "fixed" into the fail-open direction.
+func boolOrEnv(flagVal bool, envKey string) (bool, error) {
+	if flagVal {
+		return true, nil
+	}
+	raw := os.Getenv(envKey)
+	if raw == "" {
+		return false, nil
+	}
+	parsed, err := strconv.ParseBool(raw)
+	if err != nil {
+		return false, fmt.Errorf("$%s=%q is not a boolean (want true/false/1/0): %w", envKey, raw, err)
+	}
+	return parsed, nil
+}
+
 // parseEgress parses the comma-separated allowlist into a validated EgressPolicy.
 // An empty list is a valid default-deny policy (no host reachable).
 func parseEgress(csv string) (runtime.EgressPolicy, error) {

go/internal/runner/e2e_vsock_gateway_microvm_test.go

@@ -48,13 +48,15 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"connectrpc.com/connect"
 
 	compassv1 "github.com/RigelBuild/compass/go/gen/compass/v1"
+	"github.com/RigelBuild/compass/go/internal/agentuid"
 	compassv1internal "github.com/RigelBuild/compass/go/internal/gen/compass/v1"
 	"github.com/RigelBuild/compass/go/internal/microvmtest"
 	"github.com/RigelBuild/compass/go/internal/runnertest"
@@ -175,7 +177,7 @@ func w3LiveSpec(t *testing.T) runtime.AgentSpec {
 		Workspace: runtime.Workspace{
 			CheckoutDir: "/workspace",
 			HomeDir:     "/workspace",
-			UID:         1000,
+			UID:         agentuid.AgentUID,
 		},
 		Mounts: []runtime.Mount{{HostPath: t.TempDir(), ContainerPath: "/workspace"}},
 		Egress: runtime.EgressPolicy{},
@@ -453,7 +455,7 @@ func runInGuestProbe(t *testing.T, m *runtime.MicroVMRuntime, id runtime.Contain
 		t.Fatalf("marshaling probe request: %v", err)
 	}
 	script := inGuestProbeScript(string(reqBody))
-	out, err := m.Exec(ctx, id, runtime.NewExecSpec("bun", "-e", script).AsUser("1000"))
+	out, err := m.Exec(ctx, id, runtime.NewExecSpec("bun", "-e", script).AsUser(strconv.Itoa(int(agentuid.AgentUID))))
 	if err != nil {
 		t.Fatalf("in-guest bun probe exec errored (harness fault, not a round-trip verdict): %v", err)
 	}
@@ -529,7 +531,7 @@ func inGuestCanReachIP(t *testing.T, m *runtime.MicroVMRuntime, id runtime.Conta
 	ctx, cancel := context.WithTimeout(t.Context(), inGuestProbeTimeout)
 	defer cancel()
 	script := fmt.Sprintf("timeout %d bash -c 'exec 3<>/dev/tcp/%s/443 && echo connected'", guestConnectTimeoutSecs, ip)
-	out, err := m.Exec(ctx, id, runtime.NewExecSpec("sh", "-c", script).AsUser("1000"))
+	out, err := m.Exec(ctx, id, runtime.NewExecSpec("sh", "-c", script).AsUser(strconv.Itoa(int(agentuid.AgentUID))))
 	if err != nil {
 		t.Fatalf("in-guest IP connect probe to %s errored (harness fault, not a firewall verdict): %v", ip, err)
 	}