From f2861b2fd7653e5f806b8d140b6f802b933d0228 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 9 Oct 2026 09:48:35 +0800 Subject: [PATCH 01/57] feat(engine): converge live read-only volume attachments --- sandboxd/engine/installca_test.go | 4 + sandboxd/engine/volume_live.go | 182 ++++++++++++++++++++++++++++ sandboxd/engine/volume_live_test.go | 181 +++++++++++++++++++++++++++ 3 files changed, 367 insertions(+) create mode 100644 sandboxd/engine/volume_live.go create mode 100644 sandboxd/engine/volume_live_test.go diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 46362aca..8fb4a85c 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -78,6 +78,7 @@ type fakeSilkd struct { execEnv map[string]string execCode int32 execFailAt int + execHook func([]string) writeErr string writeKind string execErr string @@ -197,6 +198,9 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri if f.execFailAt > 0 && len(f.execCalls) != f.execFailAt { code, reply = 0, "" } + if code == 0 && reply == "" && f.execHook != nil { + f.execHook(argv) + } f.mu.Unlock() if reply != "" { _, _ = io.WriteString(conn, `{"type":"error","kind":"internal","message":"`+reply+`"}`+"\n") diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go new file mode 100644 index 00000000..0dca3e2a --- /dev/null +++ b/sandboxd/engine/volume_live.go @@ -0,0 +1,182 @@ +package engine + +import ( + "context" + "encoding/json/v2" + "fmt" + "path/filepath" + "slices" + "strings" + "time" +) + +const volumeMutationTimeout = 30 * time.Second + +// AttachVolume converges a read-only disk and its mount. The caller journals +// intent before calling: failures may leave a disk attached or mounted. +func (e *Engine) AttachVolume(ctx context.Context, vmName, sock string, spec VolumeSpec, mount string) error { + ctx, cancel := context.WithTimeout(ctx, volumeMutationTimeout) + defer cancel() + if spec.RW || mount == "" { + return fmt.Errorf("live attach requires a read-only mount") + } + found, err := e.attachedVolume(ctx, vmName, spec) + if err != nil { + return err + } + if !found { + if err = e.DiskAttach(ctx, vmName, spec); err != nil { + return err + } + } + device, err := e.waitForVolumeDevice(ctx, sock, spec.Name) + if err != nil { + return err + } + mounted, err := e.volumeMounted(ctx, sock, device, mount) + if err != nil || mounted { + return err + } + if err = e.MountVolume(ctx, sock, spec.Name, mount, false); err != nil { + return err + } + mounted, err = e.volumeMounted(ctx, sock, device, mount) + if err != nil { + return err + } + if !mounted { + return fmt.Errorf("volume %q did not appear at its mount point", spec.Name) + } + return nil +} + +// attachedVolume uses inspect only as positive evidence. A missing entry may +// also mean cocoon failed to query the VMM; DiskAttach then fails closed. +func (e *Engine) attachedVolume(ctx context.Context, vmName string, spec VolumeSpec) (bool, error) { + out, err := e.run(ctx, "vm", "inspect", vmName) + if err != nil { + return false, err + } + var rec struct { + AttachedDevices struct { + Disks []struct { + Name string `json:"name"` + Path string `json:"path"` + ReadOnly bool `json:"readonly"` + } `json:"disks"` + } `json:"attached_devices"` + } + if err := json.Unmarshal(out, &rec); err != nil { + return false, fmt.Errorf("parse attached disks: %w", err) + } + for _, disk := range rec.AttachedDevices.Disks { + if disk.Name != spec.Name { + continue + } + path, err := filepath.EvalSymlinks(spec.Path) + if err != nil { + return false, err + } + if !disk.ReadOnly || disk.Path != path { + return false, fmt.Errorf("attached volume %q has a different image or mode", spec.Name) + } + return true, nil + } + return false, nil +} + +// DetachVolume refuses busy or foreign mounts and confirms host removal even +// when the guest serial has disappeared. It never uses lazy or forced umount. +func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount string) error { + ctx, cancel := context.WithTimeout(ctx, volumeMutationTimeout) + defer cancel() + device, found, err := e.findVolumeDevice(ctx, sock, name) + if err != nil { + return err + } + if found { + if err = e.unmountLiveVolume(ctx, sock, device, mount); err != nil { + return err + } + } + _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name) + // Both CH and FC emit this only after checking the device set under their + // operation lock. Inspect's absent field alone is not proof of removal. + if err != nil && !strings.Contains(err.Error(), fmt.Sprintf("disk %q not attached", name)) { + return err + } + return nil +} + +func (e *Engine) unmountLiveVolume(ctx context.Context, sock, device, mount string) error { + mounted, err := e.volumeMounted(ctx, sock, device, mount) + if err != nil { + return err + } + if mounted { + if err = e.UnmountVolume(ctx, sock, mount); err != nil { + return err + } + } + // Refuse a second/bind mount of the same device before hot-unplug. + return e.volumeUnmounted(ctx, sock, device) +} + +type volumeMount struct { + device, root, path, options string +} + +func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, []volumeMount, error) { + dev, err := e.silkdReadFile(ctx, sock, "/sys/block/"+filepath.Base(device)+"/dev") + if err != nil { + return "", nil, err + } + raw, err := e.silkdReadFile(ctx, sock, "/proc/self/mountinfo") + if err != nil { + return "", nil, err + } + var mounts []volumeMount + for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { + fields := strings.Fields(line) + if len(fields) < 10 || !slices.Contains(fields[6:], "-") { + return "", nil, fmt.Errorf("invalid guest mountinfo") + } + mounts = append(mounts, volumeMount{fields[2], unescapeMount(fields[3]), unescapeMount(fields[4]), fields[5]}) + } + return strings.TrimSpace(string(dev)), mounts, nil +} + +func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string) (bool, error) { + dev, mounts, err := e.volumeMounts(ctx, sock, device) + if err != nil { + return false, err + } + matched := false + for _, entry := range mounts { + if entry.path != mount { + continue + } + if matched || entry.device != dev || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), "ro") { + return false, fmt.Errorf("volume mount %q has a different device or mode", mount) + } + matched = true + } + return matched, nil +} + +func (e *Engine) volumeUnmounted(ctx context.Context, sock, device string) error { + dev, mounts, err := e.volumeMounts(ctx, sock, device) + if err != nil { + return err + } + for _, mount := range mounts { + if mount.device == dev { + return fmt.Errorf("volume device is still mounted at %q", mount.path) + } + } + return nil +} + +func unescapeMount(s string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(s) +} diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go new file mode 100644 index 00000000..575591c2 --- /dev/null +++ b/sandboxd/engine/volume_live_test.go @@ -0,0 +1,181 @@ +package engine + +import ( + "encoding/json/v2" + "os" + "path/filepath" + "strings" + "testing" +) + +const rootMountInfo = "1 0 8:0 / / rw - ext4 /dev/vda rw\n" + +func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + // Start without a host attachment, then prove the second call reuses it. + if err := os.Remove(os.Getenv("COCOON_TEST_ATTACHED")); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + for range 2 { + if err := e.AttachVolume(t.Context(), "sbx-1", sock, spec, "/data"); err != nil { + t.Fatal(err) + } + } + if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != 3 || fake.execCalls[2][0] != "umount" { + t.Fatalf("expected one mkdir/mount/umount, got %v", fake.execCalls) + } + args, err := os.ReadFile(os.Getenv("COCOON_TEST_ATTACH_ARGS")) + if err != nil || !strings.Contains(string(args), "--readonly --directio off") { + t.Fatalf("attach args=%s error=%v", args, err) + } +} + +func TestLiveAttachRejectsForeignMounts(t *testing.T) { + for _, row := range []string{ + "2 1 8:99 / /data ro - ext4 /dev/vdz ro\n", + "2 1 8:32 / /data rw - ext4 /dev/vdc rw\n", + "2 1 8:32 /subdir /data ro - ext4 /dev/vdc ro\n", + "malformed\n", + } { + t.Run(strings.TrimSpace(row), func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + row) + fake.mu.Unlock() + if err := New("cocoon", nil, nil, false, false, "").AttachVolume(t.Context(), "sbx-1", sock, spec, "/data"); err == nil { + t.Fatal("accepted a foreign mount") + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != 0 { + t.Fatalf("modified foreign mount: %v", fake.execCalls) + } + }) + } +} + +func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { + for _, mode := range []string{"busy", "second mount", "probe error", "host error"} { + t.Run(mode, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + hostErr := "" + if mode == "host error" { + hostErr = "device removal initiated but guest has not ejected it" + } + spec := liveDiskCLI(t, hostErr) + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + switch mode { + case "busy": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + fake.execCode = 32 + case "second mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /other ro - ext4 /dev/vdc ro\n") + case "probe error": + fake.readErr["/proc/self/mountinfo"] = "read failed" + } + fake.mu.Unlock() + if err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err == nil { + t.Fatal("detach must fail closed") + } + }) + } +} + +func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { + for _, hostErr := range []string{"", `disk "imagenet" not attached`, "vm not running", "backend unavailable"} { + t.Run(hostErr, func(t *testing.T) { + sock := sockPath(t) + serveFakeSilkd(t, sock) + spec := liveDiskCLI(t, hostErr) + err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + wantErr := hostErr != "" && !strings.Contains(hostErr, "not attached") + if (err != nil) != wantErr { + t.Fatalf("got %v, want error=%v", err, wantErr) + } + }) + } +} + +func TestLiveAttachChecksHostImageAndMode(t *testing.T) { + spec := liveDiskCLI(t, "") + e := New("cocoon", nil, nil, false, false, "") + spec.Path = t.TempDir() + if _, err := e.attachedVolume(t.Context(), "sbx-1", spec); err == nil { + t.Fatal("accepted mismatched host image") + } +} + +func liveDiskCLI(t *testing.T, detachErr string) VolumeSpec { + t.Helper() + dir := t.TempDir() + image := filepath.Join(dir, "volume.img") + if err := os.WriteFile(image, nil, 0o600); err != nil { + t.Fatal(err) + } + path, err := filepath.EvalSymlinks(image) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(map[string]any{"attached_devices": map[string]any{"disks": []any{map[string]any{"name": "imagenet", "path": path, "readonly": true}}}}) + if err != nil { + t.Fatal(err) + } + inspect := filepath.Join(dir, "inspect.json") + if err := os.WriteFile(inspect, data, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("COCOON_TEST_INSPECT", inspect) + t.Setenv("COCOON_TEST_DETACH_ERROR", detachErr) + attached := filepath.Join(dir, "attached") + if err := os.WriteFile(attached, nil, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("COCOON_TEST_ATTACHED", attached) + t.Setenv("COCOON_TEST_ATTACH_ARGS", filepath.Join(dir, "args")) + fakeCocoon(t, `#!/bin/sh +if [ "$1 $2" = "vm inspect" ]; then + if [ -f "$COCOON_TEST_ATTACHED" ]; then cat "$COCOON_TEST_INSPECT"; else echo '{}'; fi + exit 0 +fi +if [ "$1 $2 $3" = "vm disk attach" ]; then + printf '%s\n' "$*" >> "$COCOON_TEST_ATTACH_ARGS" + touch "$COCOON_TEST_ATTACHED" + exit 0 +fi +if [ "$1 $2 $3" = "vm disk detach" ]; then + if [ -n "$COCOON_TEST_DETACH_ERROR" ]; then echo "$COCOON_TEST_DETACH_ERROR" >&2; exit 1; fi + exit 0 +fi +exit 2 +`) + return VolumeSpec{Name: "imagenet", Path: image} +} From c4200fa7075623817edd812d49c76371c8fdca18 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 9 Oct 2026 09:48:50 +0800 Subject: [PATCH 02/57] feat(sandboxd): journal and recover live volume mutations --- e2e/fakeengine_test.go | 17 + sandboxd/pool/checkpoint.go | 3 - sandboxd/pool/claim.go | 33 ++ sandboxd/pool/claims.go | 55 ++-- sandboxd/pool/fork.go | 6 +- sandboxd/pool/pool.go | 36 ++- sandboxd/pool/pool_test.go | 37 +++ sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/refill.go | 16 + sandboxd/pool/template.go | 3 - sandboxd/pool/volume.go | 2 +- sandboxd/pool/volume_mutation.go | 266 ++++++++++++++++ sandboxd/pool/volume_mutation_test.go | 403 ++++++++++++++++++++++++ sandboxd/server/server.go | 5 + sandboxd/server/server_test.go | 10 + sandboxd/server/volume_mutation.go | 39 +++ sandboxd/server/volume_mutation_test.go | 95 ++++++ sandboxd/types/api.go | 15 + sandboxd/types/types.go | 15 +- 19 files changed, 1003 insertions(+), 55 deletions(-) create mode 100644 sandboxd/pool/volume_mutation.go create mode 100644 sandboxd/pool/volume_mutation_test.go create mode 100644 sandboxd/server/volume_mutation.go create mode 100644 sandboxd/server/volume_mutation_test.go diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 561cf243..563fe453 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -177,6 +177,23 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } +func (f *fakeEngine) AttachVolume(ctx context.Context, vmName, sock string, spec engine.VolumeSpec, mount string) error { + if err := f.DiskAttach(ctx, vmName, spec); err != nil { + return err + } + return f.MountVolume(ctx, sock, spec.Name, mount, spec.RW) +} + +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { + if err := f.UnmountVolume(ctx, sock, mount); err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.volumeOps = append(f.volumeOps, "detach:"+name) + return nil +} + func (f *fakeEngine) volumeOpsLog() []string { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index c894d935..926388d9 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -40,9 +40,6 @@ func (m *Manager) Checkpoint(ctx context.Context, id string, cred Cred, name, te if name != "" && !types.NameRe.MatchString(name) { return types.Checkpoint{}, fmt.Errorf("%w: %q must match %s", ErrBadName, name, types.NameRe) } - if hasAppliedVolumes(sb) { - return types.Checkpoint{}, ErrVolumeCapture - } if !sb.Key.Capturable() { return types.Checkpoint{}, ErrNoEgressFork } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index a3c6dd12..394ff362 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -160,6 +160,8 @@ func (m *Manager) Renew(ctx context.Context, id string, cred Cred, ttl time.Dura if !ok { return time.Time{}, ErrUnknownSandbox } + sb.VolumeMu.Lock() + defer sb.VolumeMu.Unlock() if !cred.Operator { if err := m.tenantRemoved(ctx, sb.Tenant); err != nil { return time.Time{}, err @@ -269,6 +271,8 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. // releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { + sb.VolumeMu.Lock() + defer sb.VolumeMu.Unlock() var marked string m.mu.Lock() for { @@ -494,6 +498,14 @@ func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved } func (m *Manager) reapOnce(ctx context.Context) { + // Take volume locks outside m.mu. Busy claims are retried on the next tick; + // a volume intent must not be deleted while its device call is in flight. + locked := m.lockReapClaims() + defer func() { + for sb := range locked { + sb.VolumeMu.Unlock() + } + }() now := time.Now() type victim struct { action reapAction @@ -504,6 +516,9 @@ func (m *Manager) reapOnce(ctx context.Context) { var expired []victim var dropped, purges []string for id, sb := range m.claimed { + if !locked[sb] { + continue + } // a zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { continue @@ -623,6 +638,24 @@ func (m *Manager) reapOnce(ctx context.Context) { }) } +func (m *Manager) lockReapClaims() map[*types.Sandbox]bool { + m.mu.Lock() + var candidates []*types.Sandbox + for _, sb := range m.claimed { + if !sb.Deadline.IsZero() && time.Now().After(sb.Deadline) { + candidates = append(candidates, sb) + } + } + m.mu.Unlock() + locked := make(map[*types.Sandbox]bool, len(candidates)) + for _, sb := range candidates { + if sb.VolumeMu.TryLock() { + locked[sb] = true + } + } + return locked +} + func (m *Manager) archivesAtDeadline(sb *types.Sandbox) bool { return sb.Failed == "" && !m.tenantGone(sb.Tenant) && (sb.OnExpire == types.ExpireArchive || sb.HibernateSnap != "" && m.archiveAfterFor(sb.Key) > 0) } diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 03f1e2c7..d0fdd925 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -23,32 +23,33 @@ type claimSnapshot struct { // claimDTO is the persisted projection of a Sandbox, copied so commit marshals off m.mu. type claimDTO struct { - ID string `json:"id"` - VMName string `json:"vm_name"` - Key types.PoolKey `json:"key"` - Token string `json:"token,omitempty"` - Deadline time.Time `json:"deadline,omitzero"` - ClaimedAt time.Time `json:"claimed_at,omitzero"` - LeaseSeconds int `json:"lease_seconds,omitzero"` - Layer types.PolicyLayer `json:"policy_layer,omitempty"` - PolicySource types.PoolKey `json:"policy_source,omitzero"` - NoEgress bool `json:"no_egress,omitzero"` - Tenant string `json:"tenant,omitempty"` - EgressClass string `json:"egress_class,omitempty"` - ClaimRef string `json:"claim_ref,omitempty"` - Metadata types.Metadata `json:"metadata,omitempty"` - OnExpire types.ExpireAction `json:"on_expire,omitempty"` - Volumes []types.Volume `json:"volumes,omitempty"` - Env types.Env `json:"env,omitempty"` - VsockSocket string `json:"vsock_socket,omitempty"` - TAP string `json:"tap,omitempty"` - HibernateSnap string `json:"hibernate_snap,omitempty"` - PendingSnap string `json:"pending_snap,omitempty"` - ArchiveCk string `json:"archive_ck,omitempty"` - FromCheckpoint string `json:"from_checkpoint,omitempty"` - Restarts int `json:"restarts,omitzero"` - RestartedAt time.Time `json:"restarted_at,omitzero"` - Failed string `json:"failed,omitempty"` + ID string `json:"id"` + VMName string `json:"vm_name"` + Key types.PoolKey `json:"key"` + Token string `json:"token,omitempty"` + Deadline time.Time `json:"deadline,omitzero"` + ClaimedAt time.Time `json:"claimed_at,omitzero"` + LeaseSeconds int `json:"lease_seconds,omitzero"` + Layer types.PolicyLayer `json:"policy_layer,omitempty"` + PolicySource types.PoolKey `json:"policy_source,omitzero"` + NoEgress bool `json:"no_egress,omitzero"` + Tenant string `json:"tenant,omitempty"` + EgressClass string `json:"egress_class,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` + Metadata types.Metadata `json:"metadata,omitempty"` + OnExpire types.ExpireAction `json:"on_expire,omitempty"` + Volumes []types.Volume `json:"volumes,omitempty"` + PendingVolume *types.VolumeMutation `json:"pending_volume,omitempty"` + Env types.Env `json:"env,omitempty"` + VsockSocket string `json:"vsock_socket,omitempty"` + TAP string `json:"tap,omitempty"` + HibernateSnap string `json:"hibernate_snap,omitempty"` + PendingSnap string `json:"pending_snap,omitempty"` + ArchiveCk string `json:"archive_ck,omitempty"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + Restarts int `json:"restarts,omitzero"` + RestartedAt time.Time `json:"restarted_at,omitzero"` + Failed string `json:"failed,omitempty"` } // claimRow keeps its encoded member until set replaces the row; set never edits one. @@ -220,7 +221,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { Deadline: sb.Deadline, ClaimedAt: sb.ClaimedAt, LeaseSeconds: sb.LeaseSeconds, Layer: sb.Layer, PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, - Volumes: slices.Clone(sb.Volumes), Env: sb.Env, VsockSocket: sb.VsockSocket, + Volumes: slices.Clone(sb.Volumes), PendingVolume: sb.PendingVolume, Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index 56ba3741..83c2ad8d 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -24,9 +24,6 @@ func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl if count < 1 || count > m.maxFork { return nil, fmt.Errorf("%w: %d not in 1..%d", ErrBadCount, count, m.maxFork) } - if hasAppliedVolumes(sb) { - return nil, ErrVolumeCapture - } if !sb.Key.Capturable() { return nil, ErrNoEgressFork } @@ -78,6 +75,9 @@ func (m *Manager) forkClones(ctx context.Context, sb *types.Sandbox, count int) func (m *Manager) forkSource(ctx context.Context, sb *types.Sandbox) (create vmProvisioner, cleanup func(), inherited bool, err error) { sb.Transition.Lock() defer sb.Transition.Unlock() + if err = m.captureAllowed(sb); err != nil { + return nil, nil, false, err + } inherited = m.heldGuestEnv(sb) if sb.HibernateSnap == "" { snap, drop, snapErr := m.sourceSnap(ctx, sb) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 8ebbaff3..07a83cce 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -102,6 +102,7 @@ var ( ErrNoEgressFork = errors.New("egress-lane sandboxes cannot fork, checkpoint, or promote: a resumed guest egresses before its fresh tap can be locked") ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") + ErrVolumePending = errors.New("volume operation pending; retry it, detach its name, or release the sandbox") // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") ErrArchived = errors.New("sandbox is archived; an exec or file call wakes it first") @@ -140,6 +141,8 @@ type Engine interface { Warmup(ctx context.Context, vsockSocket string, argv []string) error TrimCow(ctx context.Context, vsockSocket string) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error + AttachVolume(ctx context.Context, vmName, sock string, spec engine.VolumeSpec, mount string) error + DetachVolume(ctx context.Context, vmName, sock, name, mount string) error MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error SyncGuest(ctx context.Context, vsockSocket string) error @@ -155,20 +158,21 @@ type SandboxSummary struct { // Token is the sandbox's own bearer token; only the root by-id read carries it. Token string `json:"token,omitempty"` // ClaimRef echoes the caller reference; empty for checkpoint branches and unprefixed forks. - ClaimRef string `json:"claim_ref,omitempty"` - Metadata types.Metadata `json:"metadata,omitempty"` - OnExpire types.ExpireAction `json:"on_expire,omitempty"` - CPUCount int `json:"cpu_count,omitzero"` - MemTotalBytes int64 `json:"mem_total_bytes,omitzero"` - Volumes []types.Volume `json:"volumes,omitempty"` - Hibernated bool `json:"hibernated"` - Archived bool `json:"archived,omitzero"` - FromCheckpoint string `json:"from_checkpoint,omitempty"` - Restarts int `json:"restarts,omitzero"` - RestartedAt time.Time `json:"restarted_at,omitzero"` - Failed string `json:"failed,omitempty"` - ClaimedAt time.Time `json:"claimed_at,omitzero"` - Deadline time.Time `json:"deadline"` + ClaimRef string `json:"claim_ref,omitempty"` + Metadata types.Metadata `json:"metadata,omitempty"` + OnExpire types.ExpireAction `json:"on_expire,omitempty"` + CPUCount int `json:"cpu_count,omitzero"` + MemTotalBytes int64 `json:"mem_total_bytes,omitzero"` + Volumes []types.Volume `json:"volumes,omitempty"` + PendingVolume *types.VolumeMutation `json:"pending_volume,omitempty"` + Hibernated bool `json:"hibernated"` + Archived bool `json:"archived,omitzero"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + Restarts int `json:"restarts,omitzero"` + RestartedAt time.Time `json:"restarted_at,omitzero"` + Failed string `json:"failed,omitempty"` + ClaimedAt time.Time `json:"claimed_at,omitzero"` + Deadline time.Time `json:"deadline"` } // PoolInfo is the ops view of one pool. @@ -688,7 +692,7 @@ func summarize(sb *types.Sandbox) SandboxSummary { FromCheckpoint: sb.FromCheckpoint, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, CPUCount: spec.CPU, MemTotalBytes: spec.MemoryBytes, - Volumes: slices.Clone(sb.Volumes), + Volumes: slices.Clone(sb.Volumes), PendingVolume: cloneVolumeMutation(sb.PendingVolume), } } @@ -711,7 +715,7 @@ func tenantOwns(tenant, owner string) bool { } func hasAppliedVolumes(sb *types.Sandbox) bool { - return len(sb.Volumes) > 0 + return len(sb.Volumes) > 0 || sb.PendingVolume != nil } // logSweepResult reports one background-sweep outcome; benign races stay silent. diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 6253b546..c19f8599 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1307,6 +1307,8 @@ type fakeEngine struct { diskAttachCancel context.CancelFunc mountVolumeErr error unmountVolumeErr error + detachVolumeErr error + liveVolumeHook func(string) error stopped map[string]bool creating map[string]bool staleOutcome engine.StaleCreateOutcome @@ -1686,6 +1688,41 @@ func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bo return f.mountVolumeErr } +func (f *fakeEngine) AttachVolume(ctx context.Context, vmName, sock string, spec engine.VolumeSpec, mount string) error { + if f.liveVolumeHook != nil { + if err := f.liveVolumeHook("attach"); err != nil { + return err + } + } + if err := ctx.Err(); err != nil { + return err + } + if err := f.DiskAttach(ctx, vmName, spec); err != nil { + return err + } + return f.MountVolume(ctx, sock, spec.Name, mount, spec.RW) +} + +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { + if f.liveVolumeHook != nil { + if err := f.liveVolumeHook("detach"); err != nil { + return err + } + } + if err := ctx.Err(); err != nil { + return err + } + if mount != "" { + if err := f.UnmountVolume(ctx, sock, mount); err != nil { + return err + } + } + f.mu.Lock() + defer f.mu.Unlock() + f.volumeOps = append(f.volumeOps, "detach:"+name) + return f.detachVolumeErr +} + func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 74449619..1bf80f74 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -66,7 +66,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { } m.claimed[id] = sb m.tenantDelta(sb.Tenant, 1) - m.adoptVolumes(sb.Volumes) + m.adoptVolumes(heldVolumes(sb)) owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 635f47c0..521f656c 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -357,6 +357,9 @@ func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { // sourceSnap picks the snapshot to export a claimed sandbox from, with its cleanup; the caller holds sb.Transition. func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, func(), error) { + if err := m.captureAllowed(sb); err != nil { + return "", nil, err + } // archive() clears VMName under the same lock, so this is the check that cannot race it if sb.ArchiveCk != "" { return "", nil, ErrArchived @@ -374,6 +377,19 @@ func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, fu return snap, func() { m.dropSnap(ctx, snap) }, nil } +// captureAllowed must run under Transition, after any live volume change. +func (m *Manager) captureAllowed(sb *types.Sandbox) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[sb.ID] != sb { + return ErrUnknownSandbox + } + if hasAppliedVolumes(sb) { + return ErrVolumeCapture + } + return nil +} + // exportSource captures a claimed sandbox into exportDir and returns its snapshot, VM name and guest-env presence. func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir string) (snap, vmName string, guestEnv bool, err error) { sb.Transition.Lock() diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 5570bb2e..8f473f2d 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -47,9 +47,6 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t if !types.NameRe.MatchString(template) { return types.PoolKey{}, "", fmt.Errorf("%w: template %q must match %s", ErrBadKey, template, types.NameRe) } - if hasAppliedVolumes(sb) { - return types.PoolKey{}, "", ErrVolumeCapture - } if !sb.Key.Capturable() { return types.PoolKey{}, "", ErrNoEgressFork } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 5054afb1..12a834bd 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -208,7 +208,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { - td := volumeTeardown{holds: sb.Volumes} + td := volumeTeardown{holds: heldVolumes(sb)} mounts := writableMounts(sb.Volumes) if mounts == 0 { return td diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go new file mode 100644 index 00000000..aeea7494 --- /dev/null +++ b/sandboxd/pool/volume_mutation.go @@ -0,0 +1,266 @@ +package pool + +import ( + "context" + "fmt" + "slices" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +// AttachVolumes adds mounted, read-only volumes to a running claim. Batches +// commit in order: retry the request after a partial failure to finish it. +func (m *Manager) AttachVolumes(ctx context.Context, id string, cred Cred, requested []types.Volume) ([]types.Volume, error) { + if types.VolumesAttachOnly(requested) || slices.ContainsFunc(requested, types.Volume.RW) { + return nil, fmt.Errorf("%w: live attach requires read-only mounts", ErrBadVolume) + } + requested, err := types.ValidateVolumes(requested, false) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + } + sb, ok := m.resolve(id, cred) + if !ok { + return nil, ErrUnknownSandbox + } + sb.VolumeMu.Lock() + defer sb.VolumeMu.Unlock() + sb.Transition.Lock() + defer sb.Transition.Unlock() + if err = m.mutableVolumes(sb); err != nil { + return nil, err + } + if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { + return nil, err + } + if pending := sb.PendingVolume; pending != nil && (pending.Detach || !slices.Contains(requested, pending.Volume)) { + return nil, ErrVolumePending + } + toAdd, err := liveVolumeAdditions(heldVolumes(sb), requested) + if err != nil { + return nil, err + } + // A pending attach already holds admission but still needs convergence. + if pending := sb.PendingVolume; pending != nil { + toAdd = append([]types.Volume{pending.Volume}, toAdd...) + } + resolved, err := m.resolveVolumes(ctx, sb.Tenant, toAdd) + if err != nil { + return nil, err + } + for _, volume := range resolved { + if err := ctx.Err(); err != nil { + return nil, err + } + if sb.PendingVolume == nil { + if err := m.startVolumeAttach(ctx, sb, volume); err != nil { + return nil, err + } + } else if err := m.confirmVolumesClean([]resolvedVolume{volume}); err != nil { + return nil, err + } + if err := m.eng.AttachVolume(ctx, sb.VMName, sb.VsockSocket, volume.disk, volume.applied.Mount); err != nil { + return nil, fmt.Errorf("attach volume %q (retry or detach to recover): %w", volume.applied.Name, err) + } + if err := m.finishVolumeMutation(ctx, sb); err != nil { + return nil, err + } + } + return slices.Clone(sb.Volumes), nil +} + +// DetachVolumes removes read-only volumes, including an interrupted attach. +// An absent valid name is a no-op; batches may partially complete on failure. +func (m *Manager) DetachVolumes(ctx context.Context, id string, cred Cred, names []string) ([]types.Volume, error) { + if err := validateDetachNames(names); err != nil { + return nil, err + } + sb, ok := m.resolve(id, cred) + if !ok { + return nil, ErrUnknownSandbox + } + sb.VolumeMu.Lock() + defer sb.VolumeMu.Unlock() + sb.Transition.Lock() + defer sb.Transition.Unlock() + if err := m.mutableVolumes(sb); err != nil { + return nil, err + } + if pending := sb.PendingVolume; pending != nil && !slices.Contains(names, pending.Volume.Name) { + return nil, ErrVolumePending + } + held := heldVolumes(sb) + for _, volume := range held { + if slices.Contains(names, volume.Name) && volume.RW() { + return nil, fmt.Errorf("%w: live detach of writable volumes is not supported", ErrBadVolume) + } + } + // Recover the pending operation first, even if it was last in the request. + if pending := sb.PendingVolume; pending != nil { + if err := m.detachLiveVolume(ctx, sb, pending.Volume); err != nil { + return nil, err + } + } + for _, volume := range slices.Clone(sb.Volumes) { + if slices.Contains(names, volume.Name) { + if err := m.detachLiveVolume(ctx, sb, volume); err != nil { + return nil, err + } + } + } + return slices.Clone(sb.Volumes), nil +} + +// mutableVolumes runs under Transition; manager-owned fields need m.mu too. +func (m *Manager) mutableVolumes(sb *types.Sandbox) error { + m.mu.Lock() + defer m.mu.Unlock() + switch { + case m.claimed[sb.ID] != sb: + return ErrUnknownSandbox + case sb.Failed != "": + return ErrFailed + case sb.ArchiveCk != "": + return ErrArchived + case sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.VsockSocket == "": + return ErrPaused + } + return nil +} + +func liveVolumeAdditions(current, requested []types.Volume) ([]types.Volume, error) { + combined := slices.Clone(current) + var added []types.Volume + for _, volume := range requested { + i := slices.IndexFunc(current, func(v types.Volume) bool { return v.Name == volume.Name }) + if i >= 0 { + if current[i] != volume { + return nil, fmt.Errorf("%w: volume %q already has a different mount or mode", ErrBadVolume, volume.Name) + } + continue + } + combined = append(combined, volume) + added = append(added, volume) + } + if len(combined) > types.MaxClaimVolumes { + return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) + } + // Validate only mounted entries, preserving existing attach-only devices. + mounted := slices.DeleteFunc(slices.Clone(combined), func(v types.Volume) bool { return v.Mount == "" }) + if _, err := types.ValidateVolumes(mounted, false); err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + } + return added, nil +} + +func validateDetachNames(names []string) error { + if len(names) > types.MaxClaimVolumes { + return fmt.Errorf("%w: too many volume names", ErrBadVolume) + } + for i, name := range names { + if !types.ValidVolumeName(name) || slices.Contains(names[:i], name) { + return fmt.Errorf("%w: invalid or duplicate volume name %q", ErrBadVolume, name) + } + } + return nil +} + +func (m *Manager) startVolumeAttach(ctx context.Context, sb *types.Sandbox, volume resolvedVolume) error { + m.mu.Lock() + if sb.OnExpire == types.ExpireArchive { + m.mu.Unlock() + return ErrVolumeCapture + } + err := m.reserveVolumes([]types.Volume{volume.applied}) + m.mu.Unlock() + if err != nil { + return err + } + if err = m.confirmVolumesClean([]resolvedVolume{volume}); err == nil { + err = m.setVolumeIntent(ctx, sb, &types.VolumeMutation{Volume: volume.applied}) + } + if err != nil { + m.unreserveVolumes([]types.Volume{volume.applied}) + } + return err +} + +func (m *Manager) detachLiveVolume(ctx context.Context, sb *types.Sandbox, volume types.Volume) error { + if err := ctx.Err(); err != nil { + return err + } + if pending := sb.PendingVolume; pending == nil || !pending.Detach { + if err := m.setVolumeIntent(ctx, sb, &types.VolumeMutation{Volume: volume, Detach: true}); err != nil { + return err + } + } + if err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, volume.Name, volume.Mount); err != nil { + return fmt.Errorf("detach volume %q (retry to recover): %w", volume.Name, err) + } + return m.finishVolumeMutation(ctx, sb) +} + +func (m *Manager) setVolumeIntent(ctx context.Context, sb *types.Sandbox, next *types.VolumeMutation) error { + m.mu.Lock() + previous := sb.PendingVolume + sb.PendingVolume = next + js := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(js); err != nil { + m.mu.Lock() + sb.PendingVolume = previous + rb := m.store.set(sb) + m.mu.Unlock() + m.recommit(ctx, rb) + return fmt.Errorf("persist volume intent: %w", err) + } + return nil +} + +func (m *Manager) finishVolumeMutation(ctx context.Context, sb *types.Sandbox) error { + m.mu.Lock() + previous, pending := sb.Volumes, sb.PendingVolume + next := slices.Clone(previous) + if pending.Detach { + next = slices.DeleteFunc(next, func(v types.Volume) bool { return v.Name == pending.Volume.Name }) + } else { + next = append(next, pending.Volume) + } + sb.Volumes, sb.PendingVolume = next, nil + js := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(js); err != nil { + // Keep both the intent and its admission hold until a retry commits. + m.mu.Lock() + sb.Volumes, sb.PendingVolume = previous, pending + rb := m.store.set(sb) + m.mu.Unlock() + m.recommit(ctx, rb) + return fmt.Errorf("persist volume result: %w", err) + } + verb := "volume_attach" + if pending.Detach { + m.unreserveVolumes([]types.Volume{pending.Volume}) + verb = "volume_detach" + } + sb.Touch() + m.recordUsage(ctx, usageEvent{Event: verb, ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant, Volumes: []string{pending.Volume.Name}}) + return nil +} + +// heldVolumes includes a possibly attached device even before it is committed. +// Callers hold m.mu or Transition (both are required for mutations). +func heldVolumes(sb *types.Sandbox) []types.Volume { + volumes := slices.Clone(sb.Volumes) + if pending := sb.PendingVolume; pending != nil && !slices.ContainsFunc(volumes, func(v types.Volume) bool { return v.Name == pending.Volume.Name }) { + volumes = append(volumes, pending.Volume) + } + return volumes +} + +func cloneVolumeMutation(op *types.VolumeMutation) *types.VolumeMutation { + if op == nil { + return nil + } + cloned := *op + return &cloned +} diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go new file mode 100644 index 00000000..8defa7da --- /dev/null +++ b/sandboxd/pool/volume_mutation_test.go @@ -0,0 +1,403 @@ +package pool + +import ( + "context" + "errors" + "os" + "slices" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveVolumesCommitAndRetry(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + request := []types.Volume{{Name: "data"}, {Name: "other", Mount: "/models"}} + for range 2 { + got, err := m.AttachVolumes(t.Context(), sb.ID, cred, request) + if err != nil || len(got) != 2 || sb.PendingVolume != nil { + t.Fatalf("attach = %v, %v; pending=%v", got, err, sb.PendingVolume) + } + got[0].Name = "caller copy" + } + if got := volumeHoldersOf(m, "data"); got.readers != 1 { + t.Fatalf("duplicate reservation: %+v", got) + } + assertLiveJournal(t, m, sb) + for range 2 { + got, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data", "other"}) + if err != nil || len(got) != 0 { + t.Fatalf("detach = %v, %v", got, err) + } + } + assertLiveJournal(t, m, sb) + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("leaked holds: %+v", got) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, cred, "after-detach", ""); err != nil { + t.Fatalf("capture after detach: %v", err) + } +} + +func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { + for _, attachOnly := range []bool{false, true} { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: "rw", AttachOnly: attachOnly}}}) + if err != nil { + t.Fatal(err) + } + original := sb.Volumes[0] + cred := Cred{Token: sb.Token} + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if sb.Volumes[0] != original { + t.Fatalf("changed existing volume: %v -> %v", original, sb.Volumes[0]) + } + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("accepted mode change: %v", err) + } + if _, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data", "other"}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { + t.Fatalf("writable detach changed the batch: %v", err) + } + } +} + +func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + for _, request := range [][]types.Volume{ + {{Name: "data", Mode: "rw"}}, + {{Name: "data", AttachOnly: true}}, + {{Name: "data", Mount: "/etc"}}, + {{Name: "data"}, {Name: "data"}}, + {{Name: "data", Mount: "/a"}, {Name: "other", Mount: "/a/b"}}, + {{Name: "missing"}}, + } { + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); !errors.Is(err, ErrBadVolume) { + t.Fatalf("request %v: %v", request, err) + } + } + if _, err := m.AttachVolumes(t.Context(), sb.ID, Cred{Token: "wrong"}, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrUnknownSandbox) { + t.Fatalf("wrong token: %v", err) + } + if _, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data", "data"}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("duplicate detach: %v", err) + } + if len(eng.volumeSpecs) != 0 || sb.PendingVolume != nil { + t.Fatal("invalid request reached engine") + } +} + +func TestLiveAttachFailureSurvivesRestartAndCanBeDetached(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + eng.mountVolumeErr = errors.New("guest unavailable after host attach") + if _, err := m.AttachVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("want mount failure") + } + assertLiveJournal(t, m, sb) + if sb.PendingVolume == nil || volumeHoldersOf(m, "data").readers != 1 { + t.Fatal("lost pending device or hold") + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if volumeHoldersOf(m2, "data").readers != 1 { + t.Fatal("restart lost pending hold") + } + cred := Cred{Token: sb.Token} + if _, err := m2.Checkpoint(t.Context(), sb.ID, cred, "pending", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("captured pending device: %v", err) + } + if _, err := m2.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumePending) { + t.Fatalf("overwrote pending operation: %v", err) + } + eng.detachVolumeErr = errors.New("host detach failed") + if _, err := m2.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}); err == nil || volumeHoldersOf(m2, "data").readers != 1 { + t.Fatal("failed detach released its hold") + } + eng.detachVolumeErr = nil + if _, err := m2.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}); err != nil { + t.Fatal(err) + } + if volumeHoldersOf(m2, "data") != (volumeHolders{}) { + t.Fatal("successful cleanup leaked hold") + } +} + +func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { + for _, stage := range []string{"intent", "attach result", "detach result"} { + t.Run(stage, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + request := []types.Volume{{Name: "data"}} + if stage == "detach result" { + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); err != nil { + t.Fatal(err) + } + } + path := m.store.path + ".tmp" + breakStore := func() { + m.store.writeMu.Lock() + defer m.store.writeMu.Unlock() + if err := os.Mkdir(path, 0o700); err != nil { + t.Fatal(err) + } + } + if stage == "intent" { + breakStore() + } else { + eng.liveVolumeHook = func(string) error { breakStore(); return nil } + } + var err error + if stage == "detach result" { + _, err = m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}) + } else { + _, err = m.AttachVolumes(t.Context(), sb.ID, cred, request) + } + if err == nil { + t.Fatal("expected persistence failure") + } + m.store.writeMu.Lock() + removeErr := os.Remove(path) + m.store.writeMu.Unlock() + if removeErr != nil { + t.Fatal(removeErr) + } + eng.liveVolumeHook = nil + if stage == "intent" { + if sb.PendingVolume != nil || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("failed intent must not reach the engine or retain a hold") + } + return + } + if sb.PendingVolume == nil || volumeHoldersOf(m, "data").readers != 1 { + t.Fatal("uncommitted physical result lost its intent or hold") + } + if stage == "detach result" { + _, err = m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}) + } else { + _, err = m.AttachVolumes(t.Context(), sb.ID, cred, request) + } + if err != nil { + t.Fatal(err) + } + assertLiveJournal(t, m, sb) + }) + } +} + +func TestLiveBatchPartialFailureRetriesWithoutDoubleHolds(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + request := []types.Volume{{Name: "data"}, {Name: "other"}} + eng.diskAttachErrFor = "other" + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); err == nil { + t.Fatal("want partial failure") + } + if len(sb.Volumes) != 1 || sb.PendingVolume == nil || sb.PendingVolume.Volume.Name != "other" { + t.Fatalf("partial state: %v %v", sb.Volumes, sb.PendingVolume) + } + eng.diskAttachErrFor = "" + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); err != nil { + t.Fatal(err) + } + for _, name := range []string{"data", "other"} { + if volumeHoldersOf(m, name).readers != 1 { + t.Fatal("retry double-reserved " + name) + } + } +} + +func TestLiveVolumeSerializesReleaseAndCapture(t *testing.T) { + for _, action := range []string{"release", "checkpoint", "fork", "promote", "hibernate", "reap"} { + t.Run(action, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + entered, resume := make(chan struct{}), make(chan struct{}) + eng.liveVolumeHook = func(string) error { close(entered); <-resume; return nil } + attached := make(chan error, 1) + go func() { + _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + attached <- err + }() + <-entered + // The durable intent must precede any physical operation. + assertLiveJournal(t, m, sb) + done := make(chan error, 1) + go func() { + var err error + switch action { + case "release": + err = m.Release(t.Context(), sb.ID, cred) + case "checkpoint": + _, err = m.Checkpoint(t.Context(), sb.ID, cred, "race", "") + case "fork": + _, err = m.Fork(t.Context(), sb.ID, cred, 1, 0, "", "") + case "promote": + _, _, err = m.Promote(t.Context(), sb.ID, cred, "race", "") + case "hibernate": + err = m.Hibernate(t.Context(), sb.ID, cred) + case "reap": + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Second) + m.mu.Unlock() + m.reapOnce(t.Context()) + } + done <- err + }() + if sb.VolumeMu.TryLock() { + sb.VolumeMu.Unlock() + t.Fatal("volume mutation did not exclude release") + } + if sb.Transition.TryLock() { + sb.Transition.Unlock() + t.Fatal("volume mutation did not exclude capture") + } + if len(eng.removedNames()) != 0 || len(eng.snapSaves) != 0 { + t.Fatal("teardown or capture overlapped device mutation") + } + close(resume) + if err := <-attached; err != nil { + t.Fatal(err) + } + err := <-done + if action != "release" && action != "reap" && !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture: %v", err) + } + if action == "release" && (err != nil || volumeHoldersOf(m, "data") != (volumeHolders{})) { + t.Fatalf("release: %v", err) + } + }) + } +} + +func TestLiveVolumeCanceledRequestRetainsIntent(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + ctx, cancel := context.WithCancel(t.Context()) + eng.liveVolumeHook = func(string) error { cancel(); return nil } + _, err := m.AttachVolumes(ctx, sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) + if !errors.Is(err, context.Canceled) || sb.PendingVolume == nil { + t.Fatalf("canceled attach: %v, pending=%v", err, sb.PendingVolume) + } + eng.liveVolumeHook = nil + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatalf("release pending claim: %v", err) + } +} + +func TestLiveVolumeAdmissionAndState(t *testing.T) { + for _, state := range []string{"paused", "failed", "archived", "archive policy", "dirty", "writer"} { + t.Run(state, func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + var want error + switch state { + case "paused": + sb.HibernateSnap, want = "sleep", ErrPaused + case "failed": + sb.Failed, want = "down", ErrFailed + case "archived": + sb.ArchiveCk, want = "ck_old", ErrArchived + case "archive policy": + sb.OnExpire, want = types.ExpireArchive, ErrVolumeCapture + case "dirty": + if err := markVolumeDirty(catalog[0].Path); err != nil { + t.Fatal(err) + } + want = ErrVolumeNeedsRecovery + case "writer": + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); err != nil { + t.Fatal(err) + } + want = ErrVolumeBusy + } + before := len(eng.volumeSpecs) + _, err := m.AttachVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) + if !errors.Is(err, want) || len(eng.volumeSpecs) != before || sb.PendingVolume != nil { + t.Fatalf("got %v want %v", err, want) + } + }) + } +} + +func TestLiveVolumesRespectOwningTenantAndRemoval(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + catalog[1].Tenants = []string{"beta"} + m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: catalog}, eng, testSecrets(t)) + if err != nil { + t.Fatal(err) + } + putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme"}) + if err != nil { + t.Fatal(err) + } + cred := Cred{Token: sb.Token} + for _, auth := range []Cred{cred, {Operator: true}} { + if _, err := m.AttachVolumes(t.Context(), sb.ID, auth, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("catalog ACL: %v", err) + } + } + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("removed tenant: %v", err) + } + if _, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}); err != nil { + t.Fatalf("cleanup after tenant removal: %v", err) + } +} + +func liveCatalog(t *testing.T) []config.VolumeSpec { + t.Helper() + return []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "data.img", "data"), Writable: true}, + {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, + } +} + +func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil { + t.Fatal(err) + } + got := claims[sb.ID] + if got == nil || !slices.Equal(got.Volumes, sb.Volumes) { + t.Fatalf("journal volumes: %+v", got) + } + if (got.PendingVolume == nil) != (sb.PendingVolume == nil) || got.PendingVolume != nil && *got.PendingVolume != *sb.PendingVolume { + t.Fatalf("journal intent: %+v, want %+v", got.PendingVolume, sb.PendingVolume) + } + if _, err := os.Stat(m.store.path); err != nil { + t.Fatal(err) + } +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 6b923dde..b3977daf 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -60,6 +60,7 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, + {pool.ErrVolumePending, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, @@ -94,6 +95,8 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo + AttachVolumes(ctx context.Context, id string, cred pool.Cred, volumes []types.Volume) ([]types.Volume, error) + DetachVolumes(ctx context.Context, id string, cred pool.Cred, names []string) ([]types.Volume, error) Sandboxes(tenant, claimRef string, metadata types.Metadata) []pool.SandboxSummary Sandbox(id string) (pool.SandboxSummary, bool) Stats(ctx context.Context, id string) (pool.SandboxStats, bool) @@ -262,6 +265,8 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) + mux.HandleFunc("POST /v1/sandboxes/{id}/volumes/attach", s.handleAttachVolumes) + mux.HandleFunc("POST /v1/sandboxes/{id}/volumes/detach", s.handleDetachVolumes) mux.HandleFunc("POST /v1/sandboxes/{id}/renew", s.handleRenew) mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index e5320de1..5de295e2 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2602,6 +2602,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { + attachVolumes func(string, pool.Cred, []types.Volume) ([]types.Volume, error) + detachVolumes func(string, pool.Cred, []string) ([]types.Volume, error) tenants []config.TenantSpec setTenants func(specs []config.TenantSpec) error putTenant func(spec config.TenantSpec) error @@ -2674,6 +2676,14 @@ type fakeManager struct { draining bool } +func (f *fakeManager) AttachVolumes(_ context.Context, id string, cred pool.Cred, volumes []types.Volume) ([]types.Volume, error) { + return f.attachVolumes(id, cred, volumes) +} + +func (f *fakeManager) DetachVolumes(_ context.Context, id string, cred pool.Cred, names []string) ([]types.Volume, error) { + return f.detachVolumes(id, cred, names) +} + func (f *fakeManager) ClaimWarm(ctx context.Context, key types.PoolKey, o pool.ClaimOptions) (*types.Sandbox, error) { f.warmCalls++ f.gotTenant, f.gotEgressClass = o.Tenant, o.EgressClass diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go new file mode 100644 index 00000000..76d1b600 --- /dev/null +++ b/sandboxd/server/volume_mutation.go @@ -0,0 +1,39 @@ +package server + +import ( + "net/http" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (s *Server) handleAttachVolumes(w http.ResponseWriter, r *http.Request) { + token, ok := sandboxToken(w, r) + if !ok { + return + } + req, ok := decodeBodyStrict[types.AttachVolumesRequest](w, r) + if !ok { + return + } + id := r.PathValue("id") + volumes, err := s.mgr.AttachVolumes(r.Context(), id, s.sandboxCred(token), req.Volumes) + writeResult(w, r, "attach volumes", id, "attach volumes failed; retry the request or detach the pending volume", err, func() { + writeJSON(w, http.StatusOK, types.SandboxVolumesResponse{Volumes: volumes}) + }) +} + +func (s *Server) handleDetachVolumes(w http.ResponseWriter, r *http.Request) { + token, ok := sandboxToken(w, r) + if !ok { + return + } + req, ok := decodeBodyStrict[types.DetachVolumesRequest](w, r) + if !ok { + return + } + id := r.PathValue("id") + volumes, err := s.mgr.DetachVolumes(r.Context(), id, s.sandboxCred(token), req.Names) + writeResult(w, r, "detach volumes", id, "detach volumes failed; retry the request", err, func() { + writeJSON(w, http.StatusOK, types.SandboxVolumesResponse{Volumes: volumes}) + }) +} diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go new file mode 100644 index 00000000..ea4ac6c0 --- /dev/null +++ b/sandboxd/server/volume_mutation_test.go @@ -0,0 +1,95 @@ +package server + +import ( + "errors" + "io" + "net/http" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveVolumeRoutesAuthorizeAndDecode(t *testing.T) { + for _, verb := range []string{"attach", "detach"} { + for _, token := range []string{"sandbox-token", "root-token"} { + t.Run(verb+"/"+token, func(t *testing.T) { + called := false + check := func(id string, cred pool.Cred) { + called = true + if id != "sb_1" || cred.Operator != (token == "root-token") || !cred.Operator && cred.Token != token { + t.Errorf("wrong credentials: %s %+v", id, cred) + } + } + mgr := &fakeManager{ + attachVolumes: func(id string, cred pool.Cred, v []types.Volume) ([]types.Volume, error) { + check(id, cred) + if !slices.Equal(v, []types.Volume{{Name: "data"}}) { + t.Errorf("volumes: %v", v) + } + return []types.Volume{{Name: "data", Mount: "/volumes/data"}}, nil + }, + detachVolumes: func(id string, cred pool.Cred, names []string) ([]types.Volume, error) { + check(id, cred) + if !slices.Equal(names, []string{"data"}) { + t.Errorf("names: %v", names) + } + return nil, nil + }, + } + ts := newTestServer(t, "root-token", mgr, nil) + body := `{"volumes":[{"name":"data"}]}` + if verb == "detach" { + body = `{"names":["data"]}` + } + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/volumes/"+verb, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK || !called { + t.Fatalf("status=%d called=%v", resp.StatusCode, called) + } + }) + } + } +} + +func TestLiveVolumeRoutesErrors(t *testing.T) { + for _, tt := range []struct { + name, token, body string + err error + code int + }{ + {"missing token", "", `{}`, nil, http.StatusUnauthorized}, + {"unknown field", "tok", `{"volumes":[],"path":"/host"}`, nil, http.StatusBadRequest}, + {"wrong sandbox", "tok", `{}`, pool.ErrUnknownSandbox, http.StatusNotFound}, + {"bad volume", "tok", `{}`, pool.ErrBadVolume, http.StatusBadRequest}, + {"pending", "tok", `{}`, pool.ErrVolumePending, http.StatusConflict}, + {"paused", "tok", `{}`, pool.ErrPaused, http.StatusConflict}, + {"failed", "tok", `{}`, pool.ErrFailed, http.StatusConflict}, + {"engine", "tok", `{}`, errors.New("host /private/image failed"), http.StatusInternalServerError}, + } { + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{attachVolumes: func(string, pool.Cred, []types.Volume) ([]types.Volume, error) { return nil, tt.err }} + ts := newTestServer(t, "root", mgr, nil) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/volumes/attach", strings.NewReader(tt.body)) + if tt.token != "" { + req.Header.Set("Authorization", "Bearer "+tt.token) + } + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { + t.Fatalf("status=%d body=%s", resp.StatusCode, body) + } + }) + } +} diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index f397db44..50068ed0 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -77,6 +77,21 @@ type VolumeListResponse struct { Volumes []VolumeInfo `json:"volumes"` } +// AttachVolumesRequest adds read-only mounts to a running sandbox. +type AttachVolumesRequest struct { + Volumes []Volume `json:"volumes"` +} + +// DetachVolumesRequest removes read-only volumes by catalog name. +type DetachVolumesRequest struct { + Names []string `json:"names"` +} + +// SandboxVolumesResponse reports the full committed set after a mutation. +type SandboxVolumesResponse struct { + Volumes []Volume `json:"volumes"` +} + // ForkRequest is the wire body of POST /v1/sandboxes/{id}/fork. type ForkRequest struct { Token string `json:"token"` diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 699db9cc..e99dc61a 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -559,6 +559,9 @@ type Sandbox struct { OnExpire ExpireAction `json:"on_expire,omitempty"` // Volumes records the volumes successfully applied to this claim. Volumes []Volume `json:"volumes,omitempty"` + // PendingVolume is written before a live device change and cleared only + // after its result is committed. Retry it, detach its name, or release. + PendingVolume *VolumeMutation `json:"pending_volume,omitempty"` // Env is replaced whole through SetEnv under the manager mutex; host-only values are never served back. Env Env `json:"env,omitempty"` @@ -597,10 +600,20 @@ type Sandbox struct { // live is the env as of the last SetEnv, read by the egress proxy without the manager mutex. live atomic.Pointer[envView] - // Transition serializes hibernate/wake; lock it before (never under) the manager mutex. + // VolumeMu keeps release/reap and expiry-policy changes out of a live volume + // operation. Lock it before Transition, and never under the manager mutex. + VolumeMu sync.Mutex `json:"-"` + // Transition serializes captures and live volume changes; lock it before (never under) the manager mutex. Transition sync.Mutex `json:"-"` } +// VolumeMutation is a recoverable, node-local live volume operation. +// Values are immutable once published; they carry no operator filesystem paths. +type VolumeMutation struct { + Volume Volume `json:"volume"` + Detach bool `json:"detach,omitzero"` +} + // Touch stamps last data-plane activity; lock-free, called on the relay hot path. func (s *Sandbox) Touch() { s.lastActivity.Store(time.Now().UnixNano()) } From 88c2d69e06bc3ae22055374a633d3ca1f20c9af6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 9 Oct 2026 09:48:50 +0800 Subject: [PATCH 03/57] feat(sdk): expose live volume mounts and document recovery --- docs/deploy.md | 9 +++++ docs/sandboxd-api.md | 73 +++++++++++++++++++++++++++++++++- docs/sdk.md | 19 +++++++++ e2e/cmd/volumesmoke/main.go | 42 +++++++++++++++++++ e2e/volume_mutation_test.go | 38 ++++++++++++++++++ sdk/go/info.go | 1 + sdk/go/volume_mutation_test.go | 66 ++++++++++++++++++++++++++++++ sdk/go/volumes.go | 49 +++++++++++++++++++++++ 8 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 e2e/volume_mutation_test.go create mode 100644 sdk/go/volume_mutation_test.go create mode 100644 sdk/go/volumes.go diff --git a/docs/deploy.md b/docs/deploy.md index 00b1155d..18e97382 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -156,6 +156,15 @@ fragment the warm pools): ### Dataset volumes +Running claims can add and remove read-only catalog volumes through the +[live volume API](sandboxd-api.md#post-v1sandboxesidvolumesattach) and Go SDK. +The image must already exist on the owner node. Interrupted operations keep +a `pending_volume` journal entry and admission hold until retried, detached +or released; drain these entries before downgrading sandboxd. The existing +`e2e/cmd/volumesmoke` command also exercises live attach/detach and retries +on a new claim without initial volumes. Run it on the target VMM/guest; +unit tests do not validate guest hotplug support. + Each catalog path must name a disk image containing a mountable whole-device filesystem. A read-only entry's image must stay immutable: do not replace, truncate, or delete it while attached; publish a new catalog name or path diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 15918865..a093f4a1 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -255,6 +255,77 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. +## POST /v1/sandboxes/{id}/volumes/attach + +Adds read-only catalog volumes to an already running sandbox without creating +a new claim. Auth: the sandbox's own bearer token, or the root token by id. +The owning tenant's catalog access list still applies, including for root; +an attach is refused if that tenant has been removed. Images must be present +on the owner node; this operation does not redirect or move the sandbox. + +```json +{"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} +``` + +The claim-time name, mount and eight-volume limits apply to the resulting +set. An omitted mount defaults to `/volumes/`. Only mounted read-only +entries are accepted (`mode` omitted or `ro`); existing writable and +attach-only entries are preserved. An identical attached name/mount/mode is +a no-op; changing an existing name's mount or mode is a 400 error. +Claims with `on_expire: archive` must change that policy before attaching. + +200 returns the **full committed set**: + +```json +{"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} +``` + +The sandbox must be running. Paused, archived and failed sandboxes return +409 without waking. Missing bearer returns 401; unknown id or wrong token +returns 404; an invalid, inaccessible or non-local volume returns 400; +a removed tenant returns 403; a writer holding the image or a dirty image +needing recovery returns 409. + +## POST /v1/sandboxes/{id}/volumes/detach + +Auth and successful response match attach. Unmounts the named read-only +volumes and waits for cocoon to confirm hot-unplug; backing files are kept. + +```json +{"names": ["imagenet"]} +``` + +An absent valid name is a no-op. Duplicate or invalid names, and requests +to detach a writable volume, return 400 before changing any devices. +An attach-only read-only disk can be detached after its workload has +unmounted it. A busy or foreign mount is refused; no lazy or forced unmount +is used. Detach remains available after the owning tenant is removed. + +### Partial failure and recovery + +Both operations validate their batch first, then commit one volume at a +time. They are **not batch transactions**: an error may leave earlier +entries completed and the current entry pending. Retry the same request +to converge, or detach the pending name to cancel an interrupted attach. +An unrelated mutation while an operation is pending returns 409. +Release is also available and waits for an in-flight volume call to finish. + +Before each device change, sandboxd writes `pending_volume` to the claim +journal. The sandbox index exposes it as +`{"volume":{"name":"imagenet","mount":"/datasets/training"},"detach":true}` +(`detach` omitted means attach). Pending devices retain their admission +holds and block hibernate, fork, checkpoint and promote. Startup adopts +these holds; the caller retries or detaches to finish recovery. There is no +automatic attach replay during startup. `volumes` contains committed +entries; during a pending operation the actual device state may differ. + +Each engine operation has a 30-second ceiling and honors cancellation. +An engine or journal failure returns 500 with a recovery hint and logs the +underlying error server-side. Journal writes are atomic; host-crash +durability follows the existing `sync_claims` setting. Complete or cancel +pending operations before downgrading to a version without this journal +field. Once every volume is detached, normal capture operations are allowed. + ## POST /v1/sandboxes/{id}/release Auth: the sandbox's own token, or the root token for operator cleanup by id. @@ -669,7 +740,7 @@ an SDK caller should set. Auth: node API token. Root sees every live claim; a tenant sees only its own. The index is `{"sandboxes": [{id, key, tenant?, deadline, claimed_at?, hibernated, archived?, from_checkpoint?, claim_ref?, metadata?, on_expire?, cpu_count, mem_total_bytes, -volumes?: [{name, mount, mode?}], restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the +volumes?: [{name, mount, mode?}], pending_volume?: {volume, detach?}, restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the size tier's allocation; `restarts` counts the cold boots that replaced an exited VMM and `restarted_at` stamps the last one (each loses guest memory and the instance-metadata document, so a caller re-establishes guest state when diff --git a/docs/sdk.md b/docs/sdk.md index 6f4fee36..d90a907d 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -234,6 +234,25 @@ checkpoint branches support neither volumes nor a claim reference in this versio `Checkpoint.New`: the labels follow the [claim bounds](sandboxd-api.md#post-v1claim), and a branch never inherits its source's labels or expiry action. +For datasets chosen after the sandbox starts, add or remove read-only mounts +on the same claim: + +```go +volumes, err := sb.AttachVolumes(ctx, + sandbox.Volume{Name: "imagenet", Mount: "/datasets/training"}) +// Use the dataset in the running guest, then detach it. +volumes, err = sb.DetachVolumes(ctx, "imagenet") +``` + +Both methods use the sandbox token on its owner node, return the full set, +and update `sb.Volumes` on success. Serialize updates to the handle's public +fields, as with `Renew`. On error `sb.Volumes` remains the last successful +response and may be stale: a batch can partially complete. Retry the same +request, or detach the pending name to recover. `Client.Sandboxes` exposes +`PendingVolume` for inspection with an API token. A paused sandbox is not +woken, writable mutations are refused, and images must exist in the owner +node's catalog. See the [API recovery contract](sandboxd-api.md#partial-failure-and-recovery). + `WithVolumesAttachOnly()` claims the same volumes without mounting them: the entries in `Sandbox.Volumes` carry an empty `Mount`, and the workload finds each device by polling `/sys/block/*/serial` for the catalog name — not diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index 0696b4d7..70b80cba 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -118,12 +118,54 @@ func run(addr, token, template, volume, rwVolume, probe string) error { } released = true fmt.Printf("VOLUME PASS concurrent_read_bytes=%d read_only=true released=true\n", len(outputs[0])) + if err := runLive(ctx, client, template, volume, probe, outputs[0]); err != nil { + return err + } if rwVolume == "" { return nil } return runWritable(ctx, client, template, rwVolume) } +// runLive starts without volumes, adds a dataset, then removes it without +// replacing the claim. Run this on the target VMM before deploying live mounts. +func runLive(ctx context.Context, client *sandbox.Client, template, volume, probe, want string) error { + const mount = "/datasets/live" + sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone)) + if err != nil { + return err + } + defer func() { _ = sb.Close() }() + for range 2 { + if _, err = sb.AttachVolumes(ctx, sandbox.Volume{Name: volume, Mount: mount}); err != nil { + return fmt.Errorf("live attach: %w", err) + } + } + got, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) + if err != nil { + return fmt.Errorf("live volume read: %w", err) + } + if got != want { + return fmt.Errorf("live volume read: bytes=%d want=%d", len(got), len(want)) + } + if _, err = sb.Exec(ctx, "touch", path.Join(mount, ".sandboxd-write-probe")); err == nil { + return errors.New("live read-only mount accepted a write") + } + if exitErr, ok := errors.AsType[*sandbox.ExitError](err); !ok || !strings.Contains(strings.ToLower(exitErr.Stderr), "read-only file system") { + return fmt.Errorf("live write probe failed without EROFS: %w", err) + } + for range 2 { + if _, err = sb.DetachVolumes(ctx, volume); err != nil { + return fmt.Errorf("live detach: %w", err) + } + } + if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { + return fmt.Errorf("detached volume probe should be absent: %w", err) + } + fmt.Printf("LIVE VOLUME PASS read_bytes=%d repeated_attach=true repeated_detach=true\n", len(got)) + return nil +} + // runWritable proves the publish workflow: a writer's bytes survive its own // release, it excludes concurrent claims, and the next reader sees them. func runWritable(ctx context.Context, client *sandbox.Client, template, volume string) error { diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go new file mode 100644 index 00000000..8a335ade --- /dev/null +++ b/e2e/volume_mutation_test.go @@ -0,0 +1,38 @@ +package e2e + +import ( + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + sandbox "github.com/cocoonstack/sandbox/sdk/go" +) + +func TestLiveVolumesEndToEnd(t *testing.T) { + image := writeVolumeImage(t, "dataset.img", "dataset-bytes") + stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "dataset", Path: image}}) + sb, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer sb.Close() + for range 2 { + got, err := sb.AttachVolumes(t.Context(), sandbox.Volume{Name: "dataset"}) + if err != nil || !slices.Equal(got, []sandbox.Volume{{Name: "dataset", Mount: "/volumes/dataset"}}) { + t.Fatalf("attach: %v %v", got, err) + } + } + if err := sb.Hibernate(t.Context()); err == nil { + t.Fatal("captured an attached volume") + } + for range 2 { + got, err := sb.DetachVolumes(t.Context(), "dataset") + if err != nil || len(got) != 0 { + t.Fatalf("detach: %v %v", got, err) + } + } + want := []string{"attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "umount:/volumes/dataset", "detach:dataset"} + if ops := stack.eng.volumeOpsLog(); !slices.Equal(ops, want) { + t.Fatalf("device operations=%v want=%v", ops, want) + } +} diff --git a/sdk/go/info.go b/sdk/go/info.go index b3a657ea..79ea84c1 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -62,6 +62,7 @@ type SandboxSummary struct { Archived bool `json:"archived,omitzero"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []Volume `json:"volumes,omitempty"` + PendingVolume *VolumeMutation `json:"pending_volume,omitempty"` ClaimRef string `json:"claim_ref,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` CPUCount int `json:"cpu_count,omitzero"` diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go new file mode 100644 index 00000000..96d705a6 --- /dev/null +++ b/sdk/go/volume_mutation_test.go @@ -0,0 +1,66 @@ +package sandbox + +import ( + "io" + "net/http" + "net/http/httptest" + "slices" + "testing" +) + +func TestLiveVolumesUseOwnerAndSandboxToken(t *testing.T) { + for _, verb := range []string{"attach", "detach"} { + t.Run(verb, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/sandboxes/sb_1/volumes/"+verb || r.Header.Get("Authorization") != "Bearer sandbox-token" { + t.Errorf("request: %s %s %s", r.Method, r.URL.Path, r.Header.Get("Authorization")) + } + body, _ := io.ReadAll(r.Body) + want := `{"volumes":[{"name":"data"}]}` + if verb == "detach" { + want = `{"names":["data"]}` + } + if string(body) != want { + t.Errorf("body=%s want=%s", body, want) + } + _, _ = io.WriteString(w, `{"volumes":[{"name":"remaining","mount":"/data"}]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts, WithAPIToken("operator-token")) + sb := &Sandbox{ID: "sb_1", token: "sandbox-token", owner: c.addr, c: c} + var got []Volume + var err error + if verb == "attach" { + got, err = sb.AttachVolumes(t.Context(), Volume{Name: "data"}) + } else { + got, err = sb.DetachVolumes(t.Context(), "data") + } + if err != nil || !slices.Equal(got, []Volume{{Name: "remaining", Mount: "/data"}}) { + t.Fatalf("got=%v err=%v", got, err) + } + got[0].Name = "caller copy" + if sb.Volumes[0].Name != "remaining" { + t.Fatal("returned volumes alias public state") + } + }) + } +} + +func TestLiveVolumeFailurePreservesLastKnownSet(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = io.WriteString(w, `{"error":"pending"}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c, Volumes: []Volume{{Name: "old"}}} + if _, err := sb.AttachVolumes(t.Context(), Volume{Name: "new"}); err == nil { + t.Fatal("missing attach error") + } + if _, err := sb.DetachVolumes(t.Context(), "old"); err == nil { + t.Fatal("missing detach error") + } + if !slices.Equal(sb.Volumes, []Volume{{Name: "old"}}) { + t.Fatal("failed request replaced last known set") + } +} diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go new file mode 100644 index 00000000..c99883e1 --- /dev/null +++ b/sdk/go/volumes.go @@ -0,0 +1,49 @@ +package sandbox + +import ( + "bytes" + "context" + "net/http" + "slices" +) + +// VolumeMutation describes an interrupted operation in a SandboxSummary. +// Retry the attach, detach this name, or release the sandbox to recover. +type VolumeMutation struct { + Volume Volume `json:"volume"` + Detach bool `json:"detach,omitzero"` +} + +// AttachVolumes adds read-only catalog mounts to this running sandbox. +// Retry the same request after a partial failure, or detach the pending name. +// On success it replaces Volumes and returns a detached copy of the full set. +// Like Renew, callers must serialize updates to this handle's public fields. +func (s *Sandbox) AttachVolumes(ctx context.Context, volumes ...Volume) ([]Volume, error) { + return s.mutateVolumes(ctx, "attach", struct { + Volumes []Volume `json:"volumes"` + }{Volumes: volumes}) +} + +// DetachVolumes removes read-only volumes, including a pending attach. +// Already absent names are harmless. Batches may partially complete on error; +// retry the request to finish it. It replaces Volumes only on success. +func (s *Sandbox) DetachVolumes(ctx context.Context, names ...string) ([]Volume, error) { + return s.mutateVolumes(ctx, "detach", struct { + Names []string `json:"names"` + }{Names: names}) +} + +func (s *Sandbox) mutateVolumes(ctx context.Context, verb string, request any) ([]Volume, error) { + body, err := encodeBody(verb+" volumes", request) + if err != nil { + return nil, err + } + response, err := s.c.doJSON[struct { + Volumes []Volume `json:"volumes"` + }](ctx, http.MethodPost, s.owner, "/v1/sandboxes/"+s.ID+"/volumes/"+verb, bytes.NewReader(body), s.token, verb+" volumes") + if err != nil { + return nil, err + } + s.Volumes = slices.Clone(response.Volumes) + return slices.Clone(s.Volumes), nil +} From 645e49756492e7d8e9337377a958185e615804fc Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 03:29:58 +0000 Subject: [PATCH 04/57] refactor: reconcile live volumes from desired state --- docs/deploy.md | 26 +- docs/sandboxd-api.md | 124 ++--- docs/sdk.md | 51 ++- e2e/cmd/volumesmoke/main.go | 41 +- e2e/fakeengine_test.go | 7 - e2e/volume_mutation_test.go | 10 +- sandboxd/engine/volume.go | 15 +- sandboxd/engine/volume_live.go | 98 +--- sandboxd/engine/volume_live_test.go | 36 +- sandboxd/engine/volume_test.go | 13 +- sandboxd/pool/claim.go | 48 +- sandboxd/pool/claims.go | 69 +-- sandboxd/pool/egress_test.go | 6 + sandboxd/pool/hibernate.go | 7 +- sandboxd/pool/liveness.go | 13 +- sandboxd/pool/liveness_test.go | 2 +- sandboxd/pool/pool.go | 39 +- sandboxd/pool/pool_test.go | 25 +- sandboxd/pool/poolstore_test.go | 6 + sandboxd/pool/refill.go | 3 +- sandboxd/pool/volume.go | 4 + sandboxd/pool/volume_mutation.go | 541 ++++++++++++++-------- sandboxd/pool/volume_mutation_test.go | 584 +++++++++++++++++++++--- sandboxd/server/server.go | 11 +- sandboxd/server/server_test.go | 17 +- sandboxd/server/volume_mutation.go | 69 ++- sandboxd/server/volume_mutation_test.go | 67 ++- sandboxd/types/api.go | 23 +- sandboxd/types/types.go | 15 +- sdk/go/client.go | 3 +- sdk/go/info.go | 2 +- sdk/go/sandbox.go | 5 +- sdk/go/volume_mutation_test.go | 161 +++++-- sdk/go/volumes.go | 121 +++-- sdk/python/cocoonsandbox/__init__.py | 5 + sdk/python/cocoonsandbox/client.py | 34 +- sdk/python/cocoonsandbox/errors.py | 5 +- sdk/python/cocoonsandbox/sandbox.py | 63 ++- sdk/python/cocoonsandbox/volumes.py | 71 +++ sdk/python/tests/conftest.py | 6 + sdk/python/tests/test_volumes.py | 128 ++++++ 41 files changed, 1847 insertions(+), 727 deletions(-) create mode 100644 sdk/python/cocoonsandbox/volumes.py create mode 100644 sdk/python/tests/test_volumes.py diff --git a/docs/deploy.md b/docs/deploy.md index 18e97382..38d5eb34 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -156,14 +156,24 @@ fragment the warm pools): ### Dataset volumes -Running claims can add and remove read-only catalog volumes through the -[live volume API](sandboxd-api.md#post-v1sandboxesidvolumesattach) and Go SDK. -The image must already exist on the owner node. Interrupted operations keep -a `pending_volume` journal entry and admission hold until retried, detached -or released; drain these entries before downgrading sandboxd. The existing -`e2e/cmd/volumesmoke` command also exercises live attach/detach and retries -on a new claim without initial volumes. Run it on the target VMM/guest; -unit tests do not validate guest hotplug support. +Running claims can change mounted read-only catalog entries through the +[GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) +and both SDKs. The image must already exist on the owner node. Live changes +require cocoon commit `6d76309` or a build containing it for idempotent disk +attach and detach. That change was on cocoon master without a release tag +when this feature was prepared. The general v0.6.9 floor above is insufficient +for live convergence. + +The claim journal stores desired `VolumesWant` and confirmed applied `Volumes`. +Admission holds cover their union across restart and teardown. Retry an +interrupted operation with the same PUT target. Release cancels convergence +and transfers holds to teardown. Capture is refused while either set is +nonempty. Drain all volume claims before downgrading sandboxd. + +`e2e/cmd/volumesmoke` exercises GET, repeated full-set PUT, PATCH add and remove, +and clearing a new claim. Real Cloud Hypervisor hotplug, busy guest unmount, +and restart during convergence remain unverified here. The maintainer will +run these checks on the target VMM and guest testbed. Each catalog path must name a disk image containing a mountable whole-device filesystem. A read-only entry's image must stay immutable: do not replace, diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index a093f4a1..eafcdd46 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -255,76 +255,98 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. -## POST /v1/sandboxes/{id}/volumes/attach +## GET /v1/sandboxes/{id}/volumes -Adds read-only catalog volumes to an already running sandbox without creating -a new claim. Auth: the sandbox's own bearer token, or the root token by id. -The owning tenant's catalog access list still applies, including for root; -an attach is refused if that tenant has been removed. Images must be present -on the owner node; this operation does not redirect or move the sandbox. +Returns the persisted desired set (`volumes_want`), confirmed applied set +(`volumes`), and per-volume `states`. Auth is the sandbox bearer token or the +root token by id. The owning node handles the request without redirecting. ```json -{"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} +{ + "volumes_want": [{"name": "imagenet", "mount": "/datasets/training"}], + "volumes": [{"name": "imagenet", "mount": "/datasets/training"}], + "states": [{"name": "imagenet", "state": "attached"}] +} ``` -The claim-time name, mount and eight-volume limits apply to the resulting -set. An omitted mount defaults to `/volumes/`. Only mounted read-only -entries are accepted (`mode` omitted or `ro`); existing writable and -attach-only entries are preserved. An identical attached name/mount/mode is -a no-op; changing an existing name's mount or mode is a 400 error. -Claims with `on_expire: archive` must change that policy before attaching. +All three arrays are present, including when empty. A state is `attached` +when the volume is in both sets, `pending` when desired but not yet confirmed, +or `detaching` when applied but no longer desired. An interrupted device call +can leave the actual guest state different from the confirmed set. A failed +convergence also includes a top-level `error` and may include an `error` on +the affected state. The sandbox index exposes both sets for operator inspection. + +## PUT /v1/sandboxes/{id}/volumes -200 returns the **full committed set**: +Replaces the full desired set and converges synchronously. Auth matches GET. +The owner tenant's catalog access list applies, including for root. The image +must exist on the owning node. The request must include a non-null `volumes` +array. An explicit empty array removes all mutable entries. ```json {"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} ``` -The sandbox must be running. Paused, archived and failed sandboxes return -409 without waking. Missing bearer returns 401; unknown id or wrong token -returns 404; an invalid, inaccessible or non-local volume returns 400; -a removed tenant returns 403; a writer holding the image or a dirty image -needing recovery returns 409. +Only mounted read-only catalog entries can change live (`mode` omitted or +`ro`). A PUT must retain every existing writable or attach-only entry unchanged. +The claim-time name, mount, and eight-volume limits apply to the desired set. +An omitted mount defaults to `/volumes/`. To change an existing name's +mount or mode, first remove that name and wait for successful convergence. +An identical target can be retried. Claims with `on_expire: archive` must change +that policy before adding a volume. + +The sandbox must be running. Paused, archived, and failed sandboxes return 409 +without waking. Missing bearer returns 401. Unknown id or wrong token returns +404. Invalid, inaccessible, or non-local entries return 400. Removed tenants +cannot add volumes and return 403. A conflicting writer or dirty image returns +409. Removal remains available after the tenant is removed. -## POST /v1/sandboxes/{id}/volumes/detach +## PATCH /v1/sandboxes/{id}/volumes -Auth and successful response match attach. Unmounts the named read-only -volumes and waits for cocoon to confirm hot-unplug; backing files are kept. +Changes the desired set by name, then converges synchronously. Auth, validation, +and responses match PUT. Omitted `add` or `remove` arrays leave that part +unchanged. An empty patch retries the current desired set. ```json -{"names": ["imagenet"]} +{"add": [{"name": "imagenet", "mount": "/datasets/training"}], "remove": ["old-data"]} ``` -An absent valid name is a no-op. Duplicate or invalid names, and requests -to detach a writable volume, return 400 before changing any devices. -An attach-only read-only disk can be detached after its workload has -unmounted it. A busy or foreign mount is refused; no lazy or forced unmount -is used. Detach remains available after the owning tenant is removed. +Adding an identical entry or removing an absent valid name is harmless. +Duplicate names, a name in both arrays, or changes to writable or attach-only +entries return 400. Removal refuses busy or foreign mounts and uses neither +lazy nor forced unmount. ### Partial failure and recovery -Both operations validate their batch first, then commit one volume at a -time. They are **not batch transactions**: an error may leave earlier -entries completed and the current entry pending. Retry the same request -to converge, or detach the pending name to cancel an interrupted attach. -An unrelated mutation while an operation is pending returns 409. -Release is also available and waits for an in-flight volume call to finish. - -Before each device change, sandboxd writes `pending_volume` to the claim -journal. The sandbox index exposes it as -`{"volume":{"name":"imagenet","mount":"/datasets/training"},"detach":true}` -(`detach` omitted means attach). Pending devices retain their admission -holds and block hibernate, fork, checkpoint and promote. Startup adopts -these holds; the caller retries or detaches to finish recovery. There is no -automatic attach replay during startup. `volumes` contains committed -entries; during a pending operation the actual device state may differ. - -Each engine operation has a 30-second ceiling and honors cancellation. -An engine or journal failure returns 500 with a recovery hint and logs the -underlying error server-side. Journal writes are atomic; host-crash -durability follows the existing `sync_claims` setting. Complete or cancel -pending operations before downgrading to a version without this journal -field. Once every volume is detached, normal capture operations are allowed. +PUT and PATCH return the GET response shape on success. A convergence error +returns a non-2xx status with the same view, so callers can inspect desired, +applied, and per-volume state. Validation and authentication errors may carry +only `error`. A failed request may already have persisted its target and +completed some device changes. Retry the same PUT to converge, or send a new +target to abandon unfinished additions. Concurrent requests serialize per +sandbox without blocking renewals or new exec, file, and port connections. + +Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`. A +superseding request may first clean up an older desired-only attachment while +the old target stays recorded. If that cleanup fails, the response retains the +old view; retry the same PUT to finish accepting the new target. Admission +holds cover the deduplicated union of both sets, including during restart and +teardown. Failed or uncertain operations retain their holds until successful +cleanup. Startup reserves those holds without replaying attachments. Release +cancels convergence and transfers the held union to teardown without waiting +for a blocked device call. Capture operations (hibernate, fork, checkpoint, and +promote) are refused while either set is nonempty or a convergence request +still owns admitted volume holds. + +Device calls honor cancellation and have a 30-second ceiling. Journal writes +are atomic. Host-crash durability follows `sync_claims`. Drain volume claims +before downgrading to a daemon that does not preserve both journal sets. + +Live convergence requires cocoon commit `6d76309` or a build containing it, +which makes matching attach and absent detach idempotent. The change was not +in a tagged release when this API was prepared. Testbed validation of real +Cloud Hypervisor hotplug, busy unmount, and restart during convergence remains +pending with the maintainer. ## POST /v1/sandboxes/{id}/release @@ -740,7 +762,7 @@ an SDK caller should set. Auth: node API token. Root sees every live claim; a tenant sees only its own. The index is `{"sandboxes": [{id, key, tenant?, deadline, claimed_at?, hibernated, archived?, from_checkpoint?, claim_ref?, metadata?, on_expire?, cpu_count, mem_total_bytes, -volumes?: [{name, mount, mode?}], pending_volume?: {volume, detach?}, restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the +volumes?: [{name, mount, mode?}], volumes_want?: [{name, mount, mode?}], restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the size tier's allocation; `restarts` counts the cold boots that replaced an exited VMM and `restarted_at` stamps the last one (each loses guest memory and the instance-metadata document, so a caller re-establishes guest state when diff --git a/docs/sdk.md b/docs/sdk.md index d90a907d..7ed9107f 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -234,24 +234,49 @@ checkpoint branches support neither volumes nor a claim reference in this versio `Checkpoint.New`: the labels follow the [claim bounds](sandboxd-api.md#post-v1claim), and a branch never inherits its source's labels or expiry action. -For datasets chosen after the sandbox starts, add or remove read-only mounts +For datasets chosen after the sandbox starts, set the desired read-only mounts on the same claim: ```go -volumes, err := sb.AttachVolumes(ctx, - sandbox.Volume{Name: "imagenet", Mount: "/datasets/training"}) -// Use the dataset in the running guest, then detach it. -volumes, err = sb.DetachVolumes(ctx, "imagenet") +view, err := sb.SetVolumes(ctx, []sandbox.Volume{ + {Name: "imagenet", Mount: "/datasets/training"}, +}) +view, err = sb.GetVolumes(ctx) +view, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{"imagenet"}}) +// An explicit empty slice clears mutable mounts. A nil slice is rejected. +view, err = sb.SetVolumes(ctx, []sandbox.Volume{}) +``` + +Python exposes the same operations and response fields: + +```python +view = sb.set_volumes([{"name": "imagenet", "mount": "/datasets/training"}]) +view = sb.get_volumes() +view = sb.patch_volumes(remove=["imagenet"]) +view = sb.set_volumes([]) # None is rejected. ``` -Both methods use the sandbox token on its owner node, return the full set, -and update `sb.Volumes` on success. Serialize updates to the handle's public -fields, as with `Renew`. On error `sb.Volumes` remains the last successful -response and may be stale: a batch can partially complete. Retry the same -request, or detach the pending name to recover. `Client.Sandboxes` exposes -`PendingVolume` for inspection with an API token. A paused sandbox is not -woken, writable mutations are refused, and images must exist in the owner -node's catalog. See the [API recovery contract](sandboxd-api.md#partial-failure-and-recovery). +Each response contains `volumes_want`, `volumes`, and `states`. Go exposes them +as `SandboxVolumesResponse.VolumesWant`, `.Volumes`, and `.States`. A state is +`attached`, `pending`, or `detaching`, with an optional `error`. Methods use the +sandbox token on its owner node and update both public sets from each complete +response, including a partial failure. Serialize changes to one handle as with +`Renew`. Returned views do not share the handle's slices or dictionaries. + +Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. +It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. Python raises +`VolumeError`, a subclass of `APIError`, with `.response`. A response is absent +if the server returned only an authentication or validation error. In that case +the handle retains its last known sets. Retry an interrupted operation with the +same full PUT target, or set a new target to recover. `Client.Sandboxes` exposes +`VolumesWant` and `Volumes` for inspection with an API token. + +A PUT must retain existing writable and attach-only entries unchanged. PATCH +cannot remove or change them. Paused sandboxes are not woken, and images must +exist on the owner node. Admission holds cover the union of desired and applied +sets. Capture is refused while either is nonempty. Release cancels convergence +and hands held volumes to teardown. See the +[API recovery contract](sandboxd-api.md#partial-failure-and-recovery). `WithVolumesAttachOnly()` claims the same volumes without mounting them: the entries in `Sandbox.Volumes` carry an empty `Mount`, and the workload finds diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index 70b80cba..deccc233 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -137,10 +137,18 @@ func runLive(ctx context.Context, client *sandbox.Client, template, volume, prob } defer func() { _ = sb.Close() }() for range 2 { - if _, err = sb.AttachVolumes(ctx, sandbox.Volume{Name: volume, Mount: mount}); err != nil { - return fmt.Errorf("live attach: %w", err) + if _, err = sb.SetVolumes(ctx, []sandbox.Volume{{Name: volume, Mount: mount}}); err != nil { + return fmt.Errorf("live set volumes: %w", err) } } + view, err := sb.GetVolumes(ctx) + if err != nil { + return fmt.Errorf("live get volumes: %w", err) + } + wantVolumes := []sandbox.Volume{{Name: volume, Mount: mount}} + if !slices.Equal(view.VolumesWant, wantVolumes) || !slices.Equal(view.Volumes, wantVolumes) || len(view.States) != 1 || view.States[0].State != "attached" { + return fmt.Errorf("live volume view not converged: %+v", view) + } got, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) if err != nil { return fmt.Errorf("live volume read: %w", err) @@ -155,14 +163,37 @@ func runLive(ctx context.Context, client *sandbox.Client, template, volume, prob return fmt.Errorf("live write probe failed without EROFS: %w", err) } for range 2 { - if _, err = sb.DetachVolumes(ctx, volume); err != nil { - return fmt.Errorf("live detach: %w", err) + if _, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{volume}}); err != nil { + return fmt.Errorf("live patch remove: %w", err) } } if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { return fmt.Errorf("detached volume probe should be absent: %w", err) } - fmt.Printf("LIVE VOLUME PASS read_bytes=%d repeated_attach=true repeated_detach=true\n", len(got)) + for range 2 { + if _, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Add: wantVolumes}); err != nil { + return fmt.Errorf("live patch add: %w", err) + } + } + if got, err = sb.Exec(ctx, "cat", path.Join(mount, probe)); err != nil || got != want { + return fmt.Errorf("live patch add read: bytes=%d want=%d error=%v", len(got), len(want), err) + } + for range 2 { + if _, err = sb.SetVolumes(ctx, []sandbox.Volume{}); err != nil { + return fmt.Errorf("live set empty volumes: %w", err) + } + } + view, err = sb.GetVolumes(ctx) + if err != nil { + return fmt.Errorf("live get empty volumes: %w", err) + } + if len(view.VolumesWant) != 0 || len(view.Volumes) != 0 || len(view.States) != 0 { + return fmt.Errorf("live clear did not converge: %+v", view) + } + if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { + return fmt.Errorf("cleared volume probe should be absent: %w", err) + } + fmt.Printf("LIVE VOLUME PASS read_bytes=%d repeated_put=true repeated_patch=true cleared=true\n", len(got)) return nil } diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 563fe453..e2b708a7 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -177,13 +177,6 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } -func (f *fakeEngine) AttachVolume(ctx context.Context, vmName, sock string, spec engine.VolumeSpec, mount string) error { - if err := f.DiskAttach(ctx, vmName, spec); err != nil { - return err - } - return f.MountVolume(ctx, sock, spec.Name, mount, spec.RW) -} - func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { if err := f.UnmountVolume(ctx, sock, mount); err != nil { return err diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 8a335ade..07b60b8c 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -17,8 +17,8 @@ func TestLiveVolumesEndToEnd(t *testing.T) { } defer sb.Close() for range 2 { - got, err := sb.AttachVolumes(t.Context(), sandbox.Volume{Name: "dataset"}) - if err != nil || !slices.Equal(got, []sandbox.Volume{{Name: "dataset", Mount: "/volumes/dataset"}}) { + got, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}) + if err != nil || !slices.Equal(got.Volumes, []sandbox.Volume{{Name: "dataset", Mount: "/volumes/dataset"}}) { t.Fatalf("attach: %v %v", got, err) } } @@ -26,12 +26,12 @@ func TestLiveVolumesEndToEnd(t *testing.T) { t.Fatal("captured an attached volume") } for range 2 { - got, err := sb.DetachVolumes(t.Context(), "dataset") - if err != nil || len(got) != 0 { + got, err := sb.SetVolumes(t.Context(), []sandbox.Volume{}) + if err != nil || len(got.Volumes) != 0 { t.Fatalf("detach: %v %v", got, err) } } - want := []string{"attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "umount:/volumes/dataset", "detach:dataset"} + want := []string{"attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "umount:/volumes/dataset", "detach:dataset"} if ops := stack.eng.volumeOpsLog(); !slices.Equal(ops, want) { t.Fatalf("device operations=%v want=%v", ops, want) } diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index 62daa94f..ec08dc7b 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -55,16 +55,27 @@ func (e *Engine) MountVolume(ctx context.Context, vsockSocket, name, mount strin if err != nil { return fmt.Errorf("wait for volume device %s: %w", name, err) } - if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil { + mounted, err := e.volumeMounted(ctx, vsockSocket, device, mount, rw) + if err != nil || mounted { + return err + } + if err = e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil { return fmt.Errorf("create volume mount point %s: %w", mount, err) } mode := types.VolumeModeRO if rw { mode = types.VolumeModeRW } - if err := e.silkdExec(ctx, vsockSocket, "mount", "-o", mode, "--", device, mount); err != nil { + if err = e.silkdExec(ctx, vsockSocket, "mount", "-o", mode, "--", device, mount); err != nil { return fmt.Errorf("mount volume %s: %w", name, err) } + mounted, err = e.volumeMounted(ctx, vsockSocket, device, mount, rw) + if err != nil { + return err + } + if !mounted { + return fmt.Errorf("volume %q did not appear at its mount point", name) + } return nil } diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 0dca3e2a..f011266d 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -2,7 +2,6 @@ package engine import ( "context" - "encoding/json/v2" "fmt" "path/filepath" "slices" @@ -10,85 +9,12 @@ import ( "time" ) -const volumeMutationTimeout = 30 * time.Second - -// AttachVolume converges a read-only disk and its mount. The caller journals -// intent before calling: failures may leave a disk attached or mounted. -func (e *Engine) AttachVolume(ctx context.Context, vmName, sock string, spec VolumeSpec, mount string) error { - ctx, cancel := context.WithTimeout(ctx, volumeMutationTimeout) - defer cancel() - if spec.RW || mount == "" { - return fmt.Errorf("live attach requires a read-only mount") - } - found, err := e.attachedVolume(ctx, vmName, spec) - if err != nil { - return err - } - if !found { - if err = e.DiskAttach(ctx, vmName, spec); err != nil { - return err - } - } - device, err := e.waitForVolumeDevice(ctx, sock, spec.Name) - if err != nil { - return err - } - mounted, err := e.volumeMounted(ctx, sock, device, mount) - if err != nil || mounted { - return err - } - if err = e.MountVolume(ctx, sock, spec.Name, mount, false); err != nil { - return err - } - mounted, err = e.volumeMounted(ctx, sock, device, mount) - if err != nil { - return err - } - if !mounted { - return fmt.Errorf("volume %q did not appear at its mount point", spec.Name) - } - return nil -} - -// attachedVolume uses inspect only as positive evidence. A missing entry may -// also mean cocoon failed to query the VMM; DiskAttach then fails closed. -func (e *Engine) attachedVolume(ctx context.Context, vmName string, spec VolumeSpec) (bool, error) { - out, err := e.run(ctx, "vm", "inspect", vmName) - if err != nil { - return false, err - } - var rec struct { - AttachedDevices struct { - Disks []struct { - Name string `json:"name"` - Path string `json:"path"` - ReadOnly bool `json:"readonly"` - } `json:"disks"` - } `json:"attached_devices"` - } - if err := json.Unmarshal(out, &rec); err != nil { - return false, fmt.Errorf("parse attached disks: %w", err) - } - for _, disk := range rec.AttachedDevices.Disks { - if disk.Name != spec.Name { - continue - } - path, err := filepath.EvalSymlinks(spec.Path) - if err != nil { - return false, err - } - if !disk.ReadOnly || disk.Path != path { - return false, fmt.Errorf("attached volume %q has a different image or mode", spec.Name) - } - return true, nil - } - return false, nil -} +const VolumeMutationTimeout = 30 * time.Second // DetachVolume refuses busy or foreign mounts and confirms host removal even // when the guest serial has disappeared. It never uses lazy or forced umount. func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount string) error { - ctx, cancel := context.WithTimeout(ctx, volumeMutationTimeout) + ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() device, found, err := e.findVolumeDevice(ctx, sock, name) if err != nil { @@ -100,16 +26,11 @@ func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount str } } _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name) - // Both CH and FC emit this only after checking the device set under their - // operation lock. Inspect's absent field alone is not proof of removal. - if err != nil && !strings.Contains(err.Error(), fmt.Sprintf("disk %q not attached", name)) { - return err - } - return nil + return err } func (e *Engine) unmountLiveVolume(ctx context.Context, sock, device, mount string) error { - mounted, err := e.volumeMounted(ctx, sock, device, mount) + mounted, err := e.volumeMounted(ctx, sock, device, mount, false) if err != nil { return err } @@ -146,17 +67,24 @@ func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, return strings.TrimSpace(string(dev)), mounts, nil } -func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string) (bool, error) { +func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string, rw bool) (bool, error) { dev, mounts, err := e.volumeMounts(ctx, sock, device) if err != nil { return false, err } + mode := "ro" + if rw { + mode = "rw" + } matched := false for _, entry := range mounts { if entry.path != mount { + if entry.device == dev { + return false, fmt.Errorf("volume device is already mounted at %q", entry.path) + } continue } - if matched || entry.device != dev || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), "ro") { + if matched || entry.device != dev || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), mode) { return false, fmt.Errorf("volume mount %q has a different device or mode", mount) } matched = true diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 575591c2..ff23abb5 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -1,7 +1,6 @@ package engine import ( - "encoding/json/v2" "os" "path/filepath" "strings" @@ -33,7 +32,10 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { fake.mu.Unlock() e := New("cocoon", nil, nil, false, false, "") for range 2 { - if err := e.AttachVolume(t.Context(), "sbx-1", sock, spec, "/data"); err != nil { + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + if err := e.MountVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { t.Fatal(err) } } @@ -67,7 +69,7 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + row) fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, false, "").AttachVolume(t.Context(), "sbx-1", sock, spec, "/data"); err == nil { + if err := New("cocoon", nil, nil, false, false, "").MountVolume(t.Context(), sock, spec.Name, "/data", false); err == nil { t.Fatal("accepted a foreign mount") } fake.mu.Lock() @@ -117,7 +119,7 @@ func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { serveFakeSilkd(t, sock) spec := liveDiskCLI(t, hostErr) err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") - wantErr := hostErr != "" && !strings.Contains(hostErr, "not attached") + wantErr := hostErr != "" if (err != nil) != wantErr { t.Fatalf("got %v, want error=%v", err, wantErr) } @@ -125,15 +127,6 @@ func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { } } -func TestLiveAttachChecksHostImageAndMode(t *testing.T) { - spec := liveDiskCLI(t, "") - e := New("cocoon", nil, nil, false, false, "") - spec.Path = t.TempDir() - if _, err := e.attachedVolume(t.Context(), "sbx-1", spec); err == nil { - t.Fatal("accepted mismatched host image") - } -} - func liveDiskCLI(t *testing.T, detachErr string) VolumeSpec { t.Helper() dir := t.TempDir() @@ -141,19 +134,6 @@ func liveDiskCLI(t *testing.T, detachErr string) VolumeSpec { if err := os.WriteFile(image, nil, 0o600); err != nil { t.Fatal(err) } - path, err := filepath.EvalSymlinks(image) - if err != nil { - t.Fatal(err) - } - data, err := json.Marshal(map[string]any{"attached_devices": map[string]any{"disks": []any{map[string]any{"name": "imagenet", "path": path, "readonly": true}}}}) - if err != nil { - t.Fatal(err) - } - inspect := filepath.Join(dir, "inspect.json") - if err := os.WriteFile(inspect, data, 0o600); err != nil { - t.Fatal(err) - } - t.Setenv("COCOON_TEST_INSPECT", inspect) t.Setenv("COCOON_TEST_DETACH_ERROR", detachErr) attached := filepath.Join(dir, "attached") if err := os.WriteFile(attached, nil, 0o600); err != nil { @@ -162,10 +142,6 @@ func liveDiskCLI(t *testing.T, detachErr string) VolumeSpec { t.Setenv("COCOON_TEST_ATTACHED", attached) t.Setenv("COCOON_TEST_ATTACH_ARGS", filepath.Join(dir, "args")) fakeCocoon(t, `#!/bin/sh -if [ "$1 $2" = "vm inspect" ]; then - if [ -f "$COCOON_TEST_ATTACHED" ]; then cat "$COCOON_TEST_INSPECT"; else echo '{}'; fi - exit 0 -fi if [ "$1 $2 $3" = "vm disk attach" ]; then printf '%s\n' "$*" >> "$COCOON_TEST_ATTACH_ARGS" touch "$COCOON_TEST_ATTACHED" diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index f9825acc..f2e1406a 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -3,6 +3,7 @@ package engine import ( "context" "errors" + "fmt" "slices" "strings" "testing" @@ -86,6 +87,7 @@ func TestMountVolumeUsesSysfsAndRequestedMode(t *testing.T) { "/sys/block/vda/serial", "/sys/block/vdb/serial", "/sys/block/vdc/serial", + "/sys/block/vdc/dev", "/proc/self/mountinfo", "/sys/block/vdc/dev", "/proc/self/mountinfo", } if fake.listCalls != 1 || !slices.Equal(fake.readCalls, wantReads) { t.Errorf("sysfs calls = list:%d read:%v, want list:1 read:%v", fake.listCalls, fake.readCalls, wantReads) @@ -159,8 +161,8 @@ func TestMountVolumeWaitsForDelayedSysfsSerial(t *testing.T) { if fake.listCalls != 2 { t.Errorf("fs_list calls = %d, want 2", fake.listCalls) } - if got := fake.readCalls[len(fake.readCalls)-1]; got != "/sys/block/vdc/serial" { - t.Errorf("last fs_read = %q, want target serial", got) + if !slices.Contains(fake.readCalls, "/sys/block/vdc/serial") { + t.Error("target serial was not read") } } @@ -281,4 +283,11 @@ func configureVolumeDevices(f *fakeSilkd) { f.serial["/sys/block/vda/serial"] = []byte("root\n") f.serial["/sys/block/vdc/serial"] = []byte("imagenet\n") f.stat["/dev/vdc"] = true + f.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + f.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + f.execHook = func(argv []string) { + if argv[0] == "mount" { + f.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + fmt.Sprintf("2 1 8:32 / %s %s - ext4 /dev/vdc %s\n", argv[len(argv)-1], argv[2], argv[2])) + } + } } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 394ff362..f3d6dcab 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -160,18 +160,17 @@ func (m *Manager) Renew(ctx context.Context, id string, cred Cred, ttl time.Dura if !ok { return time.Time{}, ErrUnknownSandbox } - sb.VolumeMu.Lock() - defer sb.VolumeMu.Unlock() if !cred.Operator { if err := m.tenantRemoved(ctx, sb.Tenant); err != nil { return time.Time{}, err } } - if err := archivable(sb.Key, hasAppliedVolumes(sb), onExpire); err != nil { - return time.Time{}, err - } lease := clampTTL(ttl) m.mu.Lock() + if err := archivable(sb.Key, m.hasVolumeHolds(sb), onExpire); err != nil { + m.mu.Unlock() + return time.Time{}, err + } if m.claimed[id] != sb { m.mu.Unlock() return time.Time{}, ErrUnknownSandbox @@ -271,8 +270,6 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. // releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { - sb.VolumeMu.Lock() - defer sb.VolumeMu.Unlock() var marked string m.mu.Lock() for { @@ -291,6 +288,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand marked = ck m.mu.Lock() } + volumeHolds := m.cancelVolumeRun(sb) snap, ck, vmName := sb.HibernateSnap, sb.ArchiveCk, sb.VMName if ck != "" { m.pendingCks[ck] = struct{}{} @@ -302,6 +300,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() m.claimed[id] = sb // roll back so memory matches the still-durable claim + m.restoreVolumeHolds(sb, volumeHolds) m.tenantDelta(sb.Tenant, 1) delete(m.pendingCks, ck) rb := m.store.set(sb) @@ -323,6 +322,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.untrack(m.pendingCks, ck) } td := m.quiesceVolumes(ctx, sb) + td.holds = volumeHolds var err error removed := vmName == "" switch { @@ -498,27 +498,17 @@ func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved } func (m *Manager) reapOnce(ctx context.Context) { - // Take volume locks outside m.mu. Busy claims are retried on the next tick; - // a volume intent must not be deleted while its device call is in flight. - locked := m.lockReapClaims() - defer func() { - for sb := range locked { - sb.VolumeMu.Unlock() - } - }() now := time.Now() type victim struct { action reapAction id, vmName, snap, ck, tenant string sb *types.Sandbox + holds []types.Volume } m.mu.Lock() var expired []victim var dropped, purges []string for id, sb := range m.claimed { - if !locked[sb] { - continue - } // a zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { continue @@ -539,7 +529,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } expired = append(expired, victim{action: action, id: id, sb: sb}) default: - expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb}) + expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: m.cancelVolumeRun(sb)}) delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) dropped = append(dropped, id) @@ -596,6 +586,7 @@ func (m *Manager) reapOnce(ctx context.Context) { delete(m.archiving, v.id) // never removed from m.claimed } else { m.claimed[v.id] = v.sb // roll back so memory matches the still-durable claim + m.restoreVolumeHolds(v.sb, v.holds) m.tenantDelta(v.tenant, 1) delete(m.pendingCks, v.ck) restored = append(restored, v.sb) @@ -629,6 +620,7 @@ func (m *Manager) reapOnce(ctx context.Context) { logSweepResult(ctx, logger, m.pauseExpired(ctx, v.sb), "archived expired sandbox "+v.id, "archive expired sandbox "+v.id) default: td := m.quiesceVolumes(ctx, v.sb) + td.holds = v.holds m.out.Disarm(v.id, m.removeOrRetry(ctx, v.vmName, v.id, "", td)) m.dropSnap(ctx, v.snap) m.counters.reaps.Add(1) @@ -638,24 +630,6 @@ func (m *Manager) reapOnce(ctx context.Context) { }) } -func (m *Manager) lockReapClaims() map[*types.Sandbox]bool { - m.mu.Lock() - var candidates []*types.Sandbox - for _, sb := range m.claimed { - if !sb.Deadline.IsZero() && time.Now().After(sb.Deadline) { - candidates = append(candidates, sb) - } - } - m.mu.Unlock() - locked := make(map[*types.Sandbox]bool, len(candidates)) - for _, sb := range candidates { - if sb.VolumeMu.TryLock() { - locked[sb] = true - } - } - return locked -} - func (m *Manager) archivesAtDeadline(sb *types.Sandbox) bool { return sb.Failed == "" && !m.tenantGone(sb.Tenant) && (sb.OnExpire == types.ExpireArchive || sb.HibernateSnap != "" && m.archiveAfterFor(sb.Key) > 0) } diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index d0fdd925..00211fb3 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -1,6 +1,7 @@ package pool import ( + "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -23,33 +24,33 @@ type claimSnapshot struct { // claimDTO is the persisted projection of a Sandbox, copied so commit marshals off m.mu. type claimDTO struct { - ID string `json:"id"` - VMName string `json:"vm_name"` - Key types.PoolKey `json:"key"` - Token string `json:"token,omitempty"` - Deadline time.Time `json:"deadline,omitzero"` - ClaimedAt time.Time `json:"claimed_at,omitzero"` - LeaseSeconds int `json:"lease_seconds,omitzero"` - Layer types.PolicyLayer `json:"policy_layer,omitempty"` - PolicySource types.PoolKey `json:"policy_source,omitzero"` - NoEgress bool `json:"no_egress,omitzero"` - Tenant string `json:"tenant,omitempty"` - EgressClass string `json:"egress_class,omitempty"` - ClaimRef string `json:"claim_ref,omitempty"` - Metadata types.Metadata `json:"metadata,omitempty"` - OnExpire types.ExpireAction `json:"on_expire,omitempty"` - Volumes []types.Volume `json:"volumes,omitempty"` - PendingVolume *types.VolumeMutation `json:"pending_volume,omitempty"` - Env types.Env `json:"env,omitempty"` - VsockSocket string `json:"vsock_socket,omitempty"` - TAP string `json:"tap,omitempty"` - HibernateSnap string `json:"hibernate_snap,omitempty"` - PendingSnap string `json:"pending_snap,omitempty"` - ArchiveCk string `json:"archive_ck,omitempty"` - FromCheckpoint string `json:"from_checkpoint,omitempty"` - Restarts int `json:"restarts,omitzero"` - RestartedAt time.Time `json:"restarted_at,omitzero"` - Failed string `json:"failed,omitempty"` + ID string `json:"id"` + VMName string `json:"vm_name"` + Key types.PoolKey `json:"key"` + Token string `json:"token,omitempty"` + Deadline time.Time `json:"deadline,omitzero"` + ClaimedAt time.Time `json:"claimed_at,omitzero"` + LeaseSeconds int `json:"lease_seconds,omitzero"` + Layer types.PolicyLayer `json:"policy_layer,omitempty"` + PolicySource types.PoolKey `json:"policy_source,omitzero"` + NoEgress bool `json:"no_egress,omitzero"` + Tenant string `json:"tenant,omitempty"` + EgressClass string `json:"egress_class,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` + Metadata types.Metadata `json:"metadata,omitempty"` + OnExpire types.ExpireAction `json:"on_expire,omitempty"` + Volumes []types.Volume `json:"volumes,omitempty"` + VolumesWant []types.Volume `json:"volumes_want"` + Env types.Env `json:"env,omitempty"` + VsockSocket string `json:"vsock_socket,omitempty"` + TAP string `json:"tap,omitempty"` + HibernateSnap string `json:"hibernate_snap,omitempty"` + PendingSnap string `json:"pending_snap,omitempty"` + ArchiveCk string `json:"archive_ck,omitempty"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + Restarts int `json:"restarts,omitzero"` + RestartedAt time.Time `json:"restarted_at,omitzero"` + Failed string `json:"failed,omitempty"` } // claimRow keeps its encoded member until set replaces the row; set never edits one. @@ -89,6 +90,18 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { if err := json.Unmarshal(raw, &claims); err != nil { return nil, fmt.Errorf("parse claims: %w", err) } + var fields map[string]map[string]jsontext.Value + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, err + } + for id, sb := range claims { + if _, present := fields[id]["volumes_want"]; !present { + sb.VolumesWant = slices.Clone(sb.Volumes) + } + if len(sb.VolumesWant) == 0 { + sb.VolumesWant = nil + } + } return claims, nil } @@ -221,7 +234,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { Deadline: sb.Deadline, ClaimedAt: sb.ClaimedAt, LeaseSeconds: sb.LeaseSeconds, Layer: sb.Layer, PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, - Volumes: slices.Clone(sb.Volumes), PendingVolume: sb.PendingVolume, Env: sb.Env, VsockSocket: sb.VsockSocket, + Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index bd7910f1..6a7559e9 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -359,6 +359,12 @@ func TestSetPoolsPreservesEgressPolicy(t *testing.T) { if !policyLive() { t.Fatal("drain + re-add wiped the pool egress policy") } + waitFor(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + p := m.pools[egKey] + return p != nil && !p.building && p.refilling == 0 + }) } func TestEgressLaneCannotForkOrCheckpoint(t *testing.T) { diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index ada924ed..e39b7823 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -48,7 +48,10 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error if sb.Failed != "" { return ErrFailed } - if hasAppliedVolumes(sb) { + m.mu.Lock() + volumeBlocked := m.hasVolumeHolds(sb) + m.mu.Unlock() + if volumeBlocked { return ErrVolumeCapture } if sb.Key.Net == types.NetEgress { @@ -331,6 +334,6 @@ func (m *Manager) recordHibernate(ctx context.Context, sb *types.Sandbox) { } func skipIdle(sb *types.Sandbox) bool { - return sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || + return sb.Key.Net == types.NetEgress || hasVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || sb.Failed != "" || sb.Busy() } diff --git a/sandboxd/pool/liveness.go b/sandboxd/pool/liveness.go index da653d1a..31219f45 100644 --- a/sandboxd/pool/liveness.go +++ b/sandboxd/pool/liveness.go @@ -150,7 +150,9 @@ func (m *Manager) recoverVMM(ctx context.Context, sb *types.Sandbox) { sb.ExitRecorded = true m.recordUsage(ctx, usageEvent{Event: "vmm_exit", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) } - reason := coldBootBlocker(sb, found) + m.mu.Lock() + reason := coldBootBlocker(sb, found, m.hasVolumeHolds(sb)) + m.mu.Unlock() if reason == "" && m.vmmRestart == types.VMMRestartNone { reason = "vmm exited" } @@ -191,7 +193,10 @@ func (m *Manager) restartFailed(ctx context.Context, sb *types.Sandbox) error { if err != nil { return fmt.Errorf("wake %s: %w", sb.ID, err) } - if reason := coldBootBlocker(sb, found); reason != "" { + m.mu.Lock() + reason := coldBootBlocker(sb, found, m.hasVolumeHolds(sb)) + m.mu.Unlock() + if reason != "" { return fmt.Errorf("%w: %s", ErrFailed, reason) } sb.RestartLog = nil @@ -267,14 +272,14 @@ func runningShaped(sb *types.Sandbox) bool { } // coldBootBlocker names why sb's VM cannot be cold-booted, "" when it can. -func coldBootBlocker(sb *types.Sandbox, found bool) string { +func coldBootBlocker(sb *types.Sandbox, found, volumes bool) string { switch { case !found: return "vm record gone" case sb.Key.Net == types.NetEgress: // the netdev nft chain dies with the tap, so the guest would boot with an unlocked NIC return "vmm exited; an egress-lane sandbox cannot cold-boot" - case hasAppliedVolumes(sb): + case volumes: return "vmm exited; volume mounts do not survive a cold boot" default: return "" diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 5a1a4747..682c976f 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -453,7 +453,7 @@ func TestColdBootBlocker(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := coldBootBlocker(tt.sb, tt.found) + got := coldBootBlocker(tt.sb, tt.found, hasVolumes(tt.sb)) if (tt.want == "") != (got == "") || !strings.Contains(got, tt.want) { t.Errorf("got %q, want %q", got, tt.want) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 07a83cce..de7ae63a 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -102,7 +102,6 @@ var ( ErrNoEgressFork = errors.New("egress-lane sandboxes cannot fork, checkpoint, or promote: a resumed guest egresses before its fresh tap can be locked") ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") - ErrVolumePending = errors.New("volume operation pending; retry it, detach its name, or release the sandbox") // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") ErrArchived = errors.New("sandbox is archived; an exec or file call wakes it first") @@ -141,7 +140,6 @@ type Engine interface { Warmup(ctx context.Context, vsockSocket string, argv []string) error TrimCow(ctx context.Context, vsockSocket string) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error - AttachVolume(ctx context.Context, vmName, sock string, spec engine.VolumeSpec, mount string) error DetachVolume(ctx context.Context, vmName, sock, name, mount string) error MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error @@ -158,21 +156,21 @@ type SandboxSummary struct { // Token is the sandbox's own bearer token; only the root by-id read carries it. Token string `json:"token,omitempty"` // ClaimRef echoes the caller reference; empty for checkpoint branches and unprefixed forks. - ClaimRef string `json:"claim_ref,omitempty"` - Metadata types.Metadata `json:"metadata,omitempty"` - OnExpire types.ExpireAction `json:"on_expire,omitempty"` - CPUCount int `json:"cpu_count,omitzero"` - MemTotalBytes int64 `json:"mem_total_bytes,omitzero"` - Volumes []types.Volume `json:"volumes,omitempty"` - PendingVolume *types.VolumeMutation `json:"pending_volume,omitempty"` - Hibernated bool `json:"hibernated"` - Archived bool `json:"archived,omitzero"` - FromCheckpoint string `json:"from_checkpoint,omitempty"` - Restarts int `json:"restarts,omitzero"` - RestartedAt time.Time `json:"restarted_at,omitzero"` - Failed string `json:"failed,omitempty"` - ClaimedAt time.Time `json:"claimed_at,omitzero"` - Deadline time.Time `json:"deadline"` + ClaimRef string `json:"claim_ref,omitempty"` + Metadata types.Metadata `json:"metadata,omitempty"` + OnExpire types.ExpireAction `json:"on_expire,omitempty"` + CPUCount int `json:"cpu_count,omitzero"` + MemTotalBytes int64 `json:"mem_total_bytes,omitzero"` + Volumes []types.Volume `json:"volumes,omitempty"` + VolumesWant []types.Volume `json:"volumes_want"` + Hibernated bool `json:"hibernated"` + Archived bool `json:"archived,omitzero"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + Restarts int `json:"restarts,omitzero"` + RestartedAt time.Time `json:"restarted_at,omitzero"` + Failed string `json:"failed,omitempty"` + ClaimedAt time.Time `json:"claimed_at,omitzero"` + Deadline time.Time `json:"deadline"` } // PoolInfo is the ops view of one pool. @@ -298,6 +296,7 @@ type Manager struct { volumes map[string]catalogVolume // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders + volumeRuns map[*types.Sandbox]*volumeRun poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. @@ -692,7 +691,7 @@ func summarize(sb *types.Sandbox) SandboxSummary { FromCheckpoint: sb.FromCheckpoint, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, CPUCount: spec.CPU, MemTotalBytes: spec.MemoryBytes, - Volumes: slices.Clone(sb.Volumes), PendingVolume: cloneVolumeMutation(sb.PendingVolume), + Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), } } @@ -714,8 +713,8 @@ func tenantOwns(tenant, owner string) bool { return tenant == "" || tenant == owner } -func hasAppliedVolumes(sb *types.Sandbox) bool { - return len(sb.Volumes) > 0 || sb.PendingVolume != nil +func hasVolumes(sb *types.Sandbox) bool { + return len(sb.Volumes) > 0 || len(sb.VolumesWant) > 0 } // logSweepResult reports one background-sweep outcome; benign races stay silent. diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index c19f8599..141b79dc 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1656,7 +1656,15 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { return f.trimErr } -func (f *fakeEngine) DiskAttach(_ context.Context, _ string, spec engine.VolumeSpec) error { +func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { + if f.liveVolumeHook != nil { + if err := f.liveVolumeHook("attach"); err != nil { + return err + } + } + if err := ctx.Err(); err != nil { + return err + } f.mu.Lock() f.volumeSpecs = append(f.volumeSpecs, spec) f.volumeOps = append(f.volumeOps, "attach:"+spec.Name) @@ -1688,21 +1696,6 @@ func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bo return f.mountVolumeErr } -func (f *fakeEngine) AttachVolume(ctx context.Context, vmName, sock string, spec engine.VolumeSpec, mount string) error { - if f.liveVolumeHook != nil { - if err := f.liveVolumeHook("attach"); err != nil { - return err - } - } - if err := ctx.Err(); err != nil { - return err - } - if err := f.DiskAttach(ctx, vmName, spec); err != nil { - return err - } - return f.MountVolume(ctx, sock, spec.Name, mount, spec.RW) -} - func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("detach"); err != nil { diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index bd1a8197..303c4b24 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -217,6 +217,12 @@ func TestAStaleCellVersionIsNotApplied(t *testing.T) { if floor != 3 { t.Errorf("warm floor %d after an older and a repeated version, want 3 from version 5", floor) } + waitFor(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + p := m.pools[apiKey] + return p != nil && !p.building && p.refilling == 0 + }) } func TestAPutTheCellCannotStoreChangesNothing(t *testing.T) { diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 521f656c..99dc23b7 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -377,14 +377,13 @@ func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, fu return snap, func() { m.dropSnap(ctx, snap) }, nil } -// captureAllowed must run under Transition, after any live volume change. func (m *Manager) captureAllowed(sb *types.Sandbox) error { m.mu.Lock() defer m.mu.Unlock() if m.claimed[sb.ID] != sb { return ErrUnknownSandbox } - if hasAppliedVolumes(sb) { + if m.hasVolumeHolds(sb) { return ErrVolumeCapture } return nil diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 12a834bd..505bb1a8 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -181,6 +181,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ return err } sb.Volumes = applied + sb.VolumesWant = slices.Clone(applied) return nil } @@ -200,6 +201,9 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res if attachOnly { return nil } + if err := ctx.Err(); err != nil { + return err + } if err := m.eng.MountVolume(ctx, sb.VsockSocket, volume.applied.Name, volume.applied.Mount, volume.disk.RW); err != nil { return fmt.Errorf("setup volume %q: %w", volume.applied.Name, err) } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index aeea7494..cf16c365 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -5,118 +5,227 @@ import ( "fmt" "slices" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" ) -// AttachVolumes adds mounted, read-only volumes to a running claim. Batches -// commit in order: retry the request after a partial failure to finish it. -func (m *Manager) AttachVolumes(ctx context.Context, id string, cred Cred, requested []types.Volume) ([]types.Volume, error) { - if types.VolumesAttachOnly(requested) || slices.ContainsFunc(requested, types.Volume.RW) { - return nil, fmt.Errorf("%w: live attach requires read-only mounts", ErrBadVolume) +type volumeRun struct { + cancel context.CancelFunc + done chan struct{} + holds []types.Volume +} + +func (m *Manager) GetVolumes(id string, cred Cred) (types.SandboxVolumesResponse, error) { + sb, ok := m.resolve(id, cred) + if !ok { + return types.SandboxVolumesResponse{}, ErrUnknownSandbox } - requested, err := types.ValidateVolumes(requested, false) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[id] != sb { + return types.SandboxVolumesResponse{}, ErrUnknownSandbox } + return volumeView(sb), nil +} + +func (m *Manager) SetVolumes(ctx context.Context, id string, cred Cred, want []types.Volume) (types.SandboxVolumesResponse, error) { + return m.changeVolumes(ctx, id, cred, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) +} + +func (m *Manager) PatchVolumes(ctx context.Context, id string, cred Cred, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { + return m.changeVolumes(ctx, id, cred, func(current []types.Volume) ([]types.Volume, error) { + if err := validateDetachNames(patch.Remove); err != nil { + return nil, err + } + additions, err := normalizeLiveVolumes(patch.Add, current) + if err != nil { + return nil, err + } + for _, v := range additions { + if volumeIndex(current, v.Name) < 0 && immutableVolume(v) { + return nil, fmt.Errorf("%w: live additions require read-only mounts", ErrBadVolume) + } + if slices.Contains(patch.Remove, v.Name) { + return nil, fmt.Errorf("%w: volume %q is both added and removed", ErrBadVolume, v.Name) + } + if i := volumeIndex(current, v.Name); i >= 0 { + if current[i] != v { + return nil, fmt.Errorf("%w: volume %q already has a different mount or mode", ErrBadVolume, v.Name) + } + } else { + current = append(current, v) + } + } + return slices.DeleteFunc(current, func(v types.Volume) bool { return slices.Contains(patch.Remove, v.Name) }), nil + }) +} + +func (m *Manager) changeVolumes(ctx context.Context, id string, cred Cred, change func([]types.Volume) ([]types.Volume, error)) (view types.SandboxVolumesResponse, err error) { sb, ok := m.resolve(id, cred) if !ok { - return nil, ErrUnknownSandbox + return view, ErrUnknownSandbox } - sb.VolumeMu.Lock() - defer sb.VolumeMu.Unlock() - sb.Transition.Lock() - defer sb.Transition.Unlock() - if err = m.mutableVolumes(sb); err != nil { - return nil, err + ctx, run, err := m.beginVolumeRun(ctx, sb) + if err != nil { + view, _ = m.GetVolumes(id, cred) + return view, err + } + failedName := "" + defer func() { + view = m.finishVolumeRun(sb, run) + if err != nil { + view.Error = err.Error() + for i := range view.States { + if view.States[i].Name == failedName { + view.States[i].Error = err.Error() + } + } + } + }() + m.mu.Lock() + current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) + m.mu.Unlock() + target, err := change(current) + if err != nil { + return view, err + } + target, err = normalizeLiveVolumes(target, held) + if err != nil { + return view, err } - if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { - return nil, err + if err = validateLiveTarget(held, target); err != nil { + return view, err } - if pending := sb.PendingVolume; pending != nil && (pending.Detach || !slices.Contains(requested, pending.Volume)) { - return nil, ErrVolumePending + additions := volumeDifference(target, held) + mutable := slices.DeleteFunc(slices.Clone(target), immutableVolume) + if len(mutable) != 0 { + if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { + return view, err + } } - toAdd, err := liveVolumeAdditions(heldVolumes(sb), requested) + resolved, err := m.resolveVolumes(ctx, sb.Tenant, mutable) if err != nil { - return nil, err + return view, err } - // A pending attach already holds admission but still needs convergence. - if pending := sb.PendingVolume; pending != nil { - toAdd = append([]types.Volume{pending.Volume}, toAdd...) + if err = m.checkVolumeRun(ctx, sb, run); err != nil { + return view, err } - resolved, err := m.resolveVolumes(ctx, sb.Tenant, toAdd) - if err != nil { - return nil, err + m.mu.Lock() + uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) + m.mu.Unlock() + for _, v := range uncertain { + failedName = v.Name + if err = m.detachVolumeRun(ctx, sb, run, v); err != nil { + return view, err + } } - for _, volume := range resolved { - if err := ctx.Err(); err != nil { - return nil, err + if err = m.prepareVolumeTarget(ctx, sb, run, target, additions, resolved); err != nil { + return view, err + } + m.mu.Lock() + removals := volumeDifference(sb.Volumes, target) + m.mu.Unlock() + for _, v := range removals { + failedName = v.Name + if err = m.detachVolumeRun(ctx, sb, run, v); err != nil { + return view, err } - if sb.PendingVolume == nil { - if err := m.startVolumeAttach(ctx, sb, volume); err != nil { - return nil, err - } - } else if err := m.confirmVolumesClean([]resolvedVolume{volume}); err != nil { - return nil, err + m.mu.Lock() + next := slices.DeleteFunc(slices.Clone(sb.Volumes), func(a types.Volume) bool { return a.Name == v.Name }) + m.mu.Unlock() + if err = m.commitVolumeApplied(ctx, sb, run, next); err != nil { + return view, err } - if err := m.eng.AttachVolume(ctx, sb.VMName, sb.VsockSocket, volume.disk, volume.applied.Mount); err != nil { - return nil, fmt.Errorf("attach volume %q (retry or detach to recover): %w", volume.applied.Name, err) + } + for _, v := range resolved { + failedName = v.applied.Name + if err = m.checkVolumeRun(ctx, sb, run); err != nil { + return view, err } - if err := m.finishVolumeMutation(ctx, sb); err != nil { - return nil, err + applyCtx, cancel := context.WithTimeout(ctx, engine.VolumeMutationTimeout) + err = m.applyVolume(applyCtx, sb, v) + cancel() + if err != nil { + return view, err + } + if err = m.checkVolumeRun(ctx, sb, run); err != nil { + return view, err + } + m.mu.Lock() + next := slices.Clone(sb.Volumes) + if volumeIndex(next, v.applied.Name) < 0 { + next = append(next, v.applied) + } + m.mu.Unlock() + if err = m.commitVolumeApplied(ctx, sb, run, next); err != nil { + return view, err } } - return slices.Clone(sb.Volumes), nil + sb.Touch() + return view, nil } -// DetachVolumes removes read-only volumes, including an interrupted attach. -// An absent valid name is a no-op; batches may partially complete on failure. -func (m *Manager) DetachVolumes(ctx context.Context, id string, cred Cred, names []string) ([]types.Volume, error) { - if err := validateDetachNames(names); err != nil { - return nil, err - } - sb, ok := m.resolve(id, cred) - if !ok { - return nil, ErrUnknownSandbox - } - sb.VolumeMu.Lock() - defer sb.VolumeMu.Unlock() - sb.Transition.Lock() - defer sb.Transition.Unlock() - if err := m.mutableVolumes(sb); err != nil { - return nil, err - } - if pending := sb.PendingVolume; pending != nil && !slices.Contains(names, pending.Volume.Name) { - return nil, ErrVolumePending - } - held := heldVolumes(sb) - for _, volume := range held { - if slices.Contains(names, volume.Name) && volume.RW() { - return nil, fmt.Errorf("%w: live detach of writable volumes is not supported", ErrBadVolume) +func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (context.Context, *volumeRun, error) { + for { + if err := ctx.Err(); err != nil { + return ctx, nil, err } - } - // Recover the pending operation first, even if it was last in the request. - if pending := sb.PendingVolume; pending != nil { - if err := m.detachLiveVolume(ctx, sb, pending.Volume); err != nil { - return nil, err + m.mu.Lock() + if m.claimed[sb.ID] != sb { + m.mu.Unlock() + return ctx, nil, ErrUnknownSandbox } - } - for _, volume := range slices.Clone(sb.Volumes) { - if slices.Contains(names, volume.Name) { - if err := m.detachLiveVolume(ctx, sb, volume); err != nil { - return nil, err + if active := m.volumeRuns[sb]; active != nil { + done := active.done + m.mu.Unlock() + select { + case <-ctx.Done(): + return ctx, nil, ctx.Err() + case <-done: } + continue + } + runCtx, cancel := context.WithCancel(ctx) + run := &volumeRun{cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb)} + if m.volumeRuns == nil { + m.volumeRuns = make(map[*types.Sandbox]*volumeRun) } + m.volumeRuns[sb] = run + m.mu.Unlock() + return runCtx, run, nil } - return slices.Clone(sb.Volumes), nil } -// mutableVolumes runs under Transition; manager-owned fields need m.mu too. -func (m *Manager) mutableVolumes(sb *types.Sandbox) error { +func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.SandboxVolumesResponse { + m.mu.Lock() + if m.claimed[sb.ID] == sb { + m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) + } + view := volumeView(sb) + delete(m.volumeRuns, sb) + close(run.done) + m.mu.Unlock() + run.cancel() + return view +} + +func (m *Manager) restoreVolumeHolds(sb *types.Sandbox, holds []types.Volume) { + if m.volumeRuns[sb] == nil { + m.releaseVolumes(volumeDifference(holds, heldVolumes(sb))) + } +} + +func (m *Manager) checkVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun) error { + if err := ctx.Err(); err != nil { + return err + } m.mu.Lock() defer m.mu.Unlock() - switch { - case m.claimed[sb.ID] != sb: + if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run { return ErrUnknownSandbox + } + switch { case sb.Failed != "": return ErrFailed case sb.ArchiveCk != "": @@ -127,140 +236,216 @@ func (m *Manager) mutableVolumes(sb *types.Sandbox) error { return nil } -func liveVolumeAdditions(current, requested []types.Volume) ([]types.Volume, error) { - combined := slices.Clone(current) - var added []types.Volume - for _, volume := range requested { - i := slices.IndexFunc(current, func(v types.Volume) bool { return v.Name == volume.Name }) - if i >= 0 { - if current[i] != volume { - return nil, fmt.Errorf("%w: volume %q already has a different mount or mode", ErrBadVolume, volume.Name) - } - continue - } - combined = append(combined, volume) - added = append(added, volume) +func (m *Manager) cancelVolumeRun(sb *types.Sandbox) []types.Volume { + if run := m.volumeRuns[sb]; run != nil { + run.cancel() + return slices.Clone(run.holds) } - if len(combined) > types.MaxClaimVolumes { - return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) - } - // Validate only mounted entries, preserving existing attach-only devices. - mounted := slices.DeleteFunc(slices.Clone(combined), func(v types.Volume) bool { return v.Mount == "" }) - if _, err := types.ValidateVolumes(mounted, false); err != nil { - return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) - } - return added, nil + return heldVolumes(sb) } -func validateDetachNames(names []string) error { - if len(names) > types.MaxClaimVolumes { - return fmt.Errorf("%w: too many volume names", ErrBadVolume) - } - for i, name := range names { - if !types.ValidVolumeName(name) || slices.Contains(names[:i], name) { - return fmt.Errorf("%w: invalid or duplicate volume name %q", ErrBadVolume, name) - } +func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) error { + sb.Transition.Lock() + defer sb.Transition.Unlock() + if err := m.checkVolumeRun(ctx, sb, run); err != nil { + return err } - return nil -} - -func (m *Manager) startVolumeAttach(ctx context.Context, sb *types.Sandbox, volume resolvedVolume) error { m.mu.Lock() - if sb.OnExpire == types.ExpireArchive { + switch { + case sb.Failed != "": + m.mu.Unlock() + return ErrFailed + case sb.ArchiveCk != "": + m.mu.Unlock() + return ErrArchived + case sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.VsockSocket == "": + m.mu.Unlock() + return ErrPaused + case sb.OnExpire == types.ExpireArchive && len(target) > 0: m.mu.Unlock() return ErrVolumeCapture } - err := m.reserveVolumes([]types.Volume{volume.applied}) - m.mu.Unlock() - if err != nil { + if err := m.reserveVolumes(additions); err != nil { + m.mu.Unlock() return err } - if err = m.confirmVolumesClean([]resolvedVolume{volume}); err == nil { - err = m.setVolumeIntent(ctx, sb, &types.VolumeMutation{Volume: volume.applied}) + run.holds = append(run.holds, additions...) + m.mu.Unlock() + cleanErr := m.confirmVolumesClean(resolved) + m.mu.Lock() + if m.claimed[sb.ID] != sb || ctx.Err() != nil { + m.mu.Unlock() + return context.Canceled + } + if cleanErr != nil { + m.releaseVolumes(additions) + run.holds = volumeDifference(run.holds, additions) + m.mu.Unlock() + return cleanErr + } + previous := sb.VolumesWant + sb.VolumesWant = slices.Clone(target) + js := m.store.set(sb) + m.mu.Unlock() + err := m.store.commit(js) + if err != nil && m.store.sync { + // A directory fsync may fail after rename. The new target and holds must + // survive until a later successful commit resolves that uncertainty. + return fmt.Errorf("persist volume target: %w", err) } if err != nil { - m.unreserveVolumes([]types.Volume{volume.applied}) + m.mu.Lock() + if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { + sb.VolumesWant = previous + rb := m.store.set(sb) + m.releaseVolumes(additions) + run.holds = volumeDifference(run.holds, additions) + m.mu.Unlock() + m.recommit(ctx, rb) + } else { + m.mu.Unlock() + } + return fmt.Errorf("persist volume target: %w", err) } - return err + m.mu.Lock() + if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run || ctx.Err() != nil { + m.mu.Unlock() + return context.Canceled + } + dropped := volumeDifference(run.holds, heldVolumes(sb)) + m.releaseVolumes(dropped) + run.holds = volumeDifference(run.holds, dropped) + m.mu.Unlock() + return nil } -func (m *Manager) detachLiveVolume(ctx context.Context, sb *types.Sandbox, volume types.Volume) error { - if err := ctx.Err(); err != nil { +func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { + if err := m.checkVolumeRun(ctx, sb, run); err != nil { return err } - if pending := sb.PendingVolume; pending == nil || !pending.Detach { - if err := m.setVolumeIntent(ctx, sb, &types.VolumeMutation{Volume: volume, Detach: true}); err != nil { - return err - } - } - if err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, volume.Name, volume.Mount); err != nil { - return fmt.Errorf("detach volume %q (retry to recover): %w", volume.Name, err) + if err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount); err != nil { + return fmt.Errorf("detach volume %q: %w", v.Name, err) } - return m.finishVolumeMutation(ctx, sb) + return m.checkVolumeRun(ctx, sb, run) } -func (m *Manager) setVolumeIntent(ctx context.Context, sb *types.Sandbox, next *types.VolumeMutation) error { +func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, next []types.Volume) error { m.mu.Lock() - previous := sb.PendingVolume - sb.PendingVolume = next + if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run || ctx.Err() != nil { + m.mu.Unlock() + return context.Canceled + } + previous := sb.Volumes + sb.Volumes = next js := m.store.set(sb) m.mu.Unlock() if err := m.store.commit(js); err != nil { m.mu.Lock() - sb.PendingVolume = previous - rb := m.store.set(sb) - m.mu.Unlock() - m.recommit(ctx, rb) - return fmt.Errorf("persist volume intent: %w", err) + if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { + sb.Volumes = previous + rb := m.store.set(sb) + m.mu.Unlock() + m.recommit(ctx, rb) + } else { + m.mu.Unlock() + } + return fmt.Errorf("persist volume result: %w", err) + } + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run || ctx.Err() != nil { + return context.Canceled } + dropped := volumeDifference(run.holds, heldVolumes(sb)) + m.releaseVolumes(dropped) + run.holds = volumeDifference(run.holds, dropped) return nil } -func (m *Manager) finishVolumeMutation(ctx context.Context, sb *types.Sandbox) error { - m.mu.Lock() - previous, pending := sb.Volumes, sb.PendingVolume - next := slices.Clone(previous) - if pending.Detach { - next = slices.DeleteFunc(next, func(v types.Volume) bool { return v.Name == pending.Volume.Name }) - } else { - next = append(next, pending.Volume) - } - sb.Volumes, sb.PendingVolume = next, nil - js := m.store.set(sb) - m.mu.Unlock() - if err := m.store.commit(js); err != nil { - // Keep both the intent and its admission hold until a retry commits. - m.mu.Lock() - sb.Volumes, sb.PendingVolume = previous, pending - rb := m.store.set(sb) - m.mu.Unlock() - m.recommit(ctx, rb) - return fmt.Errorf("persist volume result: %w", err) +func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) { + if len(volumes) > types.MaxClaimVolumes { + return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) } - verb := "volume_attach" - if pending.Detach { - m.unreserveVolumes([]types.Volume{pending.Volume}) - verb = "volume_detach" + out := make([]types.Volume, 0, len(volumes)) + for _, v := range volumes { + attachOnly := v.AttachOnly + if i := volumeIndex(held, v.Name); i >= 0 && held[i].Mount == "" && v.Mount == "" { + attachOnly = true + } + normalized, err := types.ValidateVolumes([]types.Volume{v}, attachOnly) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + } + if volumeIndex(out, v.Name) >= 0 { + return nil, fmt.Errorf("%w: duplicate volume %q", ErrBadVolume, v.Name) + } + if attachOnly { + normalized[0].AttachOnly = false + } + out = append(out, normalized[0]) + } + mounted := slices.DeleteFunc(slices.Clone(out), func(v types.Volume) bool { return v.Mount == "" }) + if _, err := types.ValidateVolumes(mounted, false); err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + } + return out, nil +} + +func validateLiveTarget(held, target []types.Volume) error { + for _, v := range held { + i := volumeIndex(target, v.Name) + if i >= 0 && target[i] != v { + return fmt.Errorf("%w: remove volume %q before changing its mount or mode", ErrBadVolume, v.Name) + } + if immutableVolume(v) && i < 0 { + return fmt.Errorf("%w: writable and attach-only volumes cannot change live", ErrBadVolume) + } + } + if slices.ContainsFunc(volumeDifference(target, held), immutableVolume) { + return fmt.Errorf("%w: live additions require read-only mounts", ErrBadVolume) } - sb.Touch() - m.recordUsage(ctx, usageEvent{Event: verb, ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant, Volumes: []string{pending.Volume.Name}}) return nil } -// heldVolumes includes a possibly attached device even before it is committed. -// Callers hold m.mu or Transition (both are required for mutations). -func heldVolumes(sb *types.Sandbox) []types.Volume { - volumes := slices.Clone(sb.Volumes) - if pending := sb.PendingVolume; pending != nil && !slices.ContainsFunc(volumes, func(v types.Volume) bool { return v.Name == pending.Volume.Name }) { - volumes = append(volumes, pending.Volume) +func immutableVolume(v types.Volume) bool { return v.RW() || v.AttachOnly || v.Mount == "" } +func volumeIndex(volumes []types.Volume, name string) int { + return slices.IndexFunc(volumes, func(v types.Volume) bool { return v.Name == name }) +} + +func volumeDifference(left, right []types.Volume) []types.Volume { + return slices.DeleteFunc(slices.Clone(left), func(v types.Volume) bool { return volumeIndex(right, v.Name) >= 0 }) +} + +func validateDetachNames(names []string) error { + if len(names) > types.MaxClaimVolumes { + return fmt.Errorf("%w: too many volume names", ErrBadVolume) } - return volumes + for i, name := range names { + if !types.ValidVolumeName(name) || slices.Contains(names[:i], name) { + return fmt.Errorf("%w: invalid or duplicate volume name %q", ErrBadVolume, name) + } + } + return nil } -func cloneVolumeMutation(op *types.VolumeMutation) *types.VolumeMutation { - if op == nil { - return nil +func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { + return hasVolumes(sb) || m.volumeRuns[sb] != nil && len(m.volumeRuns[sb].holds) > 0 +} + +func heldVolumes(sb *types.Sandbox) []types.Volume { + return append(slices.Clone(sb.Volumes), volumeDifference(sb.VolumesWant, sb.Volumes)...) +} + +func volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { + view := types.SandboxVolumesResponse{Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}} + for _, v := range heldVolumes(sb) { + state := "attached" + if volumeIndex(sb.VolumesWant, v.Name) < 0 { + state = "detaching" + } else if volumeIndex(sb.Volumes, v.Name) < 0 { + state = "pending" + } + view.States = append(view.States, types.VolumeState{Name: v.Name, State: state}) } - cloned := *op - return &cloned + return view } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 8defa7da..e743f478 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -3,12 +3,16 @@ package pool import ( "context" "errors" + "fmt" "os" "slices" + "strings" + "sync/atomic" "testing" "time" "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -19,19 +23,19 @@ func TestLiveVolumesCommitAndRetry(t *testing.T) { cred := Cred{Token: sb.Token} request := []types.Volume{{Name: "data"}, {Name: "other", Mount: "/models"}} for range 2 { - got, err := m.AttachVolumes(t.Context(), sb.ID, cred, request) - if err != nil || len(got) != 2 || sb.PendingVolume != nil { - t.Fatalf("attach = %v, %v; pending=%v", got, err, sb.PendingVolume) + got, err := m.SetVolumes(t.Context(), sb.ID, cred, request) + if err != nil || len(got.Volumes) != 2 || !slices.Equal(sb.VolumesWant, sb.Volumes) { + t.Fatalf("attach = %v, %v; pending=%v", got, err, sb.VolumesWant) } - got[0].Name = "caller copy" + got.Volumes[0].Name = "caller copy" } if got := volumeHoldersOf(m, "data"); got.readers != 1 { t.Fatalf("duplicate reservation: %+v", got) } assertLiveJournal(t, m, sb) for range 2 { - got, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data", "other"}) - if err != nil || len(got) != 0 { + got, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data", "other"}}) + if err != nil || len(got.Volumes) != 0 { t.Fatalf("detach = %v, %v", got, err) } } @@ -54,16 +58,16 @@ func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { } original := sb.Volumes[0] cred := Cred{Token: sb.Token} - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { t.Fatal(err) } if sb.Volumes[0] != original { t.Fatalf("changed existing volume: %v -> %v", original, sb.Volumes[0]) } - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("accepted mode change: %v", err) } - if _, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data", "other"}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { + if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { t.Fatalf("writable detach changed the batch: %v", err) } } @@ -82,17 +86,17 @@ func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { {{Name: "data", Mount: "/a"}, {Name: "other", Mount: "/a/b"}}, {{Name: "missing"}}, } { - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); !errors.Is(err, ErrBadVolume) { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); !errors.Is(err, ErrBadVolume) { t.Fatalf("request %v: %v", request, err) } } - if _, err := m.AttachVolumes(t.Context(), sb.ID, Cred{Token: "wrong"}, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrUnknownSandbox) { + if _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: "wrong"}, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrUnknownSandbox) { t.Fatalf("wrong token: %v", err) } - if _, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data", "data"}); !errors.Is(err, ErrBadVolume) { + if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data", "data"}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("duplicate detach: %v", err) } - if len(eng.volumeSpecs) != 0 || sb.PendingVolume != nil { + if len(eng.volumeSpecs) != 0 || !slices.Equal(sb.VolumesWant, sb.Volumes) { t.Fatal("invalid request reached engine") } } @@ -103,16 +107,16 @@ func TestLiveAttachFailureSurvivesRestartAndCanBeDetached(t *testing.T) { m := newVolumeManager(t, eng, catalog) sb := mustClaim(t, m, testKey) eng.mountVolumeErr = errors.New("guest unavailable after host attach") - if _, err := m.AttachVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}); err == nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}); err == nil { t.Fatal("want mount failure") } assertLiveJournal(t, m, sb) - if sb.PendingVolume == nil || volumeHoldersOf(m, "data").readers != 1 { + if slices.Equal(sb.VolumesWant, sb.Volumes) || volumeHoldersOf(m, "data").readers != 1 { t.Fatal("lost pending device or hold") } m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) - if err := m2.Reconcile(t.Context()); err != nil { - t.Fatal(err) + if reconcileErr := m2.Reconcile(t.Context()); reconcileErr != nil { + t.Fatal(reconcileErr) } if volumeHoldersOf(m2, "data").readers != 1 { t.Fatal("restart lost pending hold") @@ -121,15 +125,12 @@ func TestLiveAttachFailureSurvivesRestartAndCanBeDetached(t *testing.T) { if _, err := m2.Checkpoint(t.Context(), sb.ID, cred, "pending", ""); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("captured pending device: %v", err) } - if _, err := m2.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumePending) { - t.Fatalf("overwrote pending operation: %v", err) - } eng.detachVolumeErr = errors.New("host detach failed") - if _, err := m2.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}); err == nil || volumeHoldersOf(m2, "data").readers != 1 { + if _, err := m2.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}); err == nil || volumeHoldersOf(m2, "data").readers != 1 { t.Fatal("failed detach released its hold") } eng.detachVolumeErr = nil - if _, err := m2.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}); err != nil { + if _, err := m2.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}); err != nil { t.Fatal(err) } if volumeHoldersOf(m2, "data") != (volumeHolders{}) { @@ -146,7 +147,7 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { cred := Cred{Token: sb.Token} request := []types.Volume{{Name: "data"}} if stage == "detach result" { - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); err != nil { t.Fatal(err) } } @@ -165,9 +166,9 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { } var err error if stage == "detach result" { - _, err = m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}) + _, err = m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}) } else { - _, err = m.AttachVolumes(t.Context(), sb.ID, cred, request) + _, err = m.SetVolumes(t.Context(), sb.ID, cred, request) } if err == nil { t.Fatal("expected persistence failure") @@ -180,18 +181,18 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { } eng.liveVolumeHook = nil if stage == "intent" { - if sb.PendingVolume != nil || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { + if !slices.Equal(sb.VolumesWant, sb.Volumes) || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { t.Fatal("failed intent must not reach the engine or retain a hold") } return } - if sb.PendingVolume == nil || volumeHoldersOf(m, "data").readers != 1 { + if slices.Equal(sb.VolumesWant, sb.Volumes) || volumeHoldersOf(m, "data").readers != 1 { t.Fatal("uncommitted physical result lost its intent or hold") } if stage == "detach result" { - _, err = m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}) + _, err = m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}) } else { - _, err = m.AttachVolumes(t.Context(), sb.ID, cred, request) + _, err = m.SetVolumes(t.Context(), sb.ID, cred, request) } if err != nil { t.Fatal(err) @@ -208,14 +209,14 @@ func TestLiveBatchPartialFailureRetriesWithoutDoubleHolds(t *testing.T) { cred := Cred{Token: sb.Token} request := []types.Volume{{Name: "data"}, {Name: "other"}} eng.diskAttachErrFor = "other" - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); err == nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); err == nil { t.Fatal("want partial failure") } - if len(sb.Volumes) != 1 || sb.PendingVolume == nil || sb.PendingVolume.Volume.Name != "other" { - t.Fatalf("partial state: %v %v", sb.Volumes, sb.PendingVolume) + if len(sb.Volumes) != 1 || slices.Equal(sb.VolumesWant, sb.Volumes) || len(sb.VolumesWant) != 2 { + t.Fatalf("partial state: %v %v", sb.Volumes, sb.VolumesWant) } eng.diskAttachErrFor = "" - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, request); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); err != nil { t.Fatal(err) } for _, name := range []string{"data", "other"} { @@ -225,8 +226,8 @@ func TestLiveBatchPartialFailureRetriesWithoutDoubleHolds(t *testing.T) { } } -func TestLiveVolumeSerializesReleaseAndCapture(t *testing.T) { - for _, action := range []string{"release", "checkpoint", "fork", "promote", "hibernate", "reap"} { +func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { + for _, action := range []string{"release", "checkpoint", "fork", "promote", "hibernate", "reap", "renew", "get", "relay"} { t.Run(action, func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) @@ -236,11 +237,10 @@ func TestLiveVolumeSerializesReleaseAndCapture(t *testing.T) { eng.liveVolumeHook = func(string) error { close(entered); <-resume; return nil } attached := make(chan error, 1) go func() { - _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) attached <- err }() <-entered - // The durable intent must precede any physical operation. assertLiveJournal(t, m, sb) done := make(chan error, 1) go func() { @@ -261,30 +261,52 @@ func TestLiveVolumeSerializesReleaseAndCapture(t *testing.T) { sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() m.reapOnce(t.Context()) + case "renew": + _, err = m.Renew(t.Context(), sb.ID, cred, time.Minute, "") + case "get": + _, err = m.GetVolumes(sb.ID, cred) + case "relay": + var unhold func() + _, unhold, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if unhold != nil { + unhold() + } } done <- err }() - if sb.VolumeMu.TryLock() { - sb.VolumeMu.Unlock() - t.Fatal("volume mutation did not exclude release") - } - if sb.Transition.TryLock() { - sb.Transition.Unlock() - t.Fatal("volume mutation did not exclude capture") + var err error + select { + case err = <-done: + case <-time.After(2 * time.Second): + close(resume) + <-attached + t.Fatal("lifecycle blocked behind device call") } - if len(eng.removedNames()) != 0 || len(eng.snapSaves) != 0 { - t.Fatal("teardown or capture overlapped device mutation") + if slices.Contains([]string{"checkpoint", "fork", "promote", "hibernate"}, action) { + if !errors.Is(err, ErrVolumeCapture) { + t.Errorf("capture=%v", err) + } + } else if err != nil { + t.Errorf("lifecycle=%v", err) } close(resume) - if err := <-attached; err != nil { - t.Fatal(err) + attachErr := <-attached + if action == "reap" { + waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) } - err := <-done - if action != "release" && action != "reap" && !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("capture: %v", err) - } - if action == "release" && (err != nil || volumeHoldersOf(m, "data") != (volumeHolders{})) { - t.Fatalf("release: %v", err) + if action == "release" || action == "reap" { + if attachErr == nil { + t.Fatal("released worker reported success") + } + claims, err := m.store.load() + if err != nil || claims[sb.ID] != nil { + t.Fatalf("late worker resurrected claim: %v %v", claims, err) + } + if volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("teardown leaked holds") + } + } else if attachErr != nil { + t.Fatal(attachErr) } }) } @@ -296,9 +318,9 @@ func TestLiveVolumeCanceledRequestRetainsIntent(t *testing.T) { sb := mustClaim(t, m, testKey) ctx, cancel := context.WithCancel(t.Context()) eng.liveVolumeHook = func(string) error { cancel(); return nil } - _, err := m.AttachVolumes(ctx, sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) - if !errors.Is(err, context.Canceled) || sb.PendingVolume == nil { - t.Fatalf("canceled attach: %v, pending=%v", err, sb.PendingVolume) + _, err := m.SetVolumes(ctx, sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) + if !errors.Is(err, context.Canceled) || slices.Equal(sb.VolumesWant, sb.Volumes) { + t.Fatalf("canceled attach: %v, pending=%v", err, sb.VolumesWant) } eng.liveVolumeHook = nil if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil || volumeHoldersOf(m, "data") != (volumeHolders{}) { @@ -335,8 +357,8 @@ func TestLiveVolumeAdmissionAndState(t *testing.T) { want = ErrVolumeBusy } before := len(eng.volumeSpecs) - _, err := m.AttachVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) - if !errors.Is(err, want) || len(eng.volumeSpecs) != before || sb.PendingVolume != nil { + _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) + if !errors.Is(err, want) || len(eng.volumeSpecs) != before || !slices.Equal(sb.VolumesWant, sb.Volumes) { t.Fatalf("got %v want %v", err, want) } }) @@ -358,20 +380,20 @@ func TestLiveVolumesRespectOwningTenantAndRemoval(t *testing.T) { } cred := Cred{Token: sb.Token} for _, auth := range []Cred{cred, {Operator: true}} { - if _, err := m.AttachVolumes(t.Context(), sb.ID, auth, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumeUnavailable) { + if _, err := m.SetVolumes(t.Context(), sb.ID, auth, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumeUnavailable) { t.Fatalf("catalog ACL: %v", err) } } - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } if err := m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatal(err) } - if _, err := m.AttachVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { t.Fatalf("removed tenant: %v", err) } - if _, err := m.DetachVolumes(t.Context(), sb.ID, cred, []string{"data"}); err != nil { + if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}); err != nil { t.Fatalf("cleanup after tenant removal: %v", err) } } @@ -394,10 +416,442 @@ func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { if got == nil || !slices.Equal(got.Volumes, sb.Volumes) { t.Fatalf("journal volumes: %+v", got) } - if (got.PendingVolume == nil) != (sb.PendingVolume == nil) || got.PendingVolume != nil && *got.PendingVolume != *sb.PendingVolume { - t.Fatalf("journal intent: %+v, want %+v", got.PendingVolume, sb.PendingVolume) + if !slices.Equal(got.VolumesWant, sb.VolumesWant) { + t.Fatalf("journal target: %+v, want %+v", got.VolumesWant, sb.VolumesWant) } if _, err := os.Stat(m.store.path); err != nil { t.Fatal(err) } } + +func TestDesiredVolumesJournalMigration(t *testing.T) { + for _, tt := range []struct { + name, field string + want int + }{ + {"legacy", "", 1}, {"explicit empty", `,"volumes_want":[]`, 0}, {"desired", `,"volumes_want":[{"name":"other","mount":"/other"}]`, 1}, + } { + t.Run(tt.name, func(t *testing.T) { + store := newClaimStore(t.TempDir(), false) + raw := `{"sb_1":{"id":"sb_1","volumes":[{"name":"data","mount":"/data"}]` + tt.field + `}}` + if err := os.WriteFile(store.path, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + claims, err := store.load() + if err != nil { + t.Fatal(err) + } + sb := claims["sb_1"] + if len(sb.VolumesWant) != tt.want { + t.Fatalf("want=%v", sb.VolumesWant) + } + if tt.name == "desired" && sb.VolumesWant[0].Name != "other" { + t.Fatalf("target replaced by applied: %v", sb.VolumesWant) + } + if err = store.save(claims); err != nil { + t.Fatal(err) + } + rawBytes, err := os.ReadFile(store.path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(rawBytes), `"volumes_want":`) { + t.Fatalf("target presence lost: %s", rawBytes) + } + }) + } +} + +func TestFailedDetachReaddRepairsAppliedMountAfterRestart(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + target := []types.Volume{{Name: "data"}} + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, target); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("eject did not complete") + view, err := m.SetVolumes(t.Context(), sb.ID, cred, nil) + if err == nil || len(view.Volumes) != 1 || len(view.VolumesWant) != 0 || view.States[0].State != "detaching" || view.States[0].Error == "" { + t.Fatalf("detach state=%+v error=%v", view, err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if reconcileErr := m2.Reconcile(t.Context()); reconcileErr != nil { + t.Fatal(reconcileErr) + } + before := len(eng.volumeMounts) + eng.detachVolumeErr = nil + view, err = m2.SetVolumes(t.Context(), sb.ID, cred, target) + if err != nil || len(eng.volumeMounts) != before+1 || view.States[0].State != "attached" { + t.Fatalf("readd did not repair: %+v %v", view, err) + } + if volumeHoldersOf(m2, "data").readers != 1 { + t.Fatal("repair changed reservation count") + } +} + +func TestSupersedingTargetPrecleansUncertainAddBeforeJournaling(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + eng.mountVolumeErr = errors.New("mount interrupted") + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected attach failure") + } + eng.mountVolumeErr = nil + entered, resume := make(chan struct{}), make(chan struct{}) + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + close(entered) + <-resume + } + return nil + } + done := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other"}}); done <- err }() + <-entered + view, err := m.GetVolumes(sb.ID, cred) + if err != nil || len(view.VolumesWant) != 1 || view.VolumesWant[0].Name != "data" || volumeHoldersOf(m, "data").readers != 1 { + t.Errorf("forgot uncertain disk before detach: %+v %v", view, err) + } + assertLiveJournal(t, m, sb) + close(resume) + if err := <-done; err != nil { + t.Fatal(err) + } + if volumeHoldersOf(m, "data") != (volumeHolders{}) || volumeHoldersOf(m, "other").readers != 1 { + t.Fatal("superseding target leaked or lost holds") + } +} + +type gatedVolumeEngine struct { + *fakeEngine + entered chan struct{} + resume chan struct{} + calls atomic.Int32 +} + +func (e *gatedVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { + if e.calls.Add(1) == 1 { + close(e.entered) + <-e.resume + ctx = context.WithoutCancel(ctx) + } + return e.fakeEngine.DiskAttach(ctx, vmName, spec) +} + +func TestCanceledVolumeWriterOwnsHardwareUntilReturn(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = eng + ctx, cancel := context.WithCancel(t.Context()) + first := make(chan error, 1) + go func() { _, err := m.SetVolumes(ctx, sb.ID, cred, []types.Volume{{Name: "data"}}); first <- err }() + <-eng.entered + cancel() + waitingCtx, stopWaiting := context.WithTimeout(t.Context(), 25*time.Millisecond) + defer stopWaiting() + _, err := m.PatchVolumes(waitingCtx, sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + if !errors.Is(err, context.DeadlineExceeded) || eng.calls.Load() != 1 { + t.Fatalf("new writer overtook canceled device call: %v calls=%d", err, eng.calls.Load()) + } + close(eng.resume) + if firstErr := <-first; !errors.Is(firstErr, context.Canceled) { + t.Fatalf("canceled result=%v", firstErr) + } + if len(base.volumeMounts) != 0 { + t.Fatal("canceled writer issued another device call") + } + view, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + if err != nil || len(view.Volumes) != 2 { + t.Fatalf("retry lost prior desired state: %+v %v", view, err) + } +} + +func TestReleaseOwnsUncertainHoldsUntilVMRemoval(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = eng + base.removeErrFor = sb.VMName + changed := make(chan error, 1) + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + changed <- err + }() + <-eng.entered + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, cred) }() + select { + case err := <-released: + if err == nil { + t.Fatal("expected failed VM removal") + } + case <-time.After(time.Second): + close(eng.resume) + t.Fatal("release waited for volume writer") + } + if volumeHoldersOf(m, "data").readers != 1 { + t.Fatal("unconfirmed removal dropped hold") + } + close(eng.resume) + if err := <-changed; err == nil { + t.Fatal("removed claim accepted late device result") + } + if volumeHoldersOf(m, "data").readers != 1 { + t.Fatal("late writer freed teardown-owned hold") + } + claims, err := m.store.load() + if err != nil || claims[sb.ID] != nil { + t.Fatalf("late writer resurrected claim: %v %v", claims, err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("confirmed VM removal leaked hold") + } +} + +func TestConcurrentVolumePatchesUseLatestDesiredSet(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = eng + first := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + first <- err + }() + <-eng.entered + second := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + second <- err + }() + close(eng.resume) + if err := <-first; err != nil { + t.Fatal(err) + } + if err := <-second; err != nil { + t.Fatal(err) + } + view, err := m.GetVolumes(sb.ID, cred) + if err != nil || len(view.Volumes) != 2 || len(view.VolumesWant) != 2 { + t.Fatalf("lost concurrent patch: %+v %v", view, err) + } + for _, name := range []string{"data", "other"} { + if volumeHoldersOf(m, name).readers != 1 { + t.Fatal("duplicate hold for " + name) + } + } +} + +func TestSyncIntentFailureRetainsConservativeDesiredHold(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + m.store.sync = true + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { + t.Fatal(err) + } + view, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + if err == nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data").readers != 1 { + t.Fatalf("ambiguous write lost recovery state: %+v %v", view, err) + } + if err := os.Remove(m.store.path + ".tmp"); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, nil); err != nil { + t.Fatal(err) + } + if volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("cleanup leaked conservative hold") + } +} + +func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { + for _, action := range []string{"release", "reap"} { + for _, runFinished := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/finished=%t", action, runFinished), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + _, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + m.mu.Lock() + if reserveErr := m.reserveVolumes(transient); reserveErr != nil { + t.Fatal(reserveErr) + } + run.holds = append(run.holds, transient...) + if action == "reap" { + sb.Deadline = time.Now().Add(-time.Second) + } + m.mu.Unlock() + if mkdirErr := os.Mkdir(m.store.path+".tmp", 0o700); mkdirErr != nil { + t.Fatal(mkdirErr) + } + m.store.writeMu.Lock() + done := make(chan error, 1) + go func() { + if action == "reap" { + m.reapOnce(t.Context()) + done <- nil + } else { + done <- m.Release(t.Context(), sb.ID, cred) + } + }() + waitFor(t, func() bool { m.mu.Lock(); defer m.mu.Unlock(); return m.claimed[sb.ID] == nil }) + if runFinished { + m.finishVolumeRun(sb, run) + } + m.store.writeMu.Unlock() + err = <-done + if action == "release" && err == nil { + t.Fatal("release ignored commit failure") + } + m.mu.Lock() + restored := m.claimed[sb.ID] == sb + m.mu.Unlock() + if !restored { + t.Fatal("failed release did not restore claim") + } + if !runFinished { + m.finishVolumeRun(sb, run) + } + if volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("rollback orphaned transient admission hold") + } + m.store.writeMu.Lock() + err = os.Remove(m.store.path + ".tmp") + m.store.writeMu.Unlock() + if err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return !m.recommitting.Load() }) + if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if volumeHoldersOf(m, "data").readers != 1 { + t.Fatal("retry duplicated admission hold") + } + }) + } + } +} + +func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} + _, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + m.mu.Lock() + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + if err := m.reserveVolumes(transient); err != nil { + t.Fatal(err) + } + run.holds = append(run.holds, transient...) + m.mu.Unlock() + if _, err := m.Renew(t.Context(), sb.ID, cred, time.Minute, types.ExpireArchive); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("archive admitted during volume intent: %v", err) + } + m.finishVolumeRun(sb, run) + if _, err := m.Renew(t.Context(), sb.ID, cred, time.Minute, types.ExpireArchive); err != nil { + t.Fatalf("archive still blocked after abandoned intent: %v", err) + } +} + +type deadlineVolumeEngine struct { + *fakeEngine + deadlines chan time.Time +} + +func (e *deadlineVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { + deadline, _ := ctx.Deadline() + e.deadlines <- deadline + return e.fakeEngine.DiskAttach(ctx, vmName, spec) +} + +func TestLiveAttachBoundsWholeDeviceOperation(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := &deadlineVolumeEngine{fakeEngine: base, deadlines: make(chan time.Time, 1)} + m.eng = eng + start := time.Now() + if _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + deadline := <-eng.deadlines + if deadline.IsZero() || deadline.Before(start) || deadline.After(time.Now().Add(30*time.Second)) { + t.Fatalf("live attach deadline=%v", deadline) + } +} + +func TestLivenessRefusesAdmittedVolumeRun(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + _, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + m.mu.Lock() + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + if err := m.reserveVolumes(transient); err != nil { + t.Fatal(err) + } + run.holds = append(run.holds, transient...) + m.mu.Unlock() + killVMM(eng, sb.VMName) + m.recoverVMM(t.Context(), sb) + if len(eng.starts) != 0 || !strings.Contains(sb.Failed, "volume mounts") { + t.Fatalf("cold boot raced volume intent: starts=%v failed=%q", eng.starts, sb.Failed) + } + m.finishVolumeRun(sb, run) +} + +func TestRemovedTenantCanRemoveReadOnlyVolumeBesideImmutableVolume(t *testing.T) { + eng := newFakeEngine() + m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: liveCatalog(t)}, eng, testSecrets(t)) + if err != nil { + t.Fatal(err) + } + putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "other", Mode: "rw"}}}) + if err != nil { + t.Fatal(err) + } + cred := Cred{Token: sb.Token} + if _, err = m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatal(err) + } + if err = m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + view, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}) + if err != nil || len(view.Volumes) != 1 || view.Volumes[0].Name != "other" || !view.Volumes[0].RW() { + t.Fatalf("removed tenant cleanup=%+v %v", view, err) + } + if volumeHoldersOf(m, "data") != (volumeHolders{}) || volumeHoldersOf(m, "other").writers != 1 { + t.Fatal("cleanup altered immutable hold") + } +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index b3977daf..82f29cfa 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -60,7 +60,6 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, - {pool.ErrVolumePending, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, @@ -95,8 +94,9 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo - AttachVolumes(ctx context.Context, id string, cred pool.Cred, volumes []types.Volume) ([]types.Volume, error) - DetachVolumes(ctx context.Context, id string, cred pool.Cred, names []string) ([]types.Volume, error) + GetVolumes(id string, cred pool.Cred) (types.SandboxVolumesResponse, error) + SetVolumes(ctx context.Context, id string, cred pool.Cred, volumes []types.Volume) (types.SandboxVolumesResponse, error) + PatchVolumes(ctx context.Context, id string, cred pool.Cred, patch types.VolumePatch) (types.SandboxVolumesResponse, error) Sandboxes(tenant, claimRef string, metadata types.Metadata) []pool.SandboxSummary Sandbox(id string) (pool.SandboxSummary, bool) Stats(ctx context.Context, id string) (pool.SandboxStats, bool) @@ -265,8 +265,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) - mux.HandleFunc("POST /v1/sandboxes/{id}/volumes/attach", s.handleAttachVolumes) - mux.HandleFunc("POST /v1/sandboxes/{id}/volumes/detach", s.handleDetachVolumes) + mux.HandleFunc("GET /v1/sandboxes/{id}/volumes", s.handleSandboxVolumes) + mux.HandleFunc("PUT /v1/sandboxes/{id}/volumes", s.handleSandboxVolumes) + mux.HandleFunc("PATCH /v1/sandboxes/{id}/volumes", s.handleSandboxVolumes) mux.HandleFunc("POST /v1/sandboxes/{id}/renew", s.handleRenew) mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 5de295e2..048b05f1 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2602,8 +2602,9 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { - attachVolumes func(string, pool.Cred, []types.Volume) ([]types.Volume, error) - detachVolumes func(string, pool.Cred, []string) ([]types.Volume, error) + getVolumes func(string, pool.Cred) (types.SandboxVolumesResponse, error) + setVolumes func(string, pool.Cred, []types.Volume) (types.SandboxVolumesResponse, error) + patchVolumes func(string, pool.Cred, types.VolumePatch) (types.SandboxVolumesResponse, error) tenants []config.TenantSpec setTenants func(specs []config.TenantSpec) error putTenant func(spec config.TenantSpec) error @@ -2676,12 +2677,16 @@ type fakeManager struct { draining bool } -func (f *fakeManager) AttachVolumes(_ context.Context, id string, cred pool.Cred, volumes []types.Volume) ([]types.Volume, error) { - return f.attachVolumes(id, cred, volumes) +func (f *fakeManager) GetVolumes(id string, cred pool.Cred) (types.SandboxVolumesResponse, error) { + return f.getVolumes(id, cred) } -func (f *fakeManager) DetachVolumes(_ context.Context, id string, cred pool.Cred, names []string) ([]types.Volume, error) { - return f.detachVolumes(id, cred, names) +func (f *fakeManager) SetVolumes(_ context.Context, id string, cred pool.Cred, v []types.Volume) (types.SandboxVolumesResponse, error) { + return f.setVolumes(id, cred, v) +} + +func (f *fakeManager) PatchVolumes(_ context.Context, id string, cred pool.Cred, p types.VolumePatch) (types.SandboxVolumesResponse, error) { + return f.patchVolumes(id, cred, p) } func (f *fakeManager) ClaimWarm(ctx context.Context, key types.PoolKey, o pool.ClaimOptions) (*types.Sandbox, error) { diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index 76d1b600..e549e41f 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -1,39 +1,60 @@ package server import ( + "cmp" + "errors" "net/http" + "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/types" ) -func (s *Server) handleAttachVolumes(w http.ResponseWriter, r *http.Request) { +func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { return } - req, ok := decodeBodyStrict[types.AttachVolumesRequest](w, r) - if !ok { - return + id, cred := r.PathValue("id"), s.sandboxCred(token) + var view types.SandboxVolumesResponse + var err error + switch r.Method { + case http.MethodGet: + view, err = s.mgr.GetVolumes(id, cred) + case http.MethodPut: + req, ok := decodeBodyStrict[types.SetVolumesRequest](w, r) + if !ok { + return + } + if req.Volumes == nil { + writeErr(w, http.StatusBadRequest, "volumes must be an array") + return + } + view, err = s.mgr.SetVolumes(r.Context(), id, cred, req.Volumes) + case http.MethodPatch: + req, ok := decodeBodyStrict[types.VolumePatch](w, r) + if !ok { + return + } + view, err = s.mgr.PatchVolumes(r.Context(), id, cred, req) } - id := r.PathValue("id") - volumes, err := s.mgr.AttachVolumes(r.Context(), id, s.sandboxCred(token), req.Volumes) - writeResult(w, r, "attach volumes", id, "attach volumes failed; retry the request or detach the pending volume", err, func() { - writeJSON(w, http.StatusOK, types.SandboxVolumesResponse{Volumes: volumes}) - }) -} - -func (s *Server) handleDetachVolumes(w http.ResponseWriter, r *http.Request) { - token, ok := sandboxToken(w, r) - if !ok { - return - } - req, ok := decodeBodyStrict[types.DetachVolumesRequest](w, r) - if !ok { - return + code := http.StatusOK + if err != nil { + code, view.Error = http.StatusInternalServerError, "volume convergence failed; retry the desired set" + for _, rule := range poolErrHTTP { + if errors.Is(err, rule.err) { + code, view.Error = rule.code, cmp.Or(rule.msg, err.Error()) + break + } + } + if code == http.StatusInternalServerError { + log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) + } + for i := range view.States { + if view.States[i].Error != "" { + view.States[i].Error = view.Error + } + } } - id := r.PathValue("id") - volumes, err := s.mgr.DetachVolumes(r.Context(), id, s.sandboxCred(token), req.Names) - writeResult(w, r, "detach volumes", id, "detach volumes failed; retry the request", err, func() { - writeJSON(w, http.StatusOK, types.SandboxVolumesResponse{Volumes: volumes}) - }) + writeJSON(w, code, view) } diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index ea4ac6c0..2e32a61d 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -1,6 +1,7 @@ package server import ( + "encoding/json/v2" "errors" "io" "net/http" @@ -13,46 +14,55 @@ import ( ) func TestLiveVolumeRoutesAuthorizeAndDecode(t *testing.T) { - for _, verb := range []string{"attach", "detach"} { + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { for _, token := range []string{"sandbox-token", "root-token"} { - t.Run(verb+"/"+token, func(t *testing.T) { + t.Run(method+"/"+token, func(t *testing.T) { called := false check := func(id string, cred pool.Cred) { called = true if id != "sb_1" || cred.Operator != (token == "root-token") || !cred.Operator && cred.Token != token { - t.Errorf("wrong credentials: %s %+v", id, cred) + t.Errorf("credentials %s %+v", id, cred) } } + view := types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "data"}}, States: []types.VolumeState{{Name: "data", State: "pending"}}} mgr := &fakeManager{ - attachVolumes: func(id string, cred pool.Cred, v []types.Volume) ([]types.Volume, error) { + getVolumes: func(id string, cred pool.Cred) (types.SandboxVolumesResponse, error) { + check(id, cred) + return view, nil + }, + setVolumes: func(id string, cred pool.Cred, v []types.Volume) (types.SandboxVolumesResponse, error) { check(id, cred) if !slices.Equal(v, []types.Volume{{Name: "data"}}) { - t.Errorf("volumes: %v", v) + t.Errorf("volumes %v", v) } - return []types.Volume{{Name: "data", Mount: "/volumes/data"}}, nil + return view, nil }, - detachVolumes: func(id string, cred pool.Cred, names []string) ([]types.Volume, error) { + patchVolumes: func(id string, cred pool.Cred, p types.VolumePatch) (types.SandboxVolumesResponse, error) { check(id, cred) - if !slices.Equal(names, []string{"data"}) { - t.Errorf("names: %v", names) + if !slices.Equal(p.Remove, []string{"data"}) { + t.Errorf("patch %v", p) } - return nil, nil + return view, nil }, } ts := newTestServer(t, "root-token", mgr, nil) body := `{"volumes":[{"name":"data"}]}` - if verb == "detach" { - body = `{"names":["data"]}` + if method == http.MethodPatch { + body = `{"remove":["data"]}` } - req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/volumes/"+verb, strings.NewReader(body)) + req, _ := http.NewRequest(method, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(body)) req.Header.Set("Authorization", "Bearer "+token) resp, err := ts.Client().Do(req) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK || !called { - t.Fatalf("status=%d called=%v", resp.StatusCode, called) + var got types.SandboxVolumesResponse + if err := json.UnmarshalRead(resp.Body, &got); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK || !called || len(got.VolumesWant) != 1 || got.States[0].State != "pending" { + t.Fatalf("status=%d called=%v view=%+v", resp.StatusCode, called, got) } }) } @@ -65,19 +75,22 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { err error code int }{ - {"missing token", "", `{}`, nil, http.StatusUnauthorized}, + {"missing token", "", `{"volumes":[]}`, nil, http.StatusUnauthorized}, {"unknown field", "tok", `{"volumes":[],"path":"/host"}`, nil, http.StatusBadRequest}, - {"wrong sandbox", "tok", `{}`, pool.ErrUnknownSandbox, http.StatusNotFound}, - {"bad volume", "tok", `{}`, pool.ErrBadVolume, http.StatusBadRequest}, - {"pending", "tok", `{}`, pool.ErrVolumePending, http.StatusConflict}, - {"paused", "tok", `{}`, pool.ErrPaused, http.StatusConflict}, - {"failed", "tok", `{}`, pool.ErrFailed, http.StatusConflict}, - {"engine", "tok", `{}`, errors.New("host /private/image failed"), http.StatusInternalServerError}, + {"missing volumes", "tok", `{}`, nil, http.StatusBadRequest}, + {"null volumes", "tok", `{"volumes":null}`, nil, http.StatusBadRequest}, + {"wrong sandbox", "tok", `{"volumes":[]}`, pool.ErrUnknownSandbox, http.StatusNotFound}, + {"bad volume", "tok", `{"volumes":[]}`, pool.ErrBadVolume, http.StatusBadRequest}, + {"paused", "tok", `{"volumes":[]}`, pool.ErrPaused, http.StatusConflict}, + {"failed", "tok", `{"volumes":[]}`, pool.ErrFailed, http.StatusConflict}, + {"engine", "tok", `{"volumes":[]}`, errors.New("host /private/image failed"), http.StatusInternalServerError}, } { t.Run(tt.name, func(t *testing.T) { - mgr := &fakeManager{attachVolumes: func(string, pool.Cred, []types.Volume) ([]types.Volume, error) { return nil, tt.err }} + mgr := &fakeManager{setVolumes: func(string, pool.Cred, []types.Volume) (types.SandboxVolumesResponse, error) { + return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "data"}}, States: []types.VolumeState{{Name: "data", State: "pending", Error: "host /private/image failed"}}}, tt.err + }} ts := newTestServer(t, "root", mgr, nil) - req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/volumes/attach", strings.NewReader(tt.body)) + req, _ := http.NewRequest(http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(tt.body)) if tt.token != "" { req.Header.Set("Authorization", "Bearer "+tt.token) } @@ -90,6 +103,12 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { t.Fatalf("status=%d body=%s", resp.StatusCode, body) } + if tt.err != nil { + var got types.SandboxVolumesResponse + if err := json.Unmarshal(body, &got); err != nil || len(got.VolumesWant) != 1 || len(got.States) != 1 || got.Error == "" { + t.Fatalf("lost partial state: %s %v", body, err) + } + } }) } } diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 50068ed0..d253b441 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -77,19 +77,28 @@ type VolumeListResponse struct { Volumes []VolumeInfo `json:"volumes"` } -// AttachVolumesRequest adds read-only mounts to a running sandbox. -type AttachVolumesRequest struct { +// SetVolumesRequest replaces the desired set of volumes. +type SetVolumesRequest struct { Volumes []Volume `json:"volumes"` } -// DetachVolumesRequest removes read-only volumes by catalog name. -type DetachVolumesRequest struct { - Names []string `json:"names"` +type VolumePatch struct { + Add []Volume `json:"add,omitempty"` + Remove []string `json:"remove,omitempty"` } -// SandboxVolumesResponse reports the full committed set after a mutation. +type VolumeState struct { + Name string `json:"name"` + State string `json:"state"` + Error string `json:"error,omitempty"` +} + +// SandboxVolumesResponse retains the desired and applied sets after a failure. type SandboxVolumesResponse struct { - Volumes []Volume `json:"volumes"` + VolumesWant []Volume `json:"volumes_want"` + Volumes []Volume `json:"volumes"` + States []VolumeState `json:"states"` + Error string `json:"error,omitempty"` } // ForkRequest is the wire body of POST /v1/sandboxes/{id}/fork. diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index e99dc61a..755c312b 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -559,9 +559,8 @@ type Sandbox struct { OnExpire ExpireAction `json:"on_expire,omitempty"` // Volumes records the volumes successfully applied to this claim. Volumes []Volume `json:"volumes,omitempty"` - // PendingVolume is written before a live device change and cleared only - // after its result is committed. Retry it, detach its name, or release. - PendingVolume *VolumeMutation `json:"pending_volume,omitempty"` + // VolumesWant is the durable target, including additions not yet confirmed. + VolumesWant []Volume `json:"volumes_want"` // Env is replaced whole through SetEnv under the manager mutex; host-only values are never served back. Env Env `json:"env,omitempty"` @@ -600,20 +599,10 @@ type Sandbox struct { // live is the env as of the last SetEnv, read by the egress proxy without the manager mutex. live atomic.Pointer[envView] - // VolumeMu keeps release/reap and expiry-policy changes out of a live volume - // operation. Lock it before Transition, and never under the manager mutex. - VolumeMu sync.Mutex `json:"-"` // Transition serializes captures and live volume changes; lock it before (never under) the manager mutex. Transition sync.Mutex `json:"-"` } -// VolumeMutation is a recoverable, node-local live volume operation. -// Values are immutable once published; they carry no operator filesystem paths. -type VolumeMutation struct { - Volume Volume `json:"volume"` - Detach bool `json:"detach,omitzero"` -} - // Touch stamps last data-plane activity; lock-free, called on the relay hot path. func (s *Sandbox) Touch() { s.lastActivity.Store(time.Now().UnixNano()) } diff --git a/sdk/go/client.go b/sdk/go/client.go index ac6be60d..ccb932fd 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -12,6 +12,7 @@ import ( "io" "net/http" "net/url" + "slices" "strings" "sync" "time" @@ -250,7 +251,7 @@ func (c *Client) ownerAt(ctx context.Context, addr, id, token string) (string, e func (c *Client) handleFrom(dialed string, cr claimResponse) *Sandbox { return &Sandbox{ - ID: cr.ID, Deadline: cr.Deadline, Volumes: cr.Volumes, + ID: cr.ID, Deadline: cr.Deadline, Volumes: cr.Volumes, VolumesWant: slices.Clone(cr.Volumes), FromCheckpoint: cr.FromCheckpoint, TemplateDigest: cr.TemplateDigest, NetRoute: cr.NetRoute, c: c, token: cr.Token, owner: cmp.Or(cr.OwnerAddr, dialed), } diff --git a/sdk/go/info.go b/sdk/go/info.go index 79ea84c1..86879cd3 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -62,7 +62,7 @@ type SandboxSummary struct { Archived bool `json:"archived,omitzero"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []Volume `json:"volumes,omitempty"` - PendingVolume *VolumeMutation `json:"pending_volume,omitempty"` + VolumesWant []Volume `json:"volumes_want,omitempty"` ClaimRef string `json:"claim_ref,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` CPUCount int `json:"cpu_count,omitzero"` diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index fea6b3e1..a26fcd00 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -56,9 +56,10 @@ func (e *ExitError) Error() string { type Sandbox struct { ID string Deadline time.Time - // Volumes lists the volumes finalized on this claim; only a writable entry - // echoes a mode. + // Volumes lists confirmed applied mounts; only a writable entry echoes a mode. Volumes []Volume + // VolumesWant is the persisted desired set, which may differ after a failed convergence. + VolumesWant []Volume // TemplateDigest is the content identity of the promoted-template export // this sandbox was cloned from; empty for any other source. TemplateDigest string diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 96d705a6..c61af02b 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -1,66 +1,175 @@ package sandbox import ( + "context" + "encoding/json/v2" + "errors" "io" "net/http" "net/http/httptest" + "reflect" "slices" "testing" ) func TestLiveVolumesUseOwnerAndSandboxToken(t *testing.T) { - for _, verb := range []string{"attach", "detach"} { - t.Run(verb, func(t *testing.T) { + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { + t.Run(method, func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != "/v1/sandboxes/sb_1/volumes/"+verb || r.Header.Get("Authorization") != "Bearer sandbox-token" { + if r.Method != method || r.URL.Path != "/v1/sandboxes/sb_1/volumes" || r.Header.Get("Authorization") != "Bearer sandbox-token" { t.Errorf("request: %s %s %s", r.Method, r.URL.Path, r.Header.Get("Authorization")) } body, _ := io.ReadAll(r.Body) - want := `{"volumes":[{"name":"data"}]}` - if verb == "detach" { - want = `{"names":["data"]}` + want := "" + switch method { + case http.MethodPut: + want = `{"volumes":[{"name":"data"}]}` + case http.MethodPatch: + want = `{"add":[{"name":"data"}],"remove":["old"]}` } if string(body) != want { t.Errorf("body=%s want=%s", body, want) } - _, _ = io.WriteString(w, `{"volumes":[{"name":"remaining","mount":"/data"}]}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"states":[{"name":"data","state":"attached"}]}`) })) t.Cleanup(ts.Close) c := testClient(t, ts, WithAPIToken("operator-token")) sb := &Sandbox{ID: "sb_1", token: "sandbox-token", owner: c.addr, c: c} - var got []Volume + var got SandboxVolumesResponse var err error - if verb == "attach" { - got, err = sb.AttachVolumes(t.Context(), Volume{Name: "data"}) - } else { - got, err = sb.DetachVolumes(t.Context(), "data") + switch method { + case http.MethodGet: + got, err = sb.GetVolumes(t.Context()) + case http.MethodPut: + got, err = sb.SetVolumes(t.Context(), []Volume{{Name: "data"}}) + case http.MethodPatch: + got, err = sb.PatchVolumes(t.Context(), VolumePatch{Add: []Volume{{Name: "data"}}, Remove: []string{"old"}}) } - if err != nil || !slices.Equal(got, []Volume{{Name: "remaining", Mount: "/data"}}) { - t.Fatalf("got=%v err=%v", got, err) + if err != nil || !slices.Equal(got.Volumes, []Volume{{Name: "data", Mount: "/data"}}) || !slices.Equal(got.VolumesWant, got.Volumes) || got.States[0].State != "attached" { + t.Fatalf("got=%+v err=%v", got, err) } - got[0].Name = "caller copy" - if sb.Volumes[0].Name != "remaining" { + got.Volumes[0].Name, got.VolumesWant[0].Name = "caller copy", "caller copy" + if sb.Volumes[0].Name != "data" || sb.VolumesWant[0].Name != "data" { t.Fatal("returned volumes alias public state") } }) } } -func TestLiveVolumeFailurePreservesLastKnownSet(t *testing.T) { +func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusConflict) - _, _ = io.WriteString(w, `{"error":"pending"}`) + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mount":"/new"}],"volumes":[{"name":"old","mount":"/old"}],"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) - sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c, Volumes: []Volume{{Name: "old"}}} - if _, err := sb.AttachVolumes(t.Context(), Volume{Name: "new"}); err == nil { - t.Fatal("missing attach error") + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { + sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c} + var got SandboxVolumesResponse + var err error + switch method { + case http.MethodGet: + got, err = sb.GetVolumes(t.Context()) + case http.MethodPut: + got, err = sb.SetVolumes(t.Context(), []Volume{{Name: "new"}}) + case http.MethodPatch: + got, err = sb.PatchVolumes(t.Context(), VolumePatch{Remove: []string{"old"}}) + } + var failure *VolumeError + var api *APIError + if !errors.As(err, &failure) || !errors.As(err, &api) || api.Status != 500 || failure.Response == nil { + t.Fatalf("%s error lost response or APIError: %v", method, err) + } + if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" || sb.VolumesWant[0].Name != "new" || sb.Volumes[0].Name != "old" { + t.Fatalf("partial view=%+v handle=%+v", got, sb) + } + failure.Response.Volumes[0].Name = "error caller" + if got.Volumes[0].Name != "old" || sb.Volumes[0].Name != "old" { + t.Fatal("error response aliases returned or handle state") + } } - if _, err := sb.DetachVolumes(t.Context(), "old"); err == nil { - t.Fatal("missing detach error") +} + +func TestLiveVolumeValidationFailurePreservesLastKnownSet(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"error":"invalid volume"}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c, Volumes: []Volume{{Name: "old"}}, VolumesWant: []Volume{{Name: "wanted"}}} + _, err := sb.SetVolumes(t.Context(), []Volume{{Name: "new"}}) + var failure *VolumeError + if !errors.As(err, &failure) || failure.Response != nil || failure.Status != 400 { + t.Fatalf("error=%v", err) + } + if sb.Volumes[0].Name != "old" || sb.VolumesWant[0].Name != "wanted" { + t.Fatal("validation error replaced last known view") + } +} + +func TestLiveVolumesExplicitEmptyAndCancellation(t *testing.T) { + var bodies []map[string]any + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := json.UnmarshalRead(r.Body, &body); err != nil { + t.Error(err) + } + bodies = append(bodies, body) + _, _ = io.WriteString(w, `{"volumes_want":[],"volumes":[],"states":[]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", owner: c.addr, c: c} + if _, err := sb.SetVolumes(t.Context(), nil); err == nil { + t.Fatal("nil desired set was accepted") + } + if _, err := sb.SetVolumes(t.Context(), []Volume{}); err != nil { + t.Fatal(err) + } + if _, err := sb.PatchVolumes(t.Context(), VolumePatch{}); err != nil { + t.Fatal(err) + } + if _, err := sb.PatchVolumes(t.Context(), VolumePatch{Add: []Volume{}, Remove: []string{}}); err != nil { + t.Fatal(err) + } + if len(bodies) != 3 || !reflect.DeepEqual(bodies[0], map[string]any{"volumes": []any{}}) || len(bodies[1]) != 0 || !reflect.DeepEqual(bodies[2], map[string]any{"add": []any{}, "remove": []any{}}) { + t.Fatalf("requests=%+v", bodies) + } + if sb.Volumes == nil || sb.VolumesWant == nil { + t.Fatal("explicitly empty response became absent") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := sb.GetVolumes(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled context returned %v", err) + } +} + +func TestLiveVolumesRetrySameDesiredSet(t *testing.T) { + var requests []string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + requests = append(requests, string(body)) + if len(requests) == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[],"states":[{"name":"data","state":"pending"}],"error":"retry"}`) + return + } + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"states":[{"name":"data","state":"attached"}]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c} + target := []Volume{{Name: "data", Mount: "/data"}} + if _, err := sb.SetVolumes(t.Context(), target); err == nil || len(sb.Volumes) != 0 || !slices.Equal(sb.VolumesWant, target) { + t.Fatalf("failed convergence lost desired set: want=%v applied=%v error=%v", sb.VolumesWant, sb.Volumes, err) + } + view, err := sb.SetVolumes(t.Context(), target) + if err != nil || !slices.Equal(view.Volumes, target) || !slices.Equal(sb.Volumes, target) { + t.Fatalf("retry did not converge: %+v error=%v", view, err) } - if !slices.Equal(sb.Volumes, []Volume{{Name: "old"}}) { - t.Fatal("failed request replaced last known set") + if len(requests) != 2 || requests[0] != requests[1] { + t.Fatalf("retry changed target: %v", requests) } } diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index c99883e1..0a68f233 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -3,47 +3,110 @@ package sandbox import ( "bytes" "context" + "encoding/json/v2" + "errors" + "fmt" + "io" "net/http" "slices" ) -// VolumeMutation describes an interrupted operation in a SandboxSummary. -// Retry the attach, detach this name, or release the sandbox to recover. -type VolumeMutation struct { - Volume Volume `json:"volume"` - Detach bool `json:"detach,omitzero"` +// VolumeState reports the progress of one desired or applied volume. +type VolumeState struct { + Name string `json:"name"` + State string `json:"state"` // attached, pending, or detaching + Error string `json:"error,omitempty"` } -// AttachVolumes adds read-only catalog mounts to this running sandbox. -// Retry the same request after a partial failure, or detach the pending name. -// On success it replaces Volumes and returns a detached copy of the full set. -// Like Renew, callers must serialize updates to this handle's public fields. -func (s *Sandbox) AttachVolumes(ctx context.Context, volumes ...Volume) ([]Volume, error) { - return s.mutateVolumes(ctx, "attach", struct { - Volumes []Volume `json:"volumes"` - }{Volumes: volumes}) +// SandboxVolumesResponse separates persisted intent from confirmed applied mounts. +type SandboxVolumesResponse struct { + VolumesWant []Volume `json:"volumes_want"` + Volumes []Volume `json:"volumes"` + States []VolumeState `json:"states"` + Error string `json:"error,omitempty"` } -// DetachVolumes removes read-only volumes, including a pending attach. -// Already absent names are harmless. Batches may partially complete on error; -// retry the request to finish it. It replaces Volumes only on success. -func (s *Sandbox) DetachVolumes(ctx context.Context, names ...string) ([]Volume, error) { - return s.mutateVolumes(ctx, "detach", struct { - Names []string `json:"names"` - }{Names: names}) +// VolumePatch changes the desired set by name. Omitted slices are unchanged. +type VolumePatch struct { + Add []Volume `json:"add,omitzero"` + Remove []string `json:"remove,omitzero"` } -func (s *Sandbox) mutateVolumes(ctx context.Context, verb string, request any) ([]Volume, error) { - body, err := encodeBody(verb+" volumes", request) - if err != nil { - return nil, err +// VolumeError retains the server's view after a failed volume request. +// Response is nil if the server did not return a complete volume view. +type VolumeError struct { + *APIError + Response *SandboxVolumesResponse +} + +func (e *VolumeError) Unwrap() error { return e.APIError } + +// GetVolumes reads desired and applied volumes from the owning node. +// Like Renew, volume methods update public fields and must be serialized by callers. +func (s *Sandbox) GetVolumes(ctx context.Context) (SandboxVolumesResponse, error) { + return s.requestVolumes(ctx, http.MethodGet, nil, "get volumes") +} + +// SetVolumes replaces the full desired set and waits for convergence. +// An empty non-nil slice clears mutable mounts. A nil slice is rejected. +// Retry the same set after a partial failure, or set a new target to recover. +func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolumesResponse, error) { + if volumes == nil { + return SandboxVolumesResponse{}, errors.New("set volumes: use an explicit empty slice to clear volumes") } - response, err := s.c.doJSON[struct { + return s.requestVolumes(ctx, http.MethodPut, struct { Volumes []Volume `json:"volumes"` - }](ctx, http.MethodPost, s.owner, "/v1/sandboxes/"+s.ID+"/volumes/"+verb, bytes.NewReader(body), s.token, verb+" volumes") + }{Volumes: volumes}, "set volumes") +} + +// PatchVolumes adds or removes read-only mounted catalog entries by name. +// Existing writable and attach-only entries cannot be changed live. +func (s *Sandbox) PatchVolumes(ctx context.Context, patch VolumePatch) (SandboxVolumesResponse, error) { + return s.requestVolumes(ctx, http.MethodPatch, patch, "patch volumes") +} + +func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any, verb string) (SandboxVolumesResponse, error) { + var out SandboxVolumesResponse + var body io.Reader + if request != nil { + encoded, err := encodeBody(verb, request) + if err != nil { + return out, err + } + body = bytes.NewReader(encoded) + } + resp, err := s.c.roundTrip(ctx, method, s.owner, "/v1/sandboxes/"+s.ID+"/volumes", body, s.token) if err != nil { - return nil, err + return out, err + } + defer func() { _ = resp.Body.Close() }() + var wire struct { + VolumesWant *[]Volume `json:"volumes_want"` + Volumes *[]Volume `json:"volumes"` + States *[]VolumeState `json:"states"` + Error string `json:"error"` + } + decodeErr := json.UnmarshalRead(resp.Body, &wire) + complete := decodeErr == nil && wire.VolumesWant != nil && wire.Volumes != nil && wire.States != nil + if complete { + out = SandboxVolumesResponse{VolumesWant: *wire.VolumesWant, Volumes: *wire.Volumes, States: *wire.States, Error: wire.Error} + s.VolumesWant = slices.Clone(out.VolumesWant) + s.Volumes = slices.Clone(out.Volumes) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + failure := &VolumeError{APIError: &APIError{Verb: verb, Status: resp.StatusCode, Message: wire.Error}} + if complete { + view := out + view.VolumesWant, view.Volumes, view.States = slices.Clone(out.VolumesWant), slices.Clone(out.Volumes), slices.Clone(out.States) + failure.Response = &view + } + return out, failure + } + if decodeErr != nil { + return out, fmt.Errorf("decode %s response: %w", verb, decodeErr) + } + if !complete { + return out, fmt.Errorf("decode %s response: missing volume view", verb) } - s.Volumes = slices.Clone(response.Volumes) - return slices.Clone(s.Volumes), nil + return out, nil } diff --git a/sdk/python/cocoonsandbox/__init__.py b/sdk/python/cocoonsandbox/__init__.py index 0a14923e..7d95096e 100644 --- a/sdk/python/cocoonsandbox/__init__.py +++ b/sdk/python/cocoonsandbox/__init__.py @@ -14,6 +14,7 @@ from .errors import APIError, ExitError, ProtocolError, SandboxError, SandboxTimeout, SilkdError from .sandbox import Lsp, PortConn, Pty, Sandbox, Session, Watcher from .template import Template +from .volumes import SandboxVolumesResponse, Volume, VolumeError, VolumeState __all__ = [ "APIError", @@ -25,10 +26,14 @@ "ProtocolError", "Pty", "Sandbox", + "SandboxVolumesResponse", "SandboxError", "SandboxTimeout", "Session", "SilkdError", "Template", "Watcher", + "Volume", + "VolumeError", + "VolumeState", ] diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 6bc282c1..bc9934ec 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -17,6 +17,7 @@ from .endpoint import _endpoint_url from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox +from .volumes import volume_body T = TypeVar("T") @@ -211,8 +212,12 @@ def _request( if deadline is not None and time.monotonic() >= deadline: raise SandboxTimeout(f"{verb} timed out") from None raise APIError(verb, 0, str(exc)) from None - if status >= 400: - raise APIError(verb, status, _error_message(raw)) + if status < 200 or status >= 300: + try: + reply = json.loads(raw) + except (ValueError, TypeError): + reply = None + raise APIError(verb, status, _error_message(raw), reply=reply if isinstance(reply, dict) else None) if not raw: return {} try: @@ -291,7 +296,7 @@ def _claim_body( if ttl_seconds: claim["ttl_seconds"] = ttl_seconds if volumes: - claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] + claim["volumes"] = [volume_body(volume, mount) for volume in volumes] if not mount: claim["volumes_attach_only"] = True if claim_ref: @@ -299,29 +304,6 @@ def _claim_body( return claim -def _volume_body(volume: str | Mapping[str, str], mount: bool) -> dict[str, str]: - if isinstance(volume, str): - return {"name": volume} - if not isinstance(volume, Mapping): - raise TypeError("volume must be a name string or mapping") - unknown = sorted(set(volume) - {"name", "mount", "mode"}) - if unknown: - raise TypeError( - f"volume mapping accepts only name, mount, and mode, got unexpected key(s): {', '.join(unknown)}" - ) - if not mount and "mount" in volume: - raise TypeError("volume mount is meaningless with mount=False, which leaves mounting to the caller") - body = dict(volume) - mode = body.get("mode") - if mode in (None, "", "ro"): - body.pop("mode", None) - elif mode != "rw": - name = volume.get("name") - prefix = f"volume {name!r}: " if name is not None else "" - raise TypeError(f"{prefix}mode must be 'rw' or 'ro', got {mode!r}") - return body - - def _template_query(template: str, net: str, size: str) -> dict[str, str]: query = {"template": template} if net: diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index e81f1473..4d2159ea 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -1,5 +1,7 @@ """Error types raised by the SDK.""" +from __future__ import annotations + from typing import Any @@ -14,11 +16,12 @@ class SandboxTimeout(SandboxError, TimeoutError): class APIError(SandboxError): """A sandboxd control-plane call failed.""" - def __init__(self, verb: str, status: int, message: str) -> None: + def __init__(self, verb: str, status: int, message: str, *, reply: dict[str, Any] | None = None) -> None: super().__init__(f"{verb}: {message} (HTTP {status})") self.verb = verb self.status = status self.message = message + self.reply = reply class SilkdError(SandboxError): diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 35ba426f..c3c9bbc6 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -7,7 +7,7 @@ import threading import time from collections import deque -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, cast from .checkpoint import Checkpoint @@ -15,6 +15,7 @@ from .errors import APIError, ExitError, ProtocolError, SandboxError, SandboxTimeout, SilkdError, require_field from .frames import BULK_CHUNK, FS_CHUNK, KEEP_ALIVE_PROTO from .template import Template +from .volumes import SandboxVolumesResponse, Volume, VolumeError, volume_body, volume_response if TYPE_CHECKING: from .client import Client @@ -45,12 +46,72 @@ def __init__( self.from_checkpoint = from_checkpoint self.template_digest = template_digest self.volumes = [dict(volume) for volume in volumes or []] + self.volumes_want = [dict(volume) for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 def __enter__(self) -> Sandbox: return self + def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse: + """Read desired and applied volumes from the owning node.""" + return self._volumes_request("GET", None, "get volumes", timeout) + + def set_volumes( + self, volumes: Sequence[str | Mapping[str, str] | Volume], *, timeout: float | None = None + ) -> SandboxVolumesResponse: + """Replace the desired set and converge. An explicit empty list clears mutable mounts.""" + if volumes is None: + raise TypeError("set volumes requires an explicit list; use [] to clear volumes") + return self._volumes_request( + "PUT", {"volumes": [volume_body(volume, True) for volume in volumes]}, "set volumes", timeout + ) + + def patch_volumes( + self, + *, + add: Sequence[str | Mapping[str, str] | Volume] | None = None, + remove: Sequence[str] | None = None, + timeout: float | None = None, + ) -> SandboxVolumesResponse: + """Add or remove mounted read-only entries by name. Omitted arguments leave the set unchanged.""" + body: dict[str, Any] = {} + if add is not None: + body["add"] = [volume_body(volume, True) for volume in add] + if remove is not None: + body["remove"] = list(remove) + return self._volumes_request("PATCH", body, "patch volumes", timeout) + + def _volumes_request( + self, method: str, body: dict[str, Any] | None, verb: str, timeout: float | None + ) -> SandboxVolumesResponse: + try: + reply = self._client._request( + self.owner, + method, + f"/v1/sandboxes/{self.id}/volumes", + body, + verb, + bearer=self.token, + timeout=timeout or 0.0, + ) + except APIError as exc: + if exc.status == 0: + raise + view = volume_response(exc.reply) + if view is not None: + self._record_volumes(view) + raise VolumeError(exc, view) from exc + view = volume_response(reply) + if view is None: + raise APIError(verb, 0, "reply without complete volume view") + self._record_volumes(view) + return view + + def _record_volumes(self, view: SandboxVolumesResponse) -> None: + self.volumes_want = [cast(dict[str, str], volume.copy()) for volume in view["volumes_want"]] + self.volumes = [cast(dict[str, str], volume.copy()) for volume in view["volumes"]] + def __exit__(self, *exc: object) -> None: try: self.close() diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py new file mode 100644 index 00000000..bdee0783 --- /dev/null +++ b/sdk/python/cocoonsandbox/volumes.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Literal, TypedDict, cast + +from .errors import APIError + + +class _VolumeName(TypedDict): + name: str + + +class Volume(_VolumeName, total=False): + mount: str + mode: str + + +class _VolumeState(TypedDict): + name: str + state: Literal["attached", "pending", "detaching"] + + +class VolumeState(_VolumeState, total=False): + error: str + + +class _SandboxVolumesResponse(TypedDict): + volumes_want: list[Volume] + volumes: list[Volume] + states: list[VolumeState] + + +class SandboxVolumesResponse(_SandboxVolumesResponse, total=False): + error: str + + +class VolumeError(APIError): + """Failed convergence with the server's view, when one was returned.""" + + def __init__(self, error: APIError, response: SandboxVolumesResponse | None) -> None: + super().__init__(error.verb, error.status, error.message) + self.response = response + + +def volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | None: + if reply is None or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")): + return None + return cast(SandboxVolumesResponse, reply) + + +def volume_body(volume: str | Mapping[str, str] | Volume, mount: bool) -> dict[str, str]: + if isinstance(volume, str): + return {"name": volume} + if not isinstance(volume, Mapping): + raise TypeError("volume must be a name string or mapping") + unknown = sorted(set(volume) - {"name", "mount", "mode"}) + if unknown: + raise TypeError( + f"volume mapping accepts only name, mount, and mode, got unexpected key(s): {', '.join(unknown)}" + ) + if not mount and "mount" in volume: + raise TypeError("volume mount is meaningless with mount=False, which leaves mounting to the caller") + body = dict(cast(Mapping[str, str], volume)) + mode = body.get("mode") + if mode in (None, "", "ro"): + body.pop("mode", None) + elif mode != "rw": + name = volume.get("name") + prefix = f"volume {name!r}: " if name is not None else "" + raise TypeError(f"{prefix}mode must be 'rw' or 'ro', got {mode!r}") + return body diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index c8d1d892..171fca23 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -27,6 +27,12 @@ def do_GET(self): def do_DELETE(self): self._dispatch("DELETE") + def do_PUT(self): + self._dispatch("PUT") + + def do_PATCH(self): + self._dispatch("PATCH") + def log_message(self, *args): pass diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py new file mode 100644 index 00000000..786e820e --- /dev/null +++ b/sdk/python/tests/test_volumes.py @@ -0,0 +1,128 @@ +"""Live volume HTTP contract and partial convergence recovery.""" + +import pytest +from conftest import FakeNode, sandbox_at + +from cocoonsandbox import APIError, VolumeError + +PATH = "/v1/sandboxes/sb_1/volumes" +VIEW = { + "volumes_want": [{"name": "data", "mount": "/data"}], + "volumes": [{"name": "data", "mount": "/data"}], + "states": [{"name": "data", "state": "attached"}], +} + + +@pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) +def test_volume_requests_use_owner_and_sandbox_token(spawn_node, method): + seen = [] + + def reply(body, path): + seen.append(body) + assert FakeNode.last_headers["Authorization"] == "Bearer tok" + return 200, VIEW + + addr = spawn_node({(method, PATH): reply}) + sb = sandbox_at(addr, api_token="operator-token") + sb._client.addr = "unused.invalid:7777" + if method == "GET": + got = sb.get_volumes() + assert seen == [{}] + elif method == "PUT": + got = sb.set_volumes(["data"]) + assert seen == [{"volumes": [{"name": "data"}]}] + else: + got = sb.patch_volumes(add=["data"], remove=["old"]) + assert seen == [{"add": [{"name": "data"}], "remove": ["old"]}] + assert got == VIEW + assert sb.volumes == VIEW["volumes"] + assert sb.volumes_want == VIEW["volumes_want"] + got["volumes"][0]["name"] = "caller copy" + got["volumes_want"][0]["name"] = "caller copy" + assert sb.volumes[0]["name"] == sb.volumes_want[0]["name"] == "data" + + +@pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) +def test_volume_error_preserves_typed_partial_view(spawn_node, method): + partial = { + "volumes_want": [{"name": "new", "mount": "/new"}], + "volumes": [{"name": "old", "mount": "/old"}], + "states": [ + {"name": "new", "state": "pending", "error": "retry"}, + {"name": "old", "state": "detaching"}, + ], + "error": "convergence failed", + } + sb = sandbox_at(spawn_node({(method, PATH): lambda body, path: (500, partial)})) + with pytest.raises(VolumeError) as caught: + if method == "GET": + sb.get_volumes() + elif method == "PUT": + sb.set_volumes(["new"]) + else: + sb.patch_volumes(remove=["old"]) + assert isinstance(caught.value, APIError) + assert caught.value.status == 500 + assert caught.value.response == partial + assert sb.volumes_want == partial["volumes_want"] + assert sb.volumes == partial["volumes"] + caught.value.response["volumes"][0]["name"] = "caller copy" + assert sb.volumes[0]["name"] == "old" + + +def test_validation_error_preserves_last_view(spawn_node): + sb = sandbox_at(spawn_node({("PUT", PATH): lambda body, path: (400, {"error": "invalid volume"})})) + sb.volumes = [{"name": "old"}] + sb.volumes_want = [{"name": "wanted"}] + with pytest.raises(VolumeError) as caught: + sb.set_volumes(["new"]) + assert caught.value.response is None + assert sb.volumes == [{"name": "old"}] + assert sb.volumes_want == [{"name": "wanted"}] + + +def test_explicit_empty_set_and_omitted_patch(spawn_node): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": []} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + with pytest.raises(TypeError, match="explicit"): + sb.set_volumes(None) + sb.set_volumes([]) + sb.patch_volumes() + sb.patch_volumes(add=[], remove=[]) + assert seen == [{"volumes": []}, {}, {"add": [], "remove": []}] + assert sb.volumes == sb.volumes_want == [] + + +def test_identical_put_retries_partial_failure(spawn_node): + requests = [] + + def reply(body, path): + requests.append(body) + if len(requests) == 1: + return 500, { + "volumes_want": [{"name": "data", "mount": "/data"}], + "volumes": [], + "states": [{"name": "data", "state": "pending"}], + "error": "retry", + } + return 200, VIEW + + sb = sandbox_at(spawn_node({("PUT", PATH): reply})) + with pytest.raises(VolumeError): + sb.set_volumes([{"name": "data", "mount": "/data"}]) + assert sb.volumes == [] + got = sb.set_volumes([{"name": "data", "mount": "/data"}]) + assert got == VIEW + assert requests[0] == requests[1] + + +def test_volume_timeout_uses_client_deadline(black_hole): + sb = sandbox_at(black_hole, timeout=2) + with pytest.raises(APIError) as caught: + sb.get_volumes(timeout=0.02) + assert caught.value.status == 0 From 46ab32a597870ed52e9b5f9de750cb351369508f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:06:36 +0000 Subject: [PATCH 05/57] fix(engine): gate live volumes and classify mount conflicts --- sandboxd/engine/engine.go | 4 +- sandboxd/engine/installca.go | 11 ++++- sandboxd/engine/installca_test.go | 6 ++- sandboxd/engine/version_test.go | 55 ++++++++++++++++++++-- sandboxd/engine/volume_live.go | 72 ++++++++++++++++++++++++++--- sandboxd/engine/volume_live_test.go | 34 ++++++++++++-- 6 files changed, 164 insertions(+), 18 deletions(-) diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 7a5f5d68..d10b38e3 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -26,8 +26,8 @@ import ( ) const ( - // RequiredCocoon carries the snapshot/store baseline and the JSON form of an empty snapshot list. - RequiredCocoon = "v0.6.9" + // RequiredCocoon includes idempotent live disk attach and detach. + RequiredCocoon = "v0.7.2" StaleCreateCollected StaleCreateOutcome = "collected" StaleCreateBusy StaleCreateOutcome = "busy" diff --git a/sandboxd/engine/installca.go b/sandboxd/engine/installca.go index 1face152..761cf2dd 100644 --- a/sandboxd/engine/installca.go +++ b/sandboxd/engine/installca.go @@ -18,6 +18,15 @@ const ( guestExecPATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ) +type guestExitError struct { + Code int32 + Output string +} + +func (e *guestExitError) Error() string { + return fmt.Sprintf("exit code %d: %s", e.Code, e.Output) +} + // InstallCACert makes the guest trust the cluster root without update-ca-certificates. func (e *Engine) InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error { ctx, cancel := context.WithTimeout(ctx, cmdTimeout) @@ -88,7 +97,7 @@ func (e *Engine) silkdExec(ctx context.Context, vsockSocket string, argv ...stri out = appendCapped(out, resp.Data) case *wire.Exit: if resp.Code != 0 { - return fmt.Errorf("exit code %d: %s", resp.Code, bytes.TrimSpace(out)) + return &guestExitError{Code: resp.Code, Output: string(bytes.TrimSpace(out))} } return nil case *wire.ErrorResp: diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 8fb4a85c..2bbd2bd6 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -82,6 +82,7 @@ type fakeSilkd struct { writeErr string writeKind string execErr string + execDrop bool block []wire.DirEntry serial map[string][]byte readMisses map[string]int @@ -194,7 +195,7 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri f.execArgv = argv f.execCalls = append(f.execCalls, argv) f.execEnv = env - code, reply := f.execCode, f.execErr + code, reply, disconnect := f.execCode, f.execErr, f.execDrop if f.execFailAt > 0 && len(f.execCalls) != f.execFailAt { code, reply = 0, "" } @@ -202,6 +203,9 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri f.execHook(argv) } f.mu.Unlock() + if disconnect { + return + } if reply != "" { _, _ = io.WriteString(conn, `{"type":"error","kind":"internal","message":"`+reply+`"}`+"\n") return diff --git a/sandboxd/engine/version_test.go b/sandboxd/engine/version_test.go index b71b03bd..2345277e 100644 --- a/sandboxd/engine/version_test.go +++ b/sandboxd/engine/version_test.go @@ -1,6 +1,11 @@ package engine -import "testing" +import ( + "errors" + "fmt" + "strings" + "testing" +) func TestBelowFloor(t *testing.T) { tests := []struct { @@ -8,9 +13,9 @@ func TestBelowFloor(t *testing.T) { below bool comparable bool }{ - {"v0.6.9", false, true}, - {"v0.6.10", false, true}, - {"v0.7.0", false, true}, + {"v0.6.9", true, true}, + {"v0.6.10", true, true}, + {"v0.7.0", true, true}, {"v0.6.8", true, true}, {"v0.5.2", true, true}, {"v0.4.9", true, true}, @@ -29,3 +34,45 @@ func TestBelowFloor(t *testing.T) { }) } } + +func TestCheckLiveVolumeSupport(t *testing.T) { + for _, tt := range []struct { + version string + fail, allowed bool + }{ + {"v0.7.1", false, false}, + {"v0.7.2", false, true}, + {"v0.8.0", false, true}, + {"v1.0.0", false, true}, + {"v0.7.2-rc.1", false, false}, + {"master-abcdef", false, false}, + {"unknown", false, false}, + {"", false, false}, + {"v0.7.2", true, false}, + } { + t.Run(fmt.Sprintf("%s/fail=%v", tt.version, tt.fail), func(t *testing.T) { + t.Setenv("COCOON_TEST_VERSION", tt.version) + if tt.fail { + t.Setenv("COCOON_TEST_VERSION_FAIL", "1") + } else { + t.Setenv("COCOON_TEST_VERSION_FAIL", "") + } + fakeCocoon(t, `#!/bin/sh +if [ "$#" != 1 ] || [ "$1" != version ]; then exit 2; fi +if [ -n "$COCOON_TEST_VERSION_FAIL" ]; then echo /private/cocoon >&2; exit 1; fi +printf 'Version: %s\n' "$COCOON_TEST_VERSION" +`) + err := New("cocoon", nil, nil, false, false, "").CheckLiveVolumeSupport(t.Context()) + if tt.allowed { + if err != nil { + t.Fatal(err) + } + return + } + var unsupported *LiveVolumeUnsupported + if !errors.As(err, &unsupported) || !strings.Contains(err.Error(), RequiredCocoon) || strings.Contains(err.Error(), "/private") { + t.Fatalf("error %v", err) + } + }) + } +} diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index f011266d..0c05ddaa 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -2,6 +2,7 @@ package engine import ( "context" + "errors" "fmt" "path/filepath" "slices" @@ -11,8 +12,64 @@ import ( const VolumeMutationTimeout = 30 * time.Second -// DetachVolume refuses busy or foreign mounts and confirms host removal even -// when the guest serial has disappeared. It never uses lazy or forced umount. +// VolumeMountConflictKind identifies the guest condition that prevents a volume change. +type VolumeMountConflictKind string + +const ( + VolumeMountForeign VolumeMountConflictKind = "foreign" + VolumeMountBind VolumeMountConflictKind = "bind" + VolumeMountUnmountRefused VolumeMountConflictKind = "unmount-refused" +) + +// VolumeMountConflict reports a guest mount that must be resolved before retrying. +type VolumeMountConflict struct { + Kind VolumeMountConflictKind + Mount string +} + +func (e *VolumeMountConflict) Error() string { + switch e.Kind { + case VolumeMountForeign: + return fmt.Sprintf("volume mount %q has a different device, root, or mode; restore the expected mount before retrying", e.Mount) + case VolumeMountBind: + return fmt.Sprintf("volume device has another mount at %q; remove guest mount references before retrying", e.Mount) + default: + return fmt.Sprintf("cannot unmount %q; close guest references and retry", e.Mount) + } +} + +// LiveVolumeUnsupported reports a runtime that cannot safely change live volumes. +type LiveVolumeUnsupported struct { + Version string + Err error +} + +func (e *LiveVolumeUnsupported) Error() string { + if e.Version != "" { + return fmt.Sprintf("live volume changes require Cocoon >= %s; installed version is %s", RequiredCocoon, e.Version) + } + return "live volume changes require a verified Cocoon release >= " + RequiredCocoon + "; runtime version unavailable or unknown" +} + +func (e *LiveVolumeUnsupported) Unwrap() error { return e.Err } + +// CheckLiveVolumeSupport rejects runtimes without confirmed live disk semantics. +func (e *Engine) CheckLiveVolumeSupport(ctx context.Context) error { + version, err := e.Version(ctx) + if err != nil { + return &LiveVolumeUnsupported{Err: err} + } + below, known := belowFloor(version) + if !known { + return &LiveVolumeUnsupported{} + } + if below { + return &LiveVolumeUnsupported{Version: version} + } + return nil +} + +// DetachVolume unmounts a guest volume before confirming host removal. func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount string) error { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() @@ -36,6 +93,9 @@ func (e *Engine) unmountLiveVolume(ctx context.Context, sock, device, mount stri } if mounted { if err = e.UnmountVolume(ctx, sock, mount); err != nil { + if _, ok := errors.AsType[*guestExitError](err); ok { + return &VolumeMountConflict{Kind: VolumeMountUnmountRefused, Mount: mount} + } return err } } @@ -60,7 +120,7 @@ func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { fields := strings.Fields(line) if len(fields) < 10 || !slices.Contains(fields[6:], "-") { - return "", nil, fmt.Errorf("invalid guest mountinfo") + return "", nil, errors.New("invalid guest mountinfo") } mounts = append(mounts, volumeMount{fields[2], unescapeMount(fields[3]), unescapeMount(fields[4]), fields[5]}) } @@ -80,12 +140,12 @@ func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string, for _, entry := range mounts { if entry.path != mount { if entry.device == dev { - return false, fmt.Errorf("volume device is already mounted at %q", entry.path) + return false, &VolumeMountConflict{Kind: VolumeMountBind, Mount: entry.path} } continue } if matched || entry.device != dev || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), mode) { - return false, fmt.Errorf("volume mount %q has a different device or mode", mount) + return false, &VolumeMountConflict{Kind: VolumeMountForeign, Mount: mount} } matched = true } @@ -99,7 +159,7 @@ func (e *Engine) volumeUnmounted(ctx context.Context, sock, device string) error } for _, mount := range mounts { if mount.device == dev { - return fmt.Errorf("volume device is still mounted at %q", mount.path) + return &VolumeMountConflict{Kind: VolumeMountBind, Mount: mount.path} } } return nil diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index ff23abb5..ad3028fe 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -1,6 +1,7 @@ package engine import ( + "errors" "os" "path/filepath" "strings" @@ -14,7 +15,6 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { fake := serveFakeSilkd(t, sock) configureVolumeDevices(fake) spec := liveDiskCLI(t, "") - // Start without a host attachment, then prove the second call reuses it. if err := os.Remove(os.Getenv("COCOON_TEST_ATTACHED")); err != nil { t.Fatal(err) } @@ -69,9 +69,17 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + row) fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, false, "").MountVolume(t.Context(), sock, spec.Name, "/data", false); err == nil { + err := New("cocoon", nil, nil, false, false, "").MountVolume(t.Context(), sock, spec.Name, "/data", false) + if err == nil { t.Fatal("accepted a foreign mount") } + var conflict *VolumeMountConflict + if errors.As(err, &conflict) != (row != "malformed\n") { + t.Fatalf("conflict type: %v", err) + } + if conflict != nil && (conflict.Kind != VolumeMountForeign || conflict.Mount != "/data") { + t.Fatalf("conflict: %+v", conflict) + } fake.mu.Lock() defer fake.mu.Unlock() if len(fake.execCalls) != 0 { @@ -82,7 +90,7 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { } func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { - for _, mode := range []string{"busy", "second mount", "probe error", "host error"} { + for _, mode := range []string{"busy", "second mount", "probe error", "guest error", "transport disconnect", "host error"} { t.Run(mode, func(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -101,13 +109,31 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { fake.execCode = 32 case "second mount": fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /other ro - ext4 /dev/vdc ro\n") + case "transport disconnect": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + fake.execDrop = true + case "guest error": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + fake.execErr = "guest connection failed /private/runtime" case "probe error": fake.readErr["/proc/self/mountinfo"] = "read failed" } fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err == nil { + err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + if err == nil { t.Fatal("detach must fail closed") } + var conflict *VolumeMountConflict + conflictExpected := mode == "busy" || mode == "second mount" + if errors.As(err, &conflict) != conflictExpected { + t.Fatalf("conflict=%v error=%v", conflictExpected, err) + } + if conflictExpected && (!strings.Contains(conflict.Error(), "/data") && !strings.Contains(conflict.Error(), "/other") || strings.Contains(conflict.Error(), "/private")) { + t.Fatalf("unsafe or missing guest path: %v", conflict) + } + if mode == "busy" && conflict.Kind != VolumeMountUnmountRefused { + t.Fatalf("kind=%s", conflict.Kind) + } }) } } From af1939779bb90925cfeee50a20afb1e461ea8ece Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:16:19 +0000 Subject: [PATCH 06/57] fix(pool): preserve live volume state and require tenant API scope --- sandboxd/pool/pool.go | 2 + sandboxd/pool/pool_test.go | 2 + sandboxd/pool/refill.go | 4 + sandboxd/pool/volume_convergence_test.go | 318 +++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 209 +++++++++------ sandboxd/pool/volume_mutation_test.go | 139 +++++----- sandboxd/server/server.go | 12 +- sandboxd/server/server_test.go | 18 +- sandboxd/server/volume_mutation.go | 37 +-- sandboxd/server/volume_mutation_test.go | 132 ++++++++-- sandboxd/types/api.go | 15 +- 11 files changed, 681 insertions(+), 207 deletions(-) create mode 100644 sandboxd/pool/volume_convergence_test.go diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index de7ae63a..ec46fc06 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -139,6 +139,7 @@ type Engine interface { MarkLane(ctx context.Context, vsockSocket string, lane engine.Lane) error Warmup(ctx context.Context, vsockSocket string, argv []string) error TrimCow(ctx context.Context, vsockSocket string) error + CheckLiveVolumeSupport(ctx context.Context) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error DetachVolume(ctx context.Context, vmName, sock, name, mount string) error MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error @@ -297,6 +298,7 @@ type Manager struct { // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]volumeDiagnostics poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 141b79dc..69714460 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1656,6 +1656,8 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { return f.trimErr } +func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } + func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("attach"); err != nil { diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 99dc23b7..d4d199bc 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -392,6 +392,10 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { // exportSource captures a claimed sandbox into exportDir and returns its snapshot, VM name and guest-env presence. func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir string) (snap, vmName string, guestEnv bool, err error) { sb.Transition.Lock() + if captureErr := m.captureAllowed(sb); captureErr != nil { + sb.Transition.Unlock() + return "", "", false, captureErr + } m.trimForCapture(ctx, sb) snap, cleanup, err := m.sourceSnap(ctx, sb) if err != nil { diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go new file mode 100644 index 00000000..07abfb8a --- /dev/null +++ b/sandboxd/pool/volume_convergence_test.go @@ -0,0 +1,318 @@ +package pool + +import ( + "context" + "errors" + "os" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + delete(m.volumes, "data") + before := len(eng.volumeSpecs) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { + t.Fatal(err) + } + if len(eng.volumeSpecs) != before+1 || eng.volumeSpecs[before].Name != "other" { + t.Fatalf("retained volume reapplied: %v", eng.volumeSpecs) + } + before = len(eng.volumeSpecs) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { + t.Fatal(err) + } + if len(eng.volumeSpecs) != before { + t.Fatal("identical target reached devices") + } +} + +func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { + eng := newFakeEngine() + m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: liveCatalog(t)}, eng, testSecrets(t)) + if err != nil { + t.Fatal(err) + } + putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme"}) + if err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { + t.Fatal(err) + } + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"other"}}); err != nil { + t.Fatalf("remove other while retaining data: %v", err) + } + eng.detachVolumeErr = errors.New("detach result unknown") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected detaching state") + } + eng.detachVolumeErr = nil + entry := m.volumes["data"] + entry.tenants = []string{"other-tenant"} + m.volumes["data"] = entry + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatalf("held repair checked new authorization: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("new admission after tenant removal: %v", err) + } +} + +func TestReaddedDetachIsPendingAtTheDurableIntentBoundary(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + target := []types.Volume{{Name: "data"}} + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, target); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("guest unmounted; eject outcome unknown") + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { + t.Fatal("expected detach failure") + } + ctx, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + target = slices.Clone(sb.Volumes) + if checkErr := m.prepareVolumeTarget(ctx, sb, run, target, nil, nil); checkErr != nil { + t.Fatal(checkErr) + } + m.finishVolumeRun(sb, run) + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if checkErr := m2.Reconcile(t.Context()); checkErr != nil { + t.Fatal(checkErr) + } + view, err := m2.GetVolumes(sb.ID, sb.Tenant) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 1 || view.States[0].State != types.VolumePending || view.Converging { + t.Fatalf("reloaded reversal: %+v %v", view, err) + } + if !strings.Contains(view.States[0].Error, "interrupted") { + t.Fatalf("reloaded diagnostics: %+v", view) + } + eng.detachVolumeErr = nil + before := len(eng.volumeSpecs) + if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, target); err != nil { + t.Fatal(err) + } + if len(eng.volumeSpecs) != before+1 { + t.Fatal("readded volume was not repaired") + } +} + +func TestReleaseBeforeReservationDoesNotLeakAdmission(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + ctx, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + if err := m.checkVolumeRun(ctx, sb, run); err != nil { + t.Fatal(err) + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + target := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + if err := m.prepareVolumeTarget(ctx, sb, run, target, target, nil); err == nil { + t.Fatal("released claim admitted volume") + } + m.finishVolumeRun(sb, run) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); err != nil { + t.Fatalf("canceled request leaked hold: %v", err) + } +} + +func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = eng + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); done <- err }() + <-eng.entered + cancel() + view, err := m.GetVolumes(sb.ID, sb.Tenant) + if err != nil || !view.Converging { + t.Fatalf("canceled worker still owns hardware: %+v %v", view, err) + } + close(eng.resume) + if checkErr := <-done; !errors.Is(checkErr, context.Canceled) { + t.Fatal(checkErr) + } + view, err = m.GetVolumes(sb.ID, sb.Tenant) + if err != nil || view.Converging || view.Error == "" || view.States[0].Error == "" { + t.Fatalf("completed worker diagnostics: %+v %v", view, err) + } + view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) + if err != nil || view.Error != "" || view.States[0].Error != "" || view.Converging { + t.Fatalf("successful retry retained errors: %+v %v", view, err) + } + conflict := &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} + base.detachVolumeErr = conflict + view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}) + if !errors.Is(err, conflict) || view.States[0].Error != conflict.Error() { + t.Fatalf("conflict diagnostics: %+v %v", view, err) + } + got, err := m.GetVolumes(sb.ID, sb.Tenant) + if err != nil || got.States[0].Error != view.States[0].Error { + t.Fatalf("GET lost last conflict: %+v %v", got, err) + } +} + +func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + m.eng = &unsupportedVolumeEngine{eng} + before, err := os.ReadFile(m.store.path) + if err != nil { + t.Fatal(err) + } + for _, patch := range []bool{false, true} { + if patch { + _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + } else { + _, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) + } + if _, ok := errors.AsType[*engine.LiveVolumeUnsupported](err); !ok { + t.Fatalf("runtime gate: %v", err) + } + } + after, err := os.ReadFile(m.store.path) + if err != nil { + t.Fatal(err) + } + if string(before) != string(after) || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("unsupported runtime changed target, hold, or device") + } +} + +func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { + for _, state := range []string{"applied", "desired", "admitted"} { + t.Run(state, func(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}) + sb := mustClaim(t, m, testKey) + held := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + switch state { + case "applied": + sb.Volumes = held + case "desired": + sb.VolumesWant = held + case "admitted": + _, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + m.mu.Lock() + if err := m.reserveVolumes(held); err != nil { + t.Fatal(err) + } + run.holds = held + m.mu.Unlock() + defer m.finishVolumeRun(sb, run) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "blocked", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("checkpoint: %v", err) + } + if _, _, err := m.Promote(t.Context(), sb.ID, Cred{Token: sb.Token}, "blocked", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("promote: %v", err) + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("hibernate: %v", err) + } + if len(eng.trims) != 0 { + t.Fatalf("capture trimmed held volume: %v", eng.trims) + } + }) + } +} + +func TestUncertainTargetCommitReleasesOnlyConfirmedPrecleanedHolds(t *testing.T) { + for _, durable := range []string{"old", "new"} { + t.Run(durable, func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + eng.mountVolumeErr = errors.New("mount result unknown") + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected pending volume") + } + eng.mountVolumeErr = nil + m.store.sync = true + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { + t.Fatal(err) + } + } + return nil + } + target := []types.Volume{{Name: "other", Mount: "/volumes/data"}} + view, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, target) + if err == nil || len(view.VolumesWant) != 1 || view.VolumesWant[0].Name != "other" || len(view.Volumes) != 0 { + t.Fatalf("uncertain target evidence: %+v %v", view, err) + } + if got := volumeHoldersOf(m, "other"); got.readers != 1 { + t.Fatalf("uncertain target lost new hold: %+v", got) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("confirmed preclean retained hold: %+v", got) + } + if !slices.Contains(eng.volumeOpsLog(), "detach:data") || len(eng.volumeSpecs) != 1 { + t.Fatalf("device calls before uncertain target: attaches=%v detaches=%v", eng.volumeSpecs, eng.volumeOpsLog()) + } + if checkErr := os.Remove(m.store.path + ".tmp"); checkErr != nil { + t.Fatal(checkErr) + } + eng.liveVolumeHook = nil + if durable == "new" { + if checkErr := m.store.commit(m.store.mark()); checkErr != nil { + t.Fatal(checkErr) + } + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if checkErr := m2.Reconcile(t.Context()); checkErr != nil { + t.Fatal(checkErr) + } + view, err = m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, target) + if err != nil || len(view.VolumesWant) != 1 || view.VolumesWant[0].Name != "other" { + t.Fatalf("retry did not accept target: %+v %v", view, err) + } + if got := volumeHoldersOf(m2, "data"); got != (volumeHolders{}) { + t.Fatalf("successful retry retained discarded hold: %+v", got) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); err != nil { + t.Fatalf("writer after preclean and recovery: %v", err) + } + }) + } +} + +type unsupportedVolumeEngine struct{ *fakeEngine } + +func (e *unsupportedVolumeEngine) CheckLiveVolumeSupport(context.Context) error { + return &engine.LiveVolumeUnsupported{} +} diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index cf16c365..ab6523f4 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -2,39 +2,45 @@ package pool import ( "context" + "errors" "fmt" "slices" "github.com/cocoonstack/sandbox/sandboxd/engine" - "github.com/cocoonstack/sandbox/sandboxd/types" ) +var errVolumeTargetNotAccepted = errors.New("new volume target not accepted") + +type volumeEdit func([]types.Volume) ([]types.Volume, error) + +type volumeDiagnostics struct { + error string + volumes map[string]string +} + type volumeRun struct { cancel context.CancelFunc done chan struct{} holds []types.Volume } -func (m *Manager) GetVolumes(id string, cred Cred) (types.SandboxVolumesResponse, error) { - sb, ok := m.resolve(id, cred) - if !ok { - return types.SandboxVolumesResponse{}, ErrUnknownSandbox - } +func (m *Manager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { m.mu.Lock() defer m.mu.Unlock() - if m.claimed[id] != sb { + sb := m.claimed[id] + if sb == nil || !tenantOwns(tenant, sb.Tenant) { return types.SandboxVolumesResponse{}, ErrUnknownSandbox } - return volumeView(sb), nil + return m.volumeView(sb), nil } -func (m *Manager) SetVolumes(ctx context.Context, id string, cred Cred, want []types.Volume) (types.SandboxVolumesResponse, error) { - return m.changeVolumes(ctx, id, cred, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) +func (m *Manager) SetVolumes(ctx context.Context, id, tenant string, want []types.Volume) (types.SandboxVolumesResponse, error) { + return m.changeVolumes(ctx, id, tenant, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) } -func (m *Manager) PatchVolumes(ctx context.Context, id string, cred Cred, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { - return m.changeVolumes(ctx, id, cred, func(current []types.Volume) ([]types.Volume, error) { +func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { + return m.changeVolumes(ctx, id, tenant, func(current []types.Volume) ([]types.Volume, error) { if err := validateDetachNames(patch.Remove); err != nil { return nil, err } @@ -61,27 +67,24 @@ func (m *Manager) PatchVolumes(ctx context.Context, id string, cred Cred, patch }) } -func (m *Manager) changeVolumes(ctx context.Context, id string, cred Cred, change func([]types.Volume) ([]types.Volume, error)) (view types.SandboxVolumesResponse, err error) { - sb, ok := m.resolve(id, cred) - if !ok { +func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change volumeEdit) (view types.SandboxVolumesResponse, err error) { + sb, ok := m.byID(id) + if !ok || !tenantOwns(tenant, sb.Tenant) { return view, ErrUnknownSandbox } + if err = m.eng.CheckLiveVolumeSupport(ctx); err != nil { + view, _ = m.GetVolumes(id, tenant) + return view, err + } ctx, run, err := m.beginVolumeRun(ctx, sb) if err != nil { - view, _ = m.GetVolumes(id, cred) + view, _ = m.GetVolumes(id, tenant) return view, err } failedName := "" defer func() { + m.recordVolumeError(sb, run, failedName, err) view = m.finishVolumeRun(sb, run) - if err != nil { - view.Error = err.Error() - for i := range view.States { - if view.States[i].Name == failedName { - view.States[i].Error = err.Error() - } - } - } }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) @@ -98,13 +101,12 @@ func (m *Manager) changeVolumes(ctx context.Context, id string, cred Cred, chang return view, err } additions := volumeDifference(target, held) - mutable := slices.DeleteFunc(slices.Clone(target), immutableVolume) - if len(mutable) != 0 { + if len(additions) != 0 { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err } } - resolved, err := m.resolveVolumes(ctx, sb.Tenant, mutable) + resolved, err := m.resolveVolumes(ctx, sb.Tenant, additions) if err != nil { return view, err } @@ -117,6 +119,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id string, cred Cred, chang for _, v := range uncertain { failedName = v.Name if err = m.detachVolumeRun(ctx, sb, run, v); err != nil { + err = fmt.Errorf("%w: %w", errVolumeTargetNotAccepted, err) return view, err } } @@ -137,14 +140,23 @@ func (m *Manager) changeVolumes(ctx context.Context, id string, cred Cred, chang if err = m.commitVolumeApplied(ctx, sb, run, next); err != nil { return view, err } + m.clearVolumeError(sb, run, v.Name) } - for _, v := range resolved { - failedName = v.applied.Name + m.mu.Lock() + pending := volumeDifference(sb.VolumesWant, sb.Volumes) + m.mu.Unlock() + for _, v := range pending { + failedName = v.Name if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } + // Held entries retain access even after their tenant or ACL changes. + repair, resolveErr := m.resolveVolumes(ctx, "", []types.Volume{v}) + if resolveErr != nil { + return view, resolveErr + } applyCtx, cancel := context.WithTimeout(ctx, engine.VolumeMutationTimeout) - err = m.applyVolume(applyCtx, sb, v) + err = m.applyVolume(applyCtx, sb, repair[0]) cancel() if err != nil { return view, err @@ -153,14 +165,12 @@ func (m *Manager) changeVolumes(ctx context.Context, id string, cred Cred, chang return view, err } m.mu.Lock() - next := slices.Clone(sb.Volumes) - if volumeIndex(next, v.applied.Name) < 0 { - next = append(next, v.applied) - } + next := append(slices.Clone(sb.Volumes), v) m.mu.Unlock() if err = m.commitVolumeApplied(ctx, sb, run, next); err != nil { return view, err } + m.clearVolumeError(sb, run, v.Name) } sb.Touch() return view, nil @@ -202,8 +212,8 @@ func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.Sandb if m.claimed[sb.ID] == sb { m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) } - view := volumeView(sb) delete(m.volumeRuns, sb) + view := m.volumeView(sb) close(run.done) m.mu.Unlock() run.cancel() @@ -217,11 +227,15 @@ func (m *Manager) restoreVolumeHolds(sb *types.Sandbox, holds []types.Volume) { } func (m *Manager) checkVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.checkVolumeRunLocked(ctx, sb, run) +} + +func (m *Manager) checkVolumeRunLocked(ctx context.Context, sb *types.Sandbox, run *volumeRun) error { if err := ctx.Err(); err != nil { return err } - m.mu.Lock() - defer m.mu.Unlock() if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run { return ErrUnknownSandbox } @@ -247,21 +261,12 @@ func (m *Manager) cancelVolumeRun(sb *types.Sandbox) []types.Volume { func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) error { sb.Transition.Lock() defer sb.Transition.Unlock() - if err := m.checkVolumeRun(ctx, sb, run); err != nil { - return err - } m.mu.Lock() - switch { - case sb.Failed != "": - m.mu.Unlock() - return ErrFailed - case sb.ArchiveCk != "": - m.mu.Unlock() - return ErrArchived - case sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.VsockSocket == "": + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() - return ErrPaused - case sb.OnExpire == types.ExpireArchive && len(target) > 0: + return err + } + if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() return ErrVolumeCapture } @@ -273,9 +278,9 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru m.mu.Unlock() cleanErr := m.confirmVolumesClean(resolved) m.mu.Lock() - if m.claimed[sb.ID] != sb || ctx.Err() != nil { + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() - return context.Canceled + return err } if cleanErr != nil { m.releaseVolumes(additions) @@ -283,20 +288,24 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru m.mu.Unlock() return cleanErr } - previous := sb.VolumesWant + previous, applied := sb.VolumesWant, sb.Volumes + invalid := volumeDifference(sb.Volumes, sb.VolumesWant) + sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), func(v types.Volume) bool { + return volumeIndex(invalid, v.Name) >= 0 && volumeIndex(target, v.Name) >= 0 + }) sb.VolumesWant = slices.Clone(target) js := m.store.set(sb) m.mu.Unlock() err := m.store.commit(js) if err != nil && m.store.sync { - // A directory fsync may fail after rename. The new target and holds must - // survive until a later successful commit resolves that uncertainty. + // Fsync can fail after rename. Keep new admissions reserved; discarded + // pending volumes were confirmed detached before this write. return fmt.Errorf("persist volume target: %w", err) } if err != nil { m.mu.Lock() if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { - sb.VolumesWant = previous + sb.VolumesWant, sb.Volumes = previous, applied rb := m.store.set(sb) m.releaseVolumes(additions) run.holds = volumeDifference(run.holds, additions) @@ -362,6 +371,76 @@ func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, ru return nil } +func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { + return hasVolumes(sb) || m.volumeRuns[sb] != nil && len(m.volumeRuns[sb].holds) > 0 +} + +func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { + view := types.SandboxVolumesResponse{Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, Converging: m.volumeRuns[sb] != nil} + diagnostic := m.volumeErrors[sb] + view.Error = diagnostic.error + for _, v := range heldVolumes(sb) { + state := types.VolumeAttached + if volumeIndex(sb.VolumesWant, v.Name) < 0 { + state = types.VolumeDetaching + } else if volumeIndex(sb.Volumes, v.Name) < 0 { + state = types.VolumePending + } + message := diagnostic.volumes[v.Name] + if message == "" && state != types.VolumeAttached && !view.Converging { + message = "volume change interrupted; retry the desired target" + } + view.States = append(view.States, types.VolumeState{Name: v.Name, State: state, Error: message}) + } + return view +} + +func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name string, err error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run { + return + } + if m.volumeErrors == nil { + m.volumeErrors = make(map[*types.Sandbox]volumeDiagnostics) + } + diagnostic := m.volumeErrors[sb] + diagnostic.error = "" + if diagnostic.volumes == nil { + diagnostic.volumes = make(map[string]string) + } + if err != nil { + message := "volume change failed; retry the desired target" + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + message = "volume change interrupted; retry the desired target" + } + if conflict, ok := errors.AsType[*engine.VolumeMountConflict](err); ok { + message = conflict.Error() + } + diagnostic.error = message + if errors.Is(err, errVolumeTargetNotAccepted) { + diagnostic.error = errVolumeTargetNotAccepted.Error() + ": " + message + } + if name != "" { + diagnostic.volumes[name] = message + } + } + for name := range diagnostic.volumes { + if volumeIndex(heldVolumes(sb), name) < 0 { + delete(diagnostic.volumes, name) + } + } + m.volumeErrors[sb] = diagnostic +} + +func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name string) { + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { + delete(m.volumeErrors[sb].volumes, name) + } +} + func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) { if len(volumes) > types.MaxClaimVolumes { return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) @@ -428,24 +507,6 @@ func validateDetachNames(names []string) error { return nil } -func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { - return hasVolumes(sb) || m.volumeRuns[sb] != nil && len(m.volumeRuns[sb].holds) > 0 -} - func heldVolumes(sb *types.Sandbox) []types.Volume { return append(slices.Clone(sb.Volumes), volumeDifference(sb.VolumesWant, sb.Volumes)...) } - -func volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { - view := types.SandboxVolumesResponse{Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}} - for _, v := range heldVolumes(sb) { - state := "attached" - if volumeIndex(sb.VolumesWant, v.Name) < 0 { - state = "detaching" - } else if volumeIndex(sb.Volumes, v.Name) < 0 { - state = "pending" - } - view.States = append(view.States, types.VolumeState{Name: v.Name, State: state}) - } - return view -} diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index e743f478..428d74da 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -20,10 +20,9 @@ func TestLiveVolumesCommitAndRetry(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} request := []types.Volume{{Name: "data"}, {Name: "other", Mount: "/models"}} for range 2 { - got, err := m.SetVolumes(t.Context(), sb.ID, cred, request) + got, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) if err != nil || len(got.Volumes) != 2 || !slices.Equal(sb.VolumesWant, sb.Volumes) { t.Fatalf("attach = %v, %v; pending=%v", got, err, sb.VolumesWant) } @@ -34,7 +33,7 @@ func TestLiveVolumesCommitAndRetry(t *testing.T) { } assertLiveJournal(t, m, sb) for range 2 { - got, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data", "other"}}) + got, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}) if err != nil || len(got.Volumes) != 0 { t.Fatalf("detach = %v, %v", got, err) } @@ -43,7 +42,7 @@ func TestLiveVolumesCommitAndRetry(t *testing.T) { if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("leaked holds: %+v", got) } - if _, err := m.Checkpoint(t.Context(), sb.ID, cred, "after-detach", ""); err != nil { + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "after-detach", ""); err != nil { t.Fatalf("capture after detach: %v", err) } } @@ -57,17 +56,16 @@ func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { t.Fatal(err) } original := sb.Volumes[0] - cred := Cred{Token: sb.Token} - if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { t.Fatal(err) } if sb.Volumes[0] != original { t.Fatalf("changed existing volume: %v -> %v", original, sb.Volumes[0]) } - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("accepted mode change: %v", err) } - if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { t.Fatalf("writable detach changed the batch: %v", err) } } @@ -77,7 +75,6 @@ func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} for _, request := range [][]types.Volume{ {{Name: "data", Mode: "rw"}}, {{Name: "data", AttachOnly: true}}, @@ -86,14 +83,14 @@ func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { {{Name: "data", Mount: "/a"}, {Name: "other", Mount: "/a/b"}}, {{Name: "missing"}}, } { - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); !errors.Is(err, ErrBadVolume) { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); !errors.Is(err, ErrBadVolume) { t.Fatalf("request %v: %v", request, err) } } - if _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: "wrong"}, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrUnknownSandbox) { - t.Fatalf("wrong token: %v", err) + if _, err := m.SetVolumes(t.Context(), sb.ID, "wrong", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrUnknownSandbox) { + t.Fatalf("wrong tenant: %v", err) } - if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data", "data"}}); !errors.Is(err, ErrBadVolume) { + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "data"}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("duplicate detach: %v", err) } if len(eng.volumeSpecs) != 0 || !slices.Equal(sb.VolumesWant, sb.Volumes) { @@ -107,7 +104,7 @@ func TestLiveAttachFailureSurvivesRestartAndCanBeDetached(t *testing.T) { m := newVolumeManager(t, eng, catalog) sb := mustClaim(t, m, testKey) eng.mountVolumeErr = errors.New("guest unavailable after host attach") - if _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}); err == nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err == nil { t.Fatal("want mount failure") } assertLiveJournal(t, m, sb) @@ -121,16 +118,15 @@ func TestLiveAttachFailureSurvivesRestartAndCanBeDetached(t *testing.T) { if volumeHoldersOf(m2, "data").readers != 1 { t.Fatal("restart lost pending hold") } - cred := Cred{Token: sb.Token} - if _, err := m2.Checkpoint(t.Context(), sb.ID, cred, "pending", ""); !errors.Is(err, ErrVolumeCapture) { + if _, err := m2.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "pending", ""); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("captured pending device: %v", err) } eng.detachVolumeErr = errors.New("host detach failed") - if _, err := m2.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}); err == nil || volumeHoldersOf(m2, "data").readers != 1 { + if _, err := m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err == nil || volumeHoldersOf(m2, "data").readers != 1 { t.Fatal("failed detach released its hold") } eng.detachVolumeErr = nil - if _, err := m2.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}); err != nil { + if _, err := m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { t.Fatal(err) } if volumeHoldersOf(m2, "data") != (volumeHolders{}) { @@ -144,10 +140,9 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} request := []types.Volume{{Name: "data"}} if stage == "detach result" { - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err != nil { t.Fatal(err) } } @@ -166,9 +161,9 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { } var err error if stage == "detach result" { - _, err = m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}) + _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) } else { - _, err = m.SetVolumes(t.Context(), sb.ID, cred, request) + _, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) } if err == nil { t.Fatal("expected persistence failure") @@ -180,6 +175,7 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { t.Fatal(removeErr) } eng.liveVolumeHook = nil + waitFor(t, func() bool { return !m.recommitting.Load() }) if stage == "intent" { if !slices.Equal(sb.VolumesWant, sb.Volumes) || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { t.Fatal("failed intent must not reach the engine or retain a hold") @@ -190,9 +186,9 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { t.Fatal("uncommitted physical result lost its intent or hold") } if stage == "detach result" { - _, err = m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}) + _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) } else { - _, err = m.SetVolumes(t.Context(), sb.ID, cred, request) + _, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) } if err != nil { t.Fatal(err) @@ -206,17 +202,16 @@ func TestLiveBatchPartialFailureRetriesWithoutDoubleHolds(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} request := []types.Volume{{Name: "data"}, {Name: "other"}} eng.diskAttachErrFor = "other" - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); err == nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err == nil { t.Fatal("want partial failure") } if len(sb.Volumes) != 1 || slices.Equal(sb.VolumesWant, sb.Volumes) || len(sb.VolumesWant) != 2 { t.Fatalf("partial state: %v %v", sb.Volumes, sb.VolumesWant) } eng.diskAttachErrFor = "" - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, request); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err != nil { t.Fatal(err) } for _, name := range []string{"data", "other"} { @@ -232,12 +227,11 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} entered, resume := make(chan struct{}), make(chan struct{}) eng.liveVolumeHook = func(string) error { close(entered); <-resume; return nil } attached := make(chan error, 1) go func() { - _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) attached <- err }() <-entered @@ -247,24 +241,24 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { var err error switch action { case "release": - err = m.Release(t.Context(), sb.ID, cred) + err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) case "checkpoint": - _, err = m.Checkpoint(t.Context(), sb.ID, cred, "race", "") + _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "race", "") case "fork": - _, err = m.Fork(t.Context(), sb.ID, cred, 1, 0, "", "") + _, err = m.Fork(t.Context(), sb.ID, Cred{Token: sb.Token}, 1, 0, "", "") case "promote": - _, _, err = m.Promote(t.Context(), sb.ID, cred, "race", "") + _, _, err = m.Promote(t.Context(), sb.ID, Cred{Token: sb.Token}, "race", "") case "hibernate": - err = m.Hibernate(t.Context(), sb.ID, cred) + err = m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) case "reap": m.mu.Lock() sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() m.reapOnce(t.Context()) case "renew": - _, err = m.Renew(t.Context(), sb.ID, cred, time.Minute, "") + _, err = m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Minute, "") case "get": - _, err = m.GetVolumes(sb.ID, cred) + _, err = m.GetVolumes(sb.ID, sb.Tenant) case "relay": var unhold func() _, unhold, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) @@ -318,7 +312,7 @@ func TestLiveVolumeCanceledRequestRetainsIntent(t *testing.T) { sb := mustClaim(t, m, testKey) ctx, cancel := context.WithCancel(t.Context()) eng.liveVolumeHook = func(string) error { cancel(); return nil } - _, err := m.SetVolumes(ctx, sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) if !errors.Is(err, context.Canceled) || slices.Equal(sb.VolumesWant, sb.Volumes) { t.Fatalf("canceled attach: %v, pending=%v", err, sb.VolumesWant) } @@ -357,7 +351,7 @@ func TestLiveVolumeAdmissionAndState(t *testing.T) { want = ErrVolumeBusy } before := len(eng.volumeSpecs) - _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) if !errors.Is(err, want) || len(eng.volumeSpecs) != before || !slices.Equal(sb.VolumesWant, sb.Volumes) { t.Fatalf("got %v want %v", err, want) } @@ -378,22 +372,21 @@ func TestLiveVolumesRespectOwningTenantAndRemoval(t *testing.T) { if err != nil { t.Fatal(err) } - cred := Cred{Token: sb.Token} - for _, auth := range []Cred{cred, {Operator: true}} { + for _, auth := range []string{sb.Tenant, ""} { if _, err := m.SetVolumes(t.Context(), sb.ID, auth, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumeUnavailable) { t.Fatalf("catalog ACL: %v", err) } } - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } if err := m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { - t.Fatalf("removed tenant: %v", err) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatalf("held volume after tenant removal: %v", err) } - if _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}); err != nil { + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { t.Fatalf("cleanup after tenant removal: %v", err) } } @@ -467,13 +460,12 @@ func TestFailedDetachReaddRepairsAppliedMountAfterRestart(t *testing.T) { catalog := liveCatalog(t) m := newVolumeManager(t, eng, catalog) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} target := []types.Volume{{Name: "data"}} - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, target); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, target); err != nil { t.Fatal(err) } eng.detachVolumeErr = errors.New("eject did not complete") - view, err := m.SetVolumes(t.Context(), sb.ID, cred, nil) + view, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, nil) if err == nil || len(view.Volumes) != 1 || len(view.VolumesWant) != 0 || view.States[0].State != "detaching" || view.States[0].Error == "" { t.Fatalf("detach state=%+v error=%v", view, err) } @@ -483,7 +475,7 @@ func TestFailedDetachReaddRepairsAppliedMountAfterRestart(t *testing.T) { } before := len(eng.volumeMounts) eng.detachVolumeErr = nil - view, err = m2.SetVolumes(t.Context(), sb.ID, cred, target) + view, err = m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, target) if err != nil || len(eng.volumeMounts) != before+1 || view.States[0].State != "attached" { t.Fatalf("readd did not repair: %+v %v", view, err) } @@ -496,9 +488,8 @@ func TestSupersedingTargetPrecleansUncertainAddBeforeJournaling(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} eng.mountVolumeErr = errors.New("mount interrupted") - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err == nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err == nil { t.Fatal("expected attach failure") } eng.mountVolumeErr = nil @@ -511,9 +502,12 @@ func TestSupersedingTargetPrecleansUncertainAddBeforeJournaling(t *testing.T) { return nil } done := make(chan error, 1) - go func() { _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "other"}}); done <- err }() + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other"}}) + done <- err + }() <-entered - view, err := m.GetVolumes(sb.ID, cred) + view, err := m.GetVolumes(sb.ID, sb.Tenant) if err != nil || len(view.VolumesWant) != 1 || view.VolumesWant[0].Name != "data" || volumeHoldersOf(m, "data").readers != 1 { t.Errorf("forgot uncertain disk before detach: %+v %v", view, err) } @@ -547,17 +541,16 @@ func TestCanceledVolumeWriterOwnsHardwareUntilReturn(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} m.eng = eng ctx, cancel := context.WithCancel(t.Context()) first := make(chan error, 1) - go func() { _, err := m.SetVolumes(ctx, sb.ID, cred, []types.Volume{{Name: "data"}}); first <- err }() + go func() { _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); first <- err }() <-eng.entered cancel() waitingCtx, stopWaiting := context.WithTimeout(t.Context(), 25*time.Millisecond) defer stopWaiting() - _, err := m.PatchVolumes(waitingCtx, sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + _, err := m.PatchVolumes(waitingCtx, sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) if !errors.Is(err, context.DeadlineExceeded) || eng.calls.Load() != 1 { t.Fatalf("new writer overtook canceled device call: %v calls=%d", err, eng.calls.Load()) } @@ -568,7 +561,7 @@ func TestCanceledVolumeWriterOwnsHardwareUntilReturn(t *testing.T) { if len(base.volumeMounts) != 0 { t.Fatal("canceled writer issued another device call") } - view, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) if err != nil || len(view.Volumes) != 2 { t.Fatalf("retry lost prior desired state: %+v %v", view, err) } @@ -578,18 +571,17 @@ func TestReleaseOwnsUncertainHoldsUntilVMRemoval(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} m.eng = eng base.removeErrFor = sb.VMName changed := make(chan error, 1) go func() { - _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) changed <- err }() <-eng.entered released := make(chan error, 1) - go func() { released <- m.Release(t.Context(), sb.ID, cred) }() + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() select { case err := <-released: if err == nil { @@ -624,18 +616,17 @@ func TestConcurrentVolumePatchesUseLatestDesiredSet(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} m.eng = eng first := make(chan error, 1) go func() { - _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) first <- err }() <-eng.entered second := make(chan error, 1) go func() { - _, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) second <- err }() close(eng.resume) @@ -645,7 +636,7 @@ func TestConcurrentVolumePatchesUseLatestDesiredSet(t *testing.T) { if err := <-second; err != nil { t.Fatal(err) } - view, err := m.GetVolumes(sb.ID, cred) + view, err := m.GetVolumes(sb.ID, sb.Tenant) if err != nil || len(view.Volumes) != 2 || len(view.VolumesWant) != 2 { t.Fatalf("lost concurrent patch: %+v %v", view, err) } @@ -660,19 +651,18 @@ func TestSyncIntentFailureRetainsConservativeDesiredHold(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} m.store.sync = true if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { t.Fatal(err) } - view, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}) + view, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) if err == nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data").readers != 1 { t.Fatalf("ambiguous write lost recovery state: %+v %v", view, err) } if err := os.Remove(m.store.path + ".tmp"); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, nil); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, nil); err != nil { t.Fatal(err) } if volumeHoldersOf(m, "data") != (volumeHolders{}) { @@ -687,7 +677,6 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { t.Fatal(err) @@ -712,7 +701,7 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { m.reapOnce(t.Context()) done <- nil } else { - done <- m.Release(t.Context(), sb.ID, cred) + done <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) } }() waitFor(t, func() bool { m.mu.Lock(); defer m.mu.Unlock(); return m.claimed[sb.ID] == nil }) @@ -743,7 +732,7 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { t.Fatal(err) } waitFor(t, func() bool { return !m.recommitting.Load() }) - if _, err := m.SetVolumes(t.Context(), sb.ID, cred, []types.Volume{{Name: "data"}}); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } if volumeHoldersOf(m, "data").readers != 1 { @@ -758,7 +747,6 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - cred := Cred{Token: sb.Token} _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { t.Fatal(err) @@ -770,11 +758,11 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { } run.holds = append(run.holds, transient...) m.mu.Unlock() - if _, err := m.Renew(t.Context(), sb.ID, cred, time.Minute, types.ExpireArchive); !errors.Is(err, ErrVolumeCapture) { + if _, err := m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Minute, types.ExpireArchive); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("archive admitted during volume intent: %v", err) } m.finishVolumeRun(sb, run) - if _, err := m.Renew(t.Context(), sb.ID, cred, time.Minute, types.ExpireArchive); err != nil { + if _, err := m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Minute, types.ExpireArchive); err != nil { t.Fatalf("archive still blocked after abandoned intent: %v", err) } } @@ -797,7 +785,7 @@ func TestLiveAttachBoundsWholeDeviceOperation(t *testing.T) { eng := &deadlineVolumeEngine{fakeEngine: base, deadlines: make(chan time.Time, 1)} m.eng = eng start := time.Now() - if _, err := m.SetVolumes(t.Context(), sb.ID, Cred{Token: sb.Token}, []types.Volume{{Name: "data"}}); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } deadline := <-eng.deadlines @@ -840,14 +828,13 @@ func TestRemovedTenantCanRemoveReadOnlyVolumeBesideImmutableVolume(t *testing.T) if err != nil { t.Fatal(err) } - cred := Cred{Token: sb.Token} - if _, err = m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { + if _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { t.Fatal(err) } if err = m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatal(err) } - view, err := m.PatchVolumes(t.Context(), sb.ID, cred, types.VolumePatch{Remove: []string{"data"}}) + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) if err != nil || len(view.Volumes) != 1 || view.Volumes[0].Name != "other" || !view.Volumes[0].RW() { t.Fatalf("removed tenant cleanup=%+v %v", view, err) } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 82f29cfa..32119886 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -94,9 +94,9 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo - GetVolumes(id string, cred pool.Cred) (types.SandboxVolumesResponse, error) - SetVolumes(ctx context.Context, id string, cred pool.Cred, volumes []types.Volume) (types.SandboxVolumesResponse, error) - PatchVolumes(ctx context.Context, id string, cred pool.Cred, patch types.VolumePatch) (types.SandboxVolumesResponse, error) + GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) + SetVolumes(ctx context.Context, id, tenant string, volumes []types.Volume) (types.SandboxVolumesResponse, error) + PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) Sandboxes(tenant, claimRef string, metadata types.Metadata) []pool.SandboxSummary Sandbox(id string) (pool.SandboxSummary, bool) Stats(ctx context.Context, id string) (pool.SandboxStats, bool) @@ -265,9 +265,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) - mux.HandleFunc("GET /v1/sandboxes/{id}/volumes", s.handleSandboxVolumes) - mux.HandleFunc("PUT /v1/sandboxes/{id}/volumes", s.handleSandboxVolumes) - mux.HandleFunc("PATCH /v1/sandboxes/{id}/volumes", s.handleSandboxVolumes) + mux.HandleFunc("GET /v1/sandboxes/{id}/volumes", s.requireToken(s.handleSandboxVolumes)) + mux.HandleFunc("PUT /v1/sandboxes/{id}/volumes", s.requireToken(s.handleSandboxVolumes)) + mux.HandleFunc("PATCH /v1/sandboxes/{id}/volumes", s.requireToken(s.handleSandboxVolumes)) mux.HandleFunc("POST /v1/sandboxes/{id}/renew", s.handleRenew) mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 048b05f1..e1d560c7 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2602,9 +2602,9 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { - getVolumes func(string, pool.Cred) (types.SandboxVolumesResponse, error) - setVolumes func(string, pool.Cred, []types.Volume) (types.SandboxVolumesResponse, error) - patchVolumes func(string, pool.Cred, types.VolumePatch) (types.SandboxVolumesResponse, error) + getVolumes func(string, string) (types.SandboxVolumesResponse, error) + setVolumes func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) + patchVolumes func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) tenants []config.TenantSpec setTenants func(specs []config.TenantSpec) error putTenant func(spec config.TenantSpec) error @@ -2677,16 +2677,16 @@ type fakeManager struct { draining bool } -func (f *fakeManager) GetVolumes(id string, cred pool.Cred) (types.SandboxVolumesResponse, error) { - return f.getVolumes(id, cred) +func (f *fakeManager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { + return f.getVolumes(id, tenant) } -func (f *fakeManager) SetVolumes(_ context.Context, id string, cred pool.Cred, v []types.Volume) (types.SandboxVolumesResponse, error) { - return f.setVolumes(id, cred, v) +func (f *fakeManager) SetVolumes(_ context.Context, id, tenant string, v []types.Volume) (types.SandboxVolumesResponse, error) { + return f.setVolumes(id, tenant, v) } -func (f *fakeManager) PatchVolumes(_ context.Context, id string, cred pool.Cred, p types.VolumePatch) (types.SandboxVolumesResponse, error) { - return f.patchVolumes(id, cred, p) +func (f *fakeManager) PatchVolumes(_ context.Context, id, tenant string, p types.VolumePatch) (types.SandboxVolumesResponse, error) { + return f.patchVolumes(id, tenant, p) } func (f *fakeManager) ClaimWarm(ctx context.Context, key types.PoolKey, o pool.ClaimOptions) (*types.Sandbox, error) { diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index e549e41f..11ac4f31 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -2,25 +2,23 @@ package server import ( "cmp" + "context" "errors" "net/http" "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { - token, ok := sandboxToken(w, r) - if !ok { - return - } - id, cred := r.PathValue("id"), s.sandboxCred(token) + id, tenant := r.PathValue("id"), tenantFrom(r.Context()) var view types.SandboxVolumesResponse var err error switch r.Method { case http.MethodGet: - view, err = s.mgr.GetVolumes(id, cred) + view, err = s.mgr.GetVolumes(id, tenant) case http.MethodPut: req, ok := decodeBodyStrict[types.SetVolumesRequest](w, r) if !ok { @@ -30,30 +28,39 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { writeErr(w, http.StatusBadRequest, "volumes must be an array") return } - view, err = s.mgr.SetVolumes(r.Context(), id, cred, req.Volumes) + view, err = s.mgr.SetVolumes(r.Context(), id, tenant, req.Volumes) case http.MethodPatch: req, ok := decodeBodyStrict[types.VolumePatch](w, r) if !ok { return } - view, err = s.mgr.PatchVolumes(r.Context(), id, cred, req) + view, err = s.mgr.PatchVolumes(r.Context(), id, tenant, req) } code := http.StatusOK if err != nil { - code, view.Error = http.StatusInternalServerError, "volume convergence failed; retry the desired set" + message := view.Error + code, view.Error = http.StatusInternalServerError, "volume convergence failed; inspect the volume states before retrying" for _, rule := range poolErrHTTP { if errors.Is(err, rule.err) { code, view.Error = rule.code, cmp.Or(rule.msg, err.Error()) break } } - if code == http.StatusInternalServerError { - log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) + var conflict *engine.VolumeMountConflict + var unsupported *engine.LiveVolumeUnsupported + switch { + case errors.As(err, &conflict): + code, view.Error = http.StatusConflict, conflict.Error() + case errors.As(err, &unsupported): + code, view.Error = http.StatusServiceUnavailable, unsupported.Error() + case errors.Is(err, context.DeadlineExceeded): + code, view.Error = http.StatusGatewayTimeout, "volume change timed out; inspect the volume states before retrying" + case errors.Is(err, context.Canceled): + code, view.Error = http.StatusRequestTimeout, "volume change canceled; inspect the volume states before retrying" } - for i := range view.States { - if view.States[i].Error != "" { - view.States[i].Error = view.Error - } + view.Error = cmp.Or(message, view.Error) + if code >= http.StatusInternalServerError { + log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) } } writeJSON(w, code, view) diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 2e32a61d..3e2dd22a 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json/v2" "errors" "io" @@ -9,48 +10,53 @@ import ( "strings" "testing" + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/types" ) func TestLiveVolumeRoutesAuthorizeAndDecode(t *testing.T) { for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { - for _, token := range []string{"sandbox-token", "root-token"} { + for _, token := range []string{"acme-token", "root-token"} { t.Run(method+"/"+token, func(t *testing.T) { called := false - check := func(id string, cred pool.Cred) { + check := func(id, tenant string) { called = true - if id != "sb_1" || cred.Operator != (token == "root-token") || !cred.Operator && cred.Token != token { - t.Errorf("credentials %s %+v", id, cred) + if id != "sb_1" || tenant != map[string]string{"root-token": "", "acme-token": "acme"}[token] { + t.Errorf("scope %s %q", id, tenant) } } view := types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "data"}}, States: []types.VolumeState{{Name: "data", State: "pending"}}} mgr := &fakeManager{ - getVolumes: func(id string, cred pool.Cred) (types.SandboxVolumesResponse, error) { - check(id, cred) + getVolumes: func(id, tenant string) (types.SandboxVolumesResponse, error) { + check(id, tenant) return view, nil }, - setVolumes: func(id string, cred pool.Cred, v []types.Volume) (types.SandboxVolumesResponse, error) { - check(id, cred) + setVolumes: func(id, tenant string, v []types.Volume) (types.SandboxVolumesResponse, error) { + check(id, tenant) if !slices.Equal(v, []types.Volume{{Name: "data"}}) { t.Errorf("volumes %v", v) } return view, nil }, - patchVolumes: func(id string, cred pool.Cred, p types.VolumePatch) (types.SandboxVolumesResponse, error) { - check(id, cred) + patchVolumes: func(id, tenant string, p types.VolumePatch) (types.SandboxVolumesResponse, error) { + check(id, tenant) if !slices.Equal(p.Remove, []string{"data"}) { t.Errorf("patch %v", p) } return view, nil }, } - ts := newTestServer(t, "root-token", mgr, nil) + ts := newTenantTestServer(t, "root-token", []config.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) body := `{"volumes":[{"name":"data"}]}` if method == http.MethodPatch { body = `{"remove":["data"]}` } - req, _ := http.NewRequest(method, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(body)) + req, err := http.NewRequestWithContext(t.Context(), method, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } req.Header.Set("Authorization", "Bearer "+token) resp, err := ts.Client().Do(req) if err != nil { @@ -76,21 +82,33 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { code int }{ {"missing token", "", `{"volumes":[]}`, nil, http.StatusUnauthorized}, - {"unknown field", "tok", `{"volumes":[],"path":"/host"}`, nil, http.StatusBadRequest}, - {"missing volumes", "tok", `{}`, nil, http.StatusBadRequest}, - {"null volumes", "tok", `{"volumes":null}`, nil, http.StatusBadRequest}, - {"wrong sandbox", "tok", `{"volumes":[]}`, pool.ErrUnknownSandbox, http.StatusNotFound}, - {"bad volume", "tok", `{"volumes":[]}`, pool.ErrBadVolume, http.StatusBadRequest}, - {"paused", "tok", `{"volumes":[]}`, pool.ErrPaused, http.StatusConflict}, - {"failed", "tok", `{"volumes":[]}`, pool.ErrFailed, http.StatusConflict}, - {"engine", "tok", `{"volumes":[]}`, errors.New("host /private/image failed"), http.StatusInternalServerError}, + {"unknown field", "root", `{"volumes":[],"path":"/host"}`, nil, http.StatusBadRequest}, + {"missing volumes", "root", `{}`, nil, http.StatusBadRequest}, + {"null volumes", "root", `{"volumes":null}`, nil, http.StatusBadRequest}, + {"wrong sandbox", "root", `{"volumes":[]}`, pool.ErrUnknownSandbox, http.StatusNotFound}, + {"bad volume", "root", `{"volumes":[]}`, pool.ErrBadVolume, http.StatusBadRequest}, + {"paused", "root", `{"volumes":[]}`, pool.ErrPaused, http.StatusConflict}, + {"failed", "root", `{"volumes":[]}`, pool.ErrFailed, http.StatusConflict}, + {"sandbox token", "sandbox-token", `{"volumes":[]}`, nil, http.StatusUnauthorized}, + {"invalid token", "invalid", `{"volumes":[]}`, nil, http.StatusUnauthorized}, + {"foreign mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountForeign, Mount: "/data"}, http.StatusConflict}, + {"bind mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountBind, Mount: "/other"}, http.StatusConflict}, + {"unmount refused", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/data"}, http.StatusConflict}, + {"unsupported runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{Version: "v0.7.1"}, http.StatusServiceUnavailable}, + {"unknown runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{Err: errors.New("host /private/version failed")}, http.StatusServiceUnavailable}, + {"timeout", "root", `{"volumes":[]}`, context.DeadlineExceeded, http.StatusGatewayTimeout}, + {"cancel", "root", `{"volumes":[]}`, context.Canceled, http.StatusRequestTimeout}, + {"engine", "root", `{"volumes":[]}`, errors.New("host /private/image failed"), http.StatusInternalServerError}, } { t.Run(tt.name, func(t *testing.T) { - mgr := &fakeManager{setVolumes: func(string, pool.Cred, []types.Volume) (types.SandboxVolumesResponse, error) { - return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "data"}}, States: []types.VolumeState{{Name: "data", State: "pending", Error: "host /private/image failed"}}}, tt.err + mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { + return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "data"}}, States: []types.VolumeState{{Name: "data", State: "pending", Error: "volume operation failed; retry after resolving the runtime error"}}}, tt.err }} ts := newTestServer(t, "root", mgr, nil) - req, _ := http.NewRequest(http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(tt.body)) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(tt.body)) + if err != nil { + t.Fatal(err) + } if tt.token != "" { req.Header.Set("Authorization", "Bearer "+tt.token) } @@ -105,10 +123,76 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { } if tt.err != nil { var got types.SandboxVolumesResponse - if err := json.Unmarshal(body, &got); err != nil || len(got.VolumesWant) != 1 || len(got.States) != 1 || got.Error == "" { + if err := json.Unmarshal(body, &got); err != nil || len(got.VolumesWant) != 1 || len(got.States) != 1 || got.Error == "" || got.States[0].Error != "volume operation failed; retry after resolving the runtime error" { t.Fatalf("lost partial state: %s %v", body, err) } } }) } } + +func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { + t.Run(method, func(t *testing.T) { + check := func(id, tenant string) (types.SandboxVolumesResponse, error) { + if id != "sb_1" || tenant != "other" { + t.Fatalf("scope %s %q", id, tenant) + } + return types.SandboxVolumesResponse{}, pool.ErrUnknownSandbox + } + mgr := &fakeManager{ + getVolumes: check, + setVolumes: func(id, tenant string, _ []types.Volume) (types.SandboxVolumesResponse, error) { + return check(id, tenant) + }, + patchVolumes: func(id, tenant string, _ types.VolumePatch) (types.SandboxVolumesResponse, error) { + return check(id, tenant) + }, + } + ts := newTenantTestServer(t, "root", []config.TenantSpec{{Name: "other", Token: "other-token"}}, mgr, nil) + body := `{"volumes":[]}` + if method == http.MethodPatch { + body = `{}` + } + req, err := http.NewRequestWithContext(t.Context(), method, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer other-token") + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status %d", resp.StatusCode) + } + }) + } +} + +func TestLiveVolumeRoutesPreserveSafeTargetOutcome(t *testing.T) { + const message = "new volume target not accepted; cleanup of the previous pending volume failed" + const cause = "cannot unmount /data; close guest references and retry" + mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { + return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: message}, &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/data"} + }} + ts := newTestServer(t, "root", mgr, nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(`{"volumes":[]}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer root") + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var got types.SandboxVolumesResponse + if err := json.UnmarshalRead(resp.Body, &got); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusConflict || got.Error != message || len(got.States) != 1 || got.States[0].Error != cause { + t.Fatalf("status=%d view=%+v", resp.StatusCode, got) + } +} diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index d253b441..aac4f8ad 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -87,10 +87,18 @@ type VolumePatch struct { Remove []string `json:"remove,omitempty"` } +type VolumePhase string + +const ( + VolumeAttached VolumePhase = "attached" + VolumePending VolumePhase = "pending" + VolumeDetaching VolumePhase = "detaching" +) + type VolumeState struct { - Name string `json:"name"` - State string `json:"state"` - Error string `json:"error,omitempty"` + Name string `json:"name"` + State VolumePhase `json:"state"` + Error string `json:"error,omitempty"` } // SandboxVolumesResponse retains the desired and applied sets after a failure. @@ -98,6 +106,7 @@ type SandboxVolumesResponse struct { VolumesWant []Volume `json:"volumes_want"` Volumes []Volume `json:"volumes"` States []VolumeState `json:"states"` + Converging bool `json:"converging"` Error string `json:"error,omitempty"` } From 6c4721cfec33c43c572fbe6db4d3f61dcd7e6596 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:16:19 +0000 Subject: [PATCH 07/57] fix(pool): persist volume holds until VM retirement completes --- sandboxd/pool/claim.go | 29 ++- sandboxd/pool/claims.go | 125 +++++++++- sandboxd/pool/egress_test.go | 4 +- sandboxd/pool/liveness_test.go | 4 +- sandboxd/pool/pool.go | 7 +- sandboxd/pool/reconcile.go | 70 +++++- sandboxd/pool/remove.go | 67 +++++- sandboxd/pool/telemetry_test.go | 8 +- sandboxd/pool/volume.go | 16 +- sandboxd/pool/volume_retirement_test.go | 293 ++++++++++++++++++++++++ 10 files changed, 565 insertions(+), 58 deletions(-) create mode 100644 sandboxd/pool/volume_retirement_test.go diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index f3d6dcab..0e790cab 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -295,7 +295,8 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand } delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) - js := m.store.del(id) + js := m.store.retire(sb, volumeHolds) + delete(m.volumeErrors, sb) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() @@ -303,7 +304,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.restoreVolumeHolds(sb, volumeHolds) m.tenantDelta(sb.Tenant, 1) delete(m.pendingCks, ck) - rb := m.store.set(sb) + rb := m.store.restore(sb) m.mu.Unlock() logger := log.WithFunc("pool.releaseResolved") if ck != "" { @@ -321,13 +322,12 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.purgeArchiveCk(ctx, id, ck, sb.Tenant) // archived: no local VM m.untrack(m.pendingCks, ck) } - td := m.quiesceVolumes(ctx, sb) - td.holds = volumeHolds + td := m.quiesceRetiringVolumes(ctx, sb, volumeHolds) var err error removed := vmName == "" switch { case removed: - m.finishVolumeTeardown(ctx, td) + removed = m.completeVolumeTeardown(ctx, td) case m.releaseDelay > 0: m.queueRemoval(vmName, id, "", td, time.Now().Add(m.releaseDelay)) default: @@ -507,7 +507,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() var expired []victim - var dropped, purges []string + var purges []string for id, sb := range m.claimed { // a zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { @@ -529,15 +529,14 @@ func (m *Manager) reapOnce(ctx context.Context) { } expired = append(expired, victim{action: action, id: id, sb: sb}) default: - expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: m.cancelVolumeRun(sb)}) + holds := m.cancelVolumeRun(sb) + expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: holds}) + m.store.retire(sb, holds) + delete(m.volumeErrors, sb) delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) - dropped = append(dropped, id) } } - if len(dropped) > 0 { - m.store.del(dropped...) - } m.mu.Unlock() if len(expired) == 0 { return @@ -580,7 +579,6 @@ func (m *Manager) reapOnce(ctx context.Context) { m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() - var restored []*types.Sandbox for _, v := range expired { if v.action == reapArchive || v.action == reapPause { delete(m.archiving, v.id) // never removed from m.claimed @@ -589,10 +587,10 @@ func (m *Manager) reapOnce(ctx context.Context) { m.restoreVolumeHolds(v.sb, v.holds) m.tenantDelta(v.tenant, 1) delete(m.pendingCks, v.ck) - restored = append(restored, v.sb) + m.store.restore(v.sb) } } - rb := m.store.set(restored...) + rb := m.store.mark() m.mu.Unlock() for _, ck := range purgeCks { if clearErr := m.clearArchiveCk(ck); clearErr != nil { @@ -619,8 +617,7 @@ func (m *Manager) reapOnce(ctx context.Context) { case reapPause: logSweepResult(ctx, logger, m.pauseExpired(ctx, v.sb), "archived expired sandbox "+v.id, "archive expired sandbox "+v.id) default: - td := m.quiesceVolumes(ctx, v.sb) - td.holds = v.holds + td := m.quiesceRetiringVolumes(ctx, v.sb, v.holds) m.out.Disarm(v.id, m.removeOrRetry(ctx, v.vmName, v.id, "", td)) m.dropSnap(ctx, v.snap) m.counters.reaps.Add(1) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 00211fb3..8708d804 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -53,6 +53,20 @@ type claimDTO struct { Failed string `json:"failed,omitempty"` } +type volumeRetirementDTO struct { + SandboxID string `json:"sandbox_id"` + VMName string `json:"vm_name"` + TAP string `json:"tap,omitempty"` + Holds []types.Volume `json:"holds"` + Clears []string `json:"clears,omitempty"` +} + +type claimJournal struct { + FormatVersion uint8 `json:"format_version"` + Claims map[string]*types.Sandbox `json:"claims"` + VolumeRetirements map[string]volumeRetirementDTO `json:"volume_retirements"` +} + // claimRow keeps its encoded member until set replaces the row; set never edits one. type claimRow struct { dto claimDTO @@ -68,41 +82,123 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 + mu sync.Mutex + rows map[string]*claimRow + retirements map[string]volumeRetirementDTO + seq uint64 + written uint64 } func newClaimStore(dataDir string, sync bool) *claimStore { - return &claimStore{path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}} + return &claimStore{path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, retirements: map[string]volumeRetirementDTO{}} } func (s *claimStore) load() (map[string]*types.Sandbox, error) { + claims, _, err := s.loadJournal() + return claims, err +} + +func (s *claimStore) loadJournal() (map[string]*types.Sandbox, map[string]volumeRetirementDTO, error) { raw, err := os.ReadFile(s.path) if errors.Is(err, fs.ErrNotExist) { - return map[string]*types.Sandbox{}, nil + return map[string]*types.Sandbox{}, map[string]volumeRetirementDTO{}, nil } if err != nil { - return nil, fmt.Errorf("read claims: %w", err) + return nil, nil, fmt.Errorf("read claims: %w", err) + } + var envelope map[string]jsontext.Value + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, nil, fmt.Errorf("parse claims: %w", err) + } + retirements := map[string]volumeRetirementDTO{} + if _, versioned := envelope["format_version"]; versioned { + var journal claimJournal + if err := json.Unmarshal(raw, &journal); err != nil { + return nil, nil, fmt.Errorf("parse claims journal: %w", err) + } + if journal.FormatVersion != 2 { + return nil, nil, fmt.Errorf("unsupported claims journal format %d", journal.FormatVersion) + } + if journal.Claims == nil || journal.VolumeRetirements == nil { + return nil, nil, errors.New("claims journal requires claims and volume_retirements") + } + retirements = journal.VolumeRetirements + raw = envelope["claims"] } claims := map[string]*types.Sandbox{} if err := json.Unmarshal(raw, &claims); err != nil { - return nil, fmt.Errorf("parse claims: %w", err) + return nil, nil, fmt.Errorf("parse claims: %w", err) } var fields map[string]map[string]jsontext.Value if err := json.Unmarshal(raw, &fields); err != nil { - return nil, err + return nil, nil, fmt.Errorf("parse claim fields: %w", err) } for id, sb := range claims { + if sb == nil { + return nil, nil, fmt.Errorf("null claim %q", id) + } if _, present := fields[id]["volumes_want"]; !present { sb.VolumesWant = slices.Clone(sb.Volumes) } if len(sb.VolumesWant) == 0 { sb.VolumesWant = nil } + if _, retiring := retirements[sb.VMName]; retiring { + return nil, nil, fmt.Errorf("VM %q is both claimed and retiring", sb.VMName) + } + } + for name, retirement := range retirements { + if name == "" || retirement.VMName != name || retirement.SandboxID == "" || len(retirement.Holds) == 0 || claims[retirement.SandboxID] != nil { + return nil, nil, fmt.Errorf("invalid volume retirement %q", name) + } + } + s.mu.Lock() + s.retirements = maps.Clone(retirements) + s.mu.Unlock() + return claims, retirements, nil +} + +func (s *claimStore) retire(sb *types.Sandbox, holds []types.Volume) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.rows, sb.ID) + if len(holds) > 0 { + s.retirements[sb.VMName] = volumeRetirementDTO{SandboxID: sb.ID, VMName: sb.VMName, TAP: sb.TAP, Holds: slices.Clone(holds)} } - return claims, nil + s.seq++ + return claimSnapshot{seq: s.seq} +} + +func (s *claimStore) restore(sb *types.Sandbox) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.retirements, sb.VMName) + s.rows[sb.ID] = &claimRow{dto: dtoOf(sb)} + s.seq++ + return claimSnapshot{seq: s.seq} +} + +func (s *claimStore) finishRetirement(name string) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.retirements, name) + s.seq++ + return claimSnapshot{seq: s.seq} +} + +func (s *claimStore) setRetirement(retirement volumeRetirementDTO) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + s.retirements[retirement.VMName] = retirement + s.seq++ + return claimSnapshot{seq: s.seq} +} + +func (s *claimStore) retirement(name string) (volumeRetirementDTO, bool) { + s.mu.Lock() + defer s.mu.Unlock() + retirement, ok := s.retirements[name] + return retirement, ok } // set records each sandbox's projection and sequences the write; callers may hold m.mu. @@ -162,8 +258,9 @@ func (s *claimStore) commit(snap claimSnapshot) error { return nil } rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq + retirements := maps.Clone(s.retirements) s.mu.Unlock() - s.buf = append(s.buf[:0], '{') + s.buf = append(s.buf[:0], `{"format_version":2,"claims":{`...) for i, row := range rows { if row.raw == nil { raw, err := encodeRow(&row.dto) @@ -177,6 +274,12 @@ func (s *claimStore) commit(snap claimSnapshot) error { } s.buf = append(s.buf, row.raw...) } + s.buf = append(s.buf, `},"volume_retirements":`...) + rawRetirements, err := json.Marshal(retirements) + if err != nil { + return fmt.Errorf("encode volume retirements: %w", err) + } + s.buf = append(s.buf, rawRetirements...) s.buf = append(s.buf, '}') if err := s.replace(s.buf); err != nil { return err diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index 6a7559e9..f5e5d2ed 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -313,7 +313,9 @@ func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { var gotKeep map[string]bool withOutbound(t, m, func(o *outbound.Options) { o.Sweep = func(keep map[string]bool) error { gotKeep = keep; return nil } }) - m.resyncEgress(t.Context(), live, removed) + if err := m.resyncEgress(t.Context(), live, removed); err != nil { + t.Fatal(err) + } if removed["sbx-q1"] { t.Error("failed remove marked the VM gone; the sweep would drop its lock table") diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 682c976f..951be9f5 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -423,7 +423,9 @@ func TestResyncRecordsADownEgressLaneTap(t *testing.T) { o.Sweep = func(map[string]bool) error { return nil } }) - m.resyncEgress(t.Context(), live, map[string]bool{}) + if err := m.resyncEgress(t.Context(), live, map[string]bool{}); err != nil { + t.Fatal(err) + } m.mu.Lock() kept := m.claimed[sb.ID] == sb diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index ec46fc06..9de069ec 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -296,9 +296,10 @@ type Manager struct { store *claimStore volumes map[string]catalogVolume // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]volumeDiagnostics + volumeAdmission map[string]volumeHolders + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]volumeDiagnostics + unknownVolumeVMs map[string]bool poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 1bf80f74..c7c1772b 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -3,6 +3,7 @@ package pool import ( "context" "encoding/json/v2" + "errors" "fmt" "os" "path/filepath" @@ -19,7 +20,7 @@ import ( // Reconcile aligns claims and VMs after a restart; it runs before the server because it swaps in fresh records that would bypass in-flight Transition locks. func (m *Manager) Reconcile(ctx context.Context) error { - claims, err := m.store.load() + claims, retirements, err := m.store.loadJournal() if err != nil { return err } @@ -38,6 +39,13 @@ func (m *Manager) Reconcile(ctx context.Context) error { referenced := map[string]bool{} var adopted []*types.Sandbox m.mu.Lock() + for name, retirement := range retirements { + m.adoptVolumes(retirement.Holds) + if retirement.TAP != "" { + m.out.KeepLock(retirement.SandboxID, retirement.TAP) + } + owned[name] = true + } for id, sb := range claims { sb.SetEnv(sb.Env) rec, ok := live[sb.VMName] @@ -120,10 +128,17 @@ func (m *Manager) Reconcile(ctx context.Context) error { logger.Infof(ctx, "removed orphan snapshot %s", orphans[i]) }).Wait() } - m.resyncEgress(ctx, live, removed) + for name, retirement := range retirements { + td := volumeTeardown{holds: retirement.Holds, retirement: name} + if m.removeOrRetry(ctx, name, retirement.SandboxID, retirement.TAP, td) { + removed[name] = true + m.out.Disarm(retirement.SandboxID, true) + } + } + egressErr := m.resyncEgress(ctx, live, removed) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) - return saveErr + return errors.Join(saveErr, egressErr) } // sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. @@ -134,10 +149,21 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec stale = append(stale, name) } } + m.mu.Lock() + if m.unknownVolumeVMs == nil { + m.unknownVolumeVMs = make(map[string]bool) + } + for _, name := range stale { + m.unknownVolumeVMs[name] = true + } + m.mu.Unlock() gone := make([]bool, len(stale)) // distinct indices: no lock under the Wait barrier m.runBounded(ctx, len(stale), func(ctx context.Context, i int) { if m.removeStaleVM(ctx, stale[i], live[stale[i]]) { gone[i] = true + m.mu.Lock() + delete(m.unknownVolumeVMs, stale[i]) + m.mu.Unlock() log.WithFunc("pool.sweepStaleVMs").Infof(ctx, "removed stale VM %s", stale[i]) } }).Wait() @@ -172,7 +198,7 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. -func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) { +func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) error { logger := log.WithFunc("pool.resyncEgress") now := time.Now() locked, lockedErr := m.out.LockedTaps() @@ -211,7 +237,11 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco // quarantine before the sweep so a failed remove's still-running tap is kept. keep := make(map[string]bool, len(live)) for _, sb := range quarantine { - if m.quarantineClaim(ctx, sb) { + gone, err := m.quarantineClaim(ctx, sb) + if err != nil { + return err + } + if gone { removed[sb.VMName] = true } else if sb.TAP != "" { keep[sb.TAP] = true @@ -223,24 +253,42 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco keep[tap] = true } } + m.mu.Lock() + for _, pending := range m.pendingRemovals { + if pending.tap != "" { + keep[pending.tap] = true + } + } + m.mu.Unlock() if sweepErr := m.out.Sweep(keep); sweepErr != nil { logger.Warnf(ctx, "sweep orphan egress tables: %v", sweepErr) } + return nil } // quarantineClaim parks a claim whose teardown failed: out of service and queued until a remove succeeds. -func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { - td := m.quiesceVolumes(ctx, sb) - gone := m.removeOrRetry(ctx, sb.VMName, sb.ID, "", td) +func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) (bool, error) { m.mu.Lock() + holds := m.cancelVolumeRun(sb) delete(m.claimed, sb.ID) + delete(m.volumeErrors, sb) m.tenantDelta(sb.Tenant, -1) - js := m.store.del(sb.ID) + js := m.store.retire(sb, holds) m.mu.Unlock() if err := m.store.commit(js); err != nil { - m.recommit(ctx, js) + m.mu.Lock() + m.claimed[sb.ID] = sb + m.restoreVolumeHolds(sb, holds) + m.tenantDelta(sb.Tenant, 1) + rb := m.store.restore(sb) + m.mu.Unlock() + m.recommit(ctx, rb) + return false, fmt.Errorf("persist quarantine of %s: %w", sb.ID, err) } - return gone + td := m.quiesceRetiringVolumes(ctx, sb, holds) + gone := m.removeOrRetry(ctx, sb.VMName, sb.ID, sb.TAP, td) + m.out.Disarm(sb.ID, gone) + return gone, nil } // readoptEgressTap records and returns a live egress claim's tap, "" when there is none. diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 50fb9668..3c50d658 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -11,6 +11,7 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" ) func (m *Manager) removeVM(ctx context.Context, name string) bool { @@ -44,8 +45,10 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID, tap string, td volumeTeardown) bool { - if m.removeVM(ctx, name) { - m.finishVolumeTeardown(ctx, td) + if m.removeVM(ctx, name) && m.completeVolumeTeardown(ctx, td) { + m.mu.Lock() + delete(m.unknownVolumeVMs, name) + m.mu.Unlock() return true } m.queueRemoval(name, sandboxID, tap, td, time.Time{}) @@ -93,7 +96,7 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueStaleCreate(name, pending.tap) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) return } } @@ -101,11 +104,17 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) } -func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { - m.finishVolumeTeardown(ctx, pending.volumes) +func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { + if !m.completeVolumeTeardown(ctx, pending.volumes) { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + return + } + m.mu.Lock() + delete(m.unknownVolumeVMs, name) + m.mu.Unlock() if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { @@ -116,3 +125,49 @@ func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { func (m *Manager) destroy(ctx context.Context, name string) { m.removeOrRetry(ctx, name, "", "", volumeTeardown{}) } + +func (m *Manager) quiesceRetiringVolumes(ctx context.Context, sb *types.Sandbox, holds []types.Volume) volumeTeardown { + td := m.quiesceVolumes(ctx, sb) + td.holds = holds + if len(holds) == 0 { + return td + } + td.retirement = sb.VMName + retirement, ok := m.store.retirement(sb.VMName) + if !ok { + return td + } + if len(td.clears) > 0 { + next := retirement + next.Clears = slices.Clone(td.clears) + if err := m.store.commit(m.store.setRetirement(next)); err != nil { + log.WithFunc("pool.quiesceRetiringVolumes").Warnf(ctx, "persist clean volumes for %s: %v", sb.ID, err) + m.recommit(ctx, m.store.setRetirement(retirement)) + } + } + return td +} + +func (m *Manager) completeVolumeTeardown(ctx context.Context, td volumeTeardown) bool { + if td.retirement == "" { + m.finishVolumeTeardown(ctx, td) + return true + } + retirement, ok := m.store.retirement(td.retirement) + if !ok { + return true + } + for _, path := range retirement.Clears { + if err := clearVolumeDirty(path); err != nil { + log.WithFunc("pool.completeVolumeTeardown").Warnf(ctx, "clear volume marker %s: %v", path, err) + return false + } + } + if err := m.store.commit(m.store.finishRetirement(td.retirement)); err != nil { + log.WithFunc("pool.completeVolumeTeardown").Warnf(ctx, "finish volume retirement %s: %v", td.retirement, err) + m.recommit(ctx, m.store.setRetirement(retirement)) + return false + } + m.unreserveVolumes(retirement.Holds) + return true +} diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index e48d8df0..e73825ad 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -124,13 +124,9 @@ func TestTenantStampedInJournals(t *testing.T) { t.Fatalf("claim: %v", err) } - raw, err := os.ReadFile(filepath.Join(m.dataDir, "claims.json")) + claims, err := m.store.load() if err != nil { - t.Fatalf("read claims journal: %v", err) - } - claims := map[string]*types.Sandbox{} - if err = json.Unmarshal(raw, &claims); err != nil { - t.Fatalf("parse claims journal: %v", err) + t.Fatalf("load claims journal: %v", err) } if claims[sb.ID] == nil || claims[sb.ID].Tenant != "acme" { t.Errorf("persisted claim %+v, want tenant acme", claims[sb.ID]) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 505bb1a8..8e8c5c4b 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -35,10 +35,11 @@ type resolvedVolume struct { applied types.Volume } -// volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. +// volumeTeardown records marker paths only after a confirmed unmount. type volumeTeardown struct { - holds []types.Volume - clears []string + holds []types.Volume + clears []string + retirement string } type catalogVolume struct { @@ -187,6 +188,12 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume) error { + m.mu.Lock() + fenced := len(m.unknownVolumeVMs) > 0 + m.mu.Unlock() + if fenced { + return ErrVolumeUnavailable + } // attach-only mounts nothing, so there is no umount to verify and no marker. attachOnly := volume.applied.Mount == "" // write-ahead: the marker must be durable before any guest write can be. @@ -262,6 +269,9 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { + if len(volumes) > 0 && len(m.unknownVolumeVMs) > 0 { + return ErrVolumeUnavailable + } for _, volume := range volumes { holders := m.volumeAdmission[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go new file mode 100644 index 00000000..632defb6 --- /dev/null +++ b/sandboxd/pool/volume_retirement_test.go @@ -0,0 +1,293 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/outbound" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestRetiringVolumeHoldsSurviveRestart(t *testing.T) { + for _, action := range []string{"release", "reap", "quarantine"} { + for _, mode := range []string{"ro", "rw"} { + t.Run(action+"/"+mode, func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: mode}}}) + if err != nil { + t.Fatal(err) + } + eng.removeErrFor = sb.VMName + switch action { + case "release": + if checkErr := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); checkErr == nil { + t.Fatal("release accepted surviving VM") + } + case "reap": + sb.Deadline = time.Now().Add(-time.Second) + m.reapOnce(t.Context()) + waitFor(t, func() bool { m.mu.Lock(); defer m.mu.Unlock(); _, ok := m.pendingRemovals[sb.VMName]; return ok }) + case "quarantine": + if gone, quarantineErr := m.quarantineClaim(t.Context(), sb); gone || quarantineErr != nil { + t.Fatal("quarantine accepted surviving VM") + } + } + claims, retiring, err := m.store.loadJournal() + if err != nil || claims[sb.ID] != nil || len(retiring[sb.VMName].Holds) != 1 { + t.Fatalf("journal=%v retiring=%v err=%v", claims, retiring, err) + } + reloaded := newClaimStore(m.dataDir, false) + loaded, loadErr := reloaded.load() + if loadErr != nil { + t.Fatal(loadErr) + } + if saveErr := reloaded.save(loaded); saveErr != nil { + t.Fatal(saveErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if checkErr := m2.Reconcile(t.Context()); checkErr != nil { + t.Fatal(checkErr) + } + if _, checkErr := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); !errors.Is(checkErr, ErrVolumeBusy) { + t.Fatalf("writer while VM survives: %v", checkErr) + } + if saveErr := m2.store.save(m2.claimed); saveErr != nil { + t.Fatal(saveErr) + } + _, retiring, err = m2.store.loadJournal() + if err != nil || len(retiring) != 1 { + t.Fatalf("save lost retirement: %v %v", retiring, err) + } + eng.removeErrFor = "" + m2.retryRemovals(t.Context()).Wait() + _, retiring, err = m2.store.loadJournal() + if err != nil || len(retiring) != 0 { + t.Fatalf("completed retirement remained: %v %v", retiring, err) + } + if got := volumeHoldersOf(m2, "data"); got != (volumeHolders{}) { + t.Fatalf("retirement leaked holds: %+v", got) + } + writer, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}) + if err != nil { + t.Fatalf("writer after removal: %v", err) + } + if err := m2.Release(t.Context(), writer.ID, Cred{Token: writer.Token}); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestRetirementCompletionMustBeDurableBeforeAdmission(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if err != nil { + t.Fatal(err) + } + m.eng = &retirementWriteFailureEngine{fakeEngine: eng, afterRemove: func() { + if checkErr := os.Mkdir(m.store.path+".tmp", 0o700); checkErr != nil { + t.Fatal(checkErr) + } + }} + if checkErr := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); checkErr == nil { + t.Fatal("release accepted uncommitted retirement deletion") + } + if !eng.removed(sb.VMName) { + t.Fatal("test did not reach VM removal") + } + if _, checkErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); !errors.Is(checkErr, ErrVolumeBusy) { + t.Fatalf("writer before retirement deletion: %v", checkErr) + } + m.store.writeMu.Lock() + err = os.Remove(m.store.path + ".tmp") + m.store.writeMu.Unlock() + if err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return !m.recommitting.Load() }) + _, retiring, err := m.store.loadJournal() + if err != nil || len(retiring) != 1 { + t.Fatalf("uncertain completion lost retirement: %v %v", retiring, err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if reconcileErr := m2.Reconcile(t.Context()); reconcileErr != nil { + t.Fatal(reconcileErr) + } + if got := volumeHoldersOf(m2, "data"); got != (volumeHolders{}) { + t.Fatalf("gone VM retained hold: %+v", got) + } + _, retiring, err = m2.store.loadJournal() + if err != nil || len(retiring) != 0 { + t.Fatalf("gone VM retained retirement: %v %v", retiring, err) + } +} + +func TestRetirementRetriesFailedDirtyMarkerRemoval(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}) + if err != nil { + t.Fatal(err) + } + marker := volumeDirtyPath(catalog[0].Path) + if err := os.Remove(marker); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(marker, 0o700); err != nil { + t.Fatal(err) + } + blocker := filepath.Join(marker, "busy") + if err := os.WriteFile(blocker, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("release accepted failed marker removal") + } + if got := volumeHoldersOf(m, "data"); got.writers != 1 { + t.Fatalf("marker failure released hold: %+v", got) + } + if err := os.Remove(blocker); err != nil { + t.Fatal(err) + } + m.retryRemovals(t.Context()).Wait() + if volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("marker retry did not finish retirement") + } +} + +func TestClaimsJournalRejectsUnsupportedAndConflictingRecords(t *testing.T) { + for _, raw := range []string{ + `{"format_version":3,"claims":{},"volume_retirements":{}}`, + `{"format_version":2,"claims":{"sb_a":{"id":"sb_a","vm_name":"sbx-a"}},"volume_retirements":{"sbx-a":{"sandbox_id":"sb_a","vm_name":"sbx-a","holds":[{"name":"data"}]}}}`, + `{"format_version":2,"claims":{}}`, + } { + store := newClaimStore(t.TempDir(), false) + if err := os.WriteFile(store.path, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + if _, err := store.load(); err == nil { + t.Fatalf("accepted invalid journal: %s", raw) + } + } + store := newClaimStore(t.TempDir(), false) + if err := store.save(map[string]*types.Sandbox{}); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(store.path) + if err != nil { + t.Fatal(err) + } + var legacy map[string]*types.Sandbox + if err := json.Unmarshal(raw, &legacy); err == nil { + t.Fatal("legacy reader silently accepted versioned journal") + } +} + +func TestUnknownOrphanFencesNewAdmissionAndHeldRepair(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + eng.mountVolumeErr = errors.New("interrupted mount") + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected pending volume") + } + eng.mountVolumeErr = nil + const orphan = "sbx-unknown" + eng.vms[orphan] = "/vsock/unknown" + eng.removeErrFor = orphan + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if reconcileErr := m2.Reconcile(t.Context()); reconcileErr != nil { + t.Fatal(reconcileErr) + } + before := len(eng.volumeSpecs) + if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("held repair bypassed orphan fence: %v", err) + } + if len(eng.volumeSpecs) != before { + t.Fatal("held repair reached disk attach") + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("new claim bypassed orphan fence: %v", err) + } + if _, err := m2.GetVolumes(sb.ID, sb.Tenant); err != nil { + t.Fatal(err) + } + if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err != nil { + t.Fatalf("cleanup blocked by orphan fence: %v", err) + } + eng.removeErrFor = "" + m2.retryRemovals(t.Context()).Wait() + if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other"}}); err != nil { + t.Fatalf("confirmed orphan removal did not open admission: %v", err) + } +} + +func TestReconcileRejectsUncommittedVolumeQuarantine(t *testing.T) { + eng := newFakeEngine() + m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) + sb := &types.Sandbox{ID: "sb_quarantine", VMName: "sbx-quarantine", Key: egKey, Volumes: []types.Volume{{Name: "data", Mount: "/data"}}, VolumesWant: []types.Volume{{Name: "data", Mount: "/data"}}} + eng.vms[sb.VMName] = "/run/quarantine.sock" + eng.snapshots = []string{forkPrefix + "orphan"} + if err := m.store.save(map[string]*types.Sandbox{sb.ID: sb}); err != nil { + t.Fatal(err) + } + m.eng = &quarantineWriteFailureEngine{fakeEngine: eng, path: m.store.path} + withOutbound(t, m, func(o *outbound.Options) { o.Sweep = func(map[string]bool) error { return nil } }) + t.Cleanup(func() { + m.store.writeMu.Lock() + err := os.Remove(m.store.path + ".tmp") + m.store.writeMu.Unlock() + if err != nil { + t.Error(err) + } + waitFor(t, func() bool { return !m.recommitting.Load() }) + }) + if err := m.Reconcile(t.Context()); err == nil { + t.Error("startup accepted a claim whose quarantine could not be persisted") + } + if eng.removed(sb.VMName) { + t.Error("quarantine removed VM before its retirement was durable") + } + if m.claimed[sb.ID] == nil || volumeHoldersOf(m, "data").readers != 1 { + t.Error("failed quarantine lost live claim or hold") + } +} + +type retirementWriteFailureEngine struct { + *fakeEngine + afterRemove func() +} + +func (e *retirementWriteFailureEngine) Remove(ctx context.Context, name string) error { + err := e.fakeEngine.Remove(ctx, name) + if err == nil && e.afterRemove != nil { + e.afterRemove() + e.afterRemove = nil + } + return err +} + +type quarantineWriteFailureEngine struct { + *fakeEngine + path string +} + +func (e *quarantineWriteFailureEngine) SnapshotRemove(ctx context.Context, name string) error { + if err := os.Mkdir(e.path+".tmp", 0o700); err != nil { + return err + } + return e.fakeEngine.SnapshotRemove(ctx, name) +} From ee7610464cc17fd63c0442d595d32a9a4b630d5a Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:08:37 +0000 Subject: [PATCH 08/57] fix(sdk): authorize live volumes with client API tokens --- sdk/go/volume_mutation_test.go | 39 +++++++++--- sdk/go/volumes.go | 36 ++++++----- sdk/python/cocoonsandbox/__init__.py | 20 ++++-- sdk/python/cocoonsandbox/client.py | 4 +- sdk/python/cocoonsandbox/errors.py | 13 +++- sdk/python/cocoonsandbox/sandbox.py | 91 +++++++++++++++------------- sdk/python/cocoonsandbox/volumes.py | 30 ++++----- sdk/python/tests/test_volumes.py | 30 ++++++++- 8 files changed, 175 insertions(+), 88 deletions(-) diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index c61af02b..44ec5aad 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -12,11 +12,11 @@ import ( "testing" ) -func TestLiveVolumesUseOwnerAndSandboxToken(t *testing.T) { +func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { t.Run(method, func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != method || r.URL.Path != "/v1/sandboxes/sb_1/volumes" || r.Header.Get("Authorization") != "Bearer sandbox-token" { + if r.Method != method || r.URL.Path != "/v1/sandboxes/sb_1/volumes" || r.Header.Get("Authorization") != "Bearer operator-token" { t.Errorf("request: %s %s %s", r.Method, r.URL.Path, r.Header.Get("Authorization")) } body, _ := io.ReadAll(r.Body) @@ -30,11 +30,12 @@ func TestLiveVolumesUseOwnerAndSandboxToken(t *testing.T) { if string(body) != want { t.Errorf("body=%s want=%s", body, want) } - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"states":[{"name":"data","state":"attached"}]}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"converging":true,"states":[{"name":"data","state":"attached"}]}`) })) t.Cleanup(ts.Close) c := testClient(t, ts, WithAPIToken("operator-token")) sb := &Sandbox{ID: "sb_1", token: "sandbox-token", owner: c.addr, c: c} + c.addr = "unused.invalid:7777" var got SandboxVolumesResponse var err error switch method { @@ -45,7 +46,7 @@ func TestLiveVolumesUseOwnerAndSandboxToken(t *testing.T) { case http.MethodPatch: got, err = sb.PatchVolumes(t.Context(), VolumePatch{Add: []Volume{{Name: "data"}}, Remove: []string{"old"}}) } - if err != nil || !slices.Equal(got.Volumes, []Volume{{Name: "data", Mount: "/data"}}) || !slices.Equal(got.VolumesWant, got.Volumes) || got.States[0].State != "attached" { + if err != nil || !slices.Equal(got.Volumes, []Volume{{Name: "data", Mount: "/data"}}) || !slices.Equal(got.VolumesWant, got.Volumes) || got.States[0].State != VolumeAttached || !got.Converging { t.Fatalf("got=%+v err=%v", got, err) } got.Volumes[0].Name, got.VolumesWant[0].Name = "caller copy", "caller copy" @@ -59,7 +60,7 @@ func TestLiveVolumesUseOwnerAndSandboxToken(t *testing.T) { func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mount":"/new"}],"volumes":[{"name":"old","mount":"/old"}],"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mount":"/new"}],"volumes":[{"name":"old","mount":"/old"}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -116,7 +117,7 @@ func TestLiveVolumesExplicitEmptyAndCancellation(t *testing.T) { t.Error(err) } bodies = append(bodies, body) - _, _ = io.WriteString(w, `{"volumes_want":[],"volumes":[],"states":[]}`) + _, _ = io.WriteString(w, `{"volumes_want":[],"volumes":[],"converging":false,"states":[]}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -153,10 +154,10 @@ func TestLiveVolumesRetrySameDesiredSet(t *testing.T) { requests = append(requests, string(body)) if len(requests) == 1 { w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[],"states":[{"name":"data","state":"pending"}],"error":"retry"}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[],"converging":false,"states":[{"name":"data","state":"pending"}],"error":"retry"}`) return } - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"states":[{"name":"data","state":"attached"}]}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"converging":false,"states":[{"name":"data","state":"attached"}]}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -173,3 +174,25 @@ func TestLiveVolumesRetrySameDesiredSet(t *testing.T) { t.Fatalf("retry changed target: %v", requests) } } + +func TestLiveVolumeIncompleteViewPreservesHandle(t *testing.T) { + for _, status := range []int{http.StatusOK, http.StatusConflict} { + t.Run(http.StatusText(status), func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + _, _ = io.WriteString(w, `{"volumes_want":[],"volumes":[],"states":[]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", owner: c.addr, c: c, Volumes: []Volume{{Name: "old"}}, VolumesWant: []Volume{{Name: "wanted"}}} + _, err := sb.GetVolumes(t.Context()) + if err == nil || sb.Volumes[0].Name != "old" || sb.VolumesWant[0].Name != "wanted" { + t.Fatalf("handle overwritten: %+v, %v", sb, err) + } + var failure *VolumeError + if status == http.StatusConflict && (!errors.As(err, &failure) || failure.Response != nil) { + t.Fatalf("partial view accepted: %v", err) + } + }) + } +} diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 0a68f233..7ca24600 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -11,11 +11,20 @@ import ( "slices" ) +// VolumePhase is the progress of a desired or applied volume. +type VolumePhase string + +const ( + VolumeAttached VolumePhase = "attached" + VolumePending VolumePhase = "pending" + VolumeDetaching VolumePhase = "detaching" +) + // VolumeState reports the progress of one desired or applied volume. type VolumeState struct { - Name string `json:"name"` - State string `json:"state"` // attached, pending, or detaching - Error string `json:"error,omitempty"` + Name string `json:"name"` + State VolumePhase `json:"state"` + Error string `json:"error,omitempty"` } // SandboxVolumesResponse separates persisted intent from confirmed applied mounts. @@ -23,6 +32,7 @@ type SandboxVolumesResponse struct { VolumesWant []Volume `json:"volumes_want"` Volumes []Volume `json:"volumes"` States []VolumeState `json:"states"` + Converging bool `json:"converging"` Error string `json:"error,omitempty"` } @@ -32,8 +42,7 @@ type VolumePatch struct { Remove []string `json:"remove,omitzero"` } -// VolumeError retains the server's view after a failed volume request. -// Response is nil if the server did not return a complete volume view. +// VolumeError retains a complete server view after a failed volume request. type VolumeError struct { *APIError Response *SandboxVolumesResponse @@ -41,15 +50,12 @@ type VolumeError struct { func (e *VolumeError) Unwrap() error { return e.APIError } -// GetVolumes reads desired and applied volumes from the owning node. -// Like Renew, volume methods update public fields and must be serialized by callers. +// GetVolumes reads the owning node's view and updates public fields; callers must serialize volume methods. func (s *Sandbox) GetVolumes(ctx context.Context) (SandboxVolumesResponse, error) { return s.requestVolumes(ctx, http.MethodGet, nil, "get volumes") } -// SetVolumes replaces the full desired set and waits for convergence. -// An empty non-nil slice clears mutable mounts. A nil slice is rejected. -// Retry the same set after a partial failure, or set a new target to recover. +// SetVolumes converges the full desired set; an explicit empty slice clears mutable mounts, and nil is rejected. func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolumesResponse, error) { if volumes == nil { return SandboxVolumesResponse{}, errors.New("set volumes: use an explicit empty slice to clear volumes") @@ -59,8 +65,7 @@ func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolu }{Volumes: volumes}, "set volumes") } -// PatchVolumes adds or removes read-only mounted catalog entries by name. -// Existing writable and attach-only entries cannot be changed live. +// PatchVolumes adds or removes read-only mounted entries by name, retaining immutable entries. func (s *Sandbox) PatchVolumes(ctx context.Context, patch VolumePatch) (SandboxVolumesResponse, error) { return s.requestVolumes(ctx, http.MethodPatch, patch, "patch volumes") } @@ -75,7 +80,7 @@ func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any } body = bytes.NewReader(encoded) } - resp, err := s.c.roundTrip(ctx, method, s.owner, "/v1/sandboxes/"+s.ID+"/volumes", body, s.token) + resp, err := s.c.roundTrip(ctx, method, s.owner, "/v1/sandboxes/"+s.ID+"/volumes", body, s.c.apiToken) if err != nil { return out, err } @@ -84,12 +89,13 @@ func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any VolumesWant *[]Volume `json:"volumes_want"` Volumes *[]Volume `json:"volumes"` States *[]VolumeState `json:"states"` + Converging *bool `json:"converging"` Error string `json:"error"` } decodeErr := json.UnmarshalRead(resp.Body, &wire) - complete := decodeErr == nil && wire.VolumesWant != nil && wire.Volumes != nil && wire.States != nil + complete := decodeErr == nil && wire.VolumesWant != nil && wire.Volumes != nil && wire.States != nil && wire.Converging != nil if complete { - out = SandboxVolumesResponse{VolumesWant: *wire.VolumesWant, Volumes: *wire.Volumes, States: *wire.States, Error: wire.Error} + out = SandboxVolumesResponse{VolumesWant: *wire.VolumesWant, Volumes: *wire.Volumes, States: *wire.States, Converging: *wire.Converging, Error: wire.Error} s.VolumesWant = slices.Clone(out.VolumesWant) s.Volumes = slices.Clone(out.Volumes) } diff --git a/sdk/python/cocoonsandbox/__init__.py b/sdk/python/cocoonsandbox/__init__.py index 7d95096e..449f37de 100644 --- a/sdk/python/cocoonsandbox/__init__.py +++ b/sdk/python/cocoonsandbox/__init__.py @@ -11,10 +11,18 @@ from .checkpoint import Checkpoint from .client import Client -from .errors import APIError, ExitError, ProtocolError, SandboxError, SandboxTimeout, SilkdError +from .errors import APIError, ExitError, ProtocolError, SandboxError, SandboxTimeout, SilkdError, VolumeError from .sandbox import Lsp, PortConn, Pty, Sandbox, Session, Watcher from .template import Template -from .volumes import SandboxVolumesResponse, Volume, VolumeError, VolumeState +from .volumes import ( + VOLUME_ATTACHED, + VOLUME_DETACHING, + VOLUME_PENDING, + SandboxVolumesResponse, + Volume, + VolumePhase, + VolumeState, +) __all__ = [ "APIError", @@ -26,14 +34,18 @@ "ProtocolError", "Pty", "Sandbox", - "SandboxVolumesResponse", "SandboxError", "SandboxTimeout", + "SandboxVolumesResponse", "Session", "SilkdError", "Template", - "Watcher", + "VOLUME_ATTACHED", + "VOLUME_DETACHING", + "VOLUME_PENDING", "Volume", "VolumeError", + "VolumePhase", "VolumeState", + "Watcher", ] diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index bc9934ec..1095321d 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -17,7 +17,7 @@ from .endpoint import _endpoint_url from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox -from .volumes import volume_body +from .volumes import _volume_body T = TypeVar("T") @@ -296,7 +296,7 @@ def _claim_body( if ttl_seconds: claim["ttl_seconds"] = ttl_seconds if volumes: - claim["volumes"] = [volume_body(volume, mount) for volume in volumes] + claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] if not mount: claim["volumes_attach_only"] = True if claim_ref: diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 4d2159ea..c2c84bc9 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -2,7 +2,10 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .volumes import SandboxVolumesResponse class SandboxError(Exception): @@ -24,6 +27,14 @@ def __init__(self, verb: str, status: int, message: str, *, reply: dict[str, Any self.reply = reply +class VolumeError(APIError): + """Failed convergence with the server's complete view, when returned.""" + + def __init__(self, error: APIError, response: SandboxVolumesResponse | None) -> None: + super().__init__(error.verb, error.status, error.message, reply=error.reply) + self.response = response + + class SilkdError(SandboxError): """The in-guest daemon answered an error frame; kind is its typed wire kind.""" diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index c3c9bbc6..2f416f33 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -12,10 +12,19 @@ from .checkpoint import Checkpoint from .conn import Conn, ConnPool, _Closeable, remaining_timeout -from .errors import APIError, ExitError, ProtocolError, SandboxError, SandboxTimeout, SilkdError, require_field +from .errors import ( + APIError, + ExitError, + ProtocolError, + SandboxError, + SandboxTimeout, + SilkdError, + VolumeError, + require_field, +) from .frames import BULK_CHUNK, FS_CHUNK, KEEP_ALIVE_PROTO from .template import Template -from .volumes import SandboxVolumesResponse, Volume, VolumeError, volume_body, volume_response +from .volumes import SandboxVolumesResponse, Volume, _volume_body, _volume_response if TYPE_CHECKING: from .client import Client @@ -53,6 +62,13 @@ def __init__( def __enter__(self) -> Sandbox: return self + def __exit__(self, *exc: object) -> None: + try: + self.close() + except APIError: + if exc[0] is None: # a clean block surfaces a real release failure + raise + def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse: """Read desired and applied volumes from the owning node.""" return self._volumes_request("GET", None, "get volumes", timeout) @@ -64,7 +80,7 @@ def set_volumes( if volumes is None: raise TypeError("set volumes requires an explicit list; use [] to clear volumes") return self._volumes_request( - "PUT", {"volumes": [volume_body(volume, True) for volume in volumes]}, "set volumes", timeout + "PUT", {"volumes": [_volume_body(volume, True) for volume in volumes]}, "set volumes", timeout ) def patch_volumes( @@ -77,48 +93,11 @@ def patch_volumes( """Add or remove mounted read-only entries by name. Omitted arguments leave the set unchanged.""" body: dict[str, Any] = {} if add is not None: - body["add"] = [volume_body(volume, True) for volume in add] + body["add"] = [_volume_body(volume, True) for volume in add] if remove is not None: body["remove"] = list(remove) return self._volumes_request("PATCH", body, "patch volumes", timeout) - def _volumes_request( - self, method: str, body: dict[str, Any] | None, verb: str, timeout: float | None - ) -> SandboxVolumesResponse: - try: - reply = self._client._request( - self.owner, - method, - f"/v1/sandboxes/{self.id}/volumes", - body, - verb, - bearer=self.token, - timeout=timeout or 0.0, - ) - except APIError as exc: - if exc.status == 0: - raise - view = volume_response(exc.reply) - if view is not None: - self._record_volumes(view) - raise VolumeError(exc, view) from exc - view = volume_response(reply) - if view is None: - raise APIError(verb, 0, "reply without complete volume view") - self._record_volumes(view) - return view - - def _record_volumes(self, view: SandboxVolumesResponse) -> None: - self.volumes_want = [cast(dict[str, str], volume.copy()) for volume in view["volumes_want"]] - self.volumes = [cast(dict[str, str], volume.copy()) for volume in view["volumes"]] - - def __exit__(self, *exc: object) -> None: - try: - self.close() - except APIError: - if exc[0] is None: # a clean block surfaces a real release failure - raise - def exec( self, *argv: str, @@ -436,6 +415,36 @@ def close(self) -> None: if exc.status != 404: raise + def _volumes_request( + self, method: str, body: dict[str, Any] | None, verb: str, timeout: float | None + ) -> SandboxVolumesResponse: + try: + reply = self._client._request( + self.owner, + method, + f"/v1/sandboxes/{self.id}/volumes", + body, + verb, + bearer=self._client.api_token, + timeout=timeout or 0.0, + ) + except APIError as exc: + if exc.status == 0: + raise + view = _volume_response(exc.reply) + if view is not None: + self._record_volumes(view) + raise VolumeError(exc, view) from exc + view = _volume_response(reply) + if view is None: + raise APIError(verb, 0, "reply without complete volume view") + self._record_volumes(view) + return view + + def _record_volumes(self, view: SandboxVolumesResponse) -> None: + self.volumes_want = [cast(dict[str, str], volume.copy()) for volume in view["volumes_want"]] + self.volumes = [cast(dict[str, str], volume.copy()) for volume in view["volumes"]] + def _dial(self, deadline: float | None = None) -> Conn: return self._client._dial(self.owner, self.id, self.token, deadline) diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index bdee0783..65b9f116 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -1,9 +1,14 @@ +"""Volume descriptors and live convergence views.""" + from __future__ import annotations from collections.abc import Mapping -from typing import Any, Literal, TypedDict, cast +from typing import Any, Final, Literal, TypedDict, cast -from .errors import APIError +VolumePhase = Literal["attached", "pending", "detaching"] +VOLUME_ATTACHED: Final[VolumePhase] = "attached" +VOLUME_PENDING: Final[VolumePhase] = "pending" +VOLUME_DETACHING: Final[VolumePhase] = "detaching" class _VolumeName(TypedDict): @@ -17,7 +22,7 @@ class Volume(_VolumeName, total=False): class _VolumeState(TypedDict): name: str - state: Literal["attached", "pending", "detaching"] + state: VolumePhase class VolumeState(_VolumeState, total=False): @@ -28,27 +33,24 @@ class _SandboxVolumesResponse(TypedDict): volumes_want: list[Volume] volumes: list[Volume] states: list[VolumeState] + converging: bool class SandboxVolumesResponse(_SandboxVolumesResponse, total=False): error: str -class VolumeError(APIError): - """Failed convergence with the server's view, when one was returned.""" - - def __init__(self, error: APIError, response: SandboxVolumesResponse | None) -> None: - super().__init__(error.verb, error.status, error.message) - self.response = response - - -def volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | None: - if reply is None or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")): +def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | None: + if ( + reply is None + or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")) + or not isinstance(reply.get("converging"), bool) + ): return None return cast(SandboxVolumesResponse, reply) -def volume_body(volume: str | Mapping[str, str] | Volume, mount: bool) -> dict[str, str]: +def _volume_body(volume: str | Mapping[str, str] | Volume, mount: bool) -> dict[str, str]: if isinstance(volume, str): return {"name": volume} if not isinstance(volume, Mapping): diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 786e820e..8f921be9 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -9,17 +9,18 @@ VIEW = { "volumes_want": [{"name": "data", "mount": "/data"}], "volumes": [{"name": "data", "mount": "/data"}], + "converging": True, "states": [{"name": "data", "state": "attached"}], } @pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) -def test_volume_requests_use_owner_and_sandbox_token(spawn_node, method): +def test_volume_requests_use_owner_and_api_token(spawn_node, method): seen = [] def reply(body, path): seen.append(body) - assert FakeNode.last_headers["Authorization"] == "Bearer tok" + assert FakeNode.last_headers["Authorization"] == "Bearer operator-token" return 200, VIEW addr = spawn_node({(method, PATH): reply}) @@ -47,6 +48,7 @@ def test_volume_error_preserves_typed_partial_view(spawn_node, method): partial = { "volumes_want": [{"name": "new", "mount": "/new"}], "volumes": [{"name": "old", "mount": "/old"}], + "converging": False, "states": [ {"name": "new", "state": "pending", "error": "retry"}, {"name": "old", "state": "detaching"}, @@ -86,7 +88,7 @@ def test_explicit_empty_set_and_omitted_patch(spawn_node): def reply(body, path): seen.append(body) - return 200, {"volumes_want": [], "volumes": [], "states": []} + return 200, {"volumes_want": [], "volumes": [], "converging": False, "states": []} sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) with pytest.raises(TypeError, match="explicit"): @@ -107,6 +109,7 @@ def reply(body, path): return 500, { "volumes_want": [{"name": "data", "mount": "/data"}], "volumes": [], + "converging": False, "states": [{"name": "data", "state": "pending"}], "error": "retry", } @@ -126,3 +129,24 @@ def test_volume_timeout_uses_client_deadline(black_hole): with pytest.raises(APIError) as caught: sb.get_volumes(timeout=0.02) assert caught.value.status == 0 + + +@pytest.mark.parametrize("status", [200, 409]) +def test_incomplete_volume_view_preserves_handle(spawn_node, status): + incomplete = {"volumes_want": [], "volumes": [], "states": []} + sb = sandbox_at(spawn_node({("GET", PATH): lambda body, path: (status, incomplete)})) + sb.volumes = [{"name": "old"}] + sb.volumes_want = [{"name": "wanted"}] + with pytest.raises(APIError) as caught: + sb.get_volumes() + assert sb.volumes == [{"name": "old"}] + assert sb.volumes_want == [{"name": "wanted"}] + if status == 409: + assert isinstance(caught.value, VolumeError) + assert caught.value.response is None + + +def test_volume_error_export(): + from cocoonsandbox.errors import VolumeError as Error + + assert Error is VolumeError From 33de689a04bc1773837c180db9177b3ee6cfe09d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:16:30 +0000 Subject: [PATCH 09/57] test(e2e): prove live volume continuity and retirement on CH --- e2e/cmd/volumesmoke/README.md | 35 ++ e2e/cmd/volumesmoke/continuity.go | 391 ++++++++++++++++++ e2e/cmd/volumesmoke/main.go | 28 +- e2e/cmd/volumesmoke/testdata/fault_hook.py | 108 +++++ .../volumesmoke/testdata/runtime_observer.py | 26 ++ 5 files changed, 583 insertions(+), 5 deletions(-) create mode 100644 e2e/cmd/volumesmoke/README.md create mode 100644 e2e/cmd/volumesmoke/continuity.go create mode 100755 e2e/cmd/volumesmoke/testdata/fault_hook.py create mode 100755 e2e/cmd/volumesmoke/testdata/runtime_observer.py diff --git a/e2e/cmd/volumesmoke/README.md b/e2e/cmd/volumesmoke/README.md new file mode 100644 index 00000000..7965d15d --- /dev/null +++ b/e2e/cmd/volumesmoke/README.md @@ -0,0 +1,35 @@ +# Live volume hardware validation + +Run this command against a real Cloud Hypervisor node with Cocoon v0.7.2 or later. The guest needs silkd, mount, umount, awk, and ordinary coreutils. Fault hooks need Python 3 on the host. Register two catalog datasets with distinct nonempty contents at the same relative probe path. Keep their catalog backing paths unchanged throughout the run. + +Build from the repository's `e2e` directory. + +```sh +GOTOOLCHAIN=go1.27.1 GOWORK=off go build -o "$VOLUME_SMOKE_DIR/volumesmoke" ./cmd/volumesmoke +"$VOLUME_SMOKE_DIR/volumesmoke" -addr "$SANDBOX_ADDR" -token "$SANDBOX_API_TOKEN" \ + -template "$SANDBOX_TEMPLATE" -volume dataset-v1 -next-volume dataset-v2 -probe version +``` + +The command checks shared read-only mounts, EROFS, repeated updates, and removal. `-next-volume` adds a same-path replacement while preserving the sandbox ID, session ID, guest boot ID, PID, process start time, and session environment. It tests cwd, bind, and foreign-mount conflicts with HTTP 409, then retries after releasing the reference. `RETAINED begin/end` timestamps identify the repeated-target window. The runtime command log must contain zero disk attach or detach commands in that window. + +The optional fault hooks operate only on an isolated daemon. Use disposable backing files for `-release-hook`, which writes `.retirement-reuse` through a later writable mount. That catalog entry must allow writable attachment. The helper never prints the API token. + +1. Set `VOLUME_SMOKE_DIR` to an absolute scratch directory. Place the candidate `sandboxd` binary and its `config.json` there. The config must use a dedicated listen address and data directory. Its `cocoon_bin` must point to the absolute path of `testdata/runtime_observer.py`. +2. Set `COCOON_REAL_BIN` to the absolute path of the official Cocoon binary. Configure isolated `COCOON_ROOT_DIR`, `COCOON_RUN_DIR`, and `COCOON_LOG_DIR`, and a distinct network scope and cgroup parent. Preserve these environment variables for the daemon and smoke command. +3. Start the daemon as a child process and write its actual PID to `$VOLUME_SMOKE_DIR/daemon.pid`. The restart hook kills only this PID. The real VMM must keep running. +4. Build the existing guest listener with `CGO_ENABLED=0 GOTOOLCHAIN=go1.27.1 GOWORK=off go build -o "$VOLUME_SMOKE_DIR/guestserver" ./cmd/guestserver` from the `e2e` directory. Run the extended command with the three hooks. Set `VOLUME_HOOK` to the absolute path of `testdata/fault_hook.py`. + +```sh +"$VOLUME_SMOKE_DIR/volumesmoke" -addr "$SANDBOX_ADDR" -token "$SANDBOX_API_TOKEN" \ + -template "$SANDBOX_TEMPLATE" -volume dataset-v1 -next-volume dataset-v2 -probe version \ + -listener "$VOLUME_SMOKE_DIR/guestserver" \ + -change-hook "$VOLUME_HOOK" \ + -cancel-hook "$VOLUME_HOOK" \ + -release-hook "$VOLUME_HOOK" +``` + +The observer executes every ordinary command through the official Cocoon binary, including `version`. An attach gate pauses only after a successful real disk attach and before returning its result to sandboxd. The restart hook kills sandboxd at that gate. The cancellation hook cancels the caller at the same gate. Both save the API view and real VM PID and disks. Independent exec, file, and HTTP port operations must complete while the gate is active. Retrying the same desired target must converge. + +The retirement hook injects exit 73 before `vm rm` while its failure marker exists. It checks that a new writer is refused before and after a daemon restart. Removing the marker permits the ordinary cleanup worker to execute real Cocoon removal. Only successful admission, mounting, and writing by a later writer counts as backing reuse. Save `commands.jsonl`, `candidate-daemon.log`, smoke output, runtime versions, and the backing checksums outside the repository. + +The command runs the supplied absolute script path through `python3` with a fixed `restart`, `cancel`, or `retire` mode argument. It does not interpret a shell command. The hooks use `arm`, `wait`, and `interrupt` stages supplied in `VOLUME_HOOK_STAGE`, plus the sandbox ID in `SANDBOX_ID`. A failed run can leave fault markers behind. Stop the isolated daemon before removing its `gate-attach` or `fail-remove` marker and cleaning up its VMs. Do not apply these hooks to a shared production daemon. diff --git a/e2e/cmd/volumesmoke/continuity.go b/e2e/cmd/volumesmoke/continuity.go new file mode 100644 index 00000000..442dd517 --- /dev/null +++ b/e2e/cmd/volumesmoke/continuity.go @@ -0,0 +1,391 @@ +package main + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/cocoonstack/sandbox/e2e/internal/harness" + sandbox "github.com/cocoonstack/sandbox/sdk/go" +) + +type ( + liveOptions struct{ nextVolume, changeHook, cancelHook, releaseHook, listener string } + processIdentity struct { + Sandbox, Session, Boot string + PID uint32 + Start string + } +) + +func identity(ctx context.Context, sb *sandbox.Sandbox, session *sandbox.Session, pid uint32) (processIdentity, error) { + boot, err := sb.Exec(ctx, "cat", "/proc/sys/kernel/random/boot_id") + if err != nil { + return processIdentity{}, err + } + stat, err := sb.Exec(ctx, "cat", fmt.Sprintf("/proc/%d/stat", pid)) + if err != nil { + return processIdentity{}, err + } + _, rest, ok := strings.Cut(stat, ") ") + fields := strings.Fields(rest) + if !ok || len(fields) < 20 { + return processIdentity{}, fmt.Errorf("invalid process stat %q", stat) + } + return processIdentity{sb.ID, session.ID, strings.TrimSpace(boot), pid, fields[19]}, nil +} + +func runContinuity(ctx context.Context, sb *sandbox.Sandbox, volume, probe, want string, opts liveOptions) error { + const mount = "/datasets/current" + if opts.changeHook != "" || opts.cancelHook != "" { + if err := uploadListener(ctx, sb, opts.listener); err != nil { + return err + } + } + session, err := sb.NewSession(ctx, sandbox.WithSessionCwd("/tmp"), sandbox.WithSessionEnv(map[string]string{"DATASET_SESSION": "same"})) + if err != nil { + return err + } + defer func() { _ = session.Close(context.Background()) }() + pid, err := sb.Spawn(ctx, sandbox.Cmd{Argv: []string{"sleep", "3600"}, Cwd: "/tmp"}) + if err != nil { + return err + } + defer func() { _ = sb.Kill(context.Background(), pid, 0) }() + before, err := identity(ctx, sb, session, pid) + if err != nil { + return err + } + encoded, _ := json.Marshal(before) + fmt.Printf("CONTINUITY before=%s\n", encoded) + v1 := []sandbox.Volume{{Name: volume, Mount: mount}} + v2 := []sandbox.Volume{{Name: opts.nextVolume, Mount: mount}} + v2content, err := exerciseMountConflicts(ctx, sb, v1, v2, probe, want) + if err != nil { + return err + } + if opts.changeHook != "" { + if err = interruptedChange(ctx, sb, v1, opts.changeHook, false); err != nil { + return err + } + if _, err = sb.SetVolumes(ctx, v2); err != nil { + return err + } + } + if opts.cancelHook != "" { + if err = interruptedChange(ctx, sb, v1, opts.cancelHook, true); err != nil { + return err + } + } + view, err := sb.SetVolumes(ctx, v2) + if err != nil || !slices.Equal(view.Volumes, v2) || !slices.Equal(view.VolumesWant, v2) { + return fmt.Errorf("second dataset did not converge: view=%+v error=%v", view, err) + } + if content, readErr := sb.Exec(ctx, "cat", path.Join(mount, probe)); readErr != nil || content != v2content { + return fmt.Errorf("dataset changed after recovery: content=%q error=%v", content, readErr) + } + if err = requireSettled(view); err != nil { + return err + } + if err = sb.WriteFile(ctx, "/tmp/continuity", []byte("independent file"), nil); err != nil { + return err + } + data, err := sb.ReadFile(ctx, "/tmp/continuity") + if err != nil || string(data) != "independent file" { + return fmt.Errorf("independent file round trip: %q %v", data, err) + } + if _, err = sb.SetVolumes(ctx, []sandbox.Volume{}); err != nil { + return err + } + after, err := identity(ctx, sb, session, pid) + if err != nil || after != before { + return fmt.Errorf("process identity changed: before=%+v after=%+v error=%v", before, after, err) + } + sessions, err := sb.Sessions(ctx) + if err != nil || !slices.Contains(sessions, session.ID) { + return fmt.Errorf("session disappeared: %v %v", sessions, err) + } + state, err := session.Exec(ctx, "printenv", "DATASET_SESSION") + if err != nil || strings.TrimSpace(state) != "same" { + return fmt.Errorf("session state changed: %q %v", state, err) + } + encoded, _ = json.Marshal(after) + fmt.Printf("CONTINUITY PASS after=%s v1=%q v2=%q cwd_409=true bind_409=true foreign_409=true retained_bind=true erofs=true cleared=true\n", encoded, strings.TrimSpace(want), strings.TrimSpace(v2content)) + return nil +} + +func refuseBusy(ctx context.Context, sb *sandbox.Sandbox, target []sandbox.Volume, mount, reference string) error { + view, err := sb.SetVolumes(ctx, target) + api, ok := errors.AsType[*sandbox.APIError](err) + if !ok || api.Status != 409 || !strings.Contains(err.Error(), mount) { + return fmt.Errorf("%s busy removal must return 409 and guest path: view=%+v error=%v", reference, view, err) + } + encoded, _ := json.Marshal(view) + fmt.Printf("BUSY PASS reference=%s view=%s\n", reference, encoded) + return nil +} + +func interruptedChange(ctx context.Context, sb *sandbox.Sandbox, target []sandbox.Volume, hook string, cancelRequest bool) error { + mode := "restart" + if cancelRequest { + mode = "cancel" + } + if err := runHook(ctx, sb.ID, hook, mode, "arm"); err != nil { + return err + } + server, err := sb.Spawn(ctx, sandbox.Cmd{Argv: []string{"/tmp/volume-guestserver", "-addr", "127.0.0.1:18080"}, Cwd: "/tmp"}) + if err != nil { + return err + } + defer func() { _ = sb.Kill(context.Background(), server, 0) }() + for deadline := time.Now().Add(5 * time.Second); ; { + if _, err := harness.HTTPOverPort(ctx, sb, 18080, "GET", "/", nil); err == nil { + break + } + if time.Now().After(deadline) { + return errors.New("independent HTTP server did not start") + } + time.Sleep(20 * time.Millisecond) + } + requestCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, 1) + go func() { _, err := sb.SetVolumes(requestCtx, target); done <- err }() + if err := runHook(ctx, sb.ID, hook, mode, "wait"); err != nil { + return err + } + if _, err := sb.Exec(ctx, "true"); err != nil { + return err + } + if err := sb.WriteFile(ctx, "/tmp/during-change", []byte("still running"), nil); err != nil { + return err + } + if content, readErr := sb.ReadFile(ctx, "/tmp/during-change"); readErr != nil || string(content) != "still running" { + return fmt.Errorf("file read during change: %q %v", content, readErr) + } + if _, err := harness.HTTPOverPort(ctx, sb, 18080, "GET", "/during-change", nil); err != nil { + return err + } + if cancelRequest { + cancel() + } + if err := runHook(ctx, sb.ID, hook, mode, "interrupt"); err != nil { + return err + } + fmt.Println("ACTIVITY PASS exec=true file=true port=true") + requestErr := <-done + fmt.Printf("INTERRUPTION request_error=%v\n", requestErr) + if requestErr == nil { + return errors.New("change hook did not interrupt the request") + } + deadline := time.Now().Add(30 * time.Second) + for { + view, err := sb.GetVolumes(ctx) + if err == nil { + encoded, _ := json.Marshal(view) + fmt.Printf("INTERRUPTION recovered_view=%s\n", encoded) + break + } + if time.Now().After(deadline) { + return fmt.Errorf("daemon did not recover: %w", err) + } + time.Sleep(100 * time.Millisecond) + } + if _, err := sb.SetVolumes(ctx, target); err != nil { + return fmt.Errorf("retry after interruption: %w", err) + } + fmt.Printf("INTERRUPTION PASS sandbox=%s\n", sb.ID) + return nil +} + +func runHook(ctx context.Context, id, hook, mode, stage string) error { + if !filepath.IsAbs(hook) || filepath.Clean(hook) != hook { + return errors.New("fault hook must be a clean absolute Python script path") + } + command := exec.CommandContext(ctx, "python3") + command.Args = append(command.Args, hook, mode) + command.Env = append(os.Environ(), "SANDBOX_ID="+id, "VOLUME_HOOK_STAGE="+stage) + command.Stdout, command.Stderr = os.Stdout, os.Stderr + if err := command.Run(); err != nil { + return fmt.Errorf("change hook %s: %w", stage, err) + } + return nil +} + +func requireSettled(view sandbox.SandboxVolumesResponse) error { + encoded, err := json.Marshal(view) + if err != nil { + return err + } + var progress struct { + Converging *bool `json:"converging"` + } + if err := json.Unmarshal(encoded, &progress); err != nil { + return err + } + if progress.Converging == nil || *progress.Converging { + return fmt.Errorf("settled view must explicitly report converging=false: %s", encoded) + } + return nil +} + +func exerciseMountConflicts(ctx context.Context, sb *sandbox.Sandbox, v1, v2 []sandbox.Volume, probe, want string) (string, error) { + const mount = "/datasets/current" + var err error + if _, err = sb.SetVolumes(ctx, v1); err != nil { + return "", err + } + busy, err := sb.Spawn(ctx, sandbox.Cmd{Argv: []string{"sleep", "3600"}, Cwd: mount}) + if err != nil { + return "", err + } + defer func() { _ = sb.Kill(context.Background(), busy, 0) }() + if err = refuseBusy(ctx, sb, v2, mount, "cwd"); err != nil { + return "", err + } + got, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) + if err != nil || got != want { + return "", fmt.Errorf("busy cwd changed old dataset: content=%q error=%v", got, err) + } + if err = sb.Kill(ctx, busy, 0); err != nil { + return "", err + } + if _, err = sb.SetVolumes(ctx, v2); err != nil { + return "", fmt.Errorf("cwd retry: %w", err) + } + if _, err = sb.SetVolumes(ctx, v1); err != nil { + return "", err + } + if _, err = sb.Exec(ctx, "sh", "-c", "mkdir -p /tmp/volume-bind; mount --bind /datasets/current /tmp/volume-bind"); err != nil { + return "", err + } + defer func() { _, _ = sb.Exec(context.Background(), "umount", "/tmp/volume-bind") }() + fmt.Printf("RETAINED begin=%d\n", time.Now().UnixNano()) + if _, err = sb.SetVolumes(ctx, v1); err != nil { + return "", fmt.Errorf("retained volume with bind was reapplied: %w", err) + } + fmt.Printf("RETAINED end=%d\n", time.Now().UnixNano()) + if err = refuseBusy(ctx, sb, v2, "/tmp/volume-bind", "bind"); err != nil { + return "", err + } + if got, err = sb.Exec(ctx, "cat", path.Join("/tmp/volume-bind", probe)); err != nil || got != want { + return "", fmt.Errorf("busy bind was changed: content=%q error=%v", got, err) + } + if _, err = sb.Exec(ctx, "umount", "/tmp/volume-bind"); err != nil { + return "", err + } + if _, err = sb.SetVolumes(ctx, v2); err != nil { + return "", fmt.Errorf("same-path retry: %w", err) + } + v2content, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) + if err != nil || v2content == "" || v2content == want { + return "", fmt.Errorf("second dataset marker must differ: content=%q error=%v", v2content, err) + } + if _, err = sb.Exec(ctx, "touch", path.Join(mount, ".write-probe")); err == nil { + return "", errors.New("second dataset accepted a write") + } + if exit, ok := errors.AsType[*sandbox.ExitError](err); !ok || !strings.Contains(strings.ToLower(exit.Stderr), "read-only file system") { + return "", fmt.Errorf("second dataset write did not fail with EROFS: %w", err) + } + if _, err = sb.Exec(ctx, "sh", "-c", "awk '$2 == \"/datasets/current\" && $4 ~ /(^|,)ro(,|$)/ { found=1; print } END { exit !found }' /proc/mounts"); err != nil { + return "", fmt.Errorf("guest mount is not read-only: %w", err) + } + if _, err = sb.Exec(ctx, "mount", "-t", "tmpfs", "tmpfs", mount); err != nil { + return "", err + } + if err = refuseBusy(ctx, sb, v1, mount, "foreign"); err != nil { + return "", err + } + if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { + return "", fmt.Errorf("foreign mount was removed: %w", err) + } + if _, err = sb.Exec(ctx, "umount", mount); err != nil { + return "", err + } + if _, err = sb.SetVolumes(ctx, v1); err != nil { + return "", err + } + if _, err = sb.SetVolumes(ctx, v2); err != nil { + return "", err + } + return v2content, nil +} + +func runRetirement(ctx context.Context, client *sandbox.Client, template, volume, hook string) error { + reader, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone), sandbox.WithVolumes(sandbox.Volume{Name: volume, Mount: "/datasets/retiring"})) + if err != nil { + return err + } + defer func() { _ = reader.Close() }() + if err := runHook(ctx, reader.ID, hook, "retire", "arm"); err != nil { + return err + } + fmt.Printf("RETIREMENT release_error=%v\n", reader.Close()) + writerOptions := []sandbox.Option{sandbox.WithNetwork(sandbox.NetNone), sandbox.WithVolumes(sandbox.Volume{Name: volume, Mount: "/datasets/reused", Mode: "rw"})} + refuseWriter := func() error { + writer, err := client.New(ctx, template, writerOptions...) + if err == nil { + _ = writer.Close() + return errors.New("writer admitted while retired VM still owns backing") + } + api, ok := errors.AsType[*sandbox.APIError](err) + if !ok || api.Status != 409 { + return fmt.Errorf("retired backing must return 409: %w", err) + } + fmt.Printf("RETIREMENT writer_refused=%v\n", err) + return nil + } + if err := refuseWriter(); err != nil { + return err + } + if err := runHook(ctx, reader.ID, hook, "retire", "wait"); err != nil { + return err + } + if err := refuseWriter(); err != nil { + return err + } + if err := runHook(ctx, reader.ID, hook, "retire", "interrupt"); err != nil { + return err + } + deadline := time.Now().Add(45 * time.Second) + for { + writer, err := client.New(ctx, template, writerOptions...) + if err == nil { + defer func() { _ = writer.Close() }() + if _, err = writer.Exec(ctx, "touch", "/datasets/reused/.retirement-reuse"); err != nil { + return err + } + if err = writer.Close(); err != nil { + return err + } + fmt.Println("RETIREMENT PASS writer_blocked_before_restart=true writer_blocked_after_restart=true backing_reused=true") + return nil + } + api, ok := errors.AsType[*sandbox.APIError](err) + if !ok || api.Status != 409 || time.Now().After(deadline) { + return fmt.Errorf("backing not reusable after cleanup: %w", err) + } + time.Sleep(200 * time.Millisecond) + } +} + +func uploadListener(ctx context.Context, sb *sandbox.Sandbox, listener string) error { + root, err := os.OpenRoot(filepath.Dir(listener)) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + binary, err := root.ReadFile(filepath.Base(listener)) + if err != nil { + return err + } + return sb.WriteFile(ctx, "/tmp/volume-guestserver", binary, new(uint32(0o755))) +} diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index deccc233..2d3834c4 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -24,23 +24,31 @@ func main() { template := flag.String("template", "rt:24.04", "template ref") volume := flag.String("volume", "", "catalog volume name") rwVolume := flag.String("rw-volume", "", "writable catalog volume name (adds the writable leg)") + nextVolume := flag.String("next-volume", "", "second catalog dataset for the same-path continuity proof") + changeHook := flag.String("change-hook", "", "absolute Python hook path to interrupt a change; SANDBOX_ID is provided") + cancelHook := flag.String("cancel-hook", "", "absolute Python hook path to gate cancellation; SANDBOX_ID and VOLUME_HOOK_STAGE are provided") + releaseHook := flag.String("release-hook", "", "absolute Python hook path to test failed removal and recovery; uses VOLUME_HOOK_STAGE") + listener := flag.String("listener", "", "path to the existing guestserver binary for activity checks during faults") probe := flag.String("probe", "volume-e2e.txt", "non-empty file inside the volume") flag.Parse() - if err := run(*addr, *token, *template, *volume, *rwVolume, *probe); err != nil { + if err := run(*addr, *token, *template, *volume, *rwVolume, *probe, liveOptions{nextVolume: *nextVolume, changeHook: *changeHook, cancelHook: *cancelHook, releaseHook: *releaseHook, listener: *listener}); err != nil { fmt.Fprintln(os.Stderr, "volumesmoke:", err) os.Exit(1) } } -func run(addr, token, template, volume, rwVolume, probe string) error { +func run(addr, token, template, volume, rwVolume, probe string, live liveOptions) error { if volume == "" { return errors.New("volume is required") } + if (live.changeHook != "" || live.cancelHook != "") && live.listener == "" { + return errors.New("listener is required with change-hook or cancel-hook") + } if probe == "" || probe == "." || path.IsAbs(probe) || path.Clean(probe) != probe || probe == ".." || strings.HasPrefix(probe, "../") { return fmt.Errorf("probe %q must be a clean relative file path", probe) } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() const ( @@ -118,9 +126,14 @@ func run(addr, token, template, volume, rwVolume, probe string) error { } released = true fmt.Printf("VOLUME PASS concurrent_read_bytes=%d read_only=true released=true\n", len(outputs[0])) - if err := runLive(ctx, client, template, volume, probe, outputs[0]); err != nil { + if err := runLive(ctx, client, template, volume, probe, outputs[0], live); err != nil { return err } + if live.releaseHook != "" { + if err := runRetirement(ctx, client, template, volume, live.releaseHook); err != nil { + return err + } + } if rwVolume == "" { return nil } @@ -129,7 +142,7 @@ func run(addr, token, template, volume, rwVolume, probe string) error { // runLive starts without volumes, adds a dataset, then removes it without // replacing the claim. Run this on the target VMM before deploying live mounts. -func runLive(ctx context.Context, client *sandbox.Client, template, volume, probe, want string) error { +func runLive(ctx context.Context, client *sandbox.Client, template, volume, probe, want string, live liveOptions) error { const mount = "/datasets/live" sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone)) if err != nil { @@ -193,6 +206,11 @@ func runLive(ctx context.Context, client *sandbox.Client, template, volume, prob if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { return fmt.Errorf("cleared volume probe should be absent: %w", err) } + if live.nextVolume != "" { + if err := runContinuity(ctx, sb, volume, probe, want, live); err != nil { + return err + } + } fmt.Printf("LIVE VOLUME PASS read_bytes=%d repeated_put=true repeated_patch=true cleared=true\n", len(got)) return nil } diff --git a/e2e/cmd/volumesmoke/testdata/fault_hook.py b/e2e/cmd/volumesmoke/testdata/fault_hook.py new file mode 100755 index 00000000..94e4aaf9 --- /dev/null +++ b/e2e/cmd/volumesmoke/testdata/fault_hook.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Gate real attach completion, cancel a caller, or restart an isolated daemon.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time +import urllib.request + +root = Path(os.environ["VOLUME_SMOKE_DIR"]) +mode = sys.argv[1] +stage = os.environ["VOLUME_HOOK_STAGE"] +sandbox_id = os.environ["SANDBOX_ID"] +config = json.loads((root / "config.json").read_text()) +address = "http://" + config["listen"] + + +def view(): + request = urllib.request.Request( + f"{address}/v1/sandboxes/{sandbox_id}/volumes", + headers={"Authorization": "Bearer " + config["api_token"]}, + ) + with urllib.request.urlopen(request, timeout=3) as response: + return json.load(response) + + +def restart(): + old = int((root / "daemon.pid").read_text()) + os.kill(old, signal.SIGKILL) + (root / "gate-attach").unlink(missing_ok=True) + with (root / "candidate-daemon.log").open("a") as log: + process = subprocess.Popen( + [str(root / "sandboxd"), "-config", str(root / "config.json")], + stdout=log, + stderr=log, + start_new_session=True, + ) + (root / "daemon.pid").write_text(str(process.pid)) + deadline = time.monotonic() + 30 + while True: + try: + with urllib.request.urlopen(address + "/healthz", timeout=1): + break + except OSError: + if time.monotonic() > deadline: + raise RuntimeError("isolated daemon did not restart") from None + time.sleep(0.05) + print(f"HOOK daemon_restart old={old} new={process.pid}", flush=True) + + +def wait_for(marker): + deadline = time.monotonic() + 10 + while not marker.exists(): + if time.monotonic() > deadline: + raise RuntimeError(f"runtime never reached {marker.name}") + time.sleep(0.01) + print("HOOK runtime_event=" + marker.read_text(), flush=True) + + +def runtime_view(): + data = subprocess.check_output( + [os.environ["COCOON_REAL_BIN"], "vm", "list", "--format", "json"] + ) + running = [] + for vm in json.loads(data): + if not vm["config"]["name"].startswith("sbx-"): + continue + pid = vm.get("pid", 0) + running.append( + { + "id": vm["id"], + "name": vm["config"]["name"], + "pid": pid, + "pid_exists": pid > 0 and Path(f"/proc/{pid}").exists(), + "disks": vm.get("storage_configs"), + } + ) + print("HOOK real_vm=" + json.dumps(running), flush=True) + + +if stage == "arm": + marker = "remove-failed" if mode == "retire" else "gate-entered" + (root / marker).unlink(missing_ok=True) + (root / ("fail-remove" if mode == "retire" else "gate-attach")).touch() +elif stage == "wait": + wait_for(root / ("remove-failed" if mode == "retire" else "gate-entered")) + runtime_view() + if mode == "retire": + restart() + runtime_view() + else: + print("HOOK gated_view=" + json.dumps(view()), flush=True) +elif stage == "interrupt": + if mode == "restart": + restart() + elif mode == "cancel": + time.sleep(0.5) + print("HOOK canceled_view=" + json.dumps(view()), flush=True) + (root / "gate-attach").unlink(missing_ok=True) + elif mode == "retire": + (root / "fail-remove").unlink() + else: + raise RuntimeError("unknown hook mode " + mode) +else: + raise RuntimeError("unknown hook stage " + stage) diff --git a/e2e/cmd/volumesmoke/testdata/runtime_observer.py b/e2e/cmd/volumesmoke/testdata/runtime_observer.py new file mode 100755 index 00000000..233ed5f3 --- /dev/null +++ b/e2e/cmd/volumesmoke/testdata/runtime_observer.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Forward Cocoon commands and expose explicit hardware fault injection gates.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +root = Path(os.environ["VOLUME_SMOKE_DIR"]) +args = sys.argv[1:] +event = {"time": time.time(), "pid": os.getpid(), "args": args} +with (root / "commands.jsonl").open("a") as log: + log.write(json.dumps(event) + "\n") +if args[:2] == ["vm", "rm"] and (root / "fail-remove").exists(): + (root / "remove-failed").write_text(json.dumps(event)) + print("volumesmoke injected VM removal failure", file=sys.stderr) + sys.exit(73) +result = subprocess.run([os.environ["COCOON_REAL_BIN"], *args], check=False) +if args[:3] == ["vm", "disk", "attach"] and result.returncode == 0: + if (root / "gate-attach").exists(): + (root / "gate-entered").write_text(json.dumps(event)) + while (root / "gate-attach").exists(): + time.sleep(0.02) +sys.exit(result.returncode) From 27b2448996204b494d2d05a7a1cf8698ee48e28f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:16:31 +0000 Subject: [PATCH 10/57] docs(volumes): explain live replacement and recovery --- docs/deploy.md | 33 +++++++++++---- docs/live-volumes.md | 97 ++++++++++++++++++++++++++++++++++++++++++++ docs/sandboxd-api.md | 76 ++++++++++++++++++++++++---------- docs/sdk-python.md | 17 ++++++++ docs/sdk.md | 23 +++++++---- 5 files changed, 208 insertions(+), 38 deletions(-) create mode 100644 docs/live-volumes.md diff --git a/docs/deploy.md b/docs/deploy.md index 38d5eb34..b0b7f644 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -6,14 +6,14 @@ the cocoon CLI and needs a template image with silkd baked in. ## Prerequisites - Linux with KVM (`/dev/kvm`) -- [cocoon](https://github.com/cocoonstack/cocoon) **v0.6.9 or newer** installed +- [cocoon](https://github.com/cocoonstack/cocoon) **v0.7.2 or newer** installed and working (`cocoon vm run` boots a Cloud Hypervisor VM). v0.5.2 brought the disk hot-attach used by read-only volumes and the parallel-clone and snapshot/store performance work the [performance](performance.md) numbers assume; v0.6.9 prints `[]` for an empty `snapshot list --format json`, which sandboxd's snapshot reconcile reads. sandboxd logs a warning at startup when the detected cocoon is below - v0.6.9 (a dev/`master-` build is assumed current) + v0.7.2. Live volume mutation also rejects unknown and development builds - The sandbox boot artifact installed where cocoon finds it (`/boot/vmlinuz-sandbox`, `/boot/initrd.img-sandbox` — from `ghcr.io/cocoonstack/sandbox/boot:`) @@ -159,16 +159,28 @@ fragment the warm pools): Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes -require cocoon commit `6d76309` or a build containing it for idempotent disk -attach and detach. That change was on cocoon master without a release tag -when this feature was prepared. The general v0.6.9 floor above is insufficient -for live convergence. +require Cocoon v0.7.2 or newer for idempotent disk attach and detach. PUT and +PATCH reject older, unknown, development, or unqueryable versions with 503 +before any intent, hold, or device change. GET and release remain available. The claim journal stores desired `VolumesWant` and confirmed applied `Volumes`. -Admission holds cover their union across restart and teardown. Retry an +Admission holds cover their union across daemon restart and teardown. Version 2 +journals preserve a separate retirement record until VM deletion is confirmed +and the record deletion commits. Retry an interrupted operation with the same PUT target. Release cancels convergence and transfers holds to teardown. Capture is refused while either set is -nonempty. Drain all volume claims before downgrading sandboxd. +nonempty. Back up the journal before upgrading. Older daemons reject the new +format. Drain all volume claims and finish retirements before downgrading; +do not manually convert a journal that still has retirement records. + +A sandbox whose VMM exits while holding volumes is release-only. Release it and +create a new sandbox. Daemon restart recovery assumes the original VMM remains +alive and does not recreate guest processes. Unfinished changes report an +interrupted diagnostic after restart and require a caller retry. + +If startup cannot confirm removal of an unknown owned orphan, the node blocks +new volume admission and all disk attachment, including pending repair, until +the orphan is gone. GET, detach, and release remain available. `e2e/cmd/volumesmoke` exercises GET, repeated full-set PUT, PATCH add and remove, and clearing a new claim. Real Cloud Hypervisor hotplug, busy guest unmount, @@ -176,7 +188,10 @@ and restart during convergence remain unverified here. The maintainer will run these checks on the target VMM and guest testbed. Each catalog path must name a disk image containing a mountable whole-device -filesystem. A read-only entry's image must stay immutable: do not replace, +filesystem. Keep catalog names bound to the same image while any live or +retiring claim holds them. Use versioned names such as `dataset-v1` and +`dataset-v2`; sandboxd cannot detect a repointed catalog entry. A read-only +entry's image must stay immutable: do not replace, truncate, or delete it while attached; publish a new catalog name or path instead. A missing path produces a startup warning and fails only claims that request it, allowing images to be distributed after sandboxd starts. diff --git a/docs/live-volumes.md b/docs/live-volumes.md new file mode 100644 index 00000000..af91e7dc --- /dev/null +++ b/docs/live-volumes.md @@ -0,0 +1,97 @@ +# Replace a dataset in a running sandbox + +This example replaces `dataset-v1` with `dataset-v2` at `/datasets/current` +while preserving the sandbox, session, guest boot, and independent process. + +Register both read-only catalog entries on the owning node and grant the +sandbox's tenant access to them. Put `v1` and `v2` in each image's `/version` +file. Keep each name bound to the same immutable image while held. Use Cocoon +v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. +A sandbox token alone cannot manage volumes. + +Pause application reads before replacement. Close files, mmap mappings, cwd, +and bind mount references on the old disk. Replacement unmounts the old disk +before mounting the new one, so the path has a short unavailable interval. +The independent process below uses `/tmp` and can continue running. + +```python +import os + +from cocoonsandbox import Client + + +def process_identity(sb, session, pid): + stat = sb.exec("cat", f"/proc/{pid}/stat") + starttime = stat.rsplit(") ", 1)[1].split()[19] + boot = sb.exec("cat", "/proc/sys/kernel/random/boot_id").strip() + return sb.id, session.id, boot, pid, starttime + + +client = Client(os.environ["SANDBOX_ADDR"], api_token=os.environ["SANDBOX_API_TOKEN"]) +v1 = [{"name": "dataset-v1", "mount": "/datasets/current"}] +v2 = [{"name": "dataset-v2", "mount": "/datasets/current"}] + +with client.new("rt:24.04", ttl_seconds=900) as sb: + with sb.session(cwd="/tmp", env={"DATASET_SESSION": "same"}) as session: + pid = sb.spawn("sleep", "3600", cwd="/tmp") + try: + identity = process_identity(sb, session, pid) + sb.set_volumes(v1) + assert sb.exec("cat", "/datasets/current/version").strip() == "v1" + + sb.patch_volumes(remove=["dataset-v1"], add=v2) + sb.set_volumes(v2) + + assert sb.exec("cat", "/datasets/current/version").strip() == "v2" + assert process_identity(sb, session, pid) == identity + assert session.id in sb.sessions() + assert session.exec("printenv", "DATASET_SESSION").strip() == "same" + view = sb.get_volumes() + assert view["volumes_want"] == view["volumes"] == v2 + assert view["converging"] is False + + sb.set_volumes([]) + assert process_identity(sb, session, pid) == identity + finally: + sb.kill(pid) +``` + +A successful run reads `v1`, then `v2`, and preserves every identity assertion. +The repeated full target verifies that a settled mount can be retained. +This example starts without other volumes. If your sandbox has writable or +attach-only volumes, retain them in a full PUT or use PATCH to replace only +the dataset names. `None`, a nil Go slice, and a missing `volumes` field do not +mean an empty target. + +Go uses the same owning handle and client API token. + +```go +view, err := sb.PatchVolumes(ctx, sandbox.VolumePatch{ + Remove: []string{"dataset-v1"}, + Add: []sandbox.Volume{{Name: "dataset-v2", Mount: "/datasets/current"}}, +}) +if err != nil { + return err +} +if view.Converging { + return errors.New("volume request still running") +} +``` + +For a 409 mount conflict, use the guest path in `states[].error` to close +references or remove unexpected mounts, then retry. For a transport failure or +timeout, GET the current target and confirmed set before retrying the target. +A failed call can already have changed devices. `VolumeError` retains a +complete failure view in Go's `Response` or Python's `.response`. + +`pending` or `detaching` with `converging: false` needs a caller retry. GET does +not run device repair. After sandboxd restarts with the original VM still +alive, unfinished entries report an interrupted operation and keep their +admission holds. Detailed errors survive only within one daemon lifetime. +If the VMM exits, a sandbox holding volumes is release-only. Release it and +create a new sandbox; neither PUT with an empty target nor daemon restart +restores the original processes. + +The example's Python syntax and SDK signatures are checked locally. Preserving +these identities during real Cloud Hypervisor hotplug and daemon restart +requires the maintainer's hardware testbed. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index eafcdd46..1e29a141 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -258,14 +258,17 @@ entry's response is byte-identical to v1. ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set -(`volumes`), and per-volume `states`. Auth is the sandbox bearer token or the -root token by id. The owning node handles the request without redirecting. +(`volumes`), per-volume `states`, and `converging`. Auth requires a client API +token. Root can access every sandbox; a tenant can access only its own. +Sandbox tokens are rejected when authentication is enabled. The owning node +handles the request without redirecting. ```json { "volumes_want": [{"name": "imagenet", "mount": "/datasets/training"}], "volumes": [{"name": "imagenet", "mount": "/datasets/training"}], - "states": [{"name": "imagenet", "state": "attached"}] + "states": [{"name": "imagenet", "state": "attached"}], + "converging": false } ``` @@ -274,13 +277,19 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed convergence also includes a top-level `error` and may include an `error` on -the affected state. The sandbox index exposes both sets for operator inspection. +the affected state. `converging` is true only while a request worker is running. +`pending` with `converging: false` requires an explicit retry; there is no +background convergence. GET retains the last safe per-volume error for the +current daemon lifetime. After daemon restart, unfinished states report an +interrupted operation that needs retry. Successful retries clear resolved errors. +The sandbox index exposes both sets for operator inspection. ## PUT /v1/sandboxes/{id}/volumes Replaces the full desired set and converges synchronously. Auth matches GET. -The owner tenant's catalog access list applies, including for root. The image -must exist on the owning node. The request must include a non-null `volumes` +The owner tenant's catalog access list applies to newly admitted names, including +for root. Retaining, repairing, or removing held names does not reauthorize +them. A new image must exist on the owning node. The request must include a non-null `volumes` array. An explicit empty array removes all mutable entries. ```json @@ -296,10 +305,13 @@ An identical target can be retried. Claims with `on_expire: archive` must change that policy before adding a volume. The sandbox must be running. Paused, archived, and failed sandboxes return 409 -without waking. Missing bearer returns 401. Unknown id or wrong token returns -404. Invalid, inaccessible, or non-local entries return 400. Removed tenants -cannot add volumes and return 403. A conflicting writer or dirty image returns -409. Removal remains available after the tenant is removed. +without waking. A failed sandbox holding volumes is release-only after its VMM +exits; PUT with an empty target cannot restore its processes. Missing, invalid, +or sandbox bearer tokens return 401. Another tenant's sandbox and an unknown id +return 404. Invalid, inaccessible, or non-local entries return 400. Removed +tenant tokens are rejected at authentication. Root can still repair or remove +held entries, but cannot admit new names for a removed tenant. A conflicting +writer or dirty image returns 409. ## PATCH /v1/sandboxes/{id}/volumes @@ -314,7 +326,10 @@ unchanged. An empty patch retries the current desired set. Adding an identical entry or removing an absent valid name is harmless. Duplicate names, a name in both arrays, or changes to writable or attach-only entries return 400. Removal refuses busy or foreign mounts and uses neither -lazy nor forced unmount. +lazy nor forced unmount. A foreign mount, another bind mount, or a completed +ordinary unmount with nonzero exit returns 409 with the guest mount path and +a safe per-volume reason. Guest transport failures remain server errors; +timeouts return 504 and cancellation returns 408. ### Partial failure and recovery @@ -322,7 +337,9 @@ PUT and PATCH return the GET response shape on success. A convergence error returns a non-2xx status with the same view, so callers can inspect desired, applied, and per-volume state. Validation and authentication errors may carry only `error`. A failed request may already have persisted its target and -completed some device changes. Retry the same PUT to converge, or send a new +completed some device changes. GET reads the last confirmed set, rather than +probing the devices. Resolve guest mount references or runtime failures before +retrying. Retry the same PUT to converge, or send a new target to abandon unfinished additions. Concurrent requests serialize per sandbox without blocking renewals or new exec, file, and port connections. @@ -331,8 +348,10 @@ superseding request may first clean up an older desired-only attachment while the old target stays recorded. If that cleanup fails, the response retains the old view; retry the same PUT to finish accepting the new target. Admission holds cover the deduplicated union of both sets, including during restart and -teardown. Failed or uncertain operations retain their holds until successful -cleanup. Startup reserves those holds without replaying attachments. Release +teardown. A durable retirement record keeps teardown holds until the VM is +confirmed gone and the retirement deletion commits. Failed or uncertain +operations retain their holds until successful cleanup. Startup reserves those +holds without replaying attachments. Release cancels convergence and transfers the held union to teardown without waiting for a blocked device call. Capture operations (hibernate, fork, checkpoint, and promote) are refused while either set is nonempty or a convergence request @@ -340,13 +359,28 @@ still owns admitted volume holds. Device calls honor cancellation and have a 30-second ceiling. Journal writes are atomic. Host-crash durability follows `sync_claims`. Drain volume claims -before downgrading to a daemon that does not preserve both journal sets. - -Live convergence requires cocoon commit `6d76309` or a build containing it, -which makes matching attach and absent detach idempotent. The change was not -in a tagged release when this API was prepared. Testbed validation of real -Cloud Hypervisor hotplug, busy unmount, and restart during convergence remains -pending with the maintainer. +before downgrading. The version 2 claim journal also stores volume retirements; +older daemons reject it. Back up the journal before upgrading and do not +manually convert it to the legacy format while retirement records remain. + +Live convergence requires Cocoon v0.7.2 or newer for idempotent matching +attach and absent detach. PUT and PATCH reject older, unknown, development, +or unqueryable runtime versions with 503 before changing intent, holds, or +devices. GET and release remain available. + +Catalog names must keep the same backing image while held, including pending +and retiring holds. Publish changes under versioned names such as `dataset-v1` +and `dataset-v2`. The journal cannot detect a catalog name repointed to a new +image. On startup, an unknown owned orphan that cannot be confirmed gone blocks +new volume admission and all disk attachment, including pending repair. +Diagnosis, detach, and release remain available until cleanup succeeds. + +A replacement first unmounts the old disk, then mounts the new disk. Pause reads +and close files, mmap, cwd, and bind mount references during this gap. Existing +independent guest processes and sessions continue. See the +[same-sandbox replacement tutorial](live-volumes.md). Real Cloud Hypervisor +hotplug, busy unmount, and restart during convergence still require maintainer +testbed validation. ## POST /v1/sandboxes/{id}/release diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 212ed9a6..a328a8f6 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -161,6 +161,23 @@ peer advertising both the template and every volume; when that intersection is empty, one volume holder may self-verify access to a shared template store before provisioning. +## Changing read-only datasets + +Use `sb.get_volumes()`, `sb.set_volumes([...])`, and +`sb.patch_volumes(add=[...], remove=[...])` on the same running sandbox. These +methods use the client's API token on the owning node; a sandbox token cannot +manage datasets. `set_volumes([])` clears mutable entries; `None` is rejected. +PUT must retain writable and attach-only entries. PATCH retains omitted names. + +Responses include `volumes_want`, `volumes`, `states`, and `converging`. +`VolumePhase` and `VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` +name the three states. A pending state with `converging=False` needs an explicit +retry. `VolumeError`, exported from `cocoonsandbox` and `cocoonsandbox.errors`, +retains a complete failure view in `.response`. Complete replies update both +public sets; incomplete errors preserve the last view. Serialize calls on one +handle. See the [replacement tutorial](live-volumes.md) and +[API recovery contract](sandboxd-api.md#partial-failure-and-recovery). + ## Renewing ```python diff --git a/docs/sdk.md b/docs/sdk.md index 7ed9107f..d776881a 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -256,11 +256,13 @@ view = sb.patch_volumes(remove=["imagenet"]) view = sb.set_volumes([]) # None is rejected. ``` -Each response contains `volumes_want`, `volumes`, and `states`. Go exposes them -as `SandboxVolumesResponse.VolumesWant`, `.Volumes`, and `.States`. A state is -`attached`, `pending`, or `detaching`, with an optional `error`. Methods use the -sandbox token on its owner node and update both public sets from each complete -response, including a partial failure. Serialize changes to one handle as with +Each response contains `volumes_want`, `volumes`, `states`, and `converging`. +Go exposes them as `SandboxVolumesResponse.VolumesWant`, `.Volumes`, `.States`, +and `.Converging`. `VolumePhase` has `VolumeAttached`, `VolumePending`, and +`VolumeDetaching` constants. Python exposes `VolumePhase` and the corresponding +`VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` constants. Each state +can include an `error`. Methods use the client API token on the owning node +and update both public sets from each complete response, including a partial failure. Serialize changes to one handle as with `Renew`. Returned views do not share the handle's slices or dictionaries. Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. @@ -274,9 +276,14 @@ same full PUT target, or set a new target to recover. `Client.Sandboxes` exposes A PUT must retain existing writable and attach-only entries unchanged. PATCH cannot remove or change them. Paused sandboxes are not woken, and images must exist on the owner node. Admission holds cover the union of desired and applied -sets. Capture is refused while either is nonempty. Release cancels convergence -and hands held volumes to teardown. See the -[API recovery contract](sandboxd-api.md#partial-failure-and-recovery). +sets. A settled retained entry receives no new attach or mount calls. `converging` +is true only during an active request. Pending or detaching entries with false +require an explicit retry. Daemon restart replaces unfinished diagnostics with +an interrupted message. Capture is refused while either set is nonempty. +Release cancels convergence and hands held volumes to teardown. See the +[API recovery contract](sandboxd-api.md#partial-failure-and-recovery) and +[same-sandbox replacement tutorial](live-volumes.md). A failed sandbox with +volume holds is release-only after VMM exit; an empty target cannot recover it. `WithVolumesAttachOnly()` claims the same volumes without mounting them: the entries in `Sandbox.Volumes` carry an empty `Mount`, and the workload finds From 77d1d9aab66d87539d8d815d9aa26d30b785590e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:17:01 +0000 Subject: [PATCH 11/57] test(e2e): declare live volume support in fake runtime --- e2e/fakeengine_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index e2b708a7..75de75f8 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -40,6 +40,8 @@ func newFakeEngine(dir string) *fakeEngine { } } +func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } + func (f *fakeEngine) Clone(_ context.Context, _, name string, _ types.PoolKey) (types.VMRecord, error) { return f.createRecord(name) } From 3b87948d408e130c929c6f129f11b028d05a22d7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:23:29 +0000 Subject: [PATCH 12/57] refactor(volumes): name catalog repair and cleanup ownership --- e2e/cmd/volumesmoke/testdata/fault_hook.py | 8 +--- .../volumesmoke/testdata/runtime_observer.py | 13 ++---- sandboxd/pool/remove.go | 4 +- sandboxd/pool/volume.go | 46 +++++++++++-------- sandboxd/pool/volume_attach_test.go | 4 +- sandboxd/pool/volume_mutation.go | 11 ++--- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/e2e/cmd/volumesmoke/testdata/fault_hook.py b/e2e/cmd/volumesmoke/testdata/fault_hook.py index 94e4aaf9..b960f7c1 100755 --- a/e2e/cmd/volumesmoke/testdata/fault_hook.py +++ b/e2e/cmd/volumesmoke/testdata/fault_hook.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 -"""Gate real attach completion, cancel a caller, or restart an isolated daemon.""" - import json import os -from pathlib import Path import signal import subprocess import sys import time import urllib.request +from pathlib import Path root = Path(os.environ["VOLUME_SMOKE_DIR"]) mode = sys.argv[1] @@ -61,9 +59,7 @@ def wait_for(marker): def runtime_view(): - data = subprocess.check_output( - [os.environ["COCOON_REAL_BIN"], "vm", "list", "--format", "json"] - ) + data = subprocess.check_output([os.environ["COCOON_REAL_BIN"], "vm", "list", "--format", "json"]) running = [] for vm in json.loads(data): if not vm["config"]["name"].startswith("sbx-"): diff --git a/e2e/cmd/volumesmoke/testdata/runtime_observer.py b/e2e/cmd/volumesmoke/testdata/runtime_observer.py index 233ed5f3..4e2ec6c5 100755 --- a/e2e/cmd/volumesmoke/testdata/runtime_observer.py +++ b/e2e/cmd/volumesmoke/testdata/runtime_observer.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -"""Forward Cocoon commands and expose explicit hardware fault injection gates.""" - import json import os -from pathlib import Path import subprocess import sys import time +from pathlib import Path root = Path(os.environ["VOLUME_SMOKE_DIR"]) args = sys.argv[1:] @@ -18,9 +16,8 @@ print("volumesmoke injected VM removal failure", file=sys.stderr) sys.exit(73) result = subprocess.run([os.environ["COCOON_REAL_BIN"], *args], check=False) -if args[:3] == ["vm", "disk", "attach"] and result.returncode == 0: - if (root / "gate-attach").exists(): - (root / "gate-entered").write_text(json.dumps(event)) - while (root / "gate-attach").exists(): - time.sleep(0.02) +if args[:3] == ["vm", "disk", "attach"] and result.returncode == 0 and (root / "gate-attach").exists(): + (root / "gate-entered").write_text(json.dumps(event)) + while (root / "gate-attach").exists(): + time.sleep(0.02) sys.exit(result.returncode) diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 3c50d658..5f0a5380 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -137,9 +137,9 @@ func (m *Manager) quiesceRetiringVolumes(ctx context.Context, sb *types.Sandbox, if !ok { return td } - if len(td.clears) > 0 { + if len(td.cleanImagePaths) > 0 { next := retirement - next.Clears = slices.Clone(td.clears) + next.Clears = slices.Clone(td.cleanImagePaths) if err := m.store.commit(m.store.setRetirement(next)); err != nil { log.WithFunc("pool.quiesceRetiringVolumes").Warnf(ctx, "persist clean volumes for %s: %v", sb.ID, err) m.recommit(ctx, m.store.setRetirement(retirement)) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 8e8c5c4b..50ac6433 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -35,11 +35,10 @@ type resolvedVolume struct { applied types.Volume } -// volumeTeardown records marker paths only after a confirmed unmount. type volumeTeardown struct { - holds []types.Volume - clears []string - retirement string + holds []types.Volume + cleanImagePaths []string + retirement string } type catalogVolume struct { @@ -135,7 +134,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { @@ -143,22 +142,33 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if !ok || !entry.allowed(tenant) { return nil, ErrVolumeUnavailable } - if volume.RW() && !entry.writable { - return nil, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) + resolvedVolume, resolveErr := m.resolveCatalogVolume(ctx, volume) + if resolveErr != nil { + return nil, resolveErr } - if _, statErr := os.Stat(entry.disk.Path); statErr != nil { - // the operator's path stays server-side; the refusal matches the unknown one - log.WithFunc("pool.resolveVolumes").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) - return nil, ErrVolumeUnavailable - } - disk := entry.disk - disk.RW = volume.RW() - volume.AttachOnly = false // past validation the empty mount carries it - resolved = append(resolved, resolvedVolume{disk: disk, applied: volume}) + resolved = append(resolved, resolvedVolume) } return resolved, nil } +func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { + entry, ok := m.volumes[volume.Name] + if !ok { + return resolvedVolume{}, ErrVolumeUnavailable + } + if volume.RW() && !entry.writable { + return resolvedVolume{}, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) + } + if _, statErr := os.Stat(entry.disk.Path); statErr != nil { + log.WithFunc("pool.resolveCatalogVolume").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) + return resolvedVolume{}, ErrVolumeUnavailable + } + disk := entry.disk + disk.RW = volume.RW() + volume.AttachOnly = false + return resolvedVolume{disk: disk, applied: volume}, nil +} + // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. func (m *Manager) confirmVolumesClean(volumes []resolvedVolume) error { for _, volume := range volumes { @@ -239,7 +249,7 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT continue } if entry, ok := m.volumes[volume.Name]; ok { - td.clears = append(td.clears, entry.disk.Path) + td.cleanImagePaths = append(td.cleanImagePaths, entry.disk.Path) } } if stuck { @@ -259,7 +269,7 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { return } logger := log.WithFunc("pool.finishVolumeTeardown") - for _, path := range td.clears { + for _, path := range td.cleanImagePaths { if err := clearVolumeDirty(path); err != nil { logger.Errorf(ctx, err, "clear dirty marker %s", path) } diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index 7fc14e75..e08e4a2d 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -390,8 +390,8 @@ func TestQuiesceVolumesSkipsAttachOnlyEntries(t *testing.T) { if want := []string{"umount:/volumes/mounted"}; !slices.Equal(eng.volumeOpsLog(), want) { t.Errorf("quiesce ops=%v, want %v", eng.volumeOpsLog(), want) } - if !slices.Equal(td.clears, []string{mounted}) { - t.Errorf("marker clears=%v, want only %s", td.clears, mounted) + if !slices.Equal(td.cleanImagePaths, []string{mounted}) { + t.Errorf("marker clears=%v, want only %s", td.cleanImagePaths, mounted) } if !slices.Equal(td.holds, sb.Volumes) { t.Errorf("holds=%v, want every entry %v", td.holds, sb.Volumes) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index ab6523f4..2c5b5e46 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -150,13 +150,12 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - // Held entries retain access even after their tenant or ACL changes. - repair, resolveErr := m.resolveVolumes(ctx, "", []types.Volume{v}) + repair, resolveErr := m.resolveCatalogVolume(ctx, v) if resolveErr != nil { return view, resolveErr } applyCtx, cancel := context.WithTimeout(ctx, engine.VolumeMutationTimeout) - err = m.applyVolume(applyCtx, sb, repair[0]) + err = m.applyVolume(applyCtx, sb, repair) cancel() if err != nil { return view, err @@ -298,8 +297,6 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru m.mu.Unlock() err := m.store.commit(js) if err != nil && m.store.sync { - // Fsync can fail after rename. Keep new admissions reserved; discarded - // pending volumes were confirmed detached before this write. return fmt.Errorf("persist volume target: %w", err) } if err != nil { @@ -453,7 +450,7 @@ func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) } normalized, err := types.ValidateVolumes([]types.Volume{v}, attachOnly) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } if volumeIndex(out, v.Name) >= 0 { return nil, fmt.Errorf("%w: duplicate volume %q", ErrBadVolume, v.Name) @@ -465,7 +462,7 @@ func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) } mounted := slices.DeleteFunc(slices.Clone(out), func(v types.Volume) bool { return v.Mount == "" }) if _, err := types.ValidateVolumes(mounted, false); err != nil { - return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) + return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } return out, nil } From ceda20dd4ad944e354e6f300b72b610b3d2422d6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:23:58 +0000 Subject: [PATCH 13/57] test(e2e): expect retained volumes to skip device operations --- e2e/volume_mutation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 07b60b8c..8f3cf9de 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -31,7 +31,7 @@ func TestLiveVolumesEndToEnd(t *testing.T) { t.Fatalf("detach: %v %v", got, err) } } - want := []string{"attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "umount:/volumes/dataset", "detach:dataset"} + want := []string{"attach:dataset:ro", "mount:dataset:/volumes/dataset:ro", "umount:/volumes/dataset", "detach:dataset"} if ops := stack.eng.volumeOpsLog(); !slices.Equal(ops, want) { t.Fatalf("device operations=%v want=%v", ops, want) } From b249bd2813602eab44e498ecd6e106652e81940a Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:29:19 +0000 Subject: [PATCH 14/57] fix(pool): avoid shadowing the resolved volume type --- sandboxd/pool/volume.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 50ac6433..2cfbbec9 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -142,11 +142,11 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if !ok || !entry.allowed(tenant) { return nil, ErrVolumeUnavailable } - resolvedVolume, resolveErr := m.resolveCatalogVolume(ctx, volume) + resolvedEntry, resolveErr := m.resolveCatalogVolume(ctx, volume) if resolveErr != nil { return nil, resolveErr } - resolved = append(resolved, resolvedVolume) + resolved = append(resolved, resolvedEntry) } return resolved, nil } From cc444095b3ba000de9ac11d7ef0674e386153804 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:37:15 +0000 Subject: [PATCH 15/57] fix(pool): preserve convergence diagnostics on rejected targets --- e2e/volume_mutation_test.go | 25 +++++++++++++++++ sandboxd/pool/volume_convergence_test.go | 34 ++++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 11 +++++++- 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 8f3cf9de..9e2e7f63 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -1,7 +1,10 @@ package e2e import ( + "errors" + "net/http" "slices" + "strings" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -36,3 +39,25 @@ func TestLiveVolumesEndToEnd(t *testing.T) { t.Fatalf("device operations=%v want=%v", ops, want) } } + +func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { + image := writeVolumeImage(t, "dataset.img", "dataset-bytes") + stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "dataset", Path: image}}) + sb, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer sb.Close() + if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { + t.Fatal(err) + } + _, err = sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}, {Name: "dataset"}}) + var refusal *sandbox.VolumeError + if !errors.As(err, &refusal) || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicate volume") { + t.Errorf("invalid target explanation: %v", err) + } + view, err := sb.GetVolumes(t.Context()) + if err != nil || view.Error != "" || view.Converging || len(view.States) != 1 || view.States[0].Error != "" || view.States[0].State != sandbox.VolumeAttached { + t.Fatalf("rejected request changed settled state: %+v err=%v", view, err) + } +} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 07abfb8a..7659fa74 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -180,6 +180,40 @@ func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { } } +func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { + for _, priorConflict := range []bool{false, true} { + name := "settled" + if priorConflict { + name = "conflict" + } + t.Run(name, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if priorConflict { + eng.detachVolumeErr = &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { + t.Fatal("expected mount conflict") + } + } + before, err := m.GetVolumes(sb.ID, sb.Tenant) + if err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid target: %v", err) + } + after, err := m.GetVolumes(sb.ID, sb.Tenant) + if err != nil || after.Error != before.Error || !slices.Equal(after.States, before.States) || !slices.Equal(after.Volumes, before.Volumes) || !slices.Equal(after.VolumesWant, before.VolumesWant) { + t.Fatalf("rejected request changed convergence view: before=%+v after=%+v err=%v", before, after, err) + } + }) + } +} + func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 2c5b5e46..908e548b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -74,17 +74,25 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v } if err = m.eng.CheckLiveVolumeSupport(ctx); err != nil { view, _ = m.GetVolumes(id, tenant) + view.Error = "" return view, err } ctx, run, err := m.beginVolumeRun(ctx, sb) if err != nil { view, _ = m.GetVolumes(id, tenant) + view.Error = "" return view, err } failedName := "" + recordOutcome := false defer func() { - m.recordVolumeError(sb, run, failedName, err) + if recordOutcome { + m.recordVolumeError(sb, run, failedName, err) + } view = m.finishVolumeRun(sb, run) + if !recordOutcome { + view.Error = "" + } }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) @@ -113,6 +121,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } + recordOutcome = true m.mu.Lock() uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) m.mu.Unlock() From da56205bb31b489c15af3ea670e1cbe87905bf01 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:39:00 +0000 Subject: [PATCH 16/57] test(volumes): avoid error shadowing in refusal regressions --- e2e/volume_mutation_test.go | 4 ++-- sandboxd/pool/volume_convergence_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 9e2e7f63..776b9fd8 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -48,8 +48,8 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { t.Fatal(err) } defer sb.Close() - if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { - t.Fatal(err) + if _, checkErr := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); checkErr != nil { + t.Fatal(checkErr) } _, err = sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}, {Name: "dataset"}}) var refusal *sandbox.VolumeError diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 7659fa74..5f1a8932 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -203,8 +203,8 @@ func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { - t.Fatalf("invalid target: %v", err) + if _, checkErr := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(checkErr, ErrBadVolume) { + t.Fatalf("invalid target: %v", checkErr) } after, err := m.GetVolumes(sb.ID, sb.Tenant) if err != nil || after.Error != before.Error || !slices.Equal(after.States, before.States) || !slices.Equal(after.Volumes, before.Volumes) || !slices.Equal(after.VolumesWant, before.VolumesWant) { From a06af7032c989ca6e2ae7a076f98e6f41d34716d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:44:14 +0000 Subject: [PATCH 17/57] docs(volumes): record local CH and recovery validation --- docs/deploy.md | 7 +- docs/live-volumes-validation.md | 155 +++++ docs/live-volumes.md | 7 +- docs/sandboxd-api.md | 6 +- docs/test-results/pr339/backing-result.json | 5 + docs/test-results/pr339/baseline.log | 4 + docs/test-results/pr339/checks.json | 492 ++++++++++++++++ docs/test-results/pr339/checks.log | 543 ++++++++++++++++++ docs/test-results/pr339/decisions.tsv | 25 + .../pr339/diagnostic-regression.log | 114 ++++ .../test-results/pr339/guest-input-sha256.txt | 3 + .../pr339/hardware-binaries.sha256 | 3 + .../pr339/hardware-build-info.txt | 116 ++++ docs/test-results/pr339/hardware-cleanup.json | 17 + .../pr339/hardware-commands.jsonl | 88 +++ .../pr339/hardware-invalid-put.json | 75 +++ .../pr339/hardware-invocation.json | 23 + docs/test-results/pr339/hardware-journal.json | 1 + docs/test-results/pr339/hardware-prep.md | 29 + docs/test-results/pr339/hardware-revision.txt | 2 + docs/test-results/pr339/hardware.log | 34 ++ .../test-results/pr339/host-tool-versions.txt | 2 + docs/test-results/pr339/pending-eject.json | 10 + .../pr339/post-hardware-test-only.diff | 18 + .../pr339/retained-disk-calls.json | 5 + docs/test-results/pr339/review.md | 29 + 26 files changed, 1804 insertions(+), 9 deletions(-) create mode 100644 docs/live-volumes-validation.md create mode 100644 docs/test-results/pr339/backing-result.json create mode 100644 docs/test-results/pr339/baseline.log create mode 100644 docs/test-results/pr339/checks.json create mode 100644 docs/test-results/pr339/checks.log create mode 100644 docs/test-results/pr339/decisions.tsv create mode 100644 docs/test-results/pr339/diagnostic-regression.log create mode 100644 docs/test-results/pr339/guest-input-sha256.txt create mode 100644 docs/test-results/pr339/hardware-binaries.sha256 create mode 100644 docs/test-results/pr339/hardware-build-info.txt create mode 100644 docs/test-results/pr339/hardware-cleanup.json create mode 100644 docs/test-results/pr339/hardware-commands.jsonl create mode 100644 docs/test-results/pr339/hardware-invalid-put.json create mode 100644 docs/test-results/pr339/hardware-invocation.json create mode 100644 docs/test-results/pr339/hardware-journal.json create mode 100644 docs/test-results/pr339/hardware-prep.md create mode 100644 docs/test-results/pr339/hardware-revision.txt create mode 100644 docs/test-results/pr339/hardware.log create mode 100644 docs/test-results/pr339/host-tool-versions.txt create mode 100644 docs/test-results/pr339/pending-eject.json create mode 100644 docs/test-results/pr339/post-hardware-test-only.diff create mode 100644 docs/test-results/pr339/retained-disk-calls.json create mode 100644 docs/test-results/pr339/review.md diff --git a/docs/deploy.md b/docs/deploy.md index b0b7f644..427a7266 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -183,9 +183,10 @@ new volume admission and all disk attachment, including pending repair, until the orphan is gone. GET, detach, and release remain available. `e2e/cmd/volumesmoke` exercises GET, repeated full-set PUT, PATCH add and remove, -and clearing a new claim. Real Cloud Hypervisor hotplug, busy guest unmount, -and restart during convergence remain unverified here. The maintainer will -run these checks on the target VMM and guest testbed. +and clearing a new claim. The [local validation report](live-volumes-validation.md) +records real Cloud Hypervisor hotplug, busy guest unmount, daemon restart, +cancellation, and retirement recovery. It lists the hardware fault cases that +remain untested. Each catalog path must name a disk image containing a mountable whole-device filesystem. Keep catalog names bound to the same image while any live or diff --git a/docs/live-volumes-validation.md b/docs/live-volumes-validation.md new file mode 100644 index 00000000..b0f162cf --- /dev/null +++ b/docs/live-volumes-validation.md @@ -0,0 +1,155 @@ +# Local live volume validation + +This report records local validation for PR #339 on 2026-10-16. The baseline +is `645e49756492e7d8e9337377a958185e615804fc`. The initial full software +run used `ceda20dd4ad944e354e6f300b72b610b3d2422d6`. Independent +review then found that invalid requests overwrote convergence diagnostics. +Regressions reproduced that defect before its fix in +`cc444095b3ba000de9ac11d7ef0674e386153804`. The complete real hardware smoke +was rerun at that revision, followed by a real invalid-request probe. The final +application candidate is `da56205bb31b489c15af3ea670e1cbe87905bf01`, whose only +changes after that hardware run rename two test-local error variables. The +affected sandboxd and e2e race, Linux and Darwin lint, and format checks passed +there. Later documentation does not change the tested application code. + +The [hardware program](../e2e/cmd/volumesmoke/README.md) extends the existing +`volumesmoke` command. Its caller sequence matches the +[replacement tutorial](live-volumes.md), using the Go SDK. The literal Python +tutorial was not run against the VMM. Python SDK behavior is covered by its +tests and the Caddy transport integration. + +## Software checks + +All 29 distinct checks passed after the naming correction. The +[check manifest](test-results/pr339/checks.json) and +[complete output](test-results/pr339/checks.log) retain the failed lint attempt +and the successful reruns, with exact source revisions. The later regressions +also needed test-local shadow-name corrections. Failed suite attempts remain +in the evidence. The [red/green output](test-results/pr339/diagnostic-regression.log) +records the diagnostic defect before and after its fix. + +| Check | Result | +| --- | --- | +| `go test -race -count=1 ./...` in all five Go modules | PASS | +| Pinned golangci-lint v2.13.2 on Linux and Darwin in all five modules | PASS | +| Formatter diff in all five modules | PASS | +| Ruff format and lint | PASS | +| Strict Python SDK mypy | PASS | +| Python SDK, OpenAI adapter, and LangChain adapter | 247, 10, and 15 tests passed | +| PostgreSQL tenant and pool-set stores | PASS, no skipped PostgreSQL tests | +| PostgreSQL multi-node pool integration | PASS, no skipped PostgreSQL tests | +| Caddy v2.11.4 TLS integration through Go and Python | PASS | + +The toolchain was Go 1.27.1, as pinned by the repository. Permission tests ran +with `CAP_DAC_OVERRIDE` and `CAP_DAC_READ_SEARCH` removed because the local user +is root. Each lint run used a new cache. The stock pinned analyzer panicked +with a shared cache and passed on the same source with a fresh cache. No +analyzer was disabled and no lint suppression was added. + +The controlled tests include tenant API-token authorization and sandbox-token +denial, held-volume repair after tenant or ACL removal, no device calls for +settled entries, release before reservation, cancellation and late completion, +re-add after a failed detach, uncertain journal writes, startup adoption, +retirement deletion failure, unknown-orphan admission fencing, and capture +rejection before trim. These use fake engines or controlled filesystem faults. +They are separate from the VMM results below. + +## Real Cloud Hypervisor results + +The node used the official Cocoon v0.7.2 release, Git +`b3b40f3bdedc384c482eb81dcdc60a774dd8d560`, with Cloud Hypervisor v54.0.0. +The Cocoon archive matched its published checksum. The guest used Linux +6.18.37 and the current statically built sandbox-init. + +The daemon, data directory, images, runtime directories, network scope, and +cgroup parent were isolated. No existing local service was removed or +reconfigured. The test used two disposable 32 MiB ext4 images. Both catalog +entries permit writes for the final reuse probe. Ordinary replacement mounts +request read-only mode. + +| Caller-visible scenario | Result | +| --- | --- | +| Two sandboxes read the same dataset, and writes fail with EROFS | PASS | +| Replace v1 with v2 at the same guest path, repeat the target, then clear it | PASS | +| Preserve sandbox, session, boot ID, PID, process start time, and session environment | PASS | +| Busy cwd, bind mount, and foreign mount return 409 with a guest path | PASS | +| Close the reference and retry the same target | PASS | +| Repeat a settled target while it has a bind mount | PASS, zero attach or detach calls | +| Run exec, file read/write, and HTTP port requests while a real attach result is blocked | PASS | +| Kill sandboxd after real attach succeeds but before its result returns | PASS, restart adopts the original VMM and retry converges | +| Cancel the caller at that same boundary | PASS, desired intent survives and retry converges | +| Fail VM removal, restart sandboxd, and attempt a conflicting writer | PASS, writer receives 409 before and after restart | +| Permit real VM removal, then mount and write through a new writer | PASS | +| Reject duplicate volume names with 400 and preserve the settled GET view | PASS, no device calls or new convergence error | +| Remove all resources owned by the test | PASS, zero claims, retirements, VMs, sockets, and owned TAPs | + +The [raw smoke output](test-results/pr339/hardware.log) includes identities, +failure views, and recovery results. The +[runtime command log](test-results/pr339/hardware-commands.jsonl) contains the +actual command attempts. The observer forwards ordinary commands to the +official Cocoon binary. It blocks only after a successful real attach, or +injects the declared exit 73 before VM removal. It does not emulate version +support, attach idempotence, or successful deletion. + +The original VMM PID `1082008` survived the daemon restart. Retiring VMM PID +`1084550` survived the failed removal and restart until real cleanup succeeded. +The [cleanup snapshot](test-results/pr339/hardware-cleanup.json) confirms both +PIDs and the separate invalid-request probe's VM are gone. The +[invalid-request probe](test-results/pr339/hardware-invalid-put.json) records +HTTP 400 with the duplicate-name reason, unchanged GET and file contents, and +zero disk calls. The [final journal](test-results/pr339/hardware-journal.json) +has empty live and retirement maps. The +[retained-target window](test-results/pr339/retained-disk-calls.json) contains +zero device calls. The read-only v2 backing checksum did not change. The v1 +image received the intentional writable reuse probe. + +The [baseline hardware run](test-results/pr339/baseline.log) passed the shared +reads and EROFS checks, then returned HTTP 500 for the busy cwd case where the +new candidate returns HTTP 409. This establishes a behavioral difference on +the actual device path. + +## Reproduce the checks + +The [CI workflows](../.github/workflows/sandboxd.yml) provide the Go, +PostgreSQL, and Caddy commands. The +[Python workflow](../.github/workflows/python.yml) pins its tools and SDK test +commands. Local Go modules are `protocol/wire`, `sandboxd`, `sdk/go`, `e2e`, +and `mcp`. `GOWORK=off` keeps each module independent. The hardware command and +fault-hook setup are in the [hardware instructions](../e2e/cmd/volumesmoke/README.md). +The [local runtime preparation](test-results/pr339/hardware-prep.md) records +the guest image compatibility fixes and exact input checksums. + +## Coverage limits and operational requirements + +Missing guest eject ACK was not established. An external experiment paused +Cloud Hypervisor, but official Cocoon resumed the orphaned pause before +detach. The guest could therefore acknowledge removal. The +[experiment result](test-results/pr339/pending-eject.json) is inconclusive for +timeout and retry with a missing ACK. + +Separate hardware crashes after guest umount, after host detach, and at every +journal or fsync boundary were not performed. The hardware daemon used +`sync_claims: false`. The demonstrated recovery is a daemon-process restart, +not physical host power loss. Controlled unit tests cover additional journal +cuts without claiming hardware proof. + +Catalog names must remain bound to immutable images while held. The journal +cannot detect an operator changing a backing path after restart. Journal +format v2 includes durable retirement ownership. An older daemon rejects that +format, so downgrade requires draining claims and retirement work first. A +failed unknown-orphan cleanup fences new admissions and held repair until +the VM is confirmed gone. A sandbox whose VMM dies while holding volumes is +release-only. These constraints are documented in the deployment and API +guides. + +The [decision trail](test-results/pr339/decisions.tsv) records implementation +choices and failed attempts. Its relative evidence paths refer to the retained +local `.audit/pr339-implementation` directory. The artifacts linked in this +report provide the portable test evidence. Local paths in raw logs identify +the machine used for the run and are not installation requirements. +The build metadata has trailing whitespace removed. The recorded test-only +diff omits context lines. Test observations retain their original contents. + +The [independent follow-up review](test-results/pr339/review.md) verifies the +diagnostic fix, source revisions, test evidence, and decision trail. It found +no remaining actionable issue. The hardware coverage limits above remain. diff --git a/docs/live-volumes.md b/docs/live-volumes.md index af91e7dc..2838601c 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -92,6 +92,7 @@ If the VMM exits, a sandbox holding volumes is release-only. Release it and create a new sandbox; neither PUT with an empty target nor daemon restart restores the original processes. -The example's Python syntax and SDK signatures are checked locally. Preserving -these identities during real Cloud Hypervisor hotplug and daemon restart -requires the maintainer's hardware testbed. +The example's Python syntax and SDK signatures are checked locally. The +[local validation report](live-volumes-validation.md) records the equivalent +Go SDK sequence on real Cloud Hypervisor, including identity continuity and +daemon restart. It also lists hardware fault cases that remain untested. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 1e29a141..f1f213fa 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -378,9 +378,9 @@ Diagnosis, detach, and release remain available until cleanup succeeds. A replacement first unmounts the old disk, then mounts the new disk. Pause reads and close files, mmap, cwd, and bind mount references during this gap. Existing independent guest processes and sessions continue. See the -[same-sandbox replacement tutorial](live-volumes.md). Real Cloud Hypervisor -hotplug, busy unmount, and restart during convergence still require maintainer -testbed validation. +[same-sandbox replacement tutorial](live-volumes.md) and the +[local Cloud Hypervisor results](live-volumes-validation.md) for the tested +interruption boundaries and remaining coverage limits. ## POST /v1/sandboxes/{id}/release diff --git a/docs/test-results/pr339/backing-result.json b/docs/test-results/pr339/backing-result.json new file mode 100644 index 00000000..b6234244 --- /dev/null +++ b/docs/test-results/pr339/backing-result.json @@ -0,0 +1,5 @@ +{ + "readonly_v2_backing_unchanged": true, + "v1_expected_writable_retirement_probe": true, + "v1_changed": true +} diff --git a/docs/test-results/pr339/baseline.log b/docs/test-results/pr339/baseline.log new file mode 100644 index 00000000..b9b34e80 --- /dev/null +++ b/docs/test-results/pr339/baseline.log @@ -0,0 +1,4 @@ +volume=dataset-v1 claim_a=389.0ms claim_b=371.8ms +VOLUME PASS concurrent_read_bytes=3 read_only=true released=true +CONTINUITY before={"Sandbox":"sb_7d252ce9de32e3e7","Session":"sess-69-0","Boot":"4cd81a72-ea89-4282-a36d-2e3e4965755e","PID":177,"Start":"1738"} +volumesmoke: cwd busy removal must return 409 and guest path: view={VolumesWant:[{Name:dataset-v2 Mount:/datasets/current Mode:}] Volumes:[{Name:dataset-v1 Mount:/datasets/current Mode:}] States:[{Name:dataset-v1 State:detaching Error:volume convergence failed; retry the desired set} {Name:dataset-v2 State:pending Error:}] Error:volume convergence failed; retry the desired set} error=set volumes: volume convergence failed; retry the desired set (http 500) diff --git a/docs/test-results/pr339/checks.json b/docs/test-results/pr339/checks.json new file mode 100644 index 00000000..576fe19b --- /dev/null +++ b/docs/test-results/pr339/checks.json @@ -0,0 +1,492 @@ +[ + { + "name": "race-protocol-wire", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:24:25.395802+00:00", + "finished": "2026-10-16T14:24:37.429522+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "race-sandboxd", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:24:37.430198+00:00", + "finished": "2026-10-16T14:25:15.961593+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "race-sdk-go", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:15.962380+00:00", + "finished": "2026-10-16T14:25:21.209326+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "race-e2e", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:21.210098+00:00", + "finished": "2026-10-16T14:25:27.613063+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "race-mcp", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:27.613821+00:00", + "finished": "2026-10-16T14:25:30.863120+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-protocol-wire-linux", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:30.864419+00:00", + "finished": "2026-10-16T14:25:40.018552+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-protocol-wire-darwin", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:40.019495+00:00", + "finished": "2026-10-16T14:25:48.919626+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "format-protocol-wire", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:48.920450+00:00", + "finished": "2026-10-16T14:25:49.185320+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-linux", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:25:49.186187+00:00", + "finished": "2026-10-16T14:26:10.451654+00:00", + "exit_code": 1, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-darwin", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:26:10.452648+00:00", + "finished": "2026-10-16T14:26:31.939927+00:00", + "exit_code": 1, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "format-sandboxd", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:26:31.940741+00:00", + "finished": "2026-10-16T14:26:39.114484+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-sdk-go-linux", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:26:39.115439+00:00", + "finished": "2026-10-16T14:26:47.190396+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-sdk-go-darwin", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:26:47.191384+00:00", + "finished": "2026-10-16T14:26:54.961224+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "format-sdk-go", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:26:54.962067+00:00", + "finished": "2026-10-16T14:26:56.800159+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-e2e-linux", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:26:56.801154+00:00", + "finished": "2026-10-16T14:27:04.458941+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-e2e-darwin", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:04.459951+00:00", + "finished": "2026-10-16T14:27:11.984298+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "format-e2e", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:11.985177+00:00", + "finished": "2026-10-16T14:27:13.053430+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-mcp-linux", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:13.054456+00:00", + "finished": "2026-10-16T14:27:20.352493+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "lint-mcp-darwin", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:20.353533+00:00", + "finished": "2026-10-16T14:27:27.447703+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "format-mcp", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:27.448574+00:00", + "finished": "2026-10-16T14:27:27.764448+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "python-format", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:27.765281+00:00", + "finished": "2026-10-16T14:27:27.822550+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "python-lint", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:27.823279+00:00", + "finished": "2026-10-16T14:27:27.875617+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "python-types", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:27.876364+00:00", + "finished": "2026-10-16T14:27:29.059436+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "python-test-python", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:27:29.060261+00:00", + "finished": "2026-10-16T14:28:05.046371+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "python-test-openai", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:28:05.047233+00:00", + "finished": "2026-10-16T14:28:12.538967+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "python-test-langchain", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:28:12.539889+00:00", + "finished": "2026-10-16T14:28:13.774900+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "pg-stores", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:28:13.775735+00:00", + "finished": "2026-10-16T14:28:17.807480+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "pg-pool", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:28:17.808239+00:00", + "finished": "2026-10-16T14:28:21.567382+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "tls-caddy", + "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", + "started": "2026-10-16T14:28:21.568146+00:00", + "finished": "2026-10-16T14:28:24.579943+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-1", + "log": "checks.log" + }, + { + "name": "race-sandboxd", + "head": "b249bd2813602eab44e498ecd6e106652e81940a", + "started": "2026-10-16T14:30:02.164735+00:00", + "finished": "2026-10-16T14:30:20.892157+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-2", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-linux", + "head": "b249bd2813602eab44e498ecd6e106652e81940a", + "started": "2026-10-16T14:30:20.893215+00:00", + "finished": "2026-10-16T14:30:32.325344+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-2", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-darwin", + "head": "b249bd2813602eab44e498ecd6e106652e81940a", + "started": "2026-10-16T14:30:32.326220+00:00", + "finished": "2026-10-16T14:30:43.978479+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-2", + "log": "checks.log" + }, + { + "name": "format-sandboxd", + "head": "b249bd2813602eab44e498ecd6e106652e81940a", + "started": "2026-10-16T14:30:43.979239+00:00", + "finished": "2026-10-16T14:30:51.117715+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-2", + "log": "checks.log" + }, + { + "name": "race-sandboxd", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:37:16.170239+00:00", + "finished": "2026-10-16T14:37:23.438591+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "race-e2e", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:37:23.439227+00:00", + "finished": "2026-10-16T14:37:28.478238+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-linux", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:37:28.479446+00:00", + "finished": "2026-10-16T14:37:40.374645+00:00", + "exit_code": 1, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-darwin", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:37:40.375553+00:00", + "finished": "2026-10-16T14:37:52.129784+00:00", + "exit_code": 1, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "format-sandboxd", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:37:52.130570+00:00", + "finished": "2026-10-16T14:37:59.383820+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "lint-e2e-linux", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:37:59.384706+00:00", + "finished": "2026-10-16T14:38:07.028816+00:00", + "exit_code": 1, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "lint-e2e-darwin", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:38:07.029700+00:00", + "finished": "2026-10-16T14:38:14.531396+00:00", + "exit_code": 1, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "format-e2e", + "head": "cc444095b3ba000de9ac11d7ef0674e386153804", + "started": "2026-10-16T14:38:14.532209+00:00", + "finished": "2026-10-16T14:38:15.610496+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-3", + "log": "checks.log" + }, + { + "name": "race-sandboxd", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:39:26.986588+00:00", + "finished": "2026-10-16T14:39:40.289815+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "race-e2e", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:39:40.290425+00:00", + "finished": "2026-10-16T14:39:45.950918+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-linux", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:39:45.952135+00:00", + "finished": "2026-10-16T14:39:57.692268+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "lint-sandboxd-darwin", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:39:57.693158+00:00", + "finished": "2026-10-16T14:40:09.172169+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "format-sandboxd", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:40:09.172893+00:00", + "finished": "2026-10-16T14:40:16.339527+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "lint-e2e-linux", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:40:16.340419+00:00", + "finished": "2026-10-16T14:40:24.070525+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "lint-e2e-darwin", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:40:24.071440+00:00", + "finished": "2026-10-16T14:40:31.474597+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + }, + { + "name": "format-e2e", + "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", + "started": "2026-10-16T14:40:31.475370+00:00", + "finished": "2026-10-16T14:40:32.524473+00:00", + "exit_code": 0, + "unexpected_skip": false, + "attempt": "verification-4", + "log": "checks.log" + } +] diff --git a/docs/test-results/pr339/checks.log b/docs/test-results/pr339/checks.log new file mode 100644 index 00000000..eab40285 --- /dev/null +++ b/docs/test-results/pr339/checks.log @@ -0,0 +1,543 @@ + +verification-1/race-protocol-wire.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/protocol/wire 1.131s + +verification-1/race-sandboxd.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/sandboxd 1.075s +ok github.com/cocoonstack/sandbox/sandboxd/config 1.103s +ok github.com/cocoonstack/sandbox/sandboxd/egress 5.634s +ok github.com/cocoonstack/sandbox/sandboxd/engine 4.869s +ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.713s +? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] +? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.717s +? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/pool 3.372s +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.038s +ok github.com/cocoonstack/sandbox/sandboxd/server 3.370s +ok github.com/cocoonstack/sandbox/sandboxd/store 1.111s +ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.668s +ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.045s +ok github.com/cocoonstack/sandbox/sandboxd/store/s3 5.777s +? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.054s +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.046s +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/types 1.050s +ok github.com/cocoonstack/sandbox/sandboxd/utils 1.031s + +verification-1/race-sdk-go.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/sdk/go 2.570s +ok github.com/cocoonstack/sandbox/sdk/go/silkd 1.229s +? github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest [no test files] + +verification-1/race-e2e.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/e2e 3.509s +? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] +ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.033s + +verification-1/race-mcp.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/mcp 1.771s + +verification-1/lint-protocol-wire-linux.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/lint-protocol-wire-darwin.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/format-protocol-wire.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-1/lint-sandboxd-linux.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +sandboxd/pool/volume.go:145:3: shadow: declaration of "resolvedVolume" shadows declaration at line 33 (govet) + resolvedVolume, resolveErr := m.resolveCatalogVolume(ctx, volume) + ^ +1 issues: +* govet: 1 + +verification-1/lint-sandboxd-darwin.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +sandboxd/pool/volume.go:145:3: shadow: declaration of "resolvedVolume" shadows declaration at line 33 (govet) + resolvedVolume, resolveErr := m.resolveCatalogVolume(ctx, volume) + ^ +1 issues: +* govet: 1 + +verification-1/format-sandboxd.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-1/lint-sdk-go-linux.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/lint-sdk-go-darwin.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/format-sdk-go.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-1/lint-e2e-linux.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/lint-e2e-darwin.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/format-e2e.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-1/lint-mcp-linux.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/lint-mcp-darwin.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-1/format-mcp.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-1/python-format.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo +command=['/tmp/pr339-design/python-env/bin/ruff', 'format', '--check', '.'] +38 files already formatted + +verification-1/python-lint.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo +command=['/tmp/pr339-design/python-env/bin/ruff', 'check', '.'] +All checks passed! + +verification-1/python-types.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/python +command=['/tmp/pr339-design/python-env/bin/python', '-m', 'mypy'] +Success: no issues found in 10 source files + +verification-1/python-test-python.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/python +command=['/tmp/pr339-design/python-env/bin/python', '-m', 'pytest', '-q'] +........................................................................ [ 29%] +........................................................................ [ 58%] +........................................................................ [ 87%] +............................... [100%] +247 passed in 35.10s + +verification-1/python-test-openai.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/openai +command=['/tmp/pr339-design/python-env/bin/python', '-m', 'pytest', '-q'] +.......... [100%] +10 passed in 6.26s + +verification-1/python-test-langchain.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/langchain +command=['/tmp/pr339-design/python-env/bin/python', '-m', 'pytest', '-q'] +............... [100%] +15 passed in 0.35s + +verification-1/pg-stores.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['go', 'test', '-race', '-count=1', '-v', './tenants/...', './poolset/...'] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +=== RUN TestContract +=== RUN TestContract/put_resolve_peek +=== RUN TestContract/keep_and_rotate_tokens +=== RUN TestContract/list_pages_by_name +=== RUN TestContract/delete_and_replace +=== RUN TestContract/replace_swaps_tokens +=== RUN TestContract/lookup +--- PASS: TestContract (0.01s) + --- PASS: TestContract/put_resolve_peek (0.00s) + --- PASS: TestContract/keep_and_rotate_tokens (0.00s) + --- PASS: TestContract/list_pages_by_name (0.00s) + --- PASS: TestContract/delete_and_replace (0.00s) + --- PASS: TestContract/replace_swaps_tokens (0.00s) + --- PASS: TestContract/lookup (0.00s) +=== RUN TestTheSetSurvivesAReopen +--- PASS: TestTheSetSurvivesAReopen (0.00s) +=== RUN TestATenantsFileNeedsTheRootToken +--- PASS: TestATenantsFileNeedsTheRootToken (0.00s) +PASS +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.043s +=== RUN TestContract +=== RUN TestContract/put_resolve_peek +=== RUN TestContract/keep_and_rotate_tokens +=== RUN TestContract/list_pages_by_name +=== RUN TestContract/delete_and_replace +=== RUN TestContract/replace_swaps_tokens +=== RUN TestContract/lookup +--- PASS: TestContract (1.09s) + --- PASS: TestContract/put_resolve_peek (0.26s) + --- PASS: TestContract/keep_and_rotate_tokens (0.16s) + --- PASS: TestContract/list_pages_by_name (0.15s) + --- PASS: TestContract/delete_and_replace (0.25s) + --- PASS: TestContract/replace_swaps_tokens (0.15s) + --- PASS: TestContract/lookup (0.14s) +=== RUN TestAWriteOnOneNodeReachesTheOtherNodesCache +--- PASS: TestAWriteOnOneNodeReachesTheOtherNodesCache (0.40s) +=== RUN TestACachedTenantOutlivesTheDatabase +--- PASS: TestACachedTenantOutlivesTheDatabase (0.13s) +=== RUN TestARoleWithoutCreateServesAPreCreatedTable +--- PASS: TestARoleWithoutCreateServesAPreCreatedTable (0.23s) +=== RUN TestAnUnrelatedEvictionKeepsALookupCacheable +--- PASS: TestAnUnrelatedEvictionKeepsALookupCacheable (0.00s) +PASS +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 2.901s +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +=== RUN TestStoreVersionsTheCellsSet +--- PASS: TestStoreVersionsTheCellsSet (0.19s) +=== RUN TestWatchFiresOnConnectAndOnItsCellsWrites +--- PASS: TestWatchFiresOnConnectAndOnItsCellsWrites (0.42s) +=== RUN TestADownDatabaseIsUnavailable +--- PASS: TestADownDatabaseIsUnavailable (0.12s) +PASS +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.766s + +verification-1/pg-pool.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['go', 'test', '-race', '-count=1', '-v', '-run', 'TestTwoNodesOnOneMetaStore|TestNodesOfACell|TestAMetaStoreNodeWithoutACell', './pool/'] +=== RUN TestNodesOfACellShareThePoolSet +--- PASS: TestNodesOfACellShareThePoolSet (0.50s) +=== RUN TestAMetaStoreNodeWithoutACellKeepsItsOwnPools +--- PASS: TestAMetaStoreNodeWithoutACellKeepsItsOwnPools (0.19s) +=== RUN TestTwoNodesOnOneMetaStoreShareTheTenantSet +--- PASS: TestTwoNodesOnOneMetaStoreShareTheTenantSet (0.25s) +PASS +ok github.com/cocoonstack/sandbox/sandboxd/pool 2.120s + +verification-1/tls-caddy.log +head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['go', 'test', '-race', '-count=1', '-run', 'TestCaddyTLSCluster', '-timeout', '60s', '-v', '.'] +=== RUN TestCaddyTLSCluster +=== RUN TestCaddyTLSCluster/Go +=== RUN TestCaddyTLSCluster/Python +--- PASS: TestCaddyTLSCluster (0.42s) + --- PASS: TestCaddyTLSCluster/Go (0.10s) + --- PASS: TestCaddyTLSCluster/Python (0.19s) +PASS +ok github.com/cocoonstack/sandbox/e2e 1.477s + +verification-2/race-sandboxd.log +head=b249bd2813602eab44e498ecd6e106652e81940a +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/sandboxd 1.080s +ok github.com/cocoonstack/sandbox/sandboxd/config 1.083s +ok github.com/cocoonstack/sandbox/sandboxd/egress 5.628s +ok github.com/cocoonstack/sandbox/sandboxd/engine 4.841s +ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.633s +? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] +? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.704s +? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/pool 3.430s +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.039s +ok github.com/cocoonstack/sandbox/sandboxd/server 3.385s +ok github.com/cocoonstack/sandbox/sandboxd/store 1.112s +ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.657s +ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.040s +ok github.com/cocoonstack/sandbox/sandboxd/store/s3 9.072s +? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.059s +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.043s +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/types 1.079s +ok github.com/cocoonstack/sandbox/sandboxd/utils 1.032s + +verification-2/lint-sandboxd-linux.log +head=b249bd2813602eab44e498ecd6e106652e81940a +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-2/lint-sandboxd-darwin.log +head=b249bd2813602eab44e498ecd6e106652e81940a +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-2/format-sandboxd.log +head=b249bd2813602eab44e498ecd6e106652e81940a +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-3/race-sandboxd.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/sandboxd 1.069s +ok github.com/cocoonstack/sandbox/sandboxd/config 1.118s +ok github.com/cocoonstack/sandbox/sandboxd/egress 5.686s +ok github.com/cocoonstack/sandbox/sandboxd/engine 4.838s +ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.705s +? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] +? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.711s +? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/pool 3.520s +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.036s +ok github.com/cocoonstack/sandbox/sandboxd/server 3.435s +ok github.com/cocoonstack/sandbox/sandboxd/store 1.098s +ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.608s +ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.035s +ok github.com/cocoonstack/sandbox/sandboxd/store/s3 4.268s +? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.068s +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.053s +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/types 1.071s +ok github.com/cocoonstack/sandbox/sandboxd/utils 1.045s + +verification-3/race-e2e.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/e2e 3.528s +? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] +ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.032s + +verification-3/lint-sandboxd-linux.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +sandboxd/pool/volume_convergence_test.go:206:10: shadow: declaration of "err" shadows declaration at line 202 (govet) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { + ^ +1 issues: +* govet: 1 + +verification-3/lint-sandboxd-darwin.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +sandboxd/pool/volume_convergence_test.go:206:10: shadow: declaration of "err" shadows declaration at line 202 (govet) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { + ^ +1 issues: +* govet: 1 + +verification-3/format-sandboxd.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-3/lint-e2e-linux.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +e2e/volume_mutation_test.go:51:8: shadow: declaration of "err" shadows declaration at line 46 (govet) + if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { + ^ +1 issues: +* govet: 1 + +verification-3/lint-e2e-darwin.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +e2e/volume_mutation_test.go:51:8: shadow: declaration of "err" shadows declaration at line 46 (govet) + if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { + ^ +1 issues: +* govet: 1 + +verification-3/format-e2e.log +head=cc444095b3ba000de9ac11d7ef0674e386153804 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-4/race-sandboxd.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/sandboxd 1.061s +ok github.com/cocoonstack/sandbox/sandboxd/config 1.104s +ok github.com/cocoonstack/sandbox/sandboxd/egress 5.614s +ok github.com/cocoonstack/sandbox/sandboxd/engine 4.864s +ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.717s +? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] +? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.707s +? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/pool 3.521s +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.047s +ok github.com/cocoonstack/sandbox/sandboxd/server 3.392s +ok github.com/cocoonstack/sandbox/sandboxd/store 1.102s +ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.668s +ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.046s +ok github.com/cocoonstack/sandbox/sandboxd/store/s3 5.678s +? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.043s +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.039s +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/types 1.085s +ok github.com/cocoonstack/sandbox/sandboxd/utils 1.031s + +verification-4/race-e2e.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] +ok github.com/cocoonstack/sandbox/e2e 3.519s +? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] +ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.033s + +verification-4/lint-sandboxd-linux.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-4/lint-sandboxd-darwin.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-4/format-sandboxd.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] + +verification-4/lint-e2e-linux.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-4/lint-e2e-darwin.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] +0 issues. + +verification-4/format-e2e.log +head=da56205bb31b489c15af3ea670e1cbe87905bf01 +cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e +command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] diff --git a/docs/test-results/pr339/decisions.tsv b/docs/test-results/pr339/decisions.tsv new file mode 100644 index 00000000..de4569c1 --- /dev/null +++ b/docs/test-results/pr339/decisions.tsv @@ -0,0 +1,25 @@ +ts phase decision why evidence result +2026-10-16T13:52:18.363582+00:00 frame Implement approved design and update existing PR only after local verification User explicitly authorized implementation, local k3s cleanup and PR update ../pr339-design/design.md implementation authorized; no merge +2026-10-16T13:52:18.363618+00:00 ground Pin original fork branch at 645e497 and use separate worktrees Existing workspace is another project and must not supply stale vendored code contract.md three isolated worker worktrees created +2026-10-16T13:52:18.363625+00:00 ground Attempt real local CH validation Full host access now exposes KVM and vhost-vsock todo.md hardware available; runtime setup pending +2026-10-16T13:56:04.831383+00:00 environment Use isolated runtime on port 17777 without removing k3s services Current host has enough RAM and separate runtime state avoids interference /tmp/pr339-live official Cocoon downloaded; prep guest not yet ready +2026-10-16T13:56:04.831434+00:00 environment Rebuild current initramfs after prep guest exits Old local stack expects computer.layers while official Cocoon passes cocoon.layers /tmp/pr339-live/prep-vm.log earlier process-start report superseded; handshake required before readiness +2026-10-16T14:00:45.127418+00:00 baseline Run real CH smoke against unchanged PR HEAD and observe busy 500 Same verification must fail on old behavior before accepting 409 handling hardware-evidence/baseline-smoke.log RO read and EROFS pass; cwd replacement fails expected 409 with actual500 +2026-10-16T14:00:45.127463+00:00 baseline Run PostgreSQL and Caddy CI integration checks in isolated services Changes to claims and API must not break meta-store or TLS transport baseline-extra/summary.json all three checks pass without PostgreSQL skips +2026-10-16T14:00:45.127471+00:00 verification Build reusable CI-equivalent command runner The final evidence should be reproducible with explicit commit identity run-checks.py race/lint/Python/PG/TLS suites defined; final candidate pending +2026-10-16T14:03:40.384102+00:00 tooling Run final checks with CI-pinned Go1.27.1 Default host Go1.27.2 export format5 is unsupported by pinned golangci parser4; source toolchain1.27.1 and lint agree run-checks.py protocol/wire lint reproduces host failure and passes with CI toolchain; no lint bypass +2026-10-16T14:10:21.688297+00:00 design Release only superseded pending holds whose detach was confirmed before an uncertain target commit Merging old pending into W breaks same-path target validation; A/W remains authoritative design-clarifications.md cut regression required for both old/new snapshots; independent review pending +2026-10-16T14:13:28.709882+00:00 tooling Keep staticcheck enabled and isolate its stale fact cache Stock tool and same source panic with shared cache and pass with a fresh one lint-investigation.md full e2e lint0 issues with clean cache; no production workaround +2026-10-16T14:13:28.709930+00:00 correctness Fail startup if volume quarantine cannot be journaled Durable-before-delete must not expose an uncontained restored claim issues.md new regression red before propagation, green afterward +2026-10-16T14:28:51Z integration Integrate pool, adapters, SDK and real-runtime harness in ordered local commits Separate writers own worktrees; coordinator reviews and combines their finished slices repo git log 645e497..ceda20d; contract.md candidate ceda20d; no push +2026-10-16T14:28:51Z comments Remove six comment lines and encode catalog/cleanup ownership in names and functions Keep the required maintainer-requested module docstring; avoid empty-tenant authorization signaling for held repair comment-resolution.md; comment-final-review.md; commit 3b87948 independent scoped comment review finds zero remaining flags +2026-10-16T14:28:51Z hardware Run complete real CH smoke twice and clean all owned runtime resources Validate caller continuity and actual device effects beyond fake adapters hardware-evidence/results.md; final-candidate-smoke.log; final-cleanup.json main scenarios VERIFIED at ceda20d; missing-ACK experiment INCONCLUSIVE +2026-10-16T14:28:52Z verification Fix a local resolvedVolume variable that shadows its type after full suite completes The final CI-equivalent lint correctly rejects the name introduced during catalog resolution cleanup verification-1/lint-sandboxd-linux.log; verification-1/lint-sandboxd-darwin.log all five race suites pass; one lint finding pending name-only fix +2026-10-16T14:31:58Z verification Pass all 29 unique CI-equivalent checks after a local-variable rename Full attempt recorded27 passes and two identical shadow findings; rerun the four affected sandboxd checks after the fix verification-1/summary.json; verification-2/summary.json; commit b249bd2 race, Linux/Darwin lint/format, Python247+10+15, PostgreSQL without skips, and Caddy TLS pass +2026-10-16T14:37:17Z review Fix the independent P2 diagnostic finding after a red API and pool regression Rejected invalid targets must explain the request refusal without replacing the stored last convergence outcome final-review.md; review-fix/sandboxd-red.log; review-fix/e2e-red.log; commit 5c2 targeted regressions red before fix and green afterward; complete affected suites pending +2026-10-16T14:37:33Z correction Correct the prior review row commit pointer to cc44409 The prior row accidentally named a non-resolving commit abbreviation; the actual source commit is cc44409 repo commit cc44409; review-fix/sandboxd-green.log; review-fix/e2e-green.log supersedes the prior row evidence pointer only +2026-10-16T14:39:28Z verification Correct two regression-test error variable names and rerun affected checks The cc44409 race suites passed but both platform linters correctly found two shadowed err variables in new tests verification-3/summary.json; repo commit da56205; verification-4/ only test-local names changed; final affected check rerun active +2026-10-16T14:41:34Z verification Pass complete affected sandboxd and e2e checks at da56205 Keep original lint failures visible and prove final test code meets the same pinned gates verification-4/summary.json eight checks pass; all29 distinct local gates green across unchanged and affected modules +2026-10-16T14:41:34Z hardware Rerun full real CH smoke and invalid-target API probe at cc44409 Verify the independent review fix against actual live devices and keep exact SHA labels hardware-evidence/delivery-smoke.log; delivery-invalid-put.json; delivery-cleanup.json; repo cc44409..da56205 main scenarios and diagnostic fix pass; final delta is test names only; owned resources empty +2026-10-16T14:42:13Z review Accept independent follow-up code verdict and publish exact final evidence Keep source revisions and failed attempts accurate; replace stale pending-hardware statements with measured results final-followup-review.md; repo/docs/live-volumes-validation.md; verification-4/summary.json; hardware-evidence/delivery-results.md code VERIFIED at da56205; production hardware run cc44409; report packages29 green gates and49 attempt records +2026-10-16T14:43:05Z evidence Keep the machine-specific ad hoc probe source in the local audit A tentative public copy has hardcoded local paths and lint findings; the reusable hardware program and committed API regression already provide the maintained check hardware-evidence/delivery-invalid-put.py; repo/e2e/volume_mutation_test.go; repo/docs/test-results/pr339/hardware-invalid-put.json unneeded probe source copy removed; measured JSON and existing reusable checks retained diff --git a/docs/test-results/pr339/diagnostic-regression.log b/docs/test-results/pr339/diagnostic-regression.log new file mode 100644 index 00000000..dd4106b8 --- /dev/null +++ b/docs/test-results/pr339/diagnostic-regression.log @@ -0,0 +1,114 @@ + +review-fix/sandboxd-red.log +ok github.com/cocoonstack/sandbox/sandboxd 1.071s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/config 1.037s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/egress 1.036s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/engine 1.030s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/mesh 1.041s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] +? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.041s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] +--- FAIL: TestRejectedVolumeTargetPreservesConvergenceDiagnostics (0.01s) + --- FAIL: TestRejectedVolumeTargetPreservesConvergenceDiagnostics/settled (0.01s) + volume_convergence_test.go:211: rejected request changed convergence view: before={VolumesWant:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:attached Error:}] Converging:false Error:} after={VolumesWant:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:attached Error:}] Converging:false Error:volume change failed; retry the desired target} err= + --- FAIL: TestRejectedVolumeTargetPreservesConvergenceDiagnostics/conflict (0.00s) + volume_convergence_test.go:211: rejected request changed convergence view: before={VolumesWant:[] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:detaching Error:cannot unmount "/volumes/data"; close guest references and retry}] Converging:false Error:cannot unmount "/volumes/data"; close guest references and retry} after={VolumesWant:[] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:detaching Error:cannot unmount "/volumes/data"; close guest references and retry}] Converging:false Error:volume change failed; retry the desired target} err= +FAIL +FAIL github.com/cocoonstack/sandbox/sandboxd/pool 0.094s +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.047s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/server 1.052s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store 1.036s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.031s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store/peer 1.038s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store/s3 1.039s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.039s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.042s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/types 1.038s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/utils 1.031s [no tests to run] +FAIL + +review-fix/e2e-red.log +--- FAIL: TestRejectedVolumeTargetExplainsRefusalWithoutChangingState (0.03s) + volume_mutation_test.go:57: invalid target explanation: set volumes: volume change failed; retry the desired target (http 400) + volume_mutation_test.go:61: rejected request changed settled state: {VolumesWant:[{Name:dataset Mount:/volumes/dataset Mode:}] Volumes:[{Name:dataset Mount:/volumes/dataset Mode:}] States:[{Name:dataset State:attached Error:}] Converging:false Error:volume change failed; retry the desired target} err= +FAIL +FAIL github.com/cocoonstack/sandbox/e2e 0.088s +? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] +ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.031s [no tests to run] +FAIL + +review-fix/sandboxd-green.log +ok github.com/cocoonstack/sandbox/sandboxd 1.093s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/config 1.039s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/egress 1.066s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/engine 1.070s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/mesh 1.044s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] +? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.079s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/pool 1.204s +? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.042s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/server 1.106s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store 1.035s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.042s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store/peer 1.061s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/store/s3 1.049s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] +? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.041s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.041s [no tests to run] +? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] +ok github.com/cocoonstack/sandbox/sandboxd/types 1.041s [no tests to run] +ok github.com/cocoonstack/sandbox/sandboxd/utils 1.035s [no tests to run] + +review-fix/e2e-green.log +ok github.com/cocoonstack/sandbox/e2e 1.129s +? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] +? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] +ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.033s [no tests to run] diff --git a/docs/test-results/pr339/guest-input-sha256.txt b/docs/test-results/pr339/guest-input-sha256.txt new file mode 100644 index 00000000..9f4cc311 --- /dev/null +++ b/docs/test-results/pr339/guest-input-sha256.txt @@ -0,0 +1,3 @@ +1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7 /var/lib/composable-local-bfc70d3/guest-rootfs.tar +7afa73b01f77f4decb723be647399a7ade15bc8ff143262662048251e6603a92 /tmp/pr339-live/boot/vmlinuz +930e5e753ae51f0104b2d4ced96f107e5f55544cd91b1b968266de91a76e3c95 /tmp/pr339-live/boot/initrd.img diff --git a/docs/test-results/pr339/hardware-binaries.sha256 b/docs/test-results/pr339/hardware-binaries.sha256 new file mode 100644 index 00000000..f5941027 --- /dev/null +++ b/docs/test-results/pr339/hardware-binaries.sha256 @@ -0,0 +1,3 @@ +13f9d2b924669040200579e51f9301ab755bc0f0ea1bffd35e612eee17e765d8 /tmp/pr339-live/sandboxd +30fd2a6c7ec7e3d0ba305b847e8823e60a98c49f2f618ee9de0e391e2f35120b /tmp/pr339-live/volumesmoke +b22b4434d5b0e62c262bb99948de7e41eaeda52e9e617951b660b09cff33252d /tmp/pr339-live/guestserver diff --git a/docs/test-results/pr339/hardware-build-info.txt b/docs/test-results/pr339/hardware-build-info.txt new file mode 100644 index 00000000..3a9b5a01 --- /dev/null +++ b/docs/test-results/pr339/hardware-build-info.txt @@ -0,0 +1,116 @@ +/tmp/pr339-live/sandboxd: go1.27.1 + path github.com/cocoonstack/sandbox/sandboxd + mod github.com/cocoonstack/sandbox/sandboxd (devel) + dep github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= + dep github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= + dep github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= + dep github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= + dep github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= + dep github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.13 h1:wO7TVbywHwdpHLUiX6DnmP2RDYOACVeJCb6zMfSFViU= + dep github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= + dep github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= + dep github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= + dep github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= + dep github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= + dep github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= + dep github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= + dep github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= + dep github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= + dep github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= + dep github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= + dep github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= + dep github.com/aws/smithy-go v1.28.2 h1:myhcykQcatTul2B/zITjDk203G7t0awUAs1hVry5Bvg= + dep github.com/cockroachdb/errors v1.14.0 h1:EfdVEJpN3z8rPMo43Yit59LxoiIa470fSXpZXuEs+ZI= + dep github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILMqgNeV5jiqR4j+sTuvQNHdf2chuKj1M5k= + dep github.com/cockroachdb/redact v1.1.8 h1:8eVLLj6juKxiKrAEw2b8cJvNqWq++U8WOfQFuL7KTaA= + dep github.com/cocoonstack/sandbox/protocol/wire v0.0.0-00010101000000-000000000000 + => ../protocol/wire (devel) + + dep github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= + dep github.com/getsentry/sentry-go v0.49.0 h1:Ehejknu1l023Ub7QoRBVLAI7g3Jnhqku4oWx4B4Sh5s= + dep github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= + dep github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= + dep github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= + dep github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= + dep github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= + dep github.com/hashicorp/go-metrics v0.7.0 h1:lLWieZTcbzZT+rY0zrqKbyryXG8RIajdUjmM0+R79eg= + dep github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44= + dep github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= + dep github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= + dep github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= + dep github.com/hashicorp/memberlist v0.7.0 h1:JfqTDFUIAzDEYKMhSc3Gpwe05zvSU3/cYtiZ3yW59TM= + dep github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= + dep github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= + dep github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= + dep github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= + dep github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= + dep github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= + dep github.com/mattn/go-colorable v0.1.16 h1:+2YjeAe7xP1OQbZW1jERPWWCDLlNqFNCvjiqGiAt84A= + dep github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= + dep github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4YbNI= + dep github.com/mdlayher/socket v0.7.0 h1:qVREPVwtUMg17pwvveQxpurq0PVisMxi3FGgpsorMYQ= + dep github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= + dep github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= + dep github.com/projecteru2/core v0.1.7 h1:f2EdndjWFBYX6pqLZ68VcFYMkXZu6RivotaKUGCNwDQ= + dep github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= + dep github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= + dep github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= + dep golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= + dep golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= + dep golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= + dep golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= + dep golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= + dep golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= + dep google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= + dep google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= + dep gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= + build -buildmode=exe + build -compiler=gc + build CGO_ENABLED=1 + build CGO_CFLAGS= + build CGO_CPPFLAGS= + build CGO_CXXFLAGS= + build CGO_LDFLAGS= + build GOARCH=amd64 + build GOOS=linux + build GOAMD64=v1 + build vcs=git + build vcs.revision=cc444095b3ba000de9ac11d7ef0674e386153804 + build vcs.time=2026-10-16T14:37:15Z + build vcs.modified=false +/tmp/pr339-live/volumesmoke: go1.27.1 + path github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke + mod github.com/cocoonstack/sandbox/e2e (devel) + dep github.com/cocoonstack/sandbox/protocol/wire v0.1.16 + => ../protocol/wire (devel) + + dep github.com/cocoonstack/sandbox/sdk/go v0.0.0 + => ../sdk/go (devel) + + build -buildmode=exe + build -compiler=gc + build CGO_ENABLED=1 + build CGO_CFLAGS= + build CGO_CPPFLAGS= + build CGO_CXXFLAGS= + build CGO_LDFLAGS= + build GOARCH=amd64 + build GOOS=linux + build GOAMD64=v1 + build vcs=git + build vcs.revision=cc444095b3ba000de9ac11d7ef0674e386153804 + build vcs.time=2026-10-16T14:37:15Z + build vcs.modified=false +/tmp/pr339-live/guestserver: go1.27.1 + path github.com/cocoonstack/sandbox/e2e/cmd/guestserver + mod github.com/cocoonstack/sandbox/e2e (devel) + build -buildmode=exe + build -compiler=gc + build CGO_ENABLED=0 + build GOARCH=amd64 + build GOOS=linux + build GOAMD64=v1 + build vcs=git + build vcs.revision=cc444095b3ba000de9ac11d7ef0674e386153804 + build vcs.time=2026-10-16T14:37:15Z + build vcs.modified=false diff --git a/docs/test-results/pr339/hardware-cleanup.json b/docs/test-results/pr339/hardware-cleanup.json new file mode 100644 index 00000000..ffe1b430 --- /dev/null +++ b/docs/test-results/pr339/hardware-cleanup.json @@ -0,0 +1,17 @@ +{ + "candidate": "cc444095b3ba000de9ac11d7ef0674e386153804", + "claims": 0, + "volume_retirements": 0, + "vms": [], + "runtime_sockets": [], + "owned_taps": [], + "vm_pids_gone": { + "1082008": true, + "1084550": true, + "1087371": true + }, + "isolated_daemon_stopped": true, + "existing_services_stopped": [], + "retained_device_calls": 0, + "fault_markers_remaining": [] +} diff --git a/docs/test-results/pr339/hardware-commands.jsonl b/docs/test-results/pr339/hardware-commands.jsonl new file mode 100644 index 00000000..663ec970 --- /dev/null +++ b/docs/test-results/pr339/hardware-commands.jsonl @@ -0,0 +1,88 @@ +{"time": 1792161516.2106261, "pid": 1081257, "args": ["version"]} +{"time": 1792161516.290213, "pid": 1081298, "args": ["vm", "list", "--format", "json"]} +{"time": 1792161516.3823283, "pid": 1081340, "args": ["snapshot", "list", "--format", "json"]} +{"time": 1792161516.4694333, "pid": 1081381, "args": ["image", "list", "--format", "json"]} +{"time": 1792161531.5944746, "pid": 1081562, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-ab5e7c310318", "--pull", "--output", "json", "--no-direct-io=true"]} +{"time": 1792161531.7895525, "pid": 1081662, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-ab5e7c310318", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161531.9890335, "pid": 1081705, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-743eac818016", "--pull", "--output", "json", "--no-direct-io=true"]} +{"time": 1792161532.1801274, "pid": 1081804, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-743eac818016", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161532.4134898, "pid": 1081847, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-ab5e7c310318"]} +{"time": 1792161532.5762568, "pid": 1081909, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-743eac818016"]} +{"time": 1792161532.7404315, "pid": 1081966, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--pull", "--output", "json", "--no-direct-io=true"]} +{"time": 1792161532.9253728, "pid": 1082064, "args": ["version"]} +{"time": 1792161532.9955375, "pid": 1082074, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161533.243327, "pid": 1082118, "args": ["version"]} +{"time": 1792161533.3524957, "pid": 1082164, "args": ["version"]} +{"time": 1792161533.4615133, "pid": 1082205, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161533.6639063, "pid": 1082246, "args": ["version"]} +{"time": 1792161533.7398326, "pid": 1082256, "args": ["version"]} +{"time": 1792161533.8222344, "pid": 1082297, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161533.9386697, "pid": 1082340, "args": ["version"]} +{"time": 1792161534.0145915, "pid": 1082350, "args": ["version"]} +{"time": 1792161534.0949044, "pid": 1082360, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161534.2889814, "pid": 1082401, "args": ["version"]} +{"time": 1792161534.450274, "pid": 1082475, "args": ["version"]} +{"time": 1792161534.5280719, "pid": 1082517, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161534.6437583, "pid": 1082560, "args": ["version"]} +{"time": 1792161534.733479, "pid": 1082602, "args": ["version"]} +{"time": 1792161534.8219416, "pid": 1082643, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161535.0211015, "pid": 1082684, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} +{"time": 1792161535.1456678, "pid": 1082727, "args": ["version"]} +{"time": 1792161535.238117, "pid": 1082768, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} +{"time": 1792161535.4366887, "pid": 1082811, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161535.5785534, "pid": 1082854, "args": ["version"]} +{"time": 1792161535.6483173, "pid": 1082865, "args": ["version"]} +{"time": 1792161535.7378325, "pid": 1082906, "args": ["version"]} +{"time": 1792161535.8273516, "pid": 1082947, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161536.0251427, "pid": 1082988, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} +{"time": 1792161536.1503155, "pid": 1083031, "args": ["version"]} +{"time": 1792161536.2372422, "pid": 1083073, "args": ["version"]} +{"time": 1792161536.315691, "pid": 1083083, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} +{"time": 1792161536.5120556, "pid": 1083125, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161536.560109, "pid": 1083131, "args": ["vm", "list", "--format", "json"]} +{"time": 1792161536.6265824, "pid": 1083210, "args": ["version"]} +{"time": 1792161536.716744, "pid": 1083251, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161536.9173326, "pid": 1083292, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} +{"time": 1792161537.150615, "pid": 1083338, "args": ["version"]} +{"time": 1792161537.2409613, "pid": 1083379, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} +{"time": 1792161537.4404385, "pid": 1083422, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161537.712072, "pid": 1083544, "args": ["version"]} +{"time": 1792161537.7933095, "pid": 1083585, "args": ["vm", "list", "--format", "json"]} +{"time": 1792161537.881657, "pid": 1083626, "args": ["snapshot", "list", "--format", "json"]} +{"time": 1792161537.9697576, "pid": 1083667, "args": ["image", "list", "--format", "json"]} +{"time": 1792161538.0572329, "pid": 1083708, "args": ["vm", "list", "--format", "json"]} +{"time": 1792161538.1257093, "pid": 1083749, "args": ["version"]} +{"time": 1792161538.2046566, "pid": 1083791, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161538.34557, "pid": 1083832, "args": ["version"]} +{"time": 1792161538.4290745, "pid": 1083843, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161538.6181557, "pid": 1083884, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} +{"time": 1792161538.8523986, "pid": 1083930, "args": ["version"]} +{"time": 1792161538.9431992, "pid": 1083970, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} +{"time": 1792161539.141075, "pid": 1084011, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161539.8965445, "pid": 1084097, "args": ["version"]} +{"time": 1792161539.977518, "pid": 1084138, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161540.1214855, "pid": 1084179, "args": ["version"]} +{"time": 1792161540.211696, "pid": 1084235, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} +{"time": 1792161540.4132092, "pid": 1084292, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} +{"time": 1792161540.5535214, "pid": 1084355, "args": ["version"]} +{"time": 1792161540.6428962, "pid": 1084409, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} +{"time": 1792161540.8559349, "pid": 1084451, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53"]} +{"time": 1792161541.029107, "pid": 1084508, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "--pull", "--output", "json", "--no-direct-io=true"]} +{"time": 1792161541.2160385, "pid": 1084606, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161541.4820101, "pid": 1084709, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} +{"time": 1792161541.529443, "pid": 1084785, "args": ["vm", "inspect", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} +{"time": 1792161541.765802, "pid": 1084909, "args": ["version"]} +{"time": 1792161541.8461316, "pid": 1084950, "args": ["vm", "list", "--format", "json"]} +{"time": 1792161541.9320328, "pid": 1084991, "args": ["snapshot", "list", "--format", "json"]} +{"time": 1792161542.0202837, "pid": 1085032, "args": ["image", "list", "--format", "json"]} +{"time": 1792161542.1088488, "pid": 1085073, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} +{"time": 1792161542.1567278, "pid": 1085074, "args": ["vm", "inspect", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} +{"time": 1792161547.2556753, "pid": 1085179, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} +{"time": 1792161547.4440455, "pid": 1085234, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-07e68275055f", "--pull", "--output", "json", "--no-direct-io=true"]} +{"time": 1792161547.6333888, "pid": 1085332, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-07e68275055f", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--directio", "off"]} +{"time": 1792161547.8819134, "pid": 1085375, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-07e68275055f"]} +{"time": 1792161576.371202, "pid": 1087329, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-667b56746f08", "--pull", "--output", "json", "--no-direct-io=true"]} +{"time": 1792161576.5613878, "pid": 1087428, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-667b56746f08", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +{"time": 1792161576.8436613, "pid": 1087555, "args": ["version"]} +{"time": 1792161576.9886754, "pid": 1087636, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-667b56746f08"]} +{"time": 1792161602.268468, "pid": 1098326, "args": ["image", "list", "--format", "json"]} diff --git a/docs/test-results/pr339/hardware-invalid-put.json b/docs/test-results/pr339/hardware-invalid-put.json new file mode 100644 index 00000000..02cfaf97 --- /dev/null +++ b/docs/test-results/pr339/hardware-invalid-put.json @@ -0,0 +1,75 @@ +{ + "sandbox": "sb_d578a0cb4762e49f", + "status": 400, + "failure": { + "volumes_want": [ + { + "name": "dataset-v1", + "mount": "/datasets/invalid" + } + ], + "volumes": [ + { + "name": "dataset-v1", + "mount": "/datasets/invalid" + } + ], + "states": [ + { + "name": "dataset-v1", + "state": "attached" + } + ], + "converging": false, + "error": "invalid volume request: duplicate volume \"dataset-v1\"" + }, + "before": { + "volumes_want": [ + { + "name": "dataset-v1", + "mount": "/datasets/invalid" + } + ], + "volumes": [ + { + "name": "dataset-v1", + "mount": "/datasets/invalid" + } + ], + "states": [ + { + "name": "dataset-v1", + "state": "attached" + } + ], + "converging": false + }, + "after": { + "volumes_want": [ + { + "name": "dataset-v1", + "mount": "/datasets/invalid" + } + ], + "volumes": [ + { + "name": "dataset-v1", + "mount": "/datasets/invalid" + } + ], + "states": [ + { + "name": "dataset-v1", + "state": "attached" + } + ], + "converging": false + }, + "marker_before": "v1\n", + "marker_after": "v1\n", + "vm_pid": 1087371, + "vm_pid_exists": true, + "invalid_put_disk_calls": [], + "PASS": true, + "release_status": 204 +} diff --git a/docs/test-results/pr339/hardware-invocation.json b/docs/test-results/pr339/hardware-invocation.json new file mode 100644 index 00000000..d933b65c --- /dev/null +++ b/docs/test-results/pr339/hardware-invocation.json @@ -0,0 +1,23 @@ +[ + "/tmp/pr339-live/volumesmoke", + "-addr", + "127.0.0.1:17777", + "-token", + "", + "-template", + "pr339:local", + "-volume", + "dataset-v1", + "-next-volume", + "dataset-v2", + "-probe", + "version", + "-listener", + "/tmp/pr339-live/guestserver", + "-change-hook", + "/root/eric/composable-computer/.audit/pr339-implementation/hardware/e2e/cmd/volumesmoke/testdata/fault_hook.py", + "-cancel-hook", + "/root/eric/composable-computer/.audit/pr339-implementation/hardware/e2e/cmd/volumesmoke/testdata/fault_hook.py", + "-release-hook", + "/root/eric/composable-computer/.audit/pr339-implementation/hardware/e2e/cmd/volumesmoke/testdata/fault_hook.py" +] diff --git a/docs/test-results/pr339/hardware-journal.json b/docs/test-results/pr339/hardware-journal.json new file mode 100644 index 00000000..c66ad815 --- /dev/null +++ b/docs/test-results/pr339/hardware-journal.json @@ -0,0 +1 @@ +{"format_version":2,"claims":{},"volume_retirements":{}} \ No newline at end of file diff --git a/docs/test-results/pr339/hardware-prep.md b/docs/test-results/pr339/hardware-prep.md new file mode 100644 index 00000000..5dc80674 --- /dev/null +++ b/docs/test-results/pr339/hardware-prep.md @@ -0,0 +1,29 @@ +The local test uses the official Cocoon v0.7.2 x86_64 release from https://github.com/cocoonstack/cocoon/releases/tag/v0.7.2. Archive SHA256 is bd9e9c76f926a5005089ede61a85b1d98c5294c454d3efca0cb4bf4826dca8d4, matching the release checksums. Version reports Git b3b40f3bdedc384c482eb81dcdc60a774dd8d560. Cloud Hypervisor is the existing /opt/composable-local-bfc70d3/bin/cloud-hypervisor v54.0.0. No version output is forged or emulated. + +The test's root, runtime sockets, logs, synthetic API token, images, and daemon state are isolated below /tmp/pr339-live. The daemon listens at 127.0.0.1:17777. Cocoon uses network scope p9 and cgroup parent pr339-cocoon.slice. None of the existing composable-local services has been stopped or modified. + +Preparation reused the existing guest-rootfs.tar and vmlinuz from /var/lib/composable-local-bfc70d3. The first real VM immediately exited because its old initramfs expected computer.layers while official Cocoon supplies cocoon.layers. Rebuilding the current sandbox boot/init binary fixed that compatibility difference. The first rebuilt binary was dynamically linked and produced a kernel "No working init found" panic inside the small initramfs. Rebuilding it statically fixed that. These were environment preparation failures, not feature failures. The final guest was accepted only after a successful CONNECT 2048 handshake and a live host PID check. + +Commands for the working boot rebuild, run from the hardware worktree are below. The host's stable Rust toolchain is recorded in tool-versions.txt. The rootfs layer already contains silkd. It responds to real SDK operations in the baseline smoke. + +```sh +RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --locked --manifest-path boot/init/Cargo.toml +podman create --name pr339-boot localhost/composable-computer/sandbox/boot:6.18.37 +podman cp pr339-boot:/boot/initrd.img-sandbox /tmp/pr339-live/boot/initrd.img +cd /tmp/pr339-live/initramfs +cpio -idmu < /tmp/pr339-live/boot/initrd.img +cp /root/eric/composable-computer/.audit/pr339-implementation/hardware/boot/init/target/release/sandbox-init init +find . -print0 | cpio --null -o -H newc > /tmp/pr339-live/boot/initrd.img +cd /tmp/pr339-live/boot +tar -cf /tmp/pr339-live/boot.tar . +. /tmp/pr339-live/env +cocoon image import pr339:local /var/lib/composable-local-bfc70d3/guest-rootfs.tar /tmp/pr339-live/boot.tar +``` + +The order of the two input layers matters. The boot overlay must come last because the existing rootfs also contains boot files. Input checksums are in guest-input-sha256.txt. The resulting image is sha256:0356e79c7b82914c62e7b5db88b06369594c92f4bb95b4e1de5a721023b1ba91. It uses the real existing 6.18.37 kernel. The guest bootstrap is ordinary sandbox-init and systemd with silkd. The VM pr339-prep was created solely for environment proof. The successful preparation record and PID are in prep-vm-ready.json and prep-vm-current.json. + +Dataset fixtures are two 32 MiB ext4 images made with mkfs.ext4 -F -d. Their version files contain v1 and v2 respectively. Catalog names dataset-v1 and dataset-v2 point to those images. Both allow writable attachment to permit the later retirement/backing-reuse assertion. Ordinary smoke attachments request read-only mode. + +The baseline daemon was built from 645e49756492e7d8e9337377a958185e615804fc. baseline-smoke.log records real shared dataset reads and EROFS succeeding, followed by the expected failing cwd-busy assertion. The old daemon actually returned HTTP 500 instead of the required HTTP 409 and kept v1 applied with v2 desired. baseline-runtime-commands.jsonl and baseline-daemon.log record its real runtime calls and error. This is a hardware reproduction, not a fake-engine result. + +The committed testdata/runtime_observer.py and fault_hook.py are the complete fault injection mechanism. Ordinary commands execute the official binary. Attach interruption occurs only after the real attach succeeds. VM removal failures are explicitly injected exit 73 before the real remove command. The observer logs every attempted command. The test must separately prove final real removal and a later writable attach before claiming backing reuse. diff --git a/docs/test-results/pr339/hardware-revision.txt b/docs/test-results/pr339/hardware-revision.txt new file mode 100644 index 00000000..6505e866 --- /dev/null +++ b/docs/test-results/pr339/hardware-revision.txt @@ -0,0 +1,2 @@ +cc444095b3ba000de9ac11d7ef0674e386153804 +2f3624c394be1cea6f8f2e53be1f63764e9893f3 diff --git a/docs/test-results/pr339/hardware.log b/docs/test-results/pr339/hardware.log new file mode 100644 index 00000000..62e38367 --- /dev/null +++ b/docs/test-results/pr339/hardware.log @@ -0,0 +1,34 @@ +volume=dataset-v1 claim_a=396.7ms claim_b=394.1ms +VOLUME PASS concurrent_read_bytes=3 read_only=true released=true +CONTINUITY before={"Sandbox":"sb_13c00446eaf0a810","Session":"sess-68-1","Boot":"cd7dbea1-cbb8-4cbb-aba5-97f4e5aeab02","PID":181,"Start":"237088"} +BUSY PASS reference=cwd view={"volumes_want":[{"name":"dataset-v2","mount":"/datasets/current"}],"volumes":[{"name":"dataset-v1","mount":"/datasets/current"}],"states":[{"name":"dataset-v1","state":"detaching","error":"cannot unmount \"/datasets/current\"; close guest references and retry"},{"name":"dataset-v2","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false,"error":"cannot unmount \"/datasets/current\"; close guest references and retry"} +RETAINED begin=1792161535537904360 +RETAINED end=1792161535607563767 +BUSY PASS reference=bind view={"volumes_want":[{"name":"dataset-v2","mount":"/datasets/current"}],"volumes":[{"name":"dataset-v1","mount":"/datasets/current"}],"states":[{"name":"dataset-v1","state":"detaching","error":"volume device has another mount at \"/tmp/volume-bind\"; remove guest mount references before retrying"},{"name":"dataset-v2","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false,"error":"volume device has another mount at \"/tmp/volume-bind\"; remove guest mount references before retrying"} +BUSY PASS reference=foreign view={"volumes_want":[{"name":"dataset-v1","mount":"/datasets/current"}],"volumes":[{"name":"dataset-v2","mount":"/datasets/current"}],"states":[{"name":"dataset-v2","state":"detaching","error":"volume mount \"/datasets/current\" has a different device, root, or mode; restore the expected mount before retrying"},{"name":"dataset-v1","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false,"error":"volume mount \"/datasets/current\" has a different device, root, or mode; restore the expected mount before retrying"} +HOOK runtime_event={"time": 1792161537.4404385, "pid": 1083422, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +HOOK real_vm=[{"id": "3SMJN3MBILXGIJ2FUF4LLABHYZ", "name": "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "pid": 1082008, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/3SMJN3MBILXGIJ2FUF4LLABHYZ/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] +HOOK gated_view={"volumes_want": [{"name": "dataset-v1", "mount": "/datasets/current"}], "volumes": [], "states": [{"name": "dataset-v1", "state": "pending"}], "converging": true} +HOOK daemon_restart old=1081212 new=1083505 +ACTIVITY PASS exec=true file=true port=true +INTERRUPTION request_error=Put "http://127.0.0.1:17777/v1/sandboxes/sb_13c00446eaf0a810/volumes": EOF +INTERRUPTION recovered_view={"volumes_want":[{"name":"dataset-v1","mount":"/datasets/current"}],"volumes":[],"states":[{"name":"dataset-v1","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false} +INTERRUPTION PASS sandbox=sb_13c00446eaf0a810 +HOOK runtime_event={"time": 1792161539.141075, "pid": 1084011, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} +HOOK real_vm=[{"id": "3SMJN3MBILXGIJ2FUF4LLABHYZ", "name": "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "pid": 1082008, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/3SMJN3MBILXGIJ2FUF4LLABHYZ/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] +HOOK gated_view={"volumes_want": [{"name": "dataset-v1", "mount": "/datasets/current"}], "volumes": [], "states": [{"name": "dataset-v1", "state": "pending"}], "converging": true} +HOOK canceled_view={"volumes_want": [{"name": "dataset-v1", "mount": "/datasets/current"}], "volumes": [], "states": [{"name": "dataset-v1", "state": "pending", "error": "volume change failed; retry the desired target"}], "converging": false, "error": "volume change failed; retry the desired target"} +ACTIVITY PASS exec=true file=true port=true +INTERRUPTION request_error=Put "http://127.0.0.1:17777/v1/sandboxes/sb_13c00446eaf0a810/volumes": context canceled +INTERRUPTION recovered_view={"volumes_want":[{"name":"dataset-v1","mount":"/datasets/current"}],"volumes":[],"states":[{"name":"dataset-v1","state":"pending","error":"volume change failed; retry the desired target"}],"converging":false,"error":"volume change failed; retry the desired target"} +INTERRUPTION PASS sandbox=sb_13c00446eaf0a810 +CONTINUITY PASS after={"Sandbox":"sb_13c00446eaf0a810","Session":"sess-68-1","Boot":"cd7dbea1-cbb8-4cbb-aba5-97f4e5aeab02","PID":181,"Start":"237088"} v1="v1" v2="v2" cwd_409=true bind_409=true foreign_409=true retained_bind=true erofs=true cleared=true +LIVE VOLUME PASS read_bytes=3 repeated_put=true repeated_patch=true cleared=true +RETIREMENT release_error=release: release failed (http 500) +RETIREMENT writer_refused=claim: volume is held by another claim: volume "dataset-v1" (http 409) +HOOK runtime_event={"time": 1792161541.4820101, "pid": 1084709, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} +HOOK real_vm=[{"id": "OHF3UXRBAEFIHIABIJ7BNYWL2P", "name": "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "pid": 1084550, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/OHF3UXRBAEFIHIABIJ7BNYWL2P/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] +HOOK daemon_restart old=1083505 new=1084870 +HOOK real_vm=[{"id": "OHF3UXRBAEFIHIABIJ7BNYWL2P", "name": "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "pid": 1084550, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/OHF3UXRBAEFIHIABIJ7BNYWL2P/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] +RETIREMENT writer_refused=claim: volume is held by another claim: volume "dataset-v1" (http 409) +RETIREMENT PASS writer_blocked_before_restart=true writer_blocked_after_restart=true backing_reused=true diff --git a/docs/test-results/pr339/host-tool-versions.txt b/docs/test-results/pr339/host-tool-versions.txt new file mode 100644 index 00000000..e9096595 --- /dev/null +++ b/docs/test-results/pr339/host-tool-versions.txt @@ -0,0 +1,2 @@ +go version go1.27.2 linux/amd64 +rustc 1.99.1 (58cf8f9c3 2026-10-13) diff --git a/docs/test-results/pr339/pending-eject.json b/docs/test-results/pr339/pending-eject.json new file mode 100644 index 00000000..d37230f8 --- /dev/null +++ b/docs/test-results/pr339/pending-eject.json @@ -0,0 +1,10 @@ +{ + "ack_absence_injection_established": false, + "reason": "Official Cocoon converges orphaned pause by resuming the guest before disk detach, so guest ACK was available.", + "official_detach_exit": 0, + "official_detach_seconds": 0.145, + "timeout_and_second_pending_retry": "NOT TESTED by this pause mechanism", + "ordinary_guest_umount": true, + "backing_reused_writable_after_real_detach": true, + "initial_assertion_failure": "The probe premise that the guest stayed paused was false. This is not a runtime false-success finding." +} diff --git a/docs/test-results/pr339/post-hardware-test-only.diff b/docs/test-results/pr339/post-hardware-test-only.diff new file mode 100644 index 00000000..3107a58b --- /dev/null +++ b/docs/test-results/pr339/post-hardware-test-only.diff @@ -0,0 +1,18 @@ +diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go +index 9e2e7f6..776b9fd 100644 +--- a/e2e/volume_mutation_test.go ++++ b/e2e/volume_mutation_test.go +@@ -51,2 +51,2 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { +- if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { +- t.Fatal(err) ++ if _, checkErr := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); checkErr != nil { ++ t.Fatal(checkErr) +diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go +index 7659fa7..5f1a893 100644 +--- a/sandboxd/pool/volume_convergence_test.go ++++ b/sandboxd/pool/volume_convergence_test.go +@@ -206,2 +206,2 @@ func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { +- if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { +- t.Fatalf("invalid target: %v", err) ++ if _, checkErr := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(checkErr, ErrBadVolume) { ++ t.Fatalf("invalid target: %v", checkErr) diff --git a/docs/test-results/pr339/retained-disk-calls.json b/docs/test-results/pr339/retained-disk-calls.json new file mode 100644 index 00000000..9a377919 --- /dev/null +++ b/docs/test-results/pr339/retained-disk-calls.json @@ -0,0 +1,5 @@ +{ + "start_ns": 1792161535537904360, + "end_ns": 1792161535607563767, + "disk_calls": [] +} diff --git a/docs/test-results/pr339/review.md b/docs/test-results/pr339/review.md new file mode 100644 index 00000000..1b3cb39f --- /dev/null +++ b/docs/test-results/pr339/review.md @@ -0,0 +1,29 @@ +# Independent follow-up review for PR #339 + +**Implementation verdict: VERIFIED at `da56205bb31b489c15af3ea670e1cbe87905bf01` for the reviewed response, admission, convergence, and retirement behavior.** The earlier P2 diagnostic defect is fixed. The affected race, Linux and Darwin lint, and format checks pass at this exact HEAD. The full real VMM rerun and a separate real invalid-PUT probe pass at `cc444095b3ba000de9ac11d7ef0674e386153804`. Commit `da56205` changes only two test-local `err` names, so it does not change the production code tested by that VMM run. Keep the two SHA labels distinct. No new safety finding was established. This is an independent code and evidence review, not approval to merge. + +## Findings and delivery evidence + +No actionable finding remains from this follow-up review. The three corrected guides link `repo/docs/live-volumes-validation.md`. That report identifies the initial full software run at `ceda20d`, the fixed production hardware run at `cc44409`, and the final test-only revision `da56205` separately. It names the delivery-run VMM PIDs and the separate invalid-request probe. All 18 local links in the report resolve. The public `checks.json` has 49 attempt records under their actual SHAs, including six failed attempts; the latest result for each of 29 distinct checks passes without an unexpected skip. Its public `checks.log` contains the four verification rounds. The public hardware smoke, command log, cleanup snapshot, and invalid-request JSON match the corresponding `delivery-*` originals byte for byte. Their `.log` files exist despite the repository's ignore rule. The public decision trail matches the retained local trail byte for byte. `git diff --check` passes for the documentation changes. + +The decision trail records the `cc44409` diagnostic fix, immediately corrects a mistaken non-resolving abbreviation in an append-only row, and records the test-only `da56205` lint repair, eight final passing affected checks, the `cc44409` hardware rerun, and the public evidence package. The original run transcript confirms the user's 13:50 implementation, local-test, k3s-cleanup, and PR-update authorization. The trail covers the isolated worktrees, initial guest-readiness correction, baseline HTTP 500, Go 1.27.1, clean lint cache, uncertain target rule, startup quarantine failure, and comment cleanup. I found no unlogged scope expansion or PR write in the transcript through the reviewed point. The initial statically linked initramfs correction remains described in `hardware-evidence/runtime-prep.md` rather than a standalone trail row, a minor granularity limit. + +## Diagnostic fix + +`sandboxd/pool/volume_mutation.go:76-94` now clears the historical top-level error in a refused operation's returned view while leaving stored diagnostics intact. `recordOutcome` becomes true at line 124 only after target normalization, validation, new-addition authorization, and the current-run check. A duplicate target or unauthorized new addition therefore cannot replace the previous convergence error. The HTTP handler maps the current refusal when `view.Error` is empty (`sandboxd/server/volume_mutation.go:41-62`), while a later GET still reads the previous diagnostic. The runtime support and begin-run refusal branches clear the returned top-level error as well. + +The placement of `recordOutcome = true` is significant. The old-pending-volume preclean starts afterward at lines 125-133. If its detach fails, the code wraps `errVolumeTargetNotAccepted`, records that operation failure and the named volume diagnostic, and retains the old target outcome. This preserves the prior review's safe preclean interpretation. The code does not claim that the new target was accepted before preclean succeeds. The existing uncertain-target recovery test and other prior safety tests are unchanged by the follow-up production patch. + +The behavior tests were red for the predicted disagreement before the fix, not merely because a test process failed. `review-fix/sandboxd-red.log` shows both a settled GET gaining a false generic error and a prior 409 unmount reason being overwritten. `review-fix/e2e-red.log` shows HTTP 400 carrying that generic error. The corresponding green logs pass after `cc44409`. `hardware-evidence/delivery-invalid-put.json` additionally shows a real HTTP 400 duplicate-name explanation at `cc44409`, unchanged before and after GET views and backing marker, one surviving VMM PID, zero disk calls for the rejected PUT, and release 204. + +## Prior safety paths and verification + +The baseline-to-`da56205` scoped diff is 47 files. The changes after the earlier full safety review's `ceda20d` base are a local name-only production edit, the diagnostic edit above, and regression tests. I rechecked the prior audit's ordering and unchanged tests for A/W publication, same-lock admission holds, cancellation versus release/reap/capture, startup adoption and quarantine refusal, retirement deletion after actual VM absence, unknown-orphan fencing, held repair after ACL/catalog removal, and typed runtime and mount-conflict failures. The original `final-review.md` cites the complete code paths and their fake and real evidence. The new diagnostic code does not alter admission, journal, device, or retirement ordering. `git diff ceda20d..da56205` contains no new source comment or lint, type, or safety suppression. The earlier `comment-final-review.md` remains valid for the broad diff. + +`verification-3/summary.json` records successful sandboxd and e2e race and format checks at `cc44409`, but both Linux and Darwin linters correctly rejected two shadowed `err` variables in new tests. `da56205` changes only those local names. All eight affected checks in `verification-4/summary.json` have exit 0, no unexpected skip, and HEAD `da56205`; I read the matching raw lint logs. The prior `verification-1` and `verification-2` logs cover the other Go modules, Python suites and strict checks, PostgreSQL without skipped tests, and Caddy TLS at their stated earlier SHAs. + +The `cc44409` real delivery smoke exits 0 (`delivery-smoke-exit.json`). Its log records unchanged sandbox, session, boot ID, PID, start time, and environment across replacement; EROFS; cwd, bind, and foreign 409 with retry; work during a gated successful attach; daemon restart and explicit cancellation; failed VM removal with writer refusal across restart; and actual backing reuse after removal. `delivery-retained-zero-operations.json` contains zero disk calls in the settled-target window. `delivery-journal.json` has empty claims and retirements. `delivery-cleanup.json` shows zero owned VMs, sockets, and TAPs, all three observed VMM PIDs gone, and the isolated daemon stopped. The original local sandboxd remains on port 7777. These are evidence at `cc44409`, not at `da56205`. + +**INCONCLUSIVE hardware limits remain unchanged.** The external pause experiment did not establish a missing eject ACK because official Cocoon resumed the guest before detach. Separate real crashes at each umount, detach, journal, and fsync boundary and physical power loss were not run. The daemon smoke used `sync_claims=false`, so its restart is not power-loss durability proof. The journal also cannot detect an operator repointing a held catalog name, and format v2 requires draining before downgrade. These are documented constraints, not proven failures in this review. + +The review used the requested poteto-mode guidance. Its Prove It Works principle led me to read the raw red/green outputs, exact-HEAD summaries, real JSON responses, and actual cleanup state. Its technical-writing and unslop guidance kept the verdict and delivery action tied to named evidence. From b87115f70f24de5823a5600da6086b213e51e7ed Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 14:59:17 +0000 Subject: [PATCH 18/57] test(pool): group live-volume helpers and share held-run setup --- sandboxd/pool/volume_mutation_test.go | 158 ++++++++++++-------------- 1 file changed, 73 insertions(+), 85 deletions(-) diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 428d74da..371e353b 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -391,32 +391,6 @@ func TestLiveVolumesRespectOwningTenantAndRemoval(t *testing.T) { } } -func liveCatalog(t *testing.T) []config.VolumeSpec { - t.Helper() - return []config.VolumeSpec{ - {Name: "data", Path: writeVolumeImage(t, "data.img", "data"), Writable: true}, - {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, - } -} - -func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { - t.Helper() - claims, err := newClaimStore(m.dataDir, false).load() - if err != nil { - t.Fatal(err) - } - got := claims[sb.ID] - if got == nil || !slices.Equal(got.Volumes, sb.Volumes) { - t.Fatalf("journal volumes: %+v", got) - } - if !slices.Equal(got.VolumesWant, sb.VolumesWant) { - t.Fatalf("journal target: %+v, want %+v", got.VolumesWant, sb.VolumesWant) - } - if _, err := os.Stat(m.store.path); err != nil { - t.Fatal(err) - } -} - func TestDesiredVolumesJournalMigration(t *testing.T) { for _, tt := range []struct { name, field string @@ -521,22 +495,6 @@ func TestSupersedingTargetPrecleansUncertainAddBeforeJournaling(t *testing.T) { } } -type gatedVolumeEngine struct { - *fakeEngine - entered chan struct{} - resume chan struct{} - calls atomic.Int32 -} - -func (e *gatedVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { - if e.calls.Add(1) == 1 { - close(e.entered) - <-e.resume - ctx = context.WithoutCancel(ctx) - } - return e.fakeEngine.DiskAttach(ctx, vmName, spec) -} - func TestCanceledVolumeWriterOwnsHardwareUntilReturn(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) @@ -677,16 +635,8 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - _, run, err := m.beginVolumeRun(t.Context(), sb) - if err != nil { - t.Fatal(err) - } - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + run := beginAdmittedVolumeRun(t, m, sb) m.mu.Lock() - if reserveErr := m.reserveVolumes(transient); reserveErr != nil { - t.Fatal(reserveErr) - } - run.holds = append(run.holds, transient...) if action == "reap" { sb.Deadline = time.Now().Add(-time.Second) } @@ -709,7 +659,7 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { m.finishVolumeRun(sb, run) } m.store.writeMu.Unlock() - err = <-done + err := <-done if action == "release" && err == nil { t.Fatal("release ignored commit failure") } @@ -747,17 +697,7 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - _, run, err := m.beginVolumeRun(t.Context(), sb) - if err != nil { - t.Fatal(err) - } - m.mu.Lock() - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} - if err := m.reserveVolumes(transient); err != nil { - t.Fatal(err) - } - run.holds = append(run.holds, transient...) - m.mu.Unlock() + run := beginAdmittedVolumeRun(t, m, sb) if _, err := m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Minute, types.ExpireArchive); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("archive admitted during volume intent: %v", err) } @@ -767,17 +707,6 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { } } -type deadlineVolumeEngine struct { - *fakeEngine - deadlines chan time.Time -} - -func (e *deadlineVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { - deadline, _ := ctx.Deadline() - e.deadlines <- deadline - return e.fakeEngine.DiskAttach(ctx, vmName, spec) -} - func TestLiveAttachBoundsWholeDeviceOperation(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) @@ -798,17 +727,7 @@ func TestLivenessRefusesAdmittedVolumeRun(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - _, run, err := m.beginVolumeRun(t.Context(), sb) - if err != nil { - t.Fatal(err) - } - m.mu.Lock() - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} - if err := m.reserveVolumes(transient); err != nil { - t.Fatal(err) - } - run.holds = append(run.holds, transient...) - m.mu.Unlock() + run := beginAdmittedVolumeRun(t, m, sb) killVMM(eng, sb.VMName) m.recoverVMM(t.Context(), sb) if len(eng.starts) != 0 || !strings.Contains(sb.Failed, "volume mounts") { @@ -842,3 +761,72 @@ func TestRemovedTenantCanRemoveReadOnlyVolumeBesideImmutableVolume(t *testing.T) t.Fatal("cleanup altered immutable hold") } } + +func liveCatalog(t *testing.T) []config.VolumeSpec { + t.Helper() + return []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "data.img", "data"), Writable: true}, + {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, + } +} + +func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil { + t.Fatal(err) + } + got := claims[sb.ID] + if got == nil || !slices.Equal(got.Volumes, sb.Volumes) { + t.Fatalf("journal volumes: %+v", got) + } + if !slices.Equal(got.VolumesWant, sb.VolumesWant) { + t.Fatalf("journal target: %+v, want %+v", got.VolumesWant, sb.VolumesWant) + } + if _, err := os.Stat(m.store.path); err != nil { + t.Fatal(err) + } +} + +type gatedVolumeEngine struct { + *fakeEngine + entered chan struct{} + resume chan struct{} + calls atomic.Int32 +} + +func (e *gatedVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { + if e.calls.Add(1) == 1 { + close(e.entered) + <-e.resume + ctx = context.WithoutCancel(ctx) + } + return e.fakeEngine.DiskAttach(ctx, vmName, spec) +} + +type deadlineVolumeEngine struct { + *fakeEngine + deadlines chan time.Time +} + +func (e *deadlineVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { + deadline, _ := ctx.Deadline() + e.deadlines <- deadline + return e.fakeEngine.DiskAttach(ctx, vmName, spec) +} + +func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volumeRun { + t.Helper() + _, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + m.mu.Lock() + defer m.mu.Unlock() + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + if err = m.reserveVolumes(transient); err != nil { + t.Fatal(err) + } + run.holds = append(run.holds, transient...) + return run +} From fea35314c152081129357fd09d0fb3a60ae5cde7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Fri, 16 Oct 2026 15:03:59 +0000 Subject: [PATCH 19/57] docs(volumes): keep validation results in the PR discussion --- docs/deploy.md | 7 +- docs/live-volumes-validation.md | 155 ----- docs/live-volumes.md | 5 - docs/sandboxd-api.md | 4 +- docs/test-results/pr339/backing-result.json | 5 - docs/test-results/pr339/baseline.log | 4 - docs/test-results/pr339/checks.json | 492 ---------------- docs/test-results/pr339/checks.log | 543 ------------------ docs/test-results/pr339/decisions.tsv | 25 - .../pr339/diagnostic-regression.log | 114 ---- .../test-results/pr339/guest-input-sha256.txt | 3 - .../pr339/hardware-binaries.sha256 | 3 - .../pr339/hardware-build-info.txt | 116 ---- docs/test-results/pr339/hardware-cleanup.json | 17 - .../pr339/hardware-commands.jsonl | 88 --- .../pr339/hardware-invalid-put.json | 75 --- .../pr339/hardware-invocation.json | 23 - docs/test-results/pr339/hardware-journal.json | 1 - docs/test-results/pr339/hardware-prep.md | 29 - docs/test-results/pr339/hardware-revision.txt | 2 - docs/test-results/pr339/hardware.log | 34 -- .../test-results/pr339/host-tool-versions.txt | 2 - docs/test-results/pr339/pending-eject.json | 10 - .../pr339/post-hardware-test-only.diff | 18 - .../pr339/retained-disk-calls.json | 5 - docs/test-results/pr339/review.md | 29 - 26 files changed, 4 insertions(+), 1805 deletions(-) delete mode 100644 docs/live-volumes-validation.md delete mode 100644 docs/test-results/pr339/backing-result.json delete mode 100644 docs/test-results/pr339/baseline.log delete mode 100644 docs/test-results/pr339/checks.json delete mode 100644 docs/test-results/pr339/checks.log delete mode 100644 docs/test-results/pr339/decisions.tsv delete mode 100644 docs/test-results/pr339/diagnostic-regression.log delete mode 100644 docs/test-results/pr339/guest-input-sha256.txt delete mode 100644 docs/test-results/pr339/hardware-binaries.sha256 delete mode 100644 docs/test-results/pr339/hardware-build-info.txt delete mode 100644 docs/test-results/pr339/hardware-cleanup.json delete mode 100644 docs/test-results/pr339/hardware-commands.jsonl delete mode 100644 docs/test-results/pr339/hardware-invalid-put.json delete mode 100644 docs/test-results/pr339/hardware-invocation.json delete mode 100644 docs/test-results/pr339/hardware-journal.json delete mode 100644 docs/test-results/pr339/hardware-prep.md delete mode 100644 docs/test-results/pr339/hardware-revision.txt delete mode 100644 docs/test-results/pr339/hardware.log delete mode 100644 docs/test-results/pr339/host-tool-versions.txt delete mode 100644 docs/test-results/pr339/pending-eject.json delete mode 100644 docs/test-results/pr339/post-hardware-test-only.diff delete mode 100644 docs/test-results/pr339/retained-disk-calls.json delete mode 100644 docs/test-results/pr339/review.md diff --git a/docs/deploy.md b/docs/deploy.md index 427a7266..5a82e560 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -183,10 +183,9 @@ new volume admission and all disk attachment, including pending repair, until the orphan is gone. GET, detach, and release remain available. `e2e/cmd/volumesmoke` exercises GET, repeated full-set PUT, PATCH add and remove, -and clearing a new claim. The [local validation report](live-volumes-validation.md) -records real Cloud Hypervisor hotplug, busy guest unmount, daemon restart, -cancellation, and retirement recovery. It lists the hardware fault cases that -remain untested. +and clearing a new claim. Its [hardware validation instructions](../e2e/cmd/volumesmoke/README.md) +cover same-path replacement, guest mount conflicts, daemon restart, cancellation, +and retirement recovery on an isolated node. Each catalog path must name a disk image containing a mountable whole-device filesystem. Keep catalog names bound to the same image while any live or diff --git a/docs/live-volumes-validation.md b/docs/live-volumes-validation.md deleted file mode 100644 index b0f162cf..00000000 --- a/docs/live-volumes-validation.md +++ /dev/null @@ -1,155 +0,0 @@ -# Local live volume validation - -This report records local validation for PR #339 on 2026-10-16. The baseline -is `645e49756492e7d8e9337377a958185e615804fc`. The initial full software -run used `ceda20dd4ad944e354e6f300b72b610b3d2422d6`. Independent -review then found that invalid requests overwrote convergence diagnostics. -Regressions reproduced that defect before its fix in -`cc444095b3ba000de9ac11d7ef0674e386153804`. The complete real hardware smoke -was rerun at that revision, followed by a real invalid-request probe. The final -application candidate is `da56205bb31b489c15af3ea670e1cbe87905bf01`, whose only -changes after that hardware run rename two test-local error variables. The -affected sandboxd and e2e race, Linux and Darwin lint, and format checks passed -there. Later documentation does not change the tested application code. - -The [hardware program](../e2e/cmd/volumesmoke/README.md) extends the existing -`volumesmoke` command. Its caller sequence matches the -[replacement tutorial](live-volumes.md), using the Go SDK. The literal Python -tutorial was not run against the VMM. Python SDK behavior is covered by its -tests and the Caddy transport integration. - -## Software checks - -All 29 distinct checks passed after the naming correction. The -[check manifest](test-results/pr339/checks.json) and -[complete output](test-results/pr339/checks.log) retain the failed lint attempt -and the successful reruns, with exact source revisions. The later regressions -also needed test-local shadow-name corrections. Failed suite attempts remain -in the evidence. The [red/green output](test-results/pr339/diagnostic-regression.log) -records the diagnostic defect before and after its fix. - -| Check | Result | -| --- | --- | -| `go test -race -count=1 ./...` in all five Go modules | PASS | -| Pinned golangci-lint v2.13.2 on Linux and Darwin in all five modules | PASS | -| Formatter diff in all five modules | PASS | -| Ruff format and lint | PASS | -| Strict Python SDK mypy | PASS | -| Python SDK, OpenAI adapter, and LangChain adapter | 247, 10, and 15 tests passed | -| PostgreSQL tenant and pool-set stores | PASS, no skipped PostgreSQL tests | -| PostgreSQL multi-node pool integration | PASS, no skipped PostgreSQL tests | -| Caddy v2.11.4 TLS integration through Go and Python | PASS | - -The toolchain was Go 1.27.1, as pinned by the repository. Permission tests ran -with `CAP_DAC_OVERRIDE` and `CAP_DAC_READ_SEARCH` removed because the local user -is root. Each lint run used a new cache. The stock pinned analyzer panicked -with a shared cache and passed on the same source with a fresh cache. No -analyzer was disabled and no lint suppression was added. - -The controlled tests include tenant API-token authorization and sandbox-token -denial, held-volume repair after tenant or ACL removal, no device calls for -settled entries, release before reservation, cancellation and late completion, -re-add after a failed detach, uncertain journal writes, startup adoption, -retirement deletion failure, unknown-orphan admission fencing, and capture -rejection before trim. These use fake engines or controlled filesystem faults. -They are separate from the VMM results below. - -## Real Cloud Hypervisor results - -The node used the official Cocoon v0.7.2 release, Git -`b3b40f3bdedc384c482eb81dcdc60a774dd8d560`, with Cloud Hypervisor v54.0.0. -The Cocoon archive matched its published checksum. The guest used Linux -6.18.37 and the current statically built sandbox-init. - -The daemon, data directory, images, runtime directories, network scope, and -cgroup parent were isolated. No existing local service was removed or -reconfigured. The test used two disposable 32 MiB ext4 images. Both catalog -entries permit writes for the final reuse probe. Ordinary replacement mounts -request read-only mode. - -| Caller-visible scenario | Result | -| --- | --- | -| Two sandboxes read the same dataset, and writes fail with EROFS | PASS | -| Replace v1 with v2 at the same guest path, repeat the target, then clear it | PASS | -| Preserve sandbox, session, boot ID, PID, process start time, and session environment | PASS | -| Busy cwd, bind mount, and foreign mount return 409 with a guest path | PASS | -| Close the reference and retry the same target | PASS | -| Repeat a settled target while it has a bind mount | PASS, zero attach or detach calls | -| Run exec, file read/write, and HTTP port requests while a real attach result is blocked | PASS | -| Kill sandboxd after real attach succeeds but before its result returns | PASS, restart adopts the original VMM and retry converges | -| Cancel the caller at that same boundary | PASS, desired intent survives and retry converges | -| Fail VM removal, restart sandboxd, and attempt a conflicting writer | PASS, writer receives 409 before and after restart | -| Permit real VM removal, then mount and write through a new writer | PASS | -| Reject duplicate volume names with 400 and preserve the settled GET view | PASS, no device calls or new convergence error | -| Remove all resources owned by the test | PASS, zero claims, retirements, VMs, sockets, and owned TAPs | - -The [raw smoke output](test-results/pr339/hardware.log) includes identities, -failure views, and recovery results. The -[runtime command log](test-results/pr339/hardware-commands.jsonl) contains the -actual command attempts. The observer forwards ordinary commands to the -official Cocoon binary. It blocks only after a successful real attach, or -injects the declared exit 73 before VM removal. It does not emulate version -support, attach idempotence, or successful deletion. - -The original VMM PID `1082008` survived the daemon restart. Retiring VMM PID -`1084550` survived the failed removal and restart until real cleanup succeeded. -The [cleanup snapshot](test-results/pr339/hardware-cleanup.json) confirms both -PIDs and the separate invalid-request probe's VM are gone. The -[invalid-request probe](test-results/pr339/hardware-invalid-put.json) records -HTTP 400 with the duplicate-name reason, unchanged GET and file contents, and -zero disk calls. The [final journal](test-results/pr339/hardware-journal.json) -has empty live and retirement maps. The -[retained-target window](test-results/pr339/retained-disk-calls.json) contains -zero device calls. The read-only v2 backing checksum did not change. The v1 -image received the intentional writable reuse probe. - -The [baseline hardware run](test-results/pr339/baseline.log) passed the shared -reads and EROFS checks, then returned HTTP 500 for the busy cwd case where the -new candidate returns HTTP 409. This establishes a behavioral difference on -the actual device path. - -## Reproduce the checks - -The [CI workflows](../.github/workflows/sandboxd.yml) provide the Go, -PostgreSQL, and Caddy commands. The -[Python workflow](../.github/workflows/python.yml) pins its tools and SDK test -commands. Local Go modules are `protocol/wire`, `sandboxd`, `sdk/go`, `e2e`, -and `mcp`. `GOWORK=off` keeps each module independent. The hardware command and -fault-hook setup are in the [hardware instructions](../e2e/cmd/volumesmoke/README.md). -The [local runtime preparation](test-results/pr339/hardware-prep.md) records -the guest image compatibility fixes and exact input checksums. - -## Coverage limits and operational requirements - -Missing guest eject ACK was not established. An external experiment paused -Cloud Hypervisor, but official Cocoon resumed the orphaned pause before -detach. The guest could therefore acknowledge removal. The -[experiment result](test-results/pr339/pending-eject.json) is inconclusive for -timeout and retry with a missing ACK. - -Separate hardware crashes after guest umount, after host detach, and at every -journal or fsync boundary were not performed. The hardware daemon used -`sync_claims: false`. The demonstrated recovery is a daemon-process restart, -not physical host power loss. Controlled unit tests cover additional journal -cuts without claiming hardware proof. - -Catalog names must remain bound to immutable images while held. The journal -cannot detect an operator changing a backing path after restart. Journal -format v2 includes durable retirement ownership. An older daemon rejects that -format, so downgrade requires draining claims and retirement work first. A -failed unknown-orphan cleanup fences new admissions and held repair until -the VM is confirmed gone. A sandbox whose VMM dies while holding volumes is -release-only. These constraints are documented in the deployment and API -guides. - -The [decision trail](test-results/pr339/decisions.tsv) records implementation -choices and failed attempts. Its relative evidence paths refer to the retained -local `.audit/pr339-implementation` directory. The artifacts linked in this -report provide the portable test evidence. Local paths in raw logs identify -the machine used for the run and are not installation requirements. -The build metadata has trailing whitespace removed. The recorded test-only -diff omits context lines. Test observations retain their original contents. - -The [independent follow-up review](test-results/pr339/review.md) verifies the -diagnostic fix, source revisions, test evidence, and decision trail. It found -no remaining actionable issue. The hardware coverage limits above remain. diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 2838601c..790e0d21 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -91,8 +91,3 @@ admission holds. Detailed errors survive only within one daemon lifetime. If the VMM exits, a sandbox holding volumes is release-only. Release it and create a new sandbox; neither PUT with an empty target nor daemon restart restores the original processes. - -The example's Python syntax and SDK signatures are checked locally. The -[local validation report](live-volumes-validation.md) records the equivalent -Go SDK sequence on real Cloud Hypervisor, including identity continuity and -daemon restart. It also lists hardware fault cases that remain untested. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index f1f213fa..ddcf05b5 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -378,9 +378,7 @@ Diagnosis, detach, and release remain available until cleanup succeeds. A replacement first unmounts the old disk, then mounts the new disk. Pause reads and close files, mmap, cwd, and bind mount references during this gap. Existing independent guest processes and sessions continue. See the -[same-sandbox replacement tutorial](live-volumes.md) and the -[local Cloud Hypervisor results](live-volumes-validation.md) for the tested -interruption boundaries and remaining coverage limits. +[same-sandbox replacement tutorial](live-volumes.md). ## POST /v1/sandboxes/{id}/release diff --git a/docs/test-results/pr339/backing-result.json b/docs/test-results/pr339/backing-result.json deleted file mode 100644 index b6234244..00000000 --- a/docs/test-results/pr339/backing-result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "readonly_v2_backing_unchanged": true, - "v1_expected_writable_retirement_probe": true, - "v1_changed": true -} diff --git a/docs/test-results/pr339/baseline.log b/docs/test-results/pr339/baseline.log deleted file mode 100644 index b9b34e80..00000000 --- a/docs/test-results/pr339/baseline.log +++ /dev/null @@ -1,4 +0,0 @@ -volume=dataset-v1 claim_a=389.0ms claim_b=371.8ms -VOLUME PASS concurrent_read_bytes=3 read_only=true released=true -CONTINUITY before={"Sandbox":"sb_7d252ce9de32e3e7","Session":"sess-69-0","Boot":"4cd81a72-ea89-4282-a36d-2e3e4965755e","PID":177,"Start":"1738"} -volumesmoke: cwd busy removal must return 409 and guest path: view={VolumesWant:[{Name:dataset-v2 Mount:/datasets/current Mode:}] Volumes:[{Name:dataset-v1 Mount:/datasets/current Mode:}] States:[{Name:dataset-v1 State:detaching Error:volume convergence failed; retry the desired set} {Name:dataset-v2 State:pending Error:}] Error:volume convergence failed; retry the desired set} error=set volumes: volume convergence failed; retry the desired set (http 500) diff --git a/docs/test-results/pr339/checks.json b/docs/test-results/pr339/checks.json deleted file mode 100644 index 576fe19b..00000000 --- a/docs/test-results/pr339/checks.json +++ /dev/null @@ -1,492 +0,0 @@ -[ - { - "name": "race-protocol-wire", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:24:25.395802+00:00", - "finished": "2026-10-16T14:24:37.429522+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "race-sandboxd", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:24:37.430198+00:00", - "finished": "2026-10-16T14:25:15.961593+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "race-sdk-go", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:15.962380+00:00", - "finished": "2026-10-16T14:25:21.209326+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "race-e2e", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:21.210098+00:00", - "finished": "2026-10-16T14:25:27.613063+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "race-mcp", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:27.613821+00:00", - "finished": "2026-10-16T14:25:30.863120+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-protocol-wire-linux", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:30.864419+00:00", - "finished": "2026-10-16T14:25:40.018552+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-protocol-wire-darwin", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:40.019495+00:00", - "finished": "2026-10-16T14:25:48.919626+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "format-protocol-wire", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:48.920450+00:00", - "finished": "2026-10-16T14:25:49.185320+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-linux", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:25:49.186187+00:00", - "finished": "2026-10-16T14:26:10.451654+00:00", - "exit_code": 1, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-darwin", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:26:10.452648+00:00", - "finished": "2026-10-16T14:26:31.939927+00:00", - "exit_code": 1, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "format-sandboxd", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:26:31.940741+00:00", - "finished": "2026-10-16T14:26:39.114484+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-sdk-go-linux", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:26:39.115439+00:00", - "finished": "2026-10-16T14:26:47.190396+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-sdk-go-darwin", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:26:47.191384+00:00", - "finished": "2026-10-16T14:26:54.961224+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "format-sdk-go", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:26:54.962067+00:00", - "finished": "2026-10-16T14:26:56.800159+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-e2e-linux", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:26:56.801154+00:00", - "finished": "2026-10-16T14:27:04.458941+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-e2e-darwin", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:04.459951+00:00", - "finished": "2026-10-16T14:27:11.984298+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "format-e2e", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:11.985177+00:00", - "finished": "2026-10-16T14:27:13.053430+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-mcp-linux", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:13.054456+00:00", - "finished": "2026-10-16T14:27:20.352493+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "lint-mcp-darwin", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:20.353533+00:00", - "finished": "2026-10-16T14:27:27.447703+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "format-mcp", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:27.448574+00:00", - "finished": "2026-10-16T14:27:27.764448+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "python-format", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:27.765281+00:00", - "finished": "2026-10-16T14:27:27.822550+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "python-lint", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:27.823279+00:00", - "finished": "2026-10-16T14:27:27.875617+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "python-types", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:27.876364+00:00", - "finished": "2026-10-16T14:27:29.059436+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "python-test-python", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:27:29.060261+00:00", - "finished": "2026-10-16T14:28:05.046371+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "python-test-openai", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:28:05.047233+00:00", - "finished": "2026-10-16T14:28:12.538967+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "python-test-langchain", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:28:12.539889+00:00", - "finished": "2026-10-16T14:28:13.774900+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "pg-stores", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:28:13.775735+00:00", - "finished": "2026-10-16T14:28:17.807480+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "pg-pool", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:28:17.808239+00:00", - "finished": "2026-10-16T14:28:21.567382+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "tls-caddy", - "head": "ceda20dd4ad944e354e6f300b72b610b3d2422d6", - "started": "2026-10-16T14:28:21.568146+00:00", - "finished": "2026-10-16T14:28:24.579943+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-1", - "log": "checks.log" - }, - { - "name": "race-sandboxd", - "head": "b249bd2813602eab44e498ecd6e106652e81940a", - "started": "2026-10-16T14:30:02.164735+00:00", - "finished": "2026-10-16T14:30:20.892157+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-2", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-linux", - "head": "b249bd2813602eab44e498ecd6e106652e81940a", - "started": "2026-10-16T14:30:20.893215+00:00", - "finished": "2026-10-16T14:30:32.325344+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-2", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-darwin", - "head": "b249bd2813602eab44e498ecd6e106652e81940a", - "started": "2026-10-16T14:30:32.326220+00:00", - "finished": "2026-10-16T14:30:43.978479+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-2", - "log": "checks.log" - }, - { - "name": "format-sandboxd", - "head": "b249bd2813602eab44e498ecd6e106652e81940a", - "started": "2026-10-16T14:30:43.979239+00:00", - "finished": "2026-10-16T14:30:51.117715+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-2", - "log": "checks.log" - }, - { - "name": "race-sandboxd", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:37:16.170239+00:00", - "finished": "2026-10-16T14:37:23.438591+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "race-e2e", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:37:23.439227+00:00", - "finished": "2026-10-16T14:37:28.478238+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-linux", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:37:28.479446+00:00", - "finished": "2026-10-16T14:37:40.374645+00:00", - "exit_code": 1, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-darwin", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:37:40.375553+00:00", - "finished": "2026-10-16T14:37:52.129784+00:00", - "exit_code": 1, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "format-sandboxd", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:37:52.130570+00:00", - "finished": "2026-10-16T14:37:59.383820+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "lint-e2e-linux", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:37:59.384706+00:00", - "finished": "2026-10-16T14:38:07.028816+00:00", - "exit_code": 1, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "lint-e2e-darwin", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:38:07.029700+00:00", - "finished": "2026-10-16T14:38:14.531396+00:00", - "exit_code": 1, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "format-e2e", - "head": "cc444095b3ba000de9ac11d7ef0674e386153804", - "started": "2026-10-16T14:38:14.532209+00:00", - "finished": "2026-10-16T14:38:15.610496+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-3", - "log": "checks.log" - }, - { - "name": "race-sandboxd", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:39:26.986588+00:00", - "finished": "2026-10-16T14:39:40.289815+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "race-e2e", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:39:40.290425+00:00", - "finished": "2026-10-16T14:39:45.950918+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-linux", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:39:45.952135+00:00", - "finished": "2026-10-16T14:39:57.692268+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "lint-sandboxd-darwin", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:39:57.693158+00:00", - "finished": "2026-10-16T14:40:09.172169+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "format-sandboxd", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:40:09.172893+00:00", - "finished": "2026-10-16T14:40:16.339527+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "lint-e2e-linux", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:40:16.340419+00:00", - "finished": "2026-10-16T14:40:24.070525+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "lint-e2e-darwin", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:40:24.071440+00:00", - "finished": "2026-10-16T14:40:31.474597+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - }, - { - "name": "format-e2e", - "head": "da56205bb31b489c15af3ea670e1cbe87905bf01", - "started": "2026-10-16T14:40:31.475370+00:00", - "finished": "2026-10-16T14:40:32.524473+00:00", - "exit_code": 0, - "unexpected_skip": false, - "attempt": "verification-4", - "log": "checks.log" - } -] diff --git a/docs/test-results/pr339/checks.log b/docs/test-results/pr339/checks.log deleted file mode 100644 index eab40285..00000000 --- a/docs/test-results/pr339/checks.log +++ /dev/null @@ -1,543 +0,0 @@ - -verification-1/race-protocol-wire.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/protocol/wire 1.131s - -verification-1/race-sandboxd.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/sandboxd 1.075s -ok github.com/cocoonstack/sandbox/sandboxd/config 1.103s -ok github.com/cocoonstack/sandbox/sandboxd/egress 5.634s -ok github.com/cocoonstack/sandbox/sandboxd/engine 4.869s -ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.713s -? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] -? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.717s -? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/pool 3.372s -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.038s -ok github.com/cocoonstack/sandbox/sandboxd/server 3.370s -ok github.com/cocoonstack/sandbox/sandboxd/store 1.111s -ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.668s -ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.045s -ok github.com/cocoonstack/sandbox/sandboxd/store/s3 5.777s -? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.054s -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.046s -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/types 1.050s -ok github.com/cocoonstack/sandbox/sandboxd/utils 1.031s - -verification-1/race-sdk-go.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/sdk/go 2.570s -ok github.com/cocoonstack/sandbox/sdk/go/silkd 1.229s -? github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest [no test files] - -verification-1/race-e2e.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/e2e 3.509s -? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] -ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.033s - -verification-1/race-mcp.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/mcp 1.771s - -verification-1/lint-protocol-wire-linux.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/lint-protocol-wire-darwin.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/format-protocol-wire.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/protocol/wire -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-1/lint-sandboxd-linux.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -sandboxd/pool/volume.go:145:3: shadow: declaration of "resolvedVolume" shadows declaration at line 33 (govet) - resolvedVolume, resolveErr := m.resolveCatalogVolume(ctx, volume) - ^ -1 issues: -* govet: 1 - -verification-1/lint-sandboxd-darwin.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -sandboxd/pool/volume.go:145:3: shadow: declaration of "resolvedVolume" shadows declaration at line 33 (govet) - resolvedVolume, resolveErr := m.resolveCatalogVolume(ctx, volume) - ^ -1 issues: -* govet: 1 - -verification-1/format-sandboxd.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-1/lint-sdk-go-linux.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/lint-sdk-go-darwin.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/format-sdk-go.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/go -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-1/lint-e2e-linux.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/lint-e2e-darwin.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/format-e2e.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-1/lint-mcp-linux.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/lint-mcp-darwin.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-1/format-mcp.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/mcp -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-1/python-format.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo -command=['/tmp/pr339-design/python-env/bin/ruff', 'format', '--check', '.'] -38 files already formatted - -verification-1/python-lint.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo -command=['/tmp/pr339-design/python-env/bin/ruff', 'check', '.'] -All checks passed! - -verification-1/python-types.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/python -command=['/tmp/pr339-design/python-env/bin/python', '-m', 'mypy'] -Success: no issues found in 10 source files - -verification-1/python-test-python.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/python -command=['/tmp/pr339-design/python-env/bin/python', '-m', 'pytest', '-q'] -........................................................................ [ 29%] -........................................................................ [ 58%] -........................................................................ [ 87%] -............................... [100%] -247 passed in 35.10s - -verification-1/python-test-openai.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/openai -command=['/tmp/pr339-design/python-env/bin/python', '-m', 'pytest', '-q'] -.......... [100%] -10 passed in 6.26s - -verification-1/python-test-langchain.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sdk/langchain -command=['/tmp/pr339-design/python-env/bin/python', '-m', 'pytest', '-q'] -............... [100%] -15 passed in 0.35s - -verification-1/pg-stores.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['go', 'test', '-race', '-count=1', '-v', './tenants/...', './poolset/...'] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -=== RUN TestContract -=== RUN TestContract/put_resolve_peek -=== RUN TestContract/keep_and_rotate_tokens -=== RUN TestContract/list_pages_by_name -=== RUN TestContract/delete_and_replace -=== RUN TestContract/replace_swaps_tokens -=== RUN TestContract/lookup ---- PASS: TestContract (0.01s) - --- PASS: TestContract/put_resolve_peek (0.00s) - --- PASS: TestContract/keep_and_rotate_tokens (0.00s) - --- PASS: TestContract/list_pages_by_name (0.00s) - --- PASS: TestContract/delete_and_replace (0.00s) - --- PASS: TestContract/replace_swaps_tokens (0.00s) - --- PASS: TestContract/lookup (0.00s) -=== RUN TestTheSetSurvivesAReopen ---- PASS: TestTheSetSurvivesAReopen (0.00s) -=== RUN TestATenantsFileNeedsTheRootToken ---- PASS: TestATenantsFileNeedsTheRootToken (0.00s) -PASS -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.043s -=== RUN TestContract -=== RUN TestContract/put_resolve_peek -=== RUN TestContract/keep_and_rotate_tokens -=== RUN TestContract/list_pages_by_name -=== RUN TestContract/delete_and_replace -=== RUN TestContract/replace_swaps_tokens -=== RUN TestContract/lookup ---- PASS: TestContract (1.09s) - --- PASS: TestContract/put_resolve_peek (0.26s) - --- PASS: TestContract/keep_and_rotate_tokens (0.16s) - --- PASS: TestContract/list_pages_by_name (0.15s) - --- PASS: TestContract/delete_and_replace (0.25s) - --- PASS: TestContract/replace_swaps_tokens (0.15s) - --- PASS: TestContract/lookup (0.14s) -=== RUN TestAWriteOnOneNodeReachesTheOtherNodesCache ---- PASS: TestAWriteOnOneNodeReachesTheOtherNodesCache (0.40s) -=== RUN TestACachedTenantOutlivesTheDatabase ---- PASS: TestACachedTenantOutlivesTheDatabase (0.13s) -=== RUN TestARoleWithoutCreateServesAPreCreatedTable ---- PASS: TestARoleWithoutCreateServesAPreCreatedTable (0.23s) -=== RUN TestAnUnrelatedEvictionKeepsALookupCacheable ---- PASS: TestAnUnrelatedEvictionKeepsALookupCacheable (0.00s) -PASS -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 2.901s -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -=== RUN TestStoreVersionsTheCellsSet ---- PASS: TestStoreVersionsTheCellsSet (0.19s) -=== RUN TestWatchFiresOnConnectAndOnItsCellsWrites ---- PASS: TestWatchFiresOnConnectAndOnItsCellsWrites (0.42s) -=== RUN TestADownDatabaseIsUnavailable ---- PASS: TestADownDatabaseIsUnavailable (0.12s) -PASS -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.766s - -verification-1/pg-pool.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['go', 'test', '-race', '-count=1', '-v', '-run', 'TestTwoNodesOnOneMetaStore|TestNodesOfACell|TestAMetaStoreNodeWithoutACell', './pool/'] -=== RUN TestNodesOfACellShareThePoolSet ---- PASS: TestNodesOfACellShareThePoolSet (0.50s) -=== RUN TestAMetaStoreNodeWithoutACellKeepsItsOwnPools ---- PASS: TestAMetaStoreNodeWithoutACellKeepsItsOwnPools (0.19s) -=== RUN TestTwoNodesOnOneMetaStoreShareTheTenantSet ---- PASS: TestTwoNodesOnOneMetaStoreShareTheTenantSet (0.25s) -PASS -ok github.com/cocoonstack/sandbox/sandboxd/pool 2.120s - -verification-1/tls-caddy.log -head=ceda20dd4ad944e354e6f300b72b610b3d2422d6 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['go', 'test', '-race', '-count=1', '-run', 'TestCaddyTLSCluster', '-timeout', '60s', '-v', '.'] -=== RUN TestCaddyTLSCluster -=== RUN TestCaddyTLSCluster/Go -=== RUN TestCaddyTLSCluster/Python ---- PASS: TestCaddyTLSCluster (0.42s) - --- PASS: TestCaddyTLSCluster/Go (0.10s) - --- PASS: TestCaddyTLSCluster/Python (0.19s) -PASS -ok github.com/cocoonstack/sandbox/e2e 1.477s - -verification-2/race-sandboxd.log -head=b249bd2813602eab44e498ecd6e106652e81940a -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/sandboxd 1.080s -ok github.com/cocoonstack/sandbox/sandboxd/config 1.083s -ok github.com/cocoonstack/sandbox/sandboxd/egress 5.628s -ok github.com/cocoonstack/sandbox/sandboxd/engine 4.841s -ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.633s -? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] -? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.704s -? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/pool 3.430s -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.039s -ok github.com/cocoonstack/sandbox/sandboxd/server 3.385s -ok github.com/cocoonstack/sandbox/sandboxd/store 1.112s -ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.657s -ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.040s -ok github.com/cocoonstack/sandbox/sandboxd/store/s3 9.072s -? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.059s -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.043s -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/types 1.079s -ok github.com/cocoonstack/sandbox/sandboxd/utils 1.032s - -verification-2/lint-sandboxd-linux.log -head=b249bd2813602eab44e498ecd6e106652e81940a -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-2/lint-sandboxd-darwin.log -head=b249bd2813602eab44e498ecd6e106652e81940a -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-2/format-sandboxd.log -head=b249bd2813602eab44e498ecd6e106652e81940a -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-3/race-sandboxd.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/sandboxd 1.069s -ok github.com/cocoonstack/sandbox/sandboxd/config 1.118s -ok github.com/cocoonstack/sandbox/sandboxd/egress 5.686s -ok github.com/cocoonstack/sandbox/sandboxd/engine 4.838s -ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.705s -? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] -? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.711s -? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/pool 3.520s -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.036s -ok github.com/cocoonstack/sandbox/sandboxd/server 3.435s -ok github.com/cocoonstack/sandbox/sandboxd/store 1.098s -ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.608s -ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.035s -ok github.com/cocoonstack/sandbox/sandboxd/store/s3 4.268s -? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.068s -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.053s -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/types 1.071s -ok github.com/cocoonstack/sandbox/sandboxd/utils 1.045s - -verification-3/race-e2e.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/e2e 3.528s -? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] -ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.032s - -verification-3/lint-sandboxd-linux.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -sandboxd/pool/volume_convergence_test.go:206:10: shadow: declaration of "err" shadows declaration at line 202 (govet) - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { - ^ -1 issues: -* govet: 1 - -verification-3/lint-sandboxd-darwin.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -sandboxd/pool/volume_convergence_test.go:206:10: shadow: declaration of "err" shadows declaration at line 202 (govet) - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { - ^ -1 issues: -* govet: 1 - -verification-3/format-sandboxd.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-3/lint-e2e-linux.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -e2e/volume_mutation_test.go:51:8: shadow: declaration of "err" shadows declaration at line 46 (govet) - if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { - ^ -1 issues: -* govet: 1 - -verification-3/lint-e2e-darwin.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -e2e/volume_mutation_test.go:51:8: shadow: declaration of "err" shadows declaration at line 46 (govet) - if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { - ^ -1 issues: -* govet: 1 - -verification-3/format-e2e.log -head=cc444095b3ba000de9ac11d7ef0674e386153804 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-4/race-sandboxd.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/sandboxd 1.061s -ok github.com/cocoonstack/sandbox/sandboxd/config 1.104s -ok github.com/cocoonstack/sandbox/sandboxd/egress 5.614s -ok github.com/cocoonstack/sandbox/sandboxd/engine 4.864s -ok github.com/cocoonstack/sandbox/sandboxd/mesh 2.717s -? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] -? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.707s -? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/pool 3.521s -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.047s -ok github.com/cocoonstack/sandbox/sandboxd/server 3.392s -ok github.com/cocoonstack/sandbox/sandboxd/store 1.102s -ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.668s -ok github.com/cocoonstack/sandbox/sandboxd/store/peer 2.046s -ok github.com/cocoonstack/sandbox/sandboxd/store/s3 5.678s -? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.043s -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.039s -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/types 1.085s -ok github.com/cocoonstack/sandbox/sandboxd/utils 1.031s - -verification-4/race-e2e.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['setpriv', '--bounding-set=-dac_override,-dac_read_search', 'go', 'test', '-race', '-count=1', './...'] -ok github.com/cocoonstack/sandbox/e2e 3.519s -? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] -ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.033s - -verification-4/lint-sandboxd-linux.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-4/lint-sandboxd-darwin.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-4/format-sandboxd.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/sandboxd -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] - -verification-4/lint-e2e-linux.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-4/lint-e2e-darwin.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'run', './...'] -0 issues. - -verification-4/format-e2e.log -head=da56205bb31b489c15af3ea670e1cbe87905bf01 -cwd=/root/eric/composable-computer/.audit/pr339-implementation/repo/e2e -command=['/root/eric/composable-computer/.audit/pr339-implementation/repo/bin/golangci-lint-v2.13.2/golangci-lint', 'fmt', '--diff', './...'] diff --git a/docs/test-results/pr339/decisions.tsv b/docs/test-results/pr339/decisions.tsv deleted file mode 100644 index de4569c1..00000000 --- a/docs/test-results/pr339/decisions.tsv +++ /dev/null @@ -1,25 +0,0 @@ -ts phase decision why evidence result -2026-10-16T13:52:18.363582+00:00 frame Implement approved design and update existing PR only after local verification User explicitly authorized implementation, local k3s cleanup and PR update ../pr339-design/design.md implementation authorized; no merge -2026-10-16T13:52:18.363618+00:00 ground Pin original fork branch at 645e497 and use separate worktrees Existing workspace is another project and must not supply stale vendored code contract.md three isolated worker worktrees created -2026-10-16T13:52:18.363625+00:00 ground Attempt real local CH validation Full host access now exposes KVM and vhost-vsock todo.md hardware available; runtime setup pending -2026-10-16T13:56:04.831383+00:00 environment Use isolated runtime on port 17777 without removing k3s services Current host has enough RAM and separate runtime state avoids interference /tmp/pr339-live official Cocoon downloaded; prep guest not yet ready -2026-10-16T13:56:04.831434+00:00 environment Rebuild current initramfs after prep guest exits Old local stack expects computer.layers while official Cocoon passes cocoon.layers /tmp/pr339-live/prep-vm.log earlier process-start report superseded; handshake required before readiness -2026-10-16T14:00:45.127418+00:00 baseline Run real CH smoke against unchanged PR HEAD and observe busy 500 Same verification must fail on old behavior before accepting 409 handling hardware-evidence/baseline-smoke.log RO read and EROFS pass; cwd replacement fails expected 409 with actual500 -2026-10-16T14:00:45.127463+00:00 baseline Run PostgreSQL and Caddy CI integration checks in isolated services Changes to claims and API must not break meta-store or TLS transport baseline-extra/summary.json all three checks pass without PostgreSQL skips -2026-10-16T14:00:45.127471+00:00 verification Build reusable CI-equivalent command runner The final evidence should be reproducible with explicit commit identity run-checks.py race/lint/Python/PG/TLS suites defined; final candidate pending -2026-10-16T14:03:40.384102+00:00 tooling Run final checks with CI-pinned Go1.27.1 Default host Go1.27.2 export format5 is unsupported by pinned golangci parser4; source toolchain1.27.1 and lint agree run-checks.py protocol/wire lint reproduces host failure and passes with CI toolchain; no lint bypass -2026-10-16T14:10:21.688297+00:00 design Release only superseded pending holds whose detach was confirmed before an uncertain target commit Merging old pending into W breaks same-path target validation; A/W remains authoritative design-clarifications.md cut regression required for both old/new snapshots; independent review pending -2026-10-16T14:13:28.709882+00:00 tooling Keep staticcheck enabled and isolate its stale fact cache Stock tool and same source panic with shared cache and pass with a fresh one lint-investigation.md full e2e lint0 issues with clean cache; no production workaround -2026-10-16T14:13:28.709930+00:00 correctness Fail startup if volume quarantine cannot be journaled Durable-before-delete must not expose an uncontained restored claim issues.md new regression red before propagation, green afterward -2026-10-16T14:28:51Z integration Integrate pool, adapters, SDK and real-runtime harness in ordered local commits Separate writers own worktrees; coordinator reviews and combines their finished slices repo git log 645e497..ceda20d; contract.md candidate ceda20d; no push -2026-10-16T14:28:51Z comments Remove six comment lines and encode catalog/cleanup ownership in names and functions Keep the required maintainer-requested module docstring; avoid empty-tenant authorization signaling for held repair comment-resolution.md; comment-final-review.md; commit 3b87948 independent scoped comment review finds zero remaining flags -2026-10-16T14:28:51Z hardware Run complete real CH smoke twice and clean all owned runtime resources Validate caller continuity and actual device effects beyond fake adapters hardware-evidence/results.md; final-candidate-smoke.log; final-cleanup.json main scenarios VERIFIED at ceda20d; missing-ACK experiment INCONCLUSIVE -2026-10-16T14:28:52Z verification Fix a local resolvedVolume variable that shadows its type after full suite completes The final CI-equivalent lint correctly rejects the name introduced during catalog resolution cleanup verification-1/lint-sandboxd-linux.log; verification-1/lint-sandboxd-darwin.log all five race suites pass; one lint finding pending name-only fix -2026-10-16T14:31:58Z verification Pass all 29 unique CI-equivalent checks after a local-variable rename Full attempt recorded27 passes and two identical shadow findings; rerun the four affected sandboxd checks after the fix verification-1/summary.json; verification-2/summary.json; commit b249bd2 race, Linux/Darwin lint/format, Python247+10+15, PostgreSQL without skips, and Caddy TLS pass -2026-10-16T14:37:17Z review Fix the independent P2 diagnostic finding after a red API and pool regression Rejected invalid targets must explain the request refusal without replacing the stored last convergence outcome final-review.md; review-fix/sandboxd-red.log; review-fix/e2e-red.log; commit 5c2 targeted regressions red before fix and green afterward; complete affected suites pending -2026-10-16T14:37:33Z correction Correct the prior review row commit pointer to cc44409 The prior row accidentally named a non-resolving commit abbreviation; the actual source commit is cc44409 repo commit cc44409; review-fix/sandboxd-green.log; review-fix/e2e-green.log supersedes the prior row evidence pointer only -2026-10-16T14:39:28Z verification Correct two regression-test error variable names and rerun affected checks The cc44409 race suites passed but both platform linters correctly found two shadowed err variables in new tests verification-3/summary.json; repo commit da56205; verification-4/ only test-local names changed; final affected check rerun active -2026-10-16T14:41:34Z verification Pass complete affected sandboxd and e2e checks at da56205 Keep original lint failures visible and prove final test code meets the same pinned gates verification-4/summary.json eight checks pass; all29 distinct local gates green across unchanged and affected modules -2026-10-16T14:41:34Z hardware Rerun full real CH smoke and invalid-target API probe at cc44409 Verify the independent review fix against actual live devices and keep exact SHA labels hardware-evidence/delivery-smoke.log; delivery-invalid-put.json; delivery-cleanup.json; repo cc44409..da56205 main scenarios and diagnostic fix pass; final delta is test names only; owned resources empty -2026-10-16T14:42:13Z review Accept independent follow-up code verdict and publish exact final evidence Keep source revisions and failed attempts accurate; replace stale pending-hardware statements with measured results final-followup-review.md; repo/docs/live-volumes-validation.md; verification-4/summary.json; hardware-evidence/delivery-results.md code VERIFIED at da56205; production hardware run cc44409; report packages29 green gates and49 attempt records -2026-10-16T14:43:05Z evidence Keep the machine-specific ad hoc probe source in the local audit A tentative public copy has hardcoded local paths and lint findings; the reusable hardware program and committed API regression already provide the maintained check hardware-evidence/delivery-invalid-put.py; repo/e2e/volume_mutation_test.go; repo/docs/test-results/pr339/hardware-invalid-put.json unneeded probe source copy removed; measured JSON and existing reusable checks retained diff --git a/docs/test-results/pr339/diagnostic-regression.log b/docs/test-results/pr339/diagnostic-regression.log deleted file mode 100644 index dd4106b8..00000000 --- a/docs/test-results/pr339/diagnostic-regression.log +++ /dev/null @@ -1,114 +0,0 @@ - -review-fix/sandboxd-red.log -ok github.com/cocoonstack/sandbox/sandboxd 1.071s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/config 1.037s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/egress 1.036s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/engine 1.030s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/mesh 1.041s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] -? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.041s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] ---- FAIL: TestRejectedVolumeTargetPreservesConvergenceDiagnostics (0.01s) - --- FAIL: TestRejectedVolumeTargetPreservesConvergenceDiagnostics/settled (0.01s) - volume_convergence_test.go:211: rejected request changed convergence view: before={VolumesWant:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:attached Error:}] Converging:false Error:} after={VolumesWant:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:attached Error:}] Converging:false Error:volume change failed; retry the desired target} err= - --- FAIL: TestRejectedVolumeTargetPreservesConvergenceDiagnostics/conflict (0.00s) - volume_convergence_test.go:211: rejected request changed convergence view: before={VolumesWant:[] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:detaching Error:cannot unmount "/volumes/data"; close guest references and retry}] Converging:false Error:cannot unmount "/volumes/data"; close guest references and retry} after={VolumesWant:[] Volumes:[{Name:data Mount:/volumes/data Mode: AttachOnly:false}] States:[{Name:data State:detaching Error:cannot unmount "/volumes/data"; close guest references and retry}] Converging:false Error:volume change failed; retry the desired target} err= -FAIL -FAIL github.com/cocoonstack/sandbox/sandboxd/pool 0.094s -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.047s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/server 1.052s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store 1.036s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.031s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store/peer 1.038s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store/s3 1.039s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.039s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.042s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/types 1.038s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/utils 1.031s [no tests to run] -FAIL - -review-fix/e2e-red.log ---- FAIL: TestRejectedVolumeTargetExplainsRefusalWithoutChangingState (0.03s) - volume_mutation_test.go:57: invalid target explanation: set volumes: volume change failed; retry the desired target (http 400) - volume_mutation_test.go:61: rejected request changed settled state: {VolumesWant:[{Name:dataset Mount:/volumes/dataset Mode:}] Volumes:[{Name:dataset Mount:/volumes/dataset Mode:}] States:[{Name:dataset State:attached Error:}] Converging:false Error:volume change failed; retry the desired target} err= -FAIL -FAIL github.com/cocoonstack/sandbox/e2e 0.088s -? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] -ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.031s [no tests to run] -FAIL - -review-fix/sandboxd-green.log -ok github.com/cocoonstack/sandbox/sandboxd 1.093s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/config 1.039s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/egress 1.066s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/engine 1.070s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/mesh 1.044s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/metastore [no test files] -? github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/netfilter [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/outbound 1.079s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/pool 1.204s -? github.com/cocoonstack/sandbox/sandboxd/poolset [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/poolset/pg 1.042s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/server 1.106s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store 1.035s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store/dir 1.042s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store/peer 1.061s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/store/s3 1.049s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/store/storetest [no test files] -? github.com/cocoonstack/sandbox/sandboxd/tenants [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/file 1.041s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/tenants/pg 1.041s [no tests to run] -? github.com/cocoonstack/sandbox/sandboxd/tenants/tenantstest [no test files] -ok github.com/cocoonstack/sandbox/sandboxd/types 1.041s [no tests to run] -ok github.com/cocoonstack/sandbox/sandboxd/utils 1.035s [no tests to run] - -review-fix/e2e-green.log -ok github.com/cocoonstack/sandbox/e2e 1.129s -? github.com/cocoonstack/sandbox/e2e/cmd/androidsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/browsersmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/coldproof [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/credsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/crossnode [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/demo [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/egresssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/guestserver [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/interceptsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/lifecycle [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/meshsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/portsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pullbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/pushbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/qaab [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/ringsmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/rpcbench [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/smoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/sockssmoke [no test files] -? github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke [no test files] -ok github.com/cocoonstack/sandbox/e2e/internal/harness 1.033s [no tests to run] diff --git a/docs/test-results/pr339/guest-input-sha256.txt b/docs/test-results/pr339/guest-input-sha256.txt deleted file mode 100644 index 9f4cc311..00000000 --- a/docs/test-results/pr339/guest-input-sha256.txt +++ /dev/null @@ -1,3 +0,0 @@ -1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7 /var/lib/composable-local-bfc70d3/guest-rootfs.tar -7afa73b01f77f4decb723be647399a7ade15bc8ff143262662048251e6603a92 /tmp/pr339-live/boot/vmlinuz -930e5e753ae51f0104b2d4ced96f107e5f55544cd91b1b968266de91a76e3c95 /tmp/pr339-live/boot/initrd.img diff --git a/docs/test-results/pr339/hardware-binaries.sha256 b/docs/test-results/pr339/hardware-binaries.sha256 deleted file mode 100644 index f5941027..00000000 --- a/docs/test-results/pr339/hardware-binaries.sha256 +++ /dev/null @@ -1,3 +0,0 @@ -13f9d2b924669040200579e51f9301ab755bc0f0ea1bffd35e612eee17e765d8 /tmp/pr339-live/sandboxd -30fd2a6c7ec7e3d0ba305b847e8823e60a98c49f2f618ee9de0e391e2f35120b /tmp/pr339-live/volumesmoke -b22b4434d5b0e62c262bb99948de7e41eaeda52e9e617951b660b09cff33252d /tmp/pr339-live/guestserver diff --git a/docs/test-results/pr339/hardware-build-info.txt b/docs/test-results/pr339/hardware-build-info.txt deleted file mode 100644 index 3a9b5a01..00000000 --- a/docs/test-results/pr339/hardware-build-info.txt +++ /dev/null @@ -1,116 +0,0 @@ -/tmp/pr339-live/sandboxd: go1.27.1 - path github.com/cocoonstack/sandbox/sandboxd - mod github.com/cocoonstack/sandbox/sandboxd (devel) - dep github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= - dep github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= - dep github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= - dep github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= - dep github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= - dep github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.13 h1:wO7TVbywHwdpHLUiX6DnmP2RDYOACVeJCb6zMfSFViU= - dep github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= - dep github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= - dep github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= - dep github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= - dep github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= - dep github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= - dep github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= - dep github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= - dep github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= - dep github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= - dep github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= - dep github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= - dep github.com/aws/smithy-go v1.28.2 h1:myhcykQcatTul2B/zITjDk203G7t0awUAs1hVry5Bvg= - dep github.com/cockroachdb/errors v1.14.0 h1:EfdVEJpN3z8rPMo43Yit59LxoiIa470fSXpZXuEs+ZI= - dep github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILMqgNeV5jiqR4j+sTuvQNHdf2chuKj1M5k= - dep github.com/cockroachdb/redact v1.1.8 h1:8eVLLj6juKxiKrAEw2b8cJvNqWq++U8WOfQFuL7KTaA= - dep github.com/cocoonstack/sandbox/protocol/wire v0.0.0-00010101000000-000000000000 - => ../protocol/wire (devel) - - dep github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= - dep github.com/getsentry/sentry-go v0.49.0 h1:Ehejknu1l023Ub7QoRBVLAI7g3Jnhqku4oWx4B4Sh5s= - dep github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= - dep github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= - dep github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= - dep github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= - dep github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= - dep github.com/hashicorp/go-metrics v0.7.0 h1:lLWieZTcbzZT+rY0zrqKbyryXG8RIajdUjmM0+R79eg= - dep github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44= - dep github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= - dep github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= - dep github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= - dep github.com/hashicorp/memberlist v0.7.0 h1:JfqTDFUIAzDEYKMhSc3Gpwe05zvSU3/cYtiZ3yW59TM= - dep github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= - dep github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= - dep github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= - dep github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= - dep github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= - dep github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= - dep github.com/mattn/go-colorable v0.1.16 h1:+2YjeAe7xP1OQbZW1jERPWWCDLlNqFNCvjiqGiAt84A= - dep github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= - dep github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4YbNI= - dep github.com/mdlayher/socket v0.7.0 h1:qVREPVwtUMg17pwvveQxpurq0PVisMxi3FGgpsorMYQ= - dep github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= - dep github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= - dep github.com/projecteru2/core v0.1.7 h1:f2EdndjWFBYX6pqLZ68VcFYMkXZu6RivotaKUGCNwDQ= - dep github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= - dep github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= - dep github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= - dep golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= - dep golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= - dep golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= - dep golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= - dep golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= - dep golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= - dep google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= - dep google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= - dep gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= - build -buildmode=exe - build -compiler=gc - build CGO_ENABLED=1 - build CGO_CFLAGS= - build CGO_CPPFLAGS= - build CGO_CXXFLAGS= - build CGO_LDFLAGS= - build GOARCH=amd64 - build GOOS=linux - build GOAMD64=v1 - build vcs=git - build vcs.revision=cc444095b3ba000de9ac11d7ef0674e386153804 - build vcs.time=2026-10-16T14:37:15Z - build vcs.modified=false -/tmp/pr339-live/volumesmoke: go1.27.1 - path github.com/cocoonstack/sandbox/e2e/cmd/volumesmoke - mod github.com/cocoonstack/sandbox/e2e (devel) - dep github.com/cocoonstack/sandbox/protocol/wire v0.1.16 - => ../protocol/wire (devel) - - dep github.com/cocoonstack/sandbox/sdk/go v0.0.0 - => ../sdk/go (devel) - - build -buildmode=exe - build -compiler=gc - build CGO_ENABLED=1 - build CGO_CFLAGS= - build CGO_CPPFLAGS= - build CGO_CXXFLAGS= - build CGO_LDFLAGS= - build GOARCH=amd64 - build GOOS=linux - build GOAMD64=v1 - build vcs=git - build vcs.revision=cc444095b3ba000de9ac11d7ef0674e386153804 - build vcs.time=2026-10-16T14:37:15Z - build vcs.modified=false -/tmp/pr339-live/guestserver: go1.27.1 - path github.com/cocoonstack/sandbox/e2e/cmd/guestserver - mod github.com/cocoonstack/sandbox/e2e (devel) - build -buildmode=exe - build -compiler=gc - build CGO_ENABLED=0 - build GOARCH=amd64 - build GOOS=linux - build GOAMD64=v1 - build vcs=git - build vcs.revision=cc444095b3ba000de9ac11d7ef0674e386153804 - build vcs.time=2026-10-16T14:37:15Z - build vcs.modified=false diff --git a/docs/test-results/pr339/hardware-cleanup.json b/docs/test-results/pr339/hardware-cleanup.json deleted file mode 100644 index ffe1b430..00000000 --- a/docs/test-results/pr339/hardware-cleanup.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "candidate": "cc444095b3ba000de9ac11d7ef0674e386153804", - "claims": 0, - "volume_retirements": 0, - "vms": [], - "runtime_sockets": [], - "owned_taps": [], - "vm_pids_gone": { - "1082008": true, - "1084550": true, - "1087371": true - }, - "isolated_daemon_stopped": true, - "existing_services_stopped": [], - "retained_device_calls": 0, - "fault_markers_remaining": [] -} diff --git a/docs/test-results/pr339/hardware-commands.jsonl b/docs/test-results/pr339/hardware-commands.jsonl deleted file mode 100644 index 663ec970..00000000 --- a/docs/test-results/pr339/hardware-commands.jsonl +++ /dev/null @@ -1,88 +0,0 @@ -{"time": 1792161516.2106261, "pid": 1081257, "args": ["version"]} -{"time": 1792161516.290213, "pid": 1081298, "args": ["vm", "list", "--format", "json"]} -{"time": 1792161516.3823283, "pid": 1081340, "args": ["snapshot", "list", "--format", "json"]} -{"time": 1792161516.4694333, "pid": 1081381, "args": ["image", "list", "--format", "json"]} -{"time": 1792161531.5944746, "pid": 1081562, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-ab5e7c310318", "--pull", "--output", "json", "--no-direct-io=true"]} -{"time": 1792161531.7895525, "pid": 1081662, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-ab5e7c310318", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161531.9890335, "pid": 1081705, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-743eac818016", "--pull", "--output", "json", "--no-direct-io=true"]} -{"time": 1792161532.1801274, "pid": 1081804, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-743eac818016", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161532.4134898, "pid": 1081847, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-ab5e7c310318"]} -{"time": 1792161532.5762568, "pid": 1081909, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-743eac818016"]} -{"time": 1792161532.7404315, "pid": 1081966, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--pull", "--output", "json", "--no-direct-io=true"]} -{"time": 1792161532.9253728, "pid": 1082064, "args": ["version"]} -{"time": 1792161532.9955375, "pid": 1082074, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161533.243327, "pid": 1082118, "args": ["version"]} -{"time": 1792161533.3524957, "pid": 1082164, "args": ["version"]} -{"time": 1792161533.4615133, "pid": 1082205, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161533.6639063, "pid": 1082246, "args": ["version"]} -{"time": 1792161533.7398326, "pid": 1082256, "args": ["version"]} -{"time": 1792161533.8222344, "pid": 1082297, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161533.9386697, "pid": 1082340, "args": ["version"]} -{"time": 1792161534.0145915, "pid": 1082350, "args": ["version"]} -{"time": 1792161534.0949044, "pid": 1082360, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161534.2889814, "pid": 1082401, "args": ["version"]} -{"time": 1792161534.450274, "pid": 1082475, "args": ["version"]} -{"time": 1792161534.5280719, "pid": 1082517, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161534.6437583, "pid": 1082560, "args": ["version"]} -{"time": 1792161534.733479, "pid": 1082602, "args": ["version"]} -{"time": 1792161534.8219416, "pid": 1082643, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161535.0211015, "pid": 1082684, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} -{"time": 1792161535.1456678, "pid": 1082727, "args": ["version"]} -{"time": 1792161535.238117, "pid": 1082768, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} -{"time": 1792161535.4366887, "pid": 1082811, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161535.5785534, "pid": 1082854, "args": ["version"]} -{"time": 1792161535.6483173, "pid": 1082865, "args": ["version"]} -{"time": 1792161535.7378325, "pid": 1082906, "args": ["version"]} -{"time": 1792161535.8273516, "pid": 1082947, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161536.0251427, "pid": 1082988, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} -{"time": 1792161536.1503155, "pid": 1083031, "args": ["version"]} -{"time": 1792161536.2372422, "pid": 1083073, "args": ["version"]} -{"time": 1792161536.315691, "pid": 1083083, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} -{"time": 1792161536.5120556, "pid": 1083125, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161536.560109, "pid": 1083131, "args": ["vm", "list", "--format", "json"]} -{"time": 1792161536.6265824, "pid": 1083210, "args": ["version"]} -{"time": 1792161536.716744, "pid": 1083251, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161536.9173326, "pid": 1083292, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} -{"time": 1792161537.150615, "pid": 1083338, "args": ["version"]} -{"time": 1792161537.2409613, "pid": 1083379, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} -{"time": 1792161537.4404385, "pid": 1083422, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161537.712072, "pid": 1083544, "args": ["version"]} -{"time": 1792161537.7933095, "pid": 1083585, "args": ["vm", "list", "--format", "json"]} -{"time": 1792161537.881657, "pid": 1083626, "args": ["snapshot", "list", "--format", "json"]} -{"time": 1792161537.9697576, "pid": 1083667, "args": ["image", "list", "--format", "json"]} -{"time": 1792161538.0572329, "pid": 1083708, "args": ["vm", "list", "--format", "json"]} -{"time": 1792161538.1257093, "pid": 1083749, "args": ["version"]} -{"time": 1792161538.2046566, "pid": 1083791, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161538.34557, "pid": 1083832, "args": ["version"]} -{"time": 1792161538.4290745, "pid": 1083843, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161538.6181557, "pid": 1083884, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} -{"time": 1792161538.8523986, "pid": 1083930, "args": ["version"]} -{"time": 1792161538.9431992, "pid": 1083970, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} -{"time": 1792161539.141075, "pid": 1084011, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161539.8965445, "pid": 1084097, "args": ["version"]} -{"time": 1792161539.977518, "pid": 1084138, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161540.1214855, "pid": 1084179, "args": ["version"]} -{"time": 1792161540.211696, "pid": 1084235, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v1"]} -{"time": 1792161540.4132092, "pid": 1084292, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v2.raw", "--name", "dataset-v2", "--readonly", "--directio", "off"]} -{"time": 1792161540.5535214, "pid": 1084355, "args": ["version"]} -{"time": 1792161540.6428962, "pid": 1084409, "args": ["vm", "disk", "detach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--name", "dataset-v2"]} -{"time": 1792161540.8559349, "pid": 1084451, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53"]} -{"time": 1792161541.029107, "pid": 1084508, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "--pull", "--output", "json", "--no-direct-io=true"]} -{"time": 1792161541.2160385, "pid": 1084606, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161541.4820101, "pid": 1084709, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} -{"time": 1792161541.529443, "pid": 1084785, "args": ["vm", "inspect", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} -{"time": 1792161541.765802, "pid": 1084909, "args": ["version"]} -{"time": 1792161541.8461316, "pid": 1084950, "args": ["vm", "list", "--format", "json"]} -{"time": 1792161541.9320328, "pid": 1084991, "args": ["snapshot", "list", "--format", "json"]} -{"time": 1792161542.0202837, "pid": 1085032, "args": ["image", "list", "--format", "json"]} -{"time": 1792161542.1088488, "pid": 1085073, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} -{"time": 1792161542.1567278, "pid": 1085074, "args": ["vm", "inspect", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} -{"time": 1792161547.2556753, "pid": 1085179, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} -{"time": 1792161547.4440455, "pid": 1085234, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-07e68275055f", "--pull", "--output", "json", "--no-direct-io=true"]} -{"time": 1792161547.6333888, "pid": 1085332, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-07e68275055f", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--directio", "off"]} -{"time": 1792161547.8819134, "pid": 1085375, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-07e68275055f"]} -{"time": 1792161576.371202, "pid": 1087329, "args": ["vm", "clone", "--from-dir", "/tmp/pr339-live/state/goldens/8afde47472091862474d5edf567e6f55", "--name", "sbx-8afde47472091862474d5edf567e6f55-667b56746f08", "--pull", "--output", "json", "--no-direct-io=true"]} -{"time": 1792161576.5613878, "pid": 1087428, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-667b56746f08", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -{"time": 1792161576.8436613, "pid": 1087555, "args": ["version"]} -{"time": 1792161576.9886754, "pid": 1087636, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-667b56746f08"]} -{"time": 1792161602.268468, "pid": 1098326, "args": ["image", "list", "--format", "json"]} diff --git a/docs/test-results/pr339/hardware-invalid-put.json b/docs/test-results/pr339/hardware-invalid-put.json deleted file mode 100644 index 02cfaf97..00000000 --- a/docs/test-results/pr339/hardware-invalid-put.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "sandbox": "sb_d578a0cb4762e49f", - "status": 400, - "failure": { - "volumes_want": [ - { - "name": "dataset-v1", - "mount": "/datasets/invalid" - } - ], - "volumes": [ - { - "name": "dataset-v1", - "mount": "/datasets/invalid" - } - ], - "states": [ - { - "name": "dataset-v1", - "state": "attached" - } - ], - "converging": false, - "error": "invalid volume request: duplicate volume \"dataset-v1\"" - }, - "before": { - "volumes_want": [ - { - "name": "dataset-v1", - "mount": "/datasets/invalid" - } - ], - "volumes": [ - { - "name": "dataset-v1", - "mount": "/datasets/invalid" - } - ], - "states": [ - { - "name": "dataset-v1", - "state": "attached" - } - ], - "converging": false - }, - "after": { - "volumes_want": [ - { - "name": "dataset-v1", - "mount": "/datasets/invalid" - } - ], - "volumes": [ - { - "name": "dataset-v1", - "mount": "/datasets/invalid" - } - ], - "states": [ - { - "name": "dataset-v1", - "state": "attached" - } - ], - "converging": false - }, - "marker_before": "v1\n", - "marker_after": "v1\n", - "vm_pid": 1087371, - "vm_pid_exists": true, - "invalid_put_disk_calls": [], - "PASS": true, - "release_status": 204 -} diff --git a/docs/test-results/pr339/hardware-invocation.json b/docs/test-results/pr339/hardware-invocation.json deleted file mode 100644 index d933b65c..00000000 --- a/docs/test-results/pr339/hardware-invocation.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - "/tmp/pr339-live/volumesmoke", - "-addr", - "127.0.0.1:17777", - "-token", - "", - "-template", - "pr339:local", - "-volume", - "dataset-v1", - "-next-volume", - "dataset-v2", - "-probe", - "version", - "-listener", - "/tmp/pr339-live/guestserver", - "-change-hook", - "/root/eric/composable-computer/.audit/pr339-implementation/hardware/e2e/cmd/volumesmoke/testdata/fault_hook.py", - "-cancel-hook", - "/root/eric/composable-computer/.audit/pr339-implementation/hardware/e2e/cmd/volumesmoke/testdata/fault_hook.py", - "-release-hook", - "/root/eric/composable-computer/.audit/pr339-implementation/hardware/e2e/cmd/volumesmoke/testdata/fault_hook.py" -] diff --git a/docs/test-results/pr339/hardware-journal.json b/docs/test-results/pr339/hardware-journal.json deleted file mode 100644 index c66ad815..00000000 --- a/docs/test-results/pr339/hardware-journal.json +++ /dev/null @@ -1 +0,0 @@ -{"format_version":2,"claims":{},"volume_retirements":{}} \ No newline at end of file diff --git a/docs/test-results/pr339/hardware-prep.md b/docs/test-results/pr339/hardware-prep.md deleted file mode 100644 index 5dc80674..00000000 --- a/docs/test-results/pr339/hardware-prep.md +++ /dev/null @@ -1,29 +0,0 @@ -The local test uses the official Cocoon v0.7.2 x86_64 release from https://github.com/cocoonstack/cocoon/releases/tag/v0.7.2. Archive SHA256 is bd9e9c76f926a5005089ede61a85b1d98c5294c454d3efca0cb4bf4826dca8d4, matching the release checksums. Version reports Git b3b40f3bdedc384c482eb81dcdc60a774dd8d560. Cloud Hypervisor is the existing /opt/composable-local-bfc70d3/bin/cloud-hypervisor v54.0.0. No version output is forged or emulated. - -The test's root, runtime sockets, logs, synthetic API token, images, and daemon state are isolated below /tmp/pr339-live. The daemon listens at 127.0.0.1:17777. Cocoon uses network scope p9 and cgroup parent pr339-cocoon.slice. None of the existing composable-local services has been stopped or modified. - -Preparation reused the existing guest-rootfs.tar and vmlinuz from /var/lib/composable-local-bfc70d3. The first real VM immediately exited because its old initramfs expected computer.layers while official Cocoon supplies cocoon.layers. Rebuilding the current sandbox boot/init binary fixed that compatibility difference. The first rebuilt binary was dynamically linked and produced a kernel "No working init found" panic inside the small initramfs. Rebuilding it statically fixed that. These were environment preparation failures, not feature failures. The final guest was accepted only after a successful CONNECT 2048 handshake and a live host PID check. - -Commands for the working boot rebuild, run from the hardware worktree are below. The host's stable Rust toolchain is recorded in tool-versions.txt. The rootfs layer already contains silkd. It responds to real SDK operations in the baseline smoke. - -```sh -RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --locked --manifest-path boot/init/Cargo.toml -podman create --name pr339-boot localhost/composable-computer/sandbox/boot:6.18.37 -podman cp pr339-boot:/boot/initrd.img-sandbox /tmp/pr339-live/boot/initrd.img -cd /tmp/pr339-live/initramfs -cpio -idmu < /tmp/pr339-live/boot/initrd.img -cp /root/eric/composable-computer/.audit/pr339-implementation/hardware/boot/init/target/release/sandbox-init init -find . -print0 | cpio --null -o -H newc > /tmp/pr339-live/boot/initrd.img -cd /tmp/pr339-live/boot -tar -cf /tmp/pr339-live/boot.tar . -. /tmp/pr339-live/env -cocoon image import pr339:local /var/lib/composable-local-bfc70d3/guest-rootfs.tar /tmp/pr339-live/boot.tar -``` - -The order of the two input layers matters. The boot overlay must come last because the existing rootfs also contains boot files. Input checksums are in guest-input-sha256.txt. The resulting image is sha256:0356e79c7b82914c62e7b5db88b06369594c92f4bb95b4e1de5a721023b1ba91. It uses the real existing 6.18.37 kernel. The guest bootstrap is ordinary sandbox-init and systemd with silkd. The VM pr339-prep was created solely for environment proof. The successful preparation record and PID are in prep-vm-ready.json and prep-vm-current.json. - -Dataset fixtures are two 32 MiB ext4 images made with mkfs.ext4 -F -d. Their version files contain v1 and v2 respectively. Catalog names dataset-v1 and dataset-v2 point to those images. Both allow writable attachment to permit the later retirement/backing-reuse assertion. Ordinary smoke attachments request read-only mode. - -The baseline daemon was built from 645e49756492e7d8e9337377a958185e615804fc. baseline-smoke.log records real shared dataset reads and EROFS succeeding, followed by the expected failing cwd-busy assertion. The old daemon actually returned HTTP 500 instead of the required HTTP 409 and kept v1 applied with v2 desired. baseline-runtime-commands.jsonl and baseline-daemon.log record its real runtime calls and error. This is a hardware reproduction, not a fake-engine result. - -The committed testdata/runtime_observer.py and fault_hook.py are the complete fault injection mechanism. Ordinary commands execute the official binary. Attach interruption occurs only after the real attach succeeds. VM removal failures are explicitly injected exit 73 before the real remove command. The observer logs every attempted command. The test must separately prove final real removal and a later writable attach before claiming backing reuse. diff --git a/docs/test-results/pr339/hardware-revision.txt b/docs/test-results/pr339/hardware-revision.txt deleted file mode 100644 index 6505e866..00000000 --- a/docs/test-results/pr339/hardware-revision.txt +++ /dev/null @@ -1,2 +0,0 @@ -cc444095b3ba000de9ac11d7ef0674e386153804 -2f3624c394be1cea6f8f2e53be1f63764e9893f3 diff --git a/docs/test-results/pr339/hardware.log b/docs/test-results/pr339/hardware.log deleted file mode 100644 index 62e38367..00000000 --- a/docs/test-results/pr339/hardware.log +++ /dev/null @@ -1,34 +0,0 @@ -volume=dataset-v1 claim_a=396.7ms claim_b=394.1ms -VOLUME PASS concurrent_read_bytes=3 read_only=true released=true -CONTINUITY before={"Sandbox":"sb_13c00446eaf0a810","Session":"sess-68-1","Boot":"cd7dbea1-cbb8-4cbb-aba5-97f4e5aeab02","PID":181,"Start":"237088"} -BUSY PASS reference=cwd view={"volumes_want":[{"name":"dataset-v2","mount":"/datasets/current"}],"volumes":[{"name":"dataset-v1","mount":"/datasets/current"}],"states":[{"name":"dataset-v1","state":"detaching","error":"cannot unmount \"/datasets/current\"; close guest references and retry"},{"name":"dataset-v2","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false,"error":"cannot unmount \"/datasets/current\"; close guest references and retry"} -RETAINED begin=1792161535537904360 -RETAINED end=1792161535607563767 -BUSY PASS reference=bind view={"volumes_want":[{"name":"dataset-v2","mount":"/datasets/current"}],"volumes":[{"name":"dataset-v1","mount":"/datasets/current"}],"states":[{"name":"dataset-v1","state":"detaching","error":"volume device has another mount at \"/tmp/volume-bind\"; remove guest mount references before retrying"},{"name":"dataset-v2","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false,"error":"volume device has another mount at \"/tmp/volume-bind\"; remove guest mount references before retrying"} -BUSY PASS reference=foreign view={"volumes_want":[{"name":"dataset-v1","mount":"/datasets/current"}],"volumes":[{"name":"dataset-v2","mount":"/datasets/current"}],"states":[{"name":"dataset-v2","state":"detaching","error":"volume mount \"/datasets/current\" has a different device, root, or mode; restore the expected mount before retrying"},{"name":"dataset-v1","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false,"error":"volume mount \"/datasets/current\" has a different device, root, or mode; restore the expected mount before retrying"} -HOOK runtime_event={"time": 1792161537.4404385, "pid": 1083422, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -HOOK real_vm=[{"id": "3SMJN3MBILXGIJ2FUF4LLABHYZ", "name": "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "pid": 1082008, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/3SMJN3MBILXGIJ2FUF4LLABHYZ/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] -HOOK gated_view={"volumes_want": [{"name": "dataset-v1", "mount": "/datasets/current"}], "volumes": [], "states": [{"name": "dataset-v1", "state": "pending"}], "converging": true} -HOOK daemon_restart old=1081212 new=1083505 -ACTIVITY PASS exec=true file=true port=true -INTERRUPTION request_error=Put "http://127.0.0.1:17777/v1/sandboxes/sb_13c00446eaf0a810/volumes": EOF -INTERRUPTION recovered_view={"volumes_want":[{"name":"dataset-v1","mount":"/datasets/current"}],"volumes":[],"states":[{"name":"dataset-v1","state":"pending","error":"volume change interrupted; retry the desired target"}],"converging":false} -INTERRUPTION PASS sandbox=sb_13c00446eaf0a810 -HOOK runtime_event={"time": 1792161539.141075, "pid": 1084011, "args": ["vm", "disk", "attach", "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "--path", "/tmp/pr339-live/v1.raw", "--name", "dataset-v1", "--readonly", "--directio", "off"]} -HOOK real_vm=[{"id": "3SMJN3MBILXGIJ2FUF4LLABHYZ", "name": "sbx-8afde47472091862474d5edf567e6f55-dd724c346a53", "pid": 1082008, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/3SMJN3MBILXGIJ2FUF4LLABHYZ/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] -HOOK gated_view={"volumes_want": [{"name": "dataset-v1", "mount": "/datasets/current"}], "volumes": [], "states": [{"name": "dataset-v1", "state": "pending"}], "converging": true} -HOOK canceled_view={"volumes_want": [{"name": "dataset-v1", "mount": "/datasets/current"}], "volumes": [], "states": [{"name": "dataset-v1", "state": "pending", "error": "volume change failed; retry the desired target"}], "converging": false, "error": "volume change failed; retry the desired target"} -ACTIVITY PASS exec=true file=true port=true -INTERRUPTION request_error=Put "http://127.0.0.1:17777/v1/sandboxes/sb_13c00446eaf0a810/volumes": context canceled -INTERRUPTION recovered_view={"volumes_want":[{"name":"dataset-v1","mount":"/datasets/current"}],"volumes":[],"states":[{"name":"dataset-v1","state":"pending","error":"volume change failed; retry the desired target"}],"converging":false,"error":"volume change failed; retry the desired target"} -INTERRUPTION PASS sandbox=sb_13c00446eaf0a810 -CONTINUITY PASS after={"Sandbox":"sb_13c00446eaf0a810","Session":"sess-68-1","Boot":"cd7dbea1-cbb8-4cbb-aba5-97f4e5aeab02","PID":181,"Start":"237088"} v1="v1" v2="v2" cwd_409=true bind_409=true foreign_409=true retained_bind=true erofs=true cleared=true -LIVE VOLUME PASS read_bytes=3 repeated_put=true repeated_patch=true cleared=true -RETIREMENT release_error=release: release failed (http 500) -RETIREMENT writer_refused=claim: volume is held by another claim: volume "dataset-v1" (http 409) -HOOK runtime_event={"time": 1792161541.4820101, "pid": 1084709, "args": ["vm", "rm", "--force", "sbx-8afde47472091862474d5edf567e6f55-89e580237e51"]} -HOOK real_vm=[{"id": "OHF3UXRBAEFIHIABIJ7BNYWL2P", "name": "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "pid": 1084550, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/OHF3UXRBAEFIHIABIJ7BNYWL2P/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] -HOOK daemon_restart old=1083505 new=1084870 -HOOK real_vm=[{"id": "OHF3UXRBAEFIHIABIJ7BNYWL2P", "name": "sbx-8afde47472091862474d5edf567e6f55-89e580237e51", "pid": 1084550, "pid_exists": true, "disks": [{"path": "/tmp/pr339-live/runtime/oci/blobs/1e6452dcc4be48895222b1c62534f2f8135780f9c680764bedc5a3a9876004d7.erofs", "ro": true, "serial": "cocoon-layer0", "role": "layer"}, {"path": "/tmp/pr339-live/runtime/oci/blobs/8a2859c3087cbe7e9a649816b8d5afa4cbd55532de7982a3b6a753786a0b1a4b.erofs", "ro": true, "serial": "cocoon-layer1", "role": "layer"}, {"path": "/tmp/pr339-live/run/cloudhypervisor/OHF3UXRBAEFIHIABIJ7BNYWL2P/cow.raw", "ro": false, "serial": "cocoon-cow", "role": "cow"}]}] -RETIREMENT writer_refused=claim: volume is held by another claim: volume "dataset-v1" (http 409) -RETIREMENT PASS writer_blocked_before_restart=true writer_blocked_after_restart=true backing_reused=true diff --git a/docs/test-results/pr339/host-tool-versions.txt b/docs/test-results/pr339/host-tool-versions.txt deleted file mode 100644 index e9096595..00000000 --- a/docs/test-results/pr339/host-tool-versions.txt +++ /dev/null @@ -1,2 +0,0 @@ -go version go1.27.2 linux/amd64 -rustc 1.99.1 (58cf8f9c3 2026-10-13) diff --git a/docs/test-results/pr339/pending-eject.json b/docs/test-results/pr339/pending-eject.json deleted file mode 100644 index d37230f8..00000000 --- a/docs/test-results/pr339/pending-eject.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "ack_absence_injection_established": false, - "reason": "Official Cocoon converges orphaned pause by resuming the guest before disk detach, so guest ACK was available.", - "official_detach_exit": 0, - "official_detach_seconds": 0.145, - "timeout_and_second_pending_retry": "NOT TESTED by this pause mechanism", - "ordinary_guest_umount": true, - "backing_reused_writable_after_real_detach": true, - "initial_assertion_failure": "The probe premise that the guest stayed paused was false. This is not a runtime false-success finding." -} diff --git a/docs/test-results/pr339/post-hardware-test-only.diff b/docs/test-results/pr339/post-hardware-test-only.diff deleted file mode 100644 index 3107a58b..00000000 --- a/docs/test-results/pr339/post-hardware-test-only.diff +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go -index 9e2e7f6..776b9fd 100644 ---- a/e2e/volume_mutation_test.go -+++ b/e2e/volume_mutation_test.go -@@ -51,2 +51,2 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { -- if _, err := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); err != nil { -- t.Fatal(err) -+ if _, checkErr := sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}}); checkErr != nil { -+ t.Fatal(checkErr) -diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go -index 7659fa7..5f1a893 100644 ---- a/sandboxd/pool/volume_convergence_test.go -+++ b/sandboxd/pool/volume_convergence_test.go -@@ -206,2 +206,2 @@ func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { -- if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { -- t.Fatalf("invalid target: %v", err) -+ if _, checkErr := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(checkErr, ErrBadVolume) { -+ t.Fatalf("invalid target: %v", checkErr) diff --git a/docs/test-results/pr339/retained-disk-calls.json b/docs/test-results/pr339/retained-disk-calls.json deleted file mode 100644 index 9a377919..00000000 --- a/docs/test-results/pr339/retained-disk-calls.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "start_ns": 1792161535537904360, - "end_ns": 1792161535607563767, - "disk_calls": [] -} diff --git a/docs/test-results/pr339/review.md b/docs/test-results/pr339/review.md deleted file mode 100644 index 1b3cb39f..00000000 --- a/docs/test-results/pr339/review.md +++ /dev/null @@ -1,29 +0,0 @@ -# Independent follow-up review for PR #339 - -**Implementation verdict: VERIFIED at `da56205bb31b489c15af3ea670e1cbe87905bf01` for the reviewed response, admission, convergence, and retirement behavior.** The earlier P2 diagnostic defect is fixed. The affected race, Linux and Darwin lint, and format checks pass at this exact HEAD. The full real VMM rerun and a separate real invalid-PUT probe pass at `cc444095b3ba000de9ac11d7ef0674e386153804`. Commit `da56205` changes only two test-local `err` names, so it does not change the production code tested by that VMM run. Keep the two SHA labels distinct. No new safety finding was established. This is an independent code and evidence review, not approval to merge. - -## Findings and delivery evidence - -No actionable finding remains from this follow-up review. The three corrected guides link `repo/docs/live-volumes-validation.md`. That report identifies the initial full software run at `ceda20d`, the fixed production hardware run at `cc44409`, and the final test-only revision `da56205` separately. It names the delivery-run VMM PIDs and the separate invalid-request probe. All 18 local links in the report resolve. The public `checks.json` has 49 attempt records under their actual SHAs, including six failed attempts; the latest result for each of 29 distinct checks passes without an unexpected skip. Its public `checks.log` contains the four verification rounds. The public hardware smoke, command log, cleanup snapshot, and invalid-request JSON match the corresponding `delivery-*` originals byte for byte. Their `.log` files exist despite the repository's ignore rule. The public decision trail matches the retained local trail byte for byte. `git diff --check` passes for the documentation changes. - -The decision trail records the `cc44409` diagnostic fix, immediately corrects a mistaken non-resolving abbreviation in an append-only row, and records the test-only `da56205` lint repair, eight final passing affected checks, the `cc44409` hardware rerun, and the public evidence package. The original run transcript confirms the user's 13:50 implementation, local-test, k3s-cleanup, and PR-update authorization. The trail covers the isolated worktrees, initial guest-readiness correction, baseline HTTP 500, Go 1.27.1, clean lint cache, uncertain target rule, startup quarantine failure, and comment cleanup. I found no unlogged scope expansion or PR write in the transcript through the reviewed point. The initial statically linked initramfs correction remains described in `hardware-evidence/runtime-prep.md` rather than a standalone trail row, a minor granularity limit. - -## Diagnostic fix - -`sandboxd/pool/volume_mutation.go:76-94` now clears the historical top-level error in a refused operation's returned view while leaving stored diagnostics intact. `recordOutcome` becomes true at line 124 only after target normalization, validation, new-addition authorization, and the current-run check. A duplicate target or unauthorized new addition therefore cannot replace the previous convergence error. The HTTP handler maps the current refusal when `view.Error` is empty (`sandboxd/server/volume_mutation.go:41-62`), while a later GET still reads the previous diagnostic. The runtime support and begin-run refusal branches clear the returned top-level error as well. - -The placement of `recordOutcome = true` is significant. The old-pending-volume preclean starts afterward at lines 125-133. If its detach fails, the code wraps `errVolumeTargetNotAccepted`, records that operation failure and the named volume diagnostic, and retains the old target outcome. This preserves the prior review's safe preclean interpretation. The code does not claim that the new target was accepted before preclean succeeds. The existing uncertain-target recovery test and other prior safety tests are unchanged by the follow-up production patch. - -The behavior tests were red for the predicted disagreement before the fix, not merely because a test process failed. `review-fix/sandboxd-red.log` shows both a settled GET gaining a false generic error and a prior 409 unmount reason being overwritten. `review-fix/e2e-red.log` shows HTTP 400 carrying that generic error. The corresponding green logs pass after `cc44409`. `hardware-evidence/delivery-invalid-put.json` additionally shows a real HTTP 400 duplicate-name explanation at `cc44409`, unchanged before and after GET views and backing marker, one surviving VMM PID, zero disk calls for the rejected PUT, and release 204. - -## Prior safety paths and verification - -The baseline-to-`da56205` scoped diff is 47 files. The changes after the earlier full safety review's `ceda20d` base are a local name-only production edit, the diagnostic edit above, and regression tests. I rechecked the prior audit's ordering and unchanged tests for A/W publication, same-lock admission holds, cancellation versus release/reap/capture, startup adoption and quarantine refusal, retirement deletion after actual VM absence, unknown-orphan fencing, held repair after ACL/catalog removal, and typed runtime and mount-conflict failures. The original `final-review.md` cites the complete code paths and their fake and real evidence. The new diagnostic code does not alter admission, journal, device, or retirement ordering. `git diff ceda20d..da56205` contains no new source comment or lint, type, or safety suppression. The earlier `comment-final-review.md` remains valid for the broad diff. - -`verification-3/summary.json` records successful sandboxd and e2e race and format checks at `cc44409`, but both Linux and Darwin linters correctly rejected two shadowed `err` variables in new tests. `da56205` changes only those local names. All eight affected checks in `verification-4/summary.json` have exit 0, no unexpected skip, and HEAD `da56205`; I read the matching raw lint logs. The prior `verification-1` and `verification-2` logs cover the other Go modules, Python suites and strict checks, PostgreSQL without skipped tests, and Caddy TLS at their stated earlier SHAs. - -The `cc44409` real delivery smoke exits 0 (`delivery-smoke-exit.json`). Its log records unchanged sandbox, session, boot ID, PID, start time, and environment across replacement; EROFS; cwd, bind, and foreign 409 with retry; work during a gated successful attach; daemon restart and explicit cancellation; failed VM removal with writer refusal across restart; and actual backing reuse after removal. `delivery-retained-zero-operations.json` contains zero disk calls in the settled-target window. `delivery-journal.json` has empty claims and retirements. `delivery-cleanup.json` shows zero owned VMs, sockets, and TAPs, all three observed VMM PIDs gone, and the isolated daemon stopped. The original local sandboxd remains on port 7777. These are evidence at `cc44409`, not at `da56205`. - -**INCONCLUSIVE hardware limits remain unchanged.** The external pause experiment did not establish a missing eject ACK because official Cocoon resumed the guest before detach. Separate real crashes at each umount, detach, journal, and fsync boundary and physical power loss were not run. The daemon smoke used `sync_claims=false`, so its restart is not power-loss durability proof. The journal also cannot detect an operator repointing a held catalog name, and format v2 requires draining before downgrade. These are documented constraints, not proven failures in this review. - -The review used the requested poteto-mode guidance. Its Prove It Works principle led me to read the raw red/green outputs, exact-HEAD summaries, real JSON responses, and actual cleanup state. Its technical-writing and unslop guidance kept the verdict and delivery action tied to named evidence. From abf73523793328e90d1110ff64c00f765c03b1ae Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:40:56 +0800 Subject: [PATCH 20/57] pool: drop the claims journal v2, volume retirements and the orphan fence In-process, a teardown already keeps its holds in pendingRemovals until cocoon confirms the VM gone; after a restart, the hotplug image lock refuses a conflicting attach. The v2 envelope broke rolling sandboxd back, added a map clone and marshal to every claims commit and 2-3 full rewrites to a release with volumes, and the fence turned one stuck orphan into a node-wide 400 for every volume claim. quarantineClaim keeps main's egress handling. --- docs/deploy.md | 22 +- docs/sandboxd-api.md | 22 +- sandboxd/pool/claim.go | 29 +-- sandboxd/pool/claims.go | 125 +--------- sandboxd/pool/egress_test.go | 4 +- sandboxd/pool/liveness_test.go | 4 +- sandboxd/pool/pool.go | 7 +- sandboxd/pool/reconcile.go | 70 +----- sandboxd/pool/remove.go | 67 +----- sandboxd/pool/telemetry_test.go | 8 +- sandboxd/pool/volume.go | 19 +- sandboxd/pool/volume_attach_test.go | 4 +- sandboxd/pool/volume_retirement_test.go | 293 ------------------------ 13 files changed, 77 insertions(+), 597 deletions(-) delete mode 100644 sandboxd/pool/volume_retirement_test.go diff --git a/docs/deploy.md b/docs/deploy.md index 5a82e560..6b0172f1 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -164,32 +164,24 @@ PATCH reject older, unknown, development, or unqueryable versions with 503 before any intent, hold, or device change. GET and release remain available. The claim journal stores desired `VolumesWant` and confirmed applied `Volumes`. -Admission holds cover their union across daemon restart and teardown. Version 2 -journals preserve a separate retirement record until VM deletion is confirmed -and the record deletion commits. Retry an -interrupted operation with the same PUT target. Release cancels convergence +Admission holds cover their union across daemon restart and teardown. Retry +an interrupted operation with the same PUT target. Release cancels convergence and transfers holds to teardown. Capture is refused while either set is -nonempty. Back up the journal before upgrading. Older daemons reject the new -format. Drain all volume claims and finish retirements before downgrading; -do not manually convert a journal that still has retirement records. +nonempty. A sandbox whose VMM exits while holding volumes is release-only. Release it and create a new sandbox. Daemon restart recovery assumes the original VMM remains alive and does not recreate guest processes. Unfinished changes report an interrupted diagnostic after restart and require a caller retry. -If startup cannot confirm removal of an unknown owned orphan, the node blocks -new volume admission and all disk attachment, including pending repair, until -the orphan is gone. GET, detach, and release remain available. - `e2e/cmd/volumesmoke` exercises GET, repeated full-set PUT, PATCH add and remove, and clearing a new claim. Its [hardware validation instructions](../e2e/cmd/volumesmoke/README.md) -cover same-path replacement, guest mount conflicts, daemon restart, cancellation, -and retirement recovery on an isolated node. +cover same-path replacement, guest mount conflicts, daemon restart and +cancellation on an isolated node. Each catalog path must name a disk image containing a mountable whole-device -filesystem. Keep catalog names bound to the same image while any live or -retiring claim holds them. Use versioned names such as `dataset-v1` and +filesystem. Keep catalog names bound to the same image while any claim holds +them. Use versioned names such as `dataset-v1` and `dataset-v2`; sandboxd cannot detect a repointed catalog entry. A read-only entry's image must stay immutable: do not replace, truncate, or delete it while attached; publish a new catalog name or path diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ddcf05b5..ec56a413 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -348,20 +348,16 @@ superseding request may first clean up an older desired-only attachment while the old target stays recorded. If that cleanup fails, the response retains the old view; retry the same PUT to finish accepting the new target. Admission holds cover the deduplicated union of both sets, including during restart and -teardown. A durable retirement record keeps teardown holds until the VM is -confirmed gone and the retirement deletion commits. Failed or uncertain -operations retain their holds until successful cleanup. Startup reserves those -holds without replaying attachments. Release -cancels convergence and transfers the held union to teardown without waiting +teardown; a teardown keeps them until the VM is confirmed gone. Failed or +uncertain operations retain their holds until successful cleanup. Startup +reserves those holds without replaying attachments. Release cancels convergence and transfers the held union to teardown without waiting for a blocked device call. Capture operations (hibernate, fork, checkpoint, and promote) are refused while either set is nonempty or a convergence request still owns admitted volume holds. Device calls honor cancellation and have a 30-second ceiling. Journal writes -are atomic. Host-crash durability follows `sync_claims`. Drain volume claims -before downgrading. The version 2 claim journal also stores volume retirements; -older daemons reject it. Back up the journal before upgrading and do not -manually convert it to the legacy format while retirement records remain. +are atomic. Host-crash durability follows `sync_claims`. An older daemon +ignores `VolumesWant`; finish pending volume changes before downgrading. Live convergence requires Cocoon v0.7.2 or newer for idempotent matching attach and absent detach. PUT and PATCH reject older, unknown, development, @@ -369,11 +365,9 @@ or unqueryable runtime versions with 503 before changing intent, holds, or devices. GET and release remain available. Catalog names must keep the same backing image while held, including pending -and retiring holds. Publish changes under versioned names such as `dataset-v1` -and `dataset-v2`. The journal cannot detect a catalog name repointed to a new -image. On startup, an unknown owned orphan that cannot be confirmed gone blocks -new volume admission and all disk attachment, including pending repair. -Diagnosis, detach, and release remain available until cleanup succeeds. +holds. Publish changes under versioned names such as `dataset-v1` and +`dataset-v2`. The journal cannot detect a catalog name repointed to a new +image. A replacement first unmounts the old disk, then mounts the new disk. Pause reads and close files, mmap, cwd, and bind mount references during this gap. Existing diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 0e790cab..f3d6dcab 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -295,8 +295,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand } delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) - js := m.store.retire(sb, volumeHolds) - delete(m.volumeErrors, sb) + js := m.store.del(id) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() @@ -304,7 +303,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.restoreVolumeHolds(sb, volumeHolds) m.tenantDelta(sb.Tenant, 1) delete(m.pendingCks, ck) - rb := m.store.restore(sb) + rb := m.store.set(sb) m.mu.Unlock() logger := log.WithFunc("pool.releaseResolved") if ck != "" { @@ -322,12 +321,13 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.purgeArchiveCk(ctx, id, ck, sb.Tenant) // archived: no local VM m.untrack(m.pendingCks, ck) } - td := m.quiesceRetiringVolumes(ctx, sb, volumeHolds) + td := m.quiesceVolumes(ctx, sb) + td.holds = volumeHolds var err error removed := vmName == "" switch { case removed: - removed = m.completeVolumeTeardown(ctx, td) + m.finishVolumeTeardown(ctx, td) case m.releaseDelay > 0: m.queueRemoval(vmName, id, "", td, time.Now().Add(m.releaseDelay)) default: @@ -507,7 +507,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() var expired []victim - var purges []string + var dropped, purges []string for id, sb := range m.claimed { // a zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { @@ -529,14 +529,15 @@ func (m *Manager) reapOnce(ctx context.Context) { } expired = append(expired, victim{action: action, id: id, sb: sb}) default: - holds := m.cancelVolumeRun(sb) - expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: holds}) - m.store.retire(sb, holds) - delete(m.volumeErrors, sb) + expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: m.cancelVolumeRun(sb)}) delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) + dropped = append(dropped, id) } } + if len(dropped) > 0 { + m.store.del(dropped...) + } m.mu.Unlock() if len(expired) == 0 { return @@ -579,6 +580,7 @@ func (m *Manager) reapOnce(ctx context.Context) { m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() + var restored []*types.Sandbox for _, v := range expired { if v.action == reapArchive || v.action == reapPause { delete(m.archiving, v.id) // never removed from m.claimed @@ -587,10 +589,10 @@ func (m *Manager) reapOnce(ctx context.Context) { m.restoreVolumeHolds(v.sb, v.holds) m.tenantDelta(v.tenant, 1) delete(m.pendingCks, v.ck) - m.store.restore(v.sb) + restored = append(restored, v.sb) } } - rb := m.store.mark() + rb := m.store.set(restored...) m.mu.Unlock() for _, ck := range purgeCks { if clearErr := m.clearArchiveCk(ck); clearErr != nil { @@ -617,7 +619,8 @@ func (m *Manager) reapOnce(ctx context.Context) { case reapPause: logSweepResult(ctx, logger, m.pauseExpired(ctx, v.sb), "archived expired sandbox "+v.id, "archive expired sandbox "+v.id) default: - td := m.quiesceRetiringVolumes(ctx, v.sb, v.holds) + td := m.quiesceVolumes(ctx, v.sb) + td.holds = v.holds m.out.Disarm(v.id, m.removeOrRetry(ctx, v.vmName, v.id, "", td)) m.dropSnap(ctx, v.snap) m.counters.reaps.Add(1) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 8708d804..00211fb3 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -53,20 +53,6 @@ type claimDTO struct { Failed string `json:"failed,omitempty"` } -type volumeRetirementDTO struct { - SandboxID string `json:"sandbox_id"` - VMName string `json:"vm_name"` - TAP string `json:"tap,omitempty"` - Holds []types.Volume `json:"holds"` - Clears []string `json:"clears,omitempty"` -} - -type claimJournal struct { - FormatVersion uint8 `json:"format_version"` - Claims map[string]*types.Sandbox `json:"claims"` - VolumeRetirements map[string]volumeRetirementDTO `json:"volume_retirements"` -} - // claimRow keeps its encoded member until set replaces the row; set never edits one. type claimRow struct { dto claimDTO @@ -82,123 +68,41 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - retirements map[string]volumeRetirementDTO - seq uint64 - written uint64 + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 } func newClaimStore(dataDir string, sync bool) *claimStore { - return &claimStore{path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, retirements: map[string]volumeRetirementDTO{}} + return &claimStore{path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}} } func (s *claimStore) load() (map[string]*types.Sandbox, error) { - claims, _, err := s.loadJournal() - return claims, err -} - -func (s *claimStore) loadJournal() (map[string]*types.Sandbox, map[string]volumeRetirementDTO, error) { raw, err := os.ReadFile(s.path) if errors.Is(err, fs.ErrNotExist) { - return map[string]*types.Sandbox{}, map[string]volumeRetirementDTO{}, nil + return map[string]*types.Sandbox{}, nil } if err != nil { - return nil, nil, fmt.Errorf("read claims: %w", err) - } - var envelope map[string]jsontext.Value - if err := json.Unmarshal(raw, &envelope); err != nil { - return nil, nil, fmt.Errorf("parse claims: %w", err) - } - retirements := map[string]volumeRetirementDTO{} - if _, versioned := envelope["format_version"]; versioned { - var journal claimJournal - if err := json.Unmarshal(raw, &journal); err != nil { - return nil, nil, fmt.Errorf("parse claims journal: %w", err) - } - if journal.FormatVersion != 2 { - return nil, nil, fmt.Errorf("unsupported claims journal format %d", journal.FormatVersion) - } - if journal.Claims == nil || journal.VolumeRetirements == nil { - return nil, nil, errors.New("claims journal requires claims and volume_retirements") - } - retirements = journal.VolumeRetirements - raw = envelope["claims"] + return nil, fmt.Errorf("read claims: %w", err) } claims := map[string]*types.Sandbox{} if err := json.Unmarshal(raw, &claims); err != nil { - return nil, nil, fmt.Errorf("parse claims: %w", err) + return nil, fmt.Errorf("parse claims: %w", err) } var fields map[string]map[string]jsontext.Value if err := json.Unmarshal(raw, &fields); err != nil { - return nil, nil, fmt.Errorf("parse claim fields: %w", err) + return nil, err } for id, sb := range claims { - if sb == nil { - return nil, nil, fmt.Errorf("null claim %q", id) - } if _, present := fields[id]["volumes_want"]; !present { sb.VolumesWant = slices.Clone(sb.Volumes) } if len(sb.VolumesWant) == 0 { sb.VolumesWant = nil } - if _, retiring := retirements[sb.VMName]; retiring { - return nil, nil, fmt.Errorf("VM %q is both claimed and retiring", sb.VMName) - } - } - for name, retirement := range retirements { - if name == "" || retirement.VMName != name || retirement.SandboxID == "" || len(retirement.Holds) == 0 || claims[retirement.SandboxID] != nil { - return nil, nil, fmt.Errorf("invalid volume retirement %q", name) - } - } - s.mu.Lock() - s.retirements = maps.Clone(retirements) - s.mu.Unlock() - return claims, retirements, nil -} - -func (s *claimStore) retire(sb *types.Sandbox, holds []types.Volume) claimSnapshot { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.rows, sb.ID) - if len(holds) > 0 { - s.retirements[sb.VMName] = volumeRetirementDTO{SandboxID: sb.ID, VMName: sb.VMName, TAP: sb.TAP, Holds: slices.Clone(holds)} } - s.seq++ - return claimSnapshot{seq: s.seq} -} - -func (s *claimStore) restore(sb *types.Sandbox) claimSnapshot { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.retirements, sb.VMName) - s.rows[sb.ID] = &claimRow{dto: dtoOf(sb)} - s.seq++ - return claimSnapshot{seq: s.seq} -} - -func (s *claimStore) finishRetirement(name string) claimSnapshot { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.retirements, name) - s.seq++ - return claimSnapshot{seq: s.seq} -} - -func (s *claimStore) setRetirement(retirement volumeRetirementDTO) claimSnapshot { - s.mu.Lock() - defer s.mu.Unlock() - s.retirements[retirement.VMName] = retirement - s.seq++ - return claimSnapshot{seq: s.seq} -} - -func (s *claimStore) retirement(name string) (volumeRetirementDTO, bool) { - s.mu.Lock() - defer s.mu.Unlock() - retirement, ok := s.retirements[name] - return retirement, ok + return claims, nil } // set records each sandbox's projection and sequences the write; callers may hold m.mu. @@ -258,9 +162,8 @@ func (s *claimStore) commit(snap claimSnapshot) error { return nil } rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq - retirements := maps.Clone(s.retirements) s.mu.Unlock() - s.buf = append(s.buf[:0], `{"format_version":2,"claims":{`...) + s.buf = append(s.buf[:0], '{') for i, row := range rows { if row.raw == nil { raw, err := encodeRow(&row.dto) @@ -274,12 +177,6 @@ func (s *claimStore) commit(snap claimSnapshot) error { } s.buf = append(s.buf, row.raw...) } - s.buf = append(s.buf, `},"volume_retirements":`...) - rawRetirements, err := json.Marshal(retirements) - if err != nil { - return fmt.Errorf("encode volume retirements: %w", err) - } - s.buf = append(s.buf, rawRetirements...) s.buf = append(s.buf, '}') if err := s.replace(s.buf); err != nil { return err diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index f5e5d2ed..6a7559e9 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -313,9 +313,7 @@ func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { var gotKeep map[string]bool withOutbound(t, m, func(o *outbound.Options) { o.Sweep = func(keep map[string]bool) error { gotKeep = keep; return nil } }) - if err := m.resyncEgress(t.Context(), live, removed); err != nil { - t.Fatal(err) - } + m.resyncEgress(t.Context(), live, removed) if removed["sbx-q1"] { t.Error("failed remove marked the VM gone; the sweep would drop its lock table") diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 951be9f5..682c976f 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -423,9 +423,7 @@ func TestResyncRecordsADownEgressLaneTap(t *testing.T) { o.Sweep = func(map[string]bool) error { return nil } }) - if err := m.resyncEgress(t.Context(), live, map[string]bool{}); err != nil { - t.Fatal(err) - } + m.resyncEgress(t.Context(), live, map[string]bool{}) m.mu.Lock() kept := m.claimed[sb.ID] == sb diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9de069ec..ec46fc06 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -296,10 +296,9 @@ type Manager struct { store *claimStore volumes map[string]catalogVolume // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]volumeDiagnostics - unknownVolumeVMs map[string]bool + volumeAdmission map[string]volumeHolders + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]volumeDiagnostics poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index c7c1772b..1bf80f74 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -3,7 +3,6 @@ package pool import ( "context" "encoding/json/v2" - "errors" "fmt" "os" "path/filepath" @@ -20,7 +19,7 @@ import ( // Reconcile aligns claims and VMs after a restart; it runs before the server because it swaps in fresh records that would bypass in-flight Transition locks. func (m *Manager) Reconcile(ctx context.Context) error { - claims, retirements, err := m.store.loadJournal() + claims, err := m.store.load() if err != nil { return err } @@ -39,13 +38,6 @@ func (m *Manager) Reconcile(ctx context.Context) error { referenced := map[string]bool{} var adopted []*types.Sandbox m.mu.Lock() - for name, retirement := range retirements { - m.adoptVolumes(retirement.Holds) - if retirement.TAP != "" { - m.out.KeepLock(retirement.SandboxID, retirement.TAP) - } - owned[name] = true - } for id, sb := range claims { sb.SetEnv(sb.Env) rec, ok := live[sb.VMName] @@ -128,17 +120,10 @@ func (m *Manager) Reconcile(ctx context.Context) error { logger.Infof(ctx, "removed orphan snapshot %s", orphans[i]) }).Wait() } - for name, retirement := range retirements { - td := volumeTeardown{holds: retirement.Holds, retirement: name} - if m.removeOrRetry(ctx, name, retirement.SandboxID, retirement.TAP, td) { - removed[name] = true - m.out.Disarm(retirement.SandboxID, true) - } - } - egressErr := m.resyncEgress(ctx, live, removed) + m.resyncEgress(ctx, live, removed) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) - return errors.Join(saveErr, egressErr) + return saveErr } // sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. @@ -149,21 +134,10 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec stale = append(stale, name) } } - m.mu.Lock() - if m.unknownVolumeVMs == nil { - m.unknownVolumeVMs = make(map[string]bool) - } - for _, name := range stale { - m.unknownVolumeVMs[name] = true - } - m.mu.Unlock() gone := make([]bool, len(stale)) // distinct indices: no lock under the Wait barrier m.runBounded(ctx, len(stale), func(ctx context.Context, i int) { if m.removeStaleVM(ctx, stale[i], live[stale[i]]) { gone[i] = true - m.mu.Lock() - delete(m.unknownVolumeVMs, stale[i]) - m.mu.Unlock() log.WithFunc("pool.sweepStaleVMs").Infof(ctx, "removed stale VM %s", stale[i]) } }).Wait() @@ -198,7 +172,7 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. -func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) error { +func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) { logger := log.WithFunc("pool.resyncEgress") now := time.Now() locked, lockedErr := m.out.LockedTaps() @@ -237,11 +211,7 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco // quarantine before the sweep so a failed remove's still-running tap is kept. keep := make(map[string]bool, len(live)) for _, sb := range quarantine { - gone, err := m.quarantineClaim(ctx, sb) - if err != nil { - return err - } - if gone { + if m.quarantineClaim(ctx, sb) { removed[sb.VMName] = true } else if sb.TAP != "" { keep[sb.TAP] = true @@ -253,42 +223,24 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco keep[tap] = true } } - m.mu.Lock() - for _, pending := range m.pendingRemovals { - if pending.tap != "" { - keep[pending.tap] = true - } - } - m.mu.Unlock() if sweepErr := m.out.Sweep(keep); sweepErr != nil { logger.Warnf(ctx, "sweep orphan egress tables: %v", sweepErr) } - return nil } // quarantineClaim parks a claim whose teardown failed: out of service and queued until a remove succeeds. -func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) (bool, error) { +func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { + td := m.quiesceVolumes(ctx, sb) + gone := m.removeOrRetry(ctx, sb.VMName, sb.ID, "", td) m.mu.Lock() - holds := m.cancelVolumeRun(sb) delete(m.claimed, sb.ID) - delete(m.volumeErrors, sb) m.tenantDelta(sb.Tenant, -1) - js := m.store.retire(sb, holds) + js := m.store.del(sb.ID) m.mu.Unlock() if err := m.store.commit(js); err != nil { - m.mu.Lock() - m.claimed[sb.ID] = sb - m.restoreVolumeHolds(sb, holds) - m.tenantDelta(sb.Tenant, 1) - rb := m.store.restore(sb) - m.mu.Unlock() - m.recommit(ctx, rb) - return false, fmt.Errorf("persist quarantine of %s: %w", sb.ID, err) + m.recommit(ctx, js) } - td := m.quiesceRetiringVolumes(ctx, sb, holds) - gone := m.removeOrRetry(ctx, sb.VMName, sb.ID, sb.TAP, td) - m.out.Disarm(sb.ID, gone) - return gone, nil + return gone } // readoptEgressTap records and returns a live egress claim's tap, "" when there is none. diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 5f0a5380..50fb9668 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -11,7 +11,6 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/engine" - "github.com/cocoonstack/sandbox/sandboxd/types" ) func (m *Manager) removeVM(ctx context.Context, name string) bool { @@ -45,10 +44,8 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID, tap string, td volumeTeardown) bool { - if m.removeVM(ctx, name) && m.completeVolumeTeardown(ctx, td) { - m.mu.Lock() - delete(m.unknownVolumeVMs, name) - m.mu.Unlock() + if m.removeVM(ctx, name) { + m.finishVolumeTeardown(ctx, td) return true } m.queueRemoval(name, sandboxID, tap, td, time.Time{}) @@ -96,7 +93,7 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueStaleCreate(name, pending.tap) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) return } } @@ -104,17 +101,11 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) } -func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { - if !m.completeVolumeTeardown(ctx, pending.volumes) { - m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) - return - } - m.mu.Lock() - delete(m.unknownVolumeVMs, name) - m.mu.Unlock() +func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { + m.finishVolumeTeardown(ctx, pending.volumes) if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { @@ -125,49 +116,3 @@ func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendin func (m *Manager) destroy(ctx context.Context, name string) { m.removeOrRetry(ctx, name, "", "", volumeTeardown{}) } - -func (m *Manager) quiesceRetiringVolumes(ctx context.Context, sb *types.Sandbox, holds []types.Volume) volumeTeardown { - td := m.quiesceVolumes(ctx, sb) - td.holds = holds - if len(holds) == 0 { - return td - } - td.retirement = sb.VMName - retirement, ok := m.store.retirement(sb.VMName) - if !ok { - return td - } - if len(td.cleanImagePaths) > 0 { - next := retirement - next.Clears = slices.Clone(td.cleanImagePaths) - if err := m.store.commit(m.store.setRetirement(next)); err != nil { - log.WithFunc("pool.quiesceRetiringVolumes").Warnf(ctx, "persist clean volumes for %s: %v", sb.ID, err) - m.recommit(ctx, m.store.setRetirement(retirement)) - } - } - return td -} - -func (m *Manager) completeVolumeTeardown(ctx context.Context, td volumeTeardown) bool { - if td.retirement == "" { - m.finishVolumeTeardown(ctx, td) - return true - } - retirement, ok := m.store.retirement(td.retirement) - if !ok { - return true - } - for _, path := range retirement.Clears { - if err := clearVolumeDirty(path); err != nil { - log.WithFunc("pool.completeVolumeTeardown").Warnf(ctx, "clear volume marker %s: %v", path, err) - return false - } - } - if err := m.store.commit(m.store.finishRetirement(td.retirement)); err != nil { - log.WithFunc("pool.completeVolumeTeardown").Warnf(ctx, "finish volume retirement %s: %v", td.retirement, err) - m.recommit(ctx, m.store.setRetirement(retirement)) - return false - } - m.unreserveVolumes(retirement.Holds) - return true -} diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index e73825ad..e48d8df0 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -124,9 +124,13 @@ func TestTenantStampedInJournals(t *testing.T) { t.Fatalf("claim: %v", err) } - claims, err := m.store.load() + raw, err := os.ReadFile(filepath.Join(m.dataDir, "claims.json")) if err != nil { - t.Fatalf("load claims journal: %v", err) + t.Fatalf("read claims journal: %v", err) + } + claims := map[string]*types.Sandbox{} + if err = json.Unmarshal(raw, &claims); err != nil { + t.Fatalf("parse claims journal: %v", err) } if claims[sb.ID] == nil || claims[sb.ID].Tenant != "acme" { t.Errorf("persisted claim %+v, want tenant acme", claims[sb.ID]) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 2cfbbec9..080d3ffd 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -35,10 +35,10 @@ type resolvedVolume struct { applied types.Volume } +// volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - holds []types.Volume - cleanImagePaths []string - retirement string + holds []types.Volume + clears []string } type catalogVolume struct { @@ -198,12 +198,6 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume) error { - m.mu.Lock() - fenced := len(m.unknownVolumeVMs) > 0 - m.mu.Unlock() - if fenced { - return ErrVolumeUnavailable - } // attach-only mounts nothing, so there is no umount to verify and no marker. attachOnly := volume.applied.Mount == "" // write-ahead: the marker must be durable before any guest write can be. @@ -249,7 +243,7 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT continue } if entry, ok := m.volumes[volume.Name]; ok { - td.cleanImagePaths = append(td.cleanImagePaths, entry.disk.Path) + td.clears = append(td.clears, entry.disk.Path) } } if stuck { @@ -269,7 +263,7 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { return } logger := log.WithFunc("pool.finishVolumeTeardown") - for _, path := range td.cleanImagePaths { + for _, path := range td.clears { if err := clearVolumeDirty(path); err != nil { logger.Errorf(ctx, err, "clear dirty marker %s", path) } @@ -279,9 +273,6 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { - if len(volumes) > 0 && len(m.unknownVolumeVMs) > 0 { - return ErrVolumeUnavailable - } for _, volume := range volumes { holders := m.volumeAdmission[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index e08e4a2d..7fc14e75 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -390,8 +390,8 @@ func TestQuiesceVolumesSkipsAttachOnlyEntries(t *testing.T) { if want := []string{"umount:/volumes/mounted"}; !slices.Equal(eng.volumeOpsLog(), want) { t.Errorf("quiesce ops=%v, want %v", eng.volumeOpsLog(), want) } - if !slices.Equal(td.cleanImagePaths, []string{mounted}) { - t.Errorf("marker clears=%v, want only %s", td.cleanImagePaths, mounted) + if !slices.Equal(td.clears, []string{mounted}) { + t.Errorf("marker clears=%v, want only %s", td.clears, mounted) } if !slices.Equal(td.holds, sb.Volumes) { t.Errorf("holds=%v, want every entry %v", td.holds, sb.Volumes) diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go deleted file mode 100644 index 632defb6..00000000 --- a/sandboxd/pool/volume_retirement_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package pool - -import ( - "context" - "encoding/json/v2" - "errors" - "os" - "path/filepath" - "testing" - "time" - - "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/outbound" - "github.com/cocoonstack/sandbox/sandboxd/types" -) - -func TestRetiringVolumeHoldsSurviveRestart(t *testing.T) { - for _, action := range []string{"release", "reap", "quarantine"} { - for _, mode := range []string{"ro", "rw"} { - t.Run(action+"/"+mode, func(t *testing.T) { - eng := newFakeEngine() - catalog := liveCatalog(t) - m := newVolumeManager(t, eng, catalog) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: mode}}}) - if err != nil { - t.Fatal(err) - } - eng.removeErrFor = sb.VMName - switch action { - case "release": - if checkErr := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); checkErr == nil { - t.Fatal("release accepted surviving VM") - } - case "reap": - sb.Deadline = time.Now().Add(-time.Second) - m.reapOnce(t.Context()) - waitFor(t, func() bool { m.mu.Lock(); defer m.mu.Unlock(); _, ok := m.pendingRemovals[sb.VMName]; return ok }) - case "quarantine": - if gone, quarantineErr := m.quarantineClaim(t.Context(), sb); gone || quarantineErr != nil { - t.Fatal("quarantine accepted surviving VM") - } - } - claims, retiring, err := m.store.loadJournal() - if err != nil || claims[sb.ID] != nil || len(retiring[sb.VMName].Holds) != 1 { - t.Fatalf("journal=%v retiring=%v err=%v", claims, retiring, err) - } - reloaded := newClaimStore(m.dataDir, false) - loaded, loadErr := reloaded.load() - if loadErr != nil { - t.Fatal(loadErr) - } - if saveErr := reloaded.save(loaded); saveErr != nil { - t.Fatal(saveErr) - } - m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) - if checkErr := m2.Reconcile(t.Context()); checkErr != nil { - t.Fatal(checkErr) - } - if _, checkErr := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); !errors.Is(checkErr, ErrVolumeBusy) { - t.Fatalf("writer while VM survives: %v", checkErr) - } - if saveErr := m2.store.save(m2.claimed); saveErr != nil { - t.Fatal(saveErr) - } - _, retiring, err = m2.store.loadJournal() - if err != nil || len(retiring) != 1 { - t.Fatalf("save lost retirement: %v %v", retiring, err) - } - eng.removeErrFor = "" - m2.retryRemovals(t.Context()).Wait() - _, retiring, err = m2.store.loadJournal() - if err != nil || len(retiring) != 0 { - t.Fatalf("completed retirement remained: %v %v", retiring, err) - } - if got := volumeHoldersOf(m2, "data"); got != (volumeHolders{}) { - t.Fatalf("retirement leaked holds: %+v", got) - } - writer, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}) - if err != nil { - t.Fatalf("writer after removal: %v", err) - } - if err := m2.Release(t.Context(), writer.ID, Cred{Token: writer.Token}); err != nil { - t.Fatal(err) - } - }) - } - } -} - -func TestRetirementCompletionMustBeDurableBeforeAdmission(t *testing.T) { - eng := newFakeEngine() - catalog := liveCatalog(t) - m := newVolumeManager(t, eng, catalog) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) - if err != nil { - t.Fatal(err) - } - m.eng = &retirementWriteFailureEngine{fakeEngine: eng, afterRemove: func() { - if checkErr := os.Mkdir(m.store.path+".tmp", 0o700); checkErr != nil { - t.Fatal(checkErr) - } - }} - if checkErr := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); checkErr == nil { - t.Fatal("release accepted uncommitted retirement deletion") - } - if !eng.removed(sb.VMName) { - t.Fatal("test did not reach VM removal") - } - if _, checkErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}); !errors.Is(checkErr, ErrVolumeBusy) { - t.Fatalf("writer before retirement deletion: %v", checkErr) - } - m.store.writeMu.Lock() - err = os.Remove(m.store.path + ".tmp") - m.store.writeMu.Unlock() - if err != nil { - t.Fatal(err) - } - waitFor(t, func() bool { return !m.recommitting.Load() }) - _, retiring, err := m.store.loadJournal() - if err != nil || len(retiring) != 1 { - t.Fatalf("uncertain completion lost retirement: %v %v", retiring, err) - } - m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) - if reconcileErr := m2.Reconcile(t.Context()); reconcileErr != nil { - t.Fatal(reconcileErr) - } - if got := volumeHoldersOf(m2, "data"); got != (volumeHolders{}) { - t.Fatalf("gone VM retained hold: %+v", got) - } - _, retiring, err = m2.store.loadJournal() - if err != nil || len(retiring) != 0 { - t.Fatalf("gone VM retained retirement: %v %v", retiring, err) - } -} - -func TestRetirementRetriesFailedDirtyMarkerRemoval(t *testing.T) { - eng := newFakeEngine() - catalog := liveCatalog(t) - m := newVolumeManager(t, eng, catalog) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: "rw"}}}) - if err != nil { - t.Fatal(err) - } - marker := volumeDirtyPath(catalog[0].Path) - if err := os.Remove(marker); err != nil { - t.Fatal(err) - } - if err := os.Mkdir(marker, 0o700); err != nil { - t.Fatal(err) - } - blocker := filepath.Join(marker, "busy") - if err := os.WriteFile(blocker, nil, 0o600); err != nil { - t.Fatal(err) - } - if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { - t.Fatal("release accepted failed marker removal") - } - if got := volumeHoldersOf(m, "data"); got.writers != 1 { - t.Fatalf("marker failure released hold: %+v", got) - } - if err := os.Remove(blocker); err != nil { - t.Fatal(err) - } - m.retryRemovals(t.Context()).Wait() - if volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { - t.Fatal("marker retry did not finish retirement") - } -} - -func TestClaimsJournalRejectsUnsupportedAndConflictingRecords(t *testing.T) { - for _, raw := range []string{ - `{"format_version":3,"claims":{},"volume_retirements":{}}`, - `{"format_version":2,"claims":{"sb_a":{"id":"sb_a","vm_name":"sbx-a"}},"volume_retirements":{"sbx-a":{"sandbox_id":"sb_a","vm_name":"sbx-a","holds":[{"name":"data"}]}}}`, - `{"format_version":2,"claims":{}}`, - } { - store := newClaimStore(t.TempDir(), false) - if err := os.WriteFile(store.path, []byte(raw), 0o600); err != nil { - t.Fatal(err) - } - if _, err := store.load(); err == nil { - t.Fatalf("accepted invalid journal: %s", raw) - } - } - store := newClaimStore(t.TempDir(), false) - if err := store.save(map[string]*types.Sandbox{}); err != nil { - t.Fatal(err) - } - raw, err := os.ReadFile(store.path) - if err != nil { - t.Fatal(err) - } - var legacy map[string]*types.Sandbox - if err := json.Unmarshal(raw, &legacy); err == nil { - t.Fatal("legacy reader silently accepted versioned journal") - } -} - -func TestUnknownOrphanFencesNewAdmissionAndHeldRepair(t *testing.T) { - eng := newFakeEngine() - catalog := liveCatalog(t) - m := newVolumeManager(t, eng, catalog) - sb := mustClaim(t, m, testKey) - eng.mountVolumeErr = errors.New("interrupted mount") - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err == nil { - t.Fatal("expected pending volume") - } - eng.mountVolumeErr = nil - const orphan = "sbx-unknown" - eng.vms[orphan] = "/vsock/unknown" - eng.removeErrFor = orphan - m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) - if reconcileErr := m2.Reconcile(t.Context()); reconcileErr != nil { - t.Fatal(reconcileErr) - } - before := len(eng.volumeSpecs) - if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("held repair bypassed orphan fence: %v", err) - } - if len(eng.volumeSpecs) != before { - t.Fatal("held repair reached disk attach") - } - if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("new claim bypassed orphan fence: %v", err) - } - if _, err := m2.GetVolumes(sb.ID, sb.Tenant); err != nil { - t.Fatal(err) - } - if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err != nil { - t.Fatalf("cleanup blocked by orphan fence: %v", err) - } - eng.removeErrFor = "" - m2.retryRemovals(t.Context()).Wait() - if _, err := m2.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other"}}); err != nil { - t.Fatalf("confirmed orphan removal did not open admission: %v", err) - } -} - -func TestReconcileRejectsUncommittedVolumeQuarantine(t *testing.T) { - eng := newFakeEngine() - m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - sb := &types.Sandbox{ID: "sb_quarantine", VMName: "sbx-quarantine", Key: egKey, Volumes: []types.Volume{{Name: "data", Mount: "/data"}}, VolumesWant: []types.Volume{{Name: "data", Mount: "/data"}}} - eng.vms[sb.VMName] = "/run/quarantine.sock" - eng.snapshots = []string{forkPrefix + "orphan"} - if err := m.store.save(map[string]*types.Sandbox{sb.ID: sb}); err != nil { - t.Fatal(err) - } - m.eng = &quarantineWriteFailureEngine{fakeEngine: eng, path: m.store.path} - withOutbound(t, m, func(o *outbound.Options) { o.Sweep = func(map[string]bool) error { return nil } }) - t.Cleanup(func() { - m.store.writeMu.Lock() - err := os.Remove(m.store.path + ".tmp") - m.store.writeMu.Unlock() - if err != nil { - t.Error(err) - } - waitFor(t, func() bool { return !m.recommitting.Load() }) - }) - if err := m.Reconcile(t.Context()); err == nil { - t.Error("startup accepted a claim whose quarantine could not be persisted") - } - if eng.removed(sb.VMName) { - t.Error("quarantine removed VM before its retirement was durable") - } - if m.claimed[sb.ID] == nil || volumeHoldersOf(m, "data").readers != 1 { - t.Error("failed quarantine lost live claim or hold") - } -} - -type retirementWriteFailureEngine struct { - *fakeEngine - afterRemove func() -} - -func (e *retirementWriteFailureEngine) Remove(ctx context.Context, name string) error { - err := e.fakeEngine.Remove(ctx, name) - if err == nil && e.afterRemove != nil { - e.afterRemove() - e.afterRemove = nil - } - return err -} - -type quarantineWriteFailureEngine struct { - *fakeEngine - path string -} - -func (e *quarantineWriteFailureEngine) SnapshotRemove(ctx context.Context, name string) error { - if err := os.Mkdir(e.path+".tmp", 0o700); err != nil { - return err - } - return e.fakeEngine.SnapshotRemove(ctx, name) -} From de5f33b0191c7ac613b1422b469c2b901ae6230a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:42:40 +0800 Subject: [PATCH 21/57] pool: persist volumes_want only when it differs from volumes An absent field reads back as equal to Volumes, so a claims commit and a list row cost the same as before live volumes, a legacy journal loads without a second decode, and an older daemon still parses the file. The claim path stores one slice for both sets instead of cloning it. --- sandboxd/pool/claims.go | 27 ++++++++++++++------------- sandboxd/pool/claims_test.go | 2 +- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume.go | 3 +-- sandboxd/pool/volume_mutation_test.go | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 00211fb3..21cb4913 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -1,7 +1,6 @@ package pool import ( - "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -40,7 +39,7 @@ type claimDTO struct { Metadata types.Metadata `json:"metadata,omitempty"` OnExpire types.ExpireAction `json:"on_expire,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` - VolumesWant []types.Volume `json:"volumes_want"` + VolumesWant *[]types.Volume `json:"volumes_want,omitzero"` Env types.Env `json:"env,omitempty"` VsockSocket string `json:"vsock_socket,omitempty"` TAP string `json:"tap,omitempty"` @@ -90,16 +89,9 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { if err := json.Unmarshal(raw, &claims); err != nil { return nil, fmt.Errorf("parse claims: %w", err) } - var fields map[string]map[string]jsontext.Value - if err := json.Unmarshal(raw, &fields); err != nil { - return nil, err - } - for id, sb := range claims { - if _, present := fields[id]["volumes_want"]; !present { - sb.VolumesWant = slices.Clone(sb.Volumes) - } - if len(sb.VolumesWant) == 0 { - sb.VolumesWant = nil + for _, sb := range claims { + if sb.VolumesWant == nil { + sb.VolumesWant = sb.Volumes } } return claims, nil @@ -234,9 +226,18 @@ func dtoOf(sb *types.Sandbox) claimDTO { Deadline: sb.Deadline, ClaimedAt: sb.ClaimedAt, LeaseSeconds: sb.LeaseSeconds, Layer: sb.Layer, PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, - Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), Env: sb.Env, VsockSocket: sb.VsockSocket, + Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, } } + +// volumesWantDTO persists the desired set only when it differs; absent reads back as equal to Volumes. +func volumesWantDTO(sb *types.Sandbox) *[]types.Volume { + if slices.Equal(sb.VolumesWant, sb.Volumes) { + return nil + } + want := slices.Clone(sb.VolumesWant) + return &want +} diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 3281eecf..28c0e36f 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -78,7 +78,7 @@ func TestRecommitDrainsAChangeItWasTooBusyToSee(t *testing.T) { func TestStoreRoundTrip(t *testing.T) { claims := map[string]*types.Sandbox{ - "sb_a": {ID: "sb_a", VMName: "sbx-1", Key: testKey, Token: "t1", Deadline: time.Now().Add(time.Minute).UTC(), ClaimedAt: time.Now().Add(-time.Minute).UTC(), ClaimRef: "ns/workload", Metadata: types.Metadata{"team": "a"}, OnExpire: types.ExpireArchive, Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VsockSocket: "/v/1"}, + "sb_a": {ID: "sb_a", VMName: "sbx-1", Key: testKey, Token: "t1", Deadline: time.Now().Add(time.Minute).UTC(), ClaimedAt: time.Now().Add(-time.Minute).UTC(), ClaimRef: "ns/workload", Metadata: types.Metadata{"team": "a"}, OnExpire: types.ExpireArchive, Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VolumesWant: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VsockSocket: "/v/1"}, "sb_b": {ID: "sb_b", VMName: "sbx-2", Key: testKey, Token: "t2"}, } for _, sync := range []bool{false, true} { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index ec46fc06..fd3d0c81 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -163,7 +163,7 @@ type SandboxSummary struct { CPUCount int `json:"cpu_count,omitzero"` MemTotalBytes int64 `json:"mem_total_bytes,omitzero"` Volumes []types.Volume `json:"volumes,omitempty"` - VolumesWant []types.Volume `json:"volumes_want"` + VolumesWant []types.Volume `json:"volumes_want,omitempty"` Hibernated bool `json:"hibernated"` Archived bool `json:"archived,omitzero"` FromCheckpoint string `json:"from_checkpoint,omitempty"` diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 080d3ffd..6958b15f 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -191,8 +191,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ if err := group.Wait(); err != nil { return err } - sb.Volumes = applied - sb.VolumesWant = slices.Clone(applied) + sb.Volumes, sb.VolumesWant = applied, applied return nil } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 371e353b..80f4e635 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -422,8 +422,8 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.Contains(string(rawBytes), `"volumes_want":`) { - t.Fatalf("target presence lost: %s", rawBytes) + if strings.Contains(string(rawBytes), `"volumes_want":`) != (tt.field != "") { + t.Fatalf("persisted target %s, want field only when it differs", rawBytes) } }) } From b28ec8e0366281c302994de93e2e7e958f1a4186 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:45:52 +0800 Subject: [PATCH 22/57] engine: keep the claim path free of live-volume checks A claim mounts onto a fresh clone, so MountVolume goes back to its two guest calls; the idempotent mountinfo check moves to MountLiveVolume, used only by live repair, and the post-mount re-read goes. The live floor gets its own LiveVolumeCocoon so the startup warning stays at v0.6.9 for nodes that never change volumes live, dev builds are treated as current as at startup, and the cocoon version is queried once instead of on every PUT/PATCH (one exec costs about 26 ms on a 384-core host). --- docs/deploy.md | 11 ++++--- docs/sandboxd-api.md | 6 ++-- e2e/fakeengine_test.go | 4 +++ sandboxd/engine/engine.go | 24 ++++++++++----- sandboxd/engine/version_test.go | 37 +++++++++++++++-------- sandboxd/engine/volume.go | 40 +++++++++++-------------- sandboxd/engine/volume_live.go | 36 +++++++++++++--------- sandboxd/engine/volume_live_test.go | 4 +-- sandboxd/engine/volume_test.go | 13 ++------ sandboxd/pool/pool.go | 1 + sandboxd/pool/pool_test.go | 4 +++ sandboxd/pool/volume.go | 10 +++++-- sandboxd/pool/volume_mutation.go | 2 +- sandboxd/server/volume_mutation_test.go | 2 +- 14 files changed, 113 insertions(+), 81 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 6b0172f1..85280ef7 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -6,14 +6,15 @@ the cocoon CLI and needs a template image with silkd baked in. ## Prerequisites - Linux with KVM (`/dev/kvm`) -- [cocoon](https://github.com/cocoonstack/cocoon) **v0.7.2 or newer** installed +- [cocoon](https://github.com/cocoonstack/cocoon) **v0.6.9 or newer** installed and working (`cocoon vm run` boots a Cloud Hypervisor VM). v0.5.2 brought the disk hot-attach used by read-only volumes and the parallel-clone and snapshot/store performance work the [performance](performance.md) numbers assume; v0.6.9 prints `[]` for an empty `snapshot list --format json`, which sandboxd's snapshot reconcile reads. sandboxd logs a warning at startup when the detected cocoon is below - v0.7.2. Live volume mutation also rejects unknown and development builds + v0.6.9 (a dev/`master-` build is assumed current); changing volumes on + a running claim needs v0.7.2 - The sandbox boot artifact installed where cocoon finds it (`/boot/vmlinuz-sandbox`, `/boot/initrd.img-sandbox` — from `ghcr.io/cocoonstack/sandbox/boot:`) @@ -160,8 +161,10 @@ Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes require Cocoon v0.7.2 or newer for idempotent disk attach and detach. PUT and -PATCH reject older, unknown, development, or unqueryable versions with 503 -before any intent, hold, or device change. GET and release remain available. +PATCH reject an older release, or a runtime whose version cannot be queried, +with 503 before any intent, hold, or device change; a dev build is assumed +current. sandboxd reads the version once, so restart it after upgrading cocoon. +GET and release remain available. The claim journal stores desired `VolumesWant` and confirmed applied `Volumes`. Admission holds cover their union across daemon restart and teardown. Retry diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ec56a413..26dce20e 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -360,9 +360,9 @@ are atomic. Host-crash durability follows `sync_claims`. An older daemon ignores `VolumesWant`; finish pending volume changes before downgrading. Live convergence requires Cocoon v0.7.2 or newer for idempotent matching -attach and absent detach. PUT and PATCH reject older, unknown, development, -or unqueryable runtime versions with 503 before changing intent, holds, or -devices. GET and release remain available. +attach and absent detach. PUT and PATCH reject an older release, or a runtime +whose version cannot be queried, with 503 before changing intent, holds, or +devices; a dev build is assumed current. GET and release remain available. Catalog names must keep the same backing image while held, including pending holds. Publish changes under versioned names such as `dataset-v1` and diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 75de75f8..b266b9d7 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -165,6 +165,10 @@ func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bo return nil } +func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount string, rw bool) error { + return f.MountVolume(ctx, sock, name, mount, rw) +} + func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index d10b38e3..a2a614a8 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -15,6 +15,7 @@ import ( "os/exec" "strconv" "strings" + "sync/atomic" "time" "github.com/projecteru2/core/log" @@ -26,8 +27,10 @@ import ( ) const ( - // RequiredCocoon includes idempotent live disk attach and detach. - RequiredCocoon = "v0.7.2" + // RequiredCocoon carries the snapshot/store baseline and the JSON form of an empty snapshot list. + RequiredCocoon = "v0.6.9" + // LiveVolumeCocoon adds idempotent disk attach and detach for live volume changes. + LiveVolumeCocoon = "v0.7.2" StaleCreateCollected StaleCreateOutcome = "collected" StaleCreateBusy StaleCreateOutcome = "busy" @@ -72,6 +75,8 @@ type Engine struct { noBalloon bool restoreMode types.RestoreMode dumps chan struct{} + // version caches the first successful query; replacing cocoon takes a restart. + version atomic.Pointer[string] } // New returns a cocoon engine with node-wide network and disk policy. @@ -81,6 +86,9 @@ func New(bin string, bridges, networks []string, noDirectIO, noBalloon bool, res // Version reports cocoon's version string: a "vX.Y.Z" release or a "master-" dev build. func (e *Engine) Version(ctx context.Context) (string, error) { + if v := e.version.Load(); v != nil { + return *v, nil + } ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() out, err := e.run(ctx, "version") @@ -89,7 +97,9 @@ func (e *Engine) Version(ctx context.Context) (string, error) { } for line := range strings.SplitSeq(string(out), "\n") { if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Version:"); ok { - return strings.TrimSpace(v), nil + v = strings.TrimSpace(v) + e.version.Store(&v) + return v, nil } } return "", fmt.Errorf("cocoon version: no Version line in output") @@ -101,7 +111,7 @@ func (e *Engine) VersionWarning(ctx context.Context) (version, warning string) { if err != nil { return "", fmt.Sprintf("cannot determine cocoon version (need >= %s): %v", RequiredCocoon, err) } - if below, known := belowFloor(v); known && below { + if below, known := belowFloor(v, RequiredCocoon); known && below { return v, fmt.Sprintf("cocoon %s is below the required %s — upgrade cocoon", v, RequiredCocoon) } return v, "" @@ -493,13 +503,13 @@ func respFail(resp wire.Response) string { return "unexpected frame " + resp.RespType() } -// belowFloor reports whether v is below RequiredCocoon; known is false for a dev build. -func belowFloor(v string) (below, known bool) { +// belowFloor reports whether v is below floor; known is false for a dev build. +func belowFloor(v, floor string) (below, known bool) { v = "v" + strings.TrimPrefix(v, "v") if semver.Canonical(v) != v { return false, false } - return semver.Compare(v, RequiredCocoon) < 0, true + return semver.Compare(v, floor) < 0, true } func readBufLine(r *bufio.Reader, limit int) (string, error) { diff --git a/sandboxd/engine/version_test.go b/sandboxd/engine/version_test.go index 2345277e..dcfcbaac 100644 --- a/sandboxd/engine/version_test.go +++ b/sandboxd/engine/version_test.go @@ -13,9 +13,9 @@ func TestBelowFloor(t *testing.T) { below bool comparable bool }{ - {"v0.6.9", true, true}, - {"v0.6.10", true, true}, - {"v0.7.0", true, true}, + {"v0.6.9", false, true}, + {"v0.6.10", false, true}, + {"v0.7.0", false, true}, {"v0.6.8", true, true}, {"v0.5.2", true, true}, {"v0.4.9", true, true}, @@ -27,7 +27,7 @@ func TestBelowFloor(t *testing.T) { } for _, tt := range tests { t.Run(tt.v, func(t *testing.T) { - below, comparable := belowFloor(tt.v) + below, comparable := belowFloor(tt.v, RequiredCocoon) if below != tt.below || comparable != tt.comparable { t.Errorf("belowFloor(%q) = (%v, %v), want (%v, %v)", tt.v, below, comparable, tt.below, tt.comparable) } @@ -41,21 +41,17 @@ func TestCheckLiveVolumeSupport(t *testing.T) { fail, allowed bool }{ {"v0.7.1", false, false}, + {"v0.7.2-rc.1", false, false}, {"v0.7.2", false, true}, {"v0.8.0", false, true}, - {"v1.0.0", false, true}, - {"v0.7.2-rc.1", false, false}, - {"master-abcdef", false, false}, - {"unknown", false, false}, - {"", false, false}, + {"master-abcdef", false, true}, {"v0.7.2", true, false}, } { t.Run(fmt.Sprintf("%s/fail=%v", tt.version, tt.fail), func(t *testing.T) { t.Setenv("COCOON_TEST_VERSION", tt.version) + t.Setenv("COCOON_TEST_VERSION_FAIL", "") if tt.fail { t.Setenv("COCOON_TEST_VERSION_FAIL", "1") - } else { - t.Setenv("COCOON_TEST_VERSION_FAIL", "") } fakeCocoon(t, `#!/bin/sh if [ "$#" != 1 ] || [ "$1" != version ]; then exit 2; fi @@ -69,10 +65,25 @@ printf 'Version: %s\n' "$COCOON_TEST_VERSION" } return } - var unsupported *LiveVolumeUnsupported - if !errors.As(err, &unsupported) || !strings.Contains(err.Error(), RequiredCocoon) || strings.Contains(err.Error(), "/private") { + if _, ok := errors.AsType[*LiveVolumeUnsupported](err); !ok || !strings.Contains(err.Error(), LiveVolumeCocoon) || strings.Contains(err.Error(), "/private") { t.Fatalf("error %v", err) } }) } } + +func TestVersionQueriesCocoonOnce(t *testing.T) { + t.Setenv("COCOON_TEST_VERSION_FAIL", "") + fakeCocoon(t, `#!/bin/sh +if [ -n "$COCOON_TEST_VERSION_FAIL" ]; then exit 1; fi +printf 'Version: v0.7.2\n' +`) + eng := New("cocoon", nil, nil, false, false, "") + if _, err := eng.Version(t.Context()); err != nil { + t.Fatal(err) + } + t.Setenv("COCOON_TEST_VERSION_FAIL", "1") + if v, err := eng.Version(t.Context()); err != nil || v != "v0.7.2" { + t.Fatalf("cached version %q, %v", v, err) + } +} diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index ec08dc7b..f13e9cd4 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -15,6 +15,8 @@ import ( const ( // VolumeCallTimeout bounds one guest teardown exec. VolumeCallTimeout = 2 * time.Second + // VolumeMutationTimeout bounds one live attach or detach, guest steps included. + VolumeMutationTimeout = 30 * time.Second volumePollInterval = 10 * time.Millisecond volumeProbeTimeout = 2 * time.Second @@ -55,28 +57,7 @@ func (e *Engine) MountVolume(ctx context.Context, vsockSocket, name, mount strin if err != nil { return fmt.Errorf("wait for volume device %s: %w", name, err) } - mounted, err := e.volumeMounted(ctx, vsockSocket, device, mount, rw) - if err != nil || mounted { - return err - } - if err = e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil { - return fmt.Errorf("create volume mount point %s: %w", mount, err) - } - mode := types.VolumeModeRO - if rw { - mode = types.VolumeModeRW - } - if err = e.silkdExec(ctx, vsockSocket, "mount", "-o", mode, "--", device, mount); err != nil { - return fmt.Errorf("mount volume %s: %w", name, err) - } - mounted, err = e.volumeMounted(ctx, vsockSocket, device, mount, rw) - if err != nil { - return err - } - if !mounted { - return fmt.Errorf("volume %q did not appear at its mount point", name) - } - return nil + return e.mountDevice(ctx, vsockSocket, device, name, mount, rw) } // UnmountVolume flushes and detaches a guest mount before the VM is removed. @@ -149,6 +130,20 @@ func (e *Engine) findVolumeDevice(ctx context.Context, vsockSocket, name string) return "", false, nil } +func (e *Engine) mountDevice(ctx context.Context, vsockSocket, device, name, mount string, rw bool) error { + if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil { + return fmt.Errorf("create volume mount point %s: %w", mount, err) + } + mode := types.VolumeModeRO + if rw { + mode = types.VolumeModeRW + } + if err := e.silkdExec(ctx, vsockSocket, "mount", "-o", mode, "--", device, mount); err != nil { + return fmt.Errorf("mount volume %s: %w", name, err) + } + return nil +} + func (e *Engine) diskAttachArgs(vmName string, spec VolumeSpec) ([]string, error) { directIO, err := spec.directIO() if err != nil { @@ -169,3 +164,4 @@ func isNotFound(err error) bool { respErr, ok := errors.AsType[*wire.ErrorResp](err) return ok && respErr.Kind == wire.KindNotFound } + diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 0c05ddaa..945e1bd8 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -7,10 +7,9 @@ import ( "path/filepath" "slices" "strings" - "time" -) -const VolumeMutationTimeout = 30 * time.Second + "github.com/projecteru2/core/log" +) // VolumeMountConflictKind identifies the guest condition that prevents a volume change. type VolumeMountConflictKind string @@ -41,34 +40,43 @@ func (e *VolumeMountConflict) Error() string { // LiveVolumeUnsupported reports a runtime that cannot safely change live volumes. type LiveVolumeUnsupported struct { Version string - Err error } func (e *LiveVolumeUnsupported) Error() string { if e.Version != "" { - return fmt.Sprintf("live volume changes require Cocoon >= %s; installed version is %s", RequiredCocoon, e.Version) + return fmt.Sprintf("live volume changes require Cocoon >= %s; installed version is %s", LiveVolumeCocoon, e.Version) } - return "live volume changes require a verified Cocoon release >= " + RequiredCocoon + "; runtime version unavailable or unknown" + return "live volume changes require Cocoon >= " + LiveVolumeCocoon + "; runtime version unavailable" } -func (e *LiveVolumeUnsupported) Unwrap() error { return e.Err } - -// CheckLiveVolumeSupport rejects runtimes without confirmed live disk semantics. +// CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. func (e *Engine) CheckLiveVolumeSupport(ctx context.Context) error { version, err := e.Version(ctx) if err != nil { - return &LiveVolumeUnsupported{Err: err} - } - below, known := belowFloor(version) - if !known { + log.WithFunc("engine.CheckLiveVolumeSupport").Warnf(ctx, "query cocoon version: %v", err) return &LiveVolumeUnsupported{} } - if below { + if below, known := belowFloor(version, LiveVolumeCocoon); known && below { return &LiveVolumeUnsupported{Version: version} } return nil } +// MountLiveVolume mounts an attached disk unless the same mount already exists. +func (e *Engine) MountLiveVolume(ctx context.Context, sock, name, mount string, rw bool) error { + ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout) + defer cancel() + device, err := e.waitForVolumeDevice(ctx, sock, name) + if err != nil { + return fmt.Errorf("wait for volume device %s: %w", name, err) + } + mounted, err := e.volumeMounted(ctx, sock, device, mount, rw) + if err != nil || mounted { + return err + } + return e.mountDevice(ctx, sock, device, name, mount, rw) +} + // DetachVolume unmounts a guest volume before confirming host removal. func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount string) error { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index ad3028fe..5a8ce13c 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -35,7 +35,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { t.Fatal(err) } - if err := e.MountVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { t.Fatal(err) } } @@ -69,7 +69,7 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + row) fake.mu.Unlock() - err := New("cocoon", nil, nil, false, false, "").MountVolume(t.Context(), sock, spec.Name, "/data", false) + err := New("cocoon", nil, nil, false, false, "").MountLiveVolume(t.Context(), sock, spec.Name, "/data", false) if err == nil { t.Fatal("accepted a foreign mount") } diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index f2e1406a..f9825acc 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -3,7 +3,6 @@ package engine import ( "context" "errors" - "fmt" "slices" "strings" "testing" @@ -87,7 +86,6 @@ func TestMountVolumeUsesSysfsAndRequestedMode(t *testing.T) { "/sys/block/vda/serial", "/sys/block/vdb/serial", "/sys/block/vdc/serial", - "/sys/block/vdc/dev", "/proc/self/mountinfo", "/sys/block/vdc/dev", "/proc/self/mountinfo", } if fake.listCalls != 1 || !slices.Equal(fake.readCalls, wantReads) { t.Errorf("sysfs calls = list:%d read:%v, want list:1 read:%v", fake.listCalls, fake.readCalls, wantReads) @@ -161,8 +159,8 @@ func TestMountVolumeWaitsForDelayedSysfsSerial(t *testing.T) { if fake.listCalls != 2 { t.Errorf("fs_list calls = %d, want 2", fake.listCalls) } - if !slices.Contains(fake.readCalls, "/sys/block/vdc/serial") { - t.Error("target serial was not read") + if got := fake.readCalls[len(fake.readCalls)-1]; got != "/sys/block/vdc/serial" { + t.Errorf("last fs_read = %q, want target serial", got) } } @@ -283,11 +281,4 @@ func configureVolumeDevices(f *fakeSilkd) { f.serial["/sys/block/vda/serial"] = []byte("root\n") f.serial["/sys/block/vdc/serial"] = []byte("imagenet\n") f.stat["/dev/vdc"] = true - f.serial["/sys/block/vdc/dev"] = []byte("8:32\n") - f.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - f.execHook = func(argv []string) { - if argv[0] == "mount" { - f.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + fmt.Sprintf("2 1 8:32 / %s %s - ext4 /dev/vdc %s\n", argv[len(argv)-1], argv[2], argv[2])) - } - } } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index fd3d0c81..e428df63 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -143,6 +143,7 @@ type Engine interface { DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error DetachVolume(ctx context.Context, vmName, sock, name, mount string) error MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error + MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error SyncGuest(ctx context.Context, vsockSocket string) error WriteInstanceMetadata(ctx context.Context, vsockSocket string, doc []byte) error diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 69714460..676a092e 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1698,6 +1698,10 @@ func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bo return f.mountVolumeErr } +func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount string, rw bool) error { + return f.MountVolume(ctx, sock, name, mount, rw) +} + func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("detach"); err != nil { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 6958b15f..dbc7e296 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -186,7 +186,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ } group, groupCtx := errgroup.WithContext(ctx) for _, volume := range volumes { - group.Go(func() error { return m.applyVolume(groupCtx, sb, volume) }) + group.Go(func() error { return m.applyVolume(groupCtx, sb, volume, false) }) } if err := group.Wait(); err != nil { return err @@ -196,7 +196,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ } // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. -func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume) error { +func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume, live bool) error { // attach-only mounts nothing, so there is no umount to verify and no marker. attachOnly := volume.applied.Mount == "" // write-ahead: the marker must be durable before any guest write can be. @@ -214,7 +214,11 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res if err := ctx.Err(); err != nil { return err } - if err := m.eng.MountVolume(ctx, sb.VsockSocket, volume.applied.Name, volume.applied.Mount, volume.disk.RW); err != nil { + mount := m.eng.MountVolume + if live { + mount = m.eng.MountLiveVolume + } + if err := mount(ctx, sb.VsockSocket, volume.applied.Name, volume.applied.Mount, volume.disk.RW); err != nil { return fmt.Errorf("setup volume %q: %w", volume.applied.Name, err) } return nil diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 908e548b..f30d713e 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -164,7 +164,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, resolveErr } applyCtx, cancel := context.WithTimeout(ctx, engine.VolumeMutationTimeout) - err = m.applyVolume(applyCtx, sb, repair) + err = m.applyVolume(applyCtx, sb, repair, true) cancel() if err != nil { return view, err diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 3e2dd22a..8f26af3c 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -95,7 +95,7 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { {"bind mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountBind, Mount: "/other"}, http.StatusConflict}, {"unmount refused", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/data"}, http.StatusConflict}, {"unsupported runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{Version: "v0.7.1"}, http.StatusServiceUnavailable}, - {"unknown runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{Err: errors.New("host /private/version failed")}, http.StatusServiceUnavailable}, + {"unknown runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{}, http.StatusServiceUnavailable}, {"timeout", "root", `{"volumes":[]}`, context.DeadlineExceeded, http.StatusGatewayTimeout}, {"cancel", "root", `{"volumes":[]}`, context.Canceled, http.StatusRequestTimeout}, {"engine", "root", `{"volumes":[]}`, errors.New("host /private/image failed"), http.StatusInternalServerError}, From 9f01594d349cebb068e22c7e7c442190ed7bd777 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:48:24 +0800 Subject: [PATCH 23/57] server: report the volume refusal itself, not a recorded retry hint recordVolumeError stored a top-level "retry the desired target" that the handler preferred over the mapped error, so ErrVolumeBusy and ErrVolumeCapture came back as 409 asking for a retry that cannot succeed. The pool keeps per-volume diagnostics only and drops its entry once no held volume has one; the handler always reports the refusal it mapped. An unknown sandbox answers with the plain 404 body instead of an empty view the SDKs would record as complete. --- docs/sandboxd-api.md | 8 +-- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume_convergence_test.go | 6 +-- sandboxd/pool/volume_mutation.go | 67 ++++++++---------------- sandboxd/server/volume_mutation.go | 28 +++++----- sandboxd/server/volume_mutation_test.go | 42 +++++++-------- 6 files changed, 64 insertions(+), 89 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 26dce20e..6a059fbe 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -276,8 +276,8 @@ All three arrays are present, including when empty. A state is `attached` when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed -convergence also includes a top-level `error` and may include an `error` on -the affected state. `converging` is true only while a request worker is running. +PUT or PATCH adds a top-level `error` naming the refusal, and the affected +state may carry its own `error`. `converging` is true only while a request worker is running. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an @@ -335,8 +335,8 @@ timeouts return 504 and cancellation returns 408. PUT and PATCH return the GET response shape on success. A convergence error returns a non-2xx status with the same view, so callers can inspect desired, -applied, and per-volume state. Validation and authentication errors may carry -only `error`. A failed request may already have persisted its target and +applied, and per-volume state. Validation, authentication and unknown-sandbox +errors carry only `error`. A failed request may already have persisted its target and completed some device changes. GET reads the last confirmed set, rather than probing the devices. Resolve guest mount references or runtime failures before retrying. Retry the same PUT to converge, or send a new diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index e428df63..007effc3 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -299,7 +299,7 @@ type Manager struct { // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]volumeDiagnostics + volumeErrors map[*types.Sandbox]map[string]string poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 5f1a8932..aeee2531 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -161,11 +161,11 @@ func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { t.Fatal(checkErr) } view, err = m.GetVolumes(sb.ID, sb.Tenant) - if err != nil || view.Converging || view.Error == "" || view.States[0].Error == "" { + if err != nil || view.Converging || view.States[0].Error == "" { t.Fatalf("completed worker diagnostics: %+v %v", view, err) } view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) - if err != nil || view.Error != "" || view.States[0].Error != "" || view.Converging { + if err != nil || view.States[0].Error != "" || view.Converging { t.Fatalf("successful retry retained errors: %+v %v", view, err) } conflict := &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} @@ -207,7 +207,7 @@ func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { t.Fatalf("invalid target: %v", checkErr) } after, err := m.GetVolumes(sb.ID, sb.Tenant) - if err != nil || after.Error != before.Error || !slices.Equal(after.States, before.States) || !slices.Equal(after.Volumes, before.Volumes) || !slices.Equal(after.VolumesWant, before.VolumesWant) { + if err != nil || !slices.Equal(after.States, before.States) || !slices.Equal(after.Volumes, before.Volumes) || !slices.Equal(after.VolumesWant, before.VolumesWant) { t.Fatalf("rejected request changed convergence view: before=%+v after=%+v err=%v", before, after, err) } }) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index f30d713e..5d0dba19 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -4,21 +4,15 @@ import ( "context" "errors" "fmt" + "maps" "slices" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) -var errVolumeTargetNotAccepted = errors.New("new volume target not accepted") - type volumeEdit func([]types.Volume) ([]types.Volume, error) -type volumeDiagnostics struct { - error string - volumes map[string]string -} - type volumeRun struct { cancel context.CancelFunc done chan struct{} @@ -72,27 +66,18 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if !ok || !tenantOwns(tenant, sb.Tenant) { return view, ErrUnknownSandbox } - if err = m.eng.CheckLiveVolumeSupport(ctx); err != nil { - view, _ = m.GetVolumes(id, tenant) - view.Error = "" - return view, err + var run *volumeRun + if err = m.eng.CheckLiveVolumeSupport(ctx); err == nil { + ctx, run, err = m.beginVolumeRun(ctx, sb) } - ctx, run, err := m.beginVolumeRun(ctx, sb) if err != nil { view, _ = m.GetVolumes(id, tenant) - view.Error = "" return view, err } failedName := "" - recordOutcome := false defer func() { - if recordOutcome { - m.recordVolumeError(sb, run, failedName, err) - } + m.recordVolumeError(sb, run, failedName, err) view = m.finishVolumeRun(sb, run) - if !recordOutcome { - view.Error = "" - } }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) @@ -121,14 +106,12 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - recordOutcome = true m.mu.Lock() uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) m.mu.Unlock() for _, v := range uncertain { failedName = v.Name if err = m.detachVolumeRun(ctx, sb, run, v); err != nil { - err = fmt.Errorf("%w: %w", errVolumeTargetNotAccepted, err) return view, err } } @@ -383,8 +366,6 @@ func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { view := types.SandboxVolumesResponse{Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, Converging: m.volumeRuns[sb] != nil} - diagnostic := m.volumeErrors[sb] - view.Error = diagnostic.error for _, v := range heldVolumes(sb) { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { @@ -392,7 +373,7 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { } else if volumeIndex(sb.Volumes, v.Name) < 0 { state = types.VolumePending } - message := diagnostic.volumes[v.Name] + message := m.volumeErrors[sb][v.Name] if message == "" && state != types.VolumeAttached && !view.Converging { message = "volume change interrupted; retry the desired target" } @@ -407,15 +388,8 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run { return } - if m.volumeErrors == nil { - m.volumeErrors = make(map[*types.Sandbox]volumeDiagnostics) - } - diagnostic := m.volumeErrors[sb] - diagnostic.error = "" - if diagnostic.volumes == nil { - diagnostic.volumes = make(map[string]string) - } - if err != nil { + errs := m.volumeErrors[sb] + if err != nil && name != "" { message := "volume change failed; retry the desired target" if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { message = "volume change interrupted; retry the desired target" @@ -423,27 +397,28 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri if conflict, ok := errors.AsType[*engine.VolumeMountConflict](err); ok { message = conflict.Error() } - diagnostic.error = message - if errors.Is(err, errVolumeTargetNotAccepted) { - diagnostic.error = errVolumeTargetNotAccepted.Error() + ": " + message - } - if name != "" { - diagnostic.volumes[name] = message + if errs == nil { + errs = map[string]string{} } + errs[name] = message } - for name := range diagnostic.volumes { - if volumeIndex(heldVolumes(sb), name) < 0 { - delete(diagnostic.volumes, name) - } + held := heldVolumes(sb) + maps.DeleteFunc(errs, func(name, _ string) bool { return volumeIndex(held, name) < 0 }) + if len(errs) == 0 { + delete(m.volumeErrors, sb) + return + } + if m.volumeErrors == nil { + m.volumeErrors = map[*types.Sandbox]map[string]string{} } - m.volumeErrors[sb] = diagnostic + m.volumeErrors[sb] = errs } func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name string) { m.mu.Lock() defer m.mu.Unlock() if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { - delete(m.volumeErrors[sb].volumes, name) + delete(m.volumeErrors[sb], name) } } diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index 11ac4f31..d22c162d 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -9,6 +9,7 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -36,29 +37,32 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { } view, err = s.mgr.PatchVolumes(r.Context(), id, tenant, req) } + if errors.Is(err, pool.ErrUnknownSandbox) { + writePoolErr(w, err) + return + } code := http.StatusOK if err != nil { - message := view.Error code, view.Error = http.StatusInternalServerError, "volume convergence failed; inspect the volume states before retrying" - for _, rule := range poolErrHTTP { - if errors.Is(err, rule.err) { - code, view.Error = rule.code, cmp.Or(rule.msg, err.Error()) - break - } - } - var conflict *engine.VolumeMountConflict - var unsupported *engine.LiveVolumeUnsupported + conflict, isConflict := errors.AsType[*engine.VolumeMountConflict](err) + unsupported, isUnsupported := errors.AsType[*engine.LiveVolumeUnsupported](err) switch { - case errors.As(err, &conflict): + case isConflict: code, view.Error = http.StatusConflict, conflict.Error() - case errors.As(err, &unsupported): + case isUnsupported: code, view.Error = http.StatusServiceUnavailable, unsupported.Error() case errors.Is(err, context.DeadlineExceeded): code, view.Error = http.StatusGatewayTimeout, "volume change timed out; inspect the volume states before retrying" case errors.Is(err, context.Canceled): code, view.Error = http.StatusRequestTimeout, "volume change canceled; inspect the volume states before retrying" + default: + for _, rule := range poolErrHTTP { + if errors.Is(err, rule.err) { + code, view.Error = rule.code, cmp.Or(rule.msg, err.Error()) + break + } + } } - view.Error = cmp.Or(message, view.Error) if code >= http.StatusInternalServerError { log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) } diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 8f26af3c..47773b6b 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json/v2" "errors" + "fmt" "io" "net/http" "slices" @@ -121,7 +122,7 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { t.Fatalf("status=%d body=%s", resp.StatusCode, body) } - if tt.err != nil { + if tt.err != nil && tt.code != http.StatusNotFound { var got types.SandboxVolumesResponse if err := json.Unmarshal(body, &got); err != nil || len(got.VolumesWant) != 1 || len(got.States) != 1 || got.Error == "" || got.States[0].Error != "volume operation failed; retry after resolving the runtime error" { t.Fatalf("lost partial state: %s %v", body, err) @@ -171,28 +172,23 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { } } -func TestLiveVolumeRoutesPreserveSafeTargetOutcome(t *testing.T) { - const message = "new volume target not accepted; cleanup of the previous pending volume failed" +func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { const cause = "cannot unmount /data; close guest references and retry" - mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { - return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: message}, &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/data"} - }} - ts := newTestServer(t, "root", mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(`{"volumes":[]}`)) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Authorization", "Bearer root") - resp, err := ts.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - var got types.SandboxVolumesResponse - if err := json.UnmarshalRead(resp.Body, &got); err != nil { - t.Fatal(err) - } - if resp.StatusCode != http.StatusConflict || got.Error != message || len(got.States) != 1 || got.States[0].Error != cause { - t.Fatalf("status=%d view=%+v", resp.StatusCode, got) + for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture} { + t.Run(refusal.Error(), func(t *testing.T) { + mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { + return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: "volume change failed; retry the desired target"}, fmt.Errorf("%w: volume %q", refusal, "data") + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", `{"volumes":[]}`) + defer resp.Body.Close() + var got types.SandboxVolumesResponse + if err := json.UnmarshalRead(resp.Body, &got); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusConflict || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { + t.Fatalf("status=%d view=%+v", resp.StatusCode, got) + } + }) } } From 2822b938b7e4b1c1a48a2691f4f042151357ce40 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:49:05 +0800 Subject: [PATCH 24/57] pool: drop a claim's volume diagnostics when its run is cancelled Release, reap and destroy all take the claim's volumes through cancelVolumeRun, but its diagnostics entry stayed keyed by the dropped sandbox for the life of the daemon whenever its last change had failed. --- sandboxd/pool/volume_convergence_test.go | 22 ++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 1 + 2 files changed, 23 insertions(+) diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index aeee2531..a90e94a3 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -180,6 +180,28 @@ func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { } } +func TestReleaseDropsVolumeDiagnostics(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { + t.Fatal("expected mount conflict") + } + eng.detachVolumeErr = nil + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.mu.Lock() + defer m.mu.Unlock() + if len(m.volumeErrors) != 0 { + t.Fatalf("released claim kept diagnostics: %v", m.volumeErrors) + } +} + func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { for _, priorConflict := range []bool{false, true} { name := "settled" diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 5d0dba19..8be40d45 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -242,6 +242,7 @@ func (m *Manager) checkVolumeRunLocked(ctx context.Context, sb *types.Sandbox, r } func (m *Manager) cancelVolumeRun(sb *types.Sandbox) []types.Volume { + delete(m.volumeErrors, sb) if run := m.volumeRuns[sb]; run != nil { run.cancel() return slices.Clone(run.holds) From 165d495692f0e6c568d67ceee62d136937318315 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:50:28 +0800 Subject: [PATCH 25/57] sdk/python: keep >= 400 as the failure test and reject a non-positive volume timeout The volume change widened the shared request check to every non-2xx status, which altered every other verb. timeout=0 silently fell back to the client default, unlike run(), which rejects it. --- sdk/python/cocoonsandbox/client.py | 2 +- sdk/python/cocoonsandbox/sandbox.py | 2 ++ sdk/python/tests/test_volumes.py | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 1095321d..cbf29602 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -212,7 +212,7 @@ def _request( if deadline is not None and time.monotonic() >= deadline: raise SandboxTimeout(f"{verb} timed out") from None raise APIError(verb, 0, str(exc)) from None - if status < 200 or status >= 300: + if status >= 400: try: reply = json.loads(raw) except (ValueError, TypeError): diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 2f416f33..c8dd0ed5 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -418,6 +418,8 @@ def close(self) -> None: def _volumes_request( self, method: str, body: dict[str, Any] | None, verb: str, timeout: float | None ) -> SandboxVolumesResponse: + if timeout is not None and timeout <= 0: + raise ValueError("timeout must be positive") try: reply = self._client._request( self.owner, diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 8f921be9..6ece1ea6 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -131,6 +131,13 @@ def test_volume_timeout_uses_client_deadline(black_hole): assert caught.value.status == 0 +@pytest.mark.parametrize("timeout", [0, -1]) +def test_volume_methods_reject_non_positive_timeout(black_hole, timeout): + sb = sandbox_at(black_hole) + with pytest.raises(ValueError): + sb.get_volumes(timeout=timeout) + + @pytest.mark.parametrize("status", [200, 409]) def test_incomplete_volume_view_preserves_handle(spawn_node, status): incomplete = {"volumes_want": [], "volumes": [], "states": []} From 5b2be301d00db17f2bc0598b196aac5bf01b4b11 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:51:07 +0800 Subject: [PATCH 26/57] e2e: keep the live volume leg behind -live and drop the fault-injection harness The regular volume leg ran the live steps unconditionally, so every e2e node needed cocoon v0.7.2; VOLUME_LIVE=1 opts in now. continuity.go, the Python fault hooks and their README served one contributor hardware report; the maintained hardware procedure covers those faults. --- docs/deploy.md | 7 +- e2e/cmd/volumesmoke/README.md | 35 -- e2e/cmd/volumesmoke/continuity.go | 391 ------------------ e2e/cmd/volumesmoke/main.go | 35 +- e2e/cmd/volumesmoke/testdata/fault_hook.py | 104 ----- .../volumesmoke/testdata/runtime_observer.py | 23 -- scripts/sandboxd-e2e.sh | 4 +- 7 files changed, 15 insertions(+), 584 deletions(-) delete mode 100644 e2e/cmd/volumesmoke/README.md delete mode 100644 e2e/cmd/volumesmoke/continuity.go delete mode 100755 e2e/cmd/volumesmoke/testdata/fault_hook.py delete mode 100755 e2e/cmd/volumesmoke/testdata/runtime_observer.py diff --git a/docs/deploy.md b/docs/deploy.md index 85280ef7..289812d4 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -177,10 +177,9 @@ create a new sandbox. Daemon restart recovery assumes the original VMM remains alive and does not recreate guest processes. Unfinished changes report an interrupted diagnostic after restart and require a caller retry. -`e2e/cmd/volumesmoke` exercises GET, repeated full-set PUT, PATCH add and remove, -and clearing a new claim. Its [hardware validation instructions](../e2e/cmd/volumesmoke/README.md) -cover same-path replacement, guest mount conflicts, daemon restart and -cancellation on an isolated node. +`e2e/cmd/volumesmoke -live` (`VOLUME_LIVE=1` for `scripts/sandboxd-e2e.sh`) +exercises GET, repeated full-set PUT, PATCH add and remove, and clearing a new +claim. Each catalog path must name a disk image containing a mountable whole-device filesystem. Keep catalog names bound to the same image while any claim holds diff --git a/e2e/cmd/volumesmoke/README.md b/e2e/cmd/volumesmoke/README.md deleted file mode 100644 index 7965d15d..00000000 --- a/e2e/cmd/volumesmoke/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Live volume hardware validation - -Run this command against a real Cloud Hypervisor node with Cocoon v0.7.2 or later. The guest needs silkd, mount, umount, awk, and ordinary coreutils. Fault hooks need Python 3 on the host. Register two catalog datasets with distinct nonempty contents at the same relative probe path. Keep their catalog backing paths unchanged throughout the run. - -Build from the repository's `e2e` directory. - -```sh -GOTOOLCHAIN=go1.27.1 GOWORK=off go build -o "$VOLUME_SMOKE_DIR/volumesmoke" ./cmd/volumesmoke -"$VOLUME_SMOKE_DIR/volumesmoke" -addr "$SANDBOX_ADDR" -token "$SANDBOX_API_TOKEN" \ - -template "$SANDBOX_TEMPLATE" -volume dataset-v1 -next-volume dataset-v2 -probe version -``` - -The command checks shared read-only mounts, EROFS, repeated updates, and removal. `-next-volume` adds a same-path replacement while preserving the sandbox ID, session ID, guest boot ID, PID, process start time, and session environment. It tests cwd, bind, and foreign-mount conflicts with HTTP 409, then retries after releasing the reference. `RETAINED begin/end` timestamps identify the repeated-target window. The runtime command log must contain zero disk attach or detach commands in that window. - -The optional fault hooks operate only on an isolated daemon. Use disposable backing files for `-release-hook`, which writes `.retirement-reuse` through a later writable mount. That catalog entry must allow writable attachment. The helper never prints the API token. - -1. Set `VOLUME_SMOKE_DIR` to an absolute scratch directory. Place the candidate `sandboxd` binary and its `config.json` there. The config must use a dedicated listen address and data directory. Its `cocoon_bin` must point to the absolute path of `testdata/runtime_observer.py`. -2. Set `COCOON_REAL_BIN` to the absolute path of the official Cocoon binary. Configure isolated `COCOON_ROOT_DIR`, `COCOON_RUN_DIR`, and `COCOON_LOG_DIR`, and a distinct network scope and cgroup parent. Preserve these environment variables for the daemon and smoke command. -3. Start the daemon as a child process and write its actual PID to `$VOLUME_SMOKE_DIR/daemon.pid`. The restart hook kills only this PID. The real VMM must keep running. -4. Build the existing guest listener with `CGO_ENABLED=0 GOTOOLCHAIN=go1.27.1 GOWORK=off go build -o "$VOLUME_SMOKE_DIR/guestserver" ./cmd/guestserver` from the `e2e` directory. Run the extended command with the three hooks. Set `VOLUME_HOOK` to the absolute path of `testdata/fault_hook.py`. - -```sh -"$VOLUME_SMOKE_DIR/volumesmoke" -addr "$SANDBOX_ADDR" -token "$SANDBOX_API_TOKEN" \ - -template "$SANDBOX_TEMPLATE" -volume dataset-v1 -next-volume dataset-v2 -probe version \ - -listener "$VOLUME_SMOKE_DIR/guestserver" \ - -change-hook "$VOLUME_HOOK" \ - -cancel-hook "$VOLUME_HOOK" \ - -release-hook "$VOLUME_HOOK" -``` - -The observer executes every ordinary command through the official Cocoon binary, including `version`. An attach gate pauses only after a successful real disk attach and before returning its result to sandboxd. The restart hook kills sandboxd at that gate. The cancellation hook cancels the caller at the same gate. Both save the API view and real VM PID and disks. Independent exec, file, and HTTP port operations must complete while the gate is active. Retrying the same desired target must converge. - -The retirement hook injects exit 73 before `vm rm` while its failure marker exists. It checks that a new writer is refused before and after a daemon restart. Removing the marker permits the ordinary cleanup worker to execute real Cocoon removal. Only successful admission, mounting, and writing by a later writer counts as backing reuse. Save `commands.jsonl`, `candidate-daemon.log`, smoke output, runtime versions, and the backing checksums outside the repository. - -The command runs the supplied absolute script path through `python3` with a fixed `restart`, `cancel`, or `retire` mode argument. It does not interpret a shell command. The hooks use `arm`, `wait`, and `interrupt` stages supplied in `VOLUME_HOOK_STAGE`, plus the sandbox ID in `SANDBOX_ID`. A failed run can leave fault markers behind. Stop the isolated daemon before removing its `gate-attach` or `fail-remove` marker and cleaning up its VMs. Do not apply these hooks to a shared production daemon. diff --git a/e2e/cmd/volumesmoke/continuity.go b/e2e/cmd/volumesmoke/continuity.go deleted file mode 100644 index 442dd517..00000000 --- a/e2e/cmd/volumesmoke/continuity.go +++ /dev/null @@ -1,391 +0,0 @@ -package main - -import ( - "context" - "encoding/json/v2" - "errors" - "fmt" - "os" - "os/exec" - "path" - "path/filepath" - "slices" - "strings" - "time" - - "github.com/cocoonstack/sandbox/e2e/internal/harness" - sandbox "github.com/cocoonstack/sandbox/sdk/go" -) - -type ( - liveOptions struct{ nextVolume, changeHook, cancelHook, releaseHook, listener string } - processIdentity struct { - Sandbox, Session, Boot string - PID uint32 - Start string - } -) - -func identity(ctx context.Context, sb *sandbox.Sandbox, session *sandbox.Session, pid uint32) (processIdentity, error) { - boot, err := sb.Exec(ctx, "cat", "/proc/sys/kernel/random/boot_id") - if err != nil { - return processIdentity{}, err - } - stat, err := sb.Exec(ctx, "cat", fmt.Sprintf("/proc/%d/stat", pid)) - if err != nil { - return processIdentity{}, err - } - _, rest, ok := strings.Cut(stat, ") ") - fields := strings.Fields(rest) - if !ok || len(fields) < 20 { - return processIdentity{}, fmt.Errorf("invalid process stat %q", stat) - } - return processIdentity{sb.ID, session.ID, strings.TrimSpace(boot), pid, fields[19]}, nil -} - -func runContinuity(ctx context.Context, sb *sandbox.Sandbox, volume, probe, want string, opts liveOptions) error { - const mount = "/datasets/current" - if opts.changeHook != "" || opts.cancelHook != "" { - if err := uploadListener(ctx, sb, opts.listener); err != nil { - return err - } - } - session, err := sb.NewSession(ctx, sandbox.WithSessionCwd("/tmp"), sandbox.WithSessionEnv(map[string]string{"DATASET_SESSION": "same"})) - if err != nil { - return err - } - defer func() { _ = session.Close(context.Background()) }() - pid, err := sb.Spawn(ctx, sandbox.Cmd{Argv: []string{"sleep", "3600"}, Cwd: "/tmp"}) - if err != nil { - return err - } - defer func() { _ = sb.Kill(context.Background(), pid, 0) }() - before, err := identity(ctx, sb, session, pid) - if err != nil { - return err - } - encoded, _ := json.Marshal(before) - fmt.Printf("CONTINUITY before=%s\n", encoded) - v1 := []sandbox.Volume{{Name: volume, Mount: mount}} - v2 := []sandbox.Volume{{Name: opts.nextVolume, Mount: mount}} - v2content, err := exerciseMountConflicts(ctx, sb, v1, v2, probe, want) - if err != nil { - return err - } - if opts.changeHook != "" { - if err = interruptedChange(ctx, sb, v1, opts.changeHook, false); err != nil { - return err - } - if _, err = sb.SetVolumes(ctx, v2); err != nil { - return err - } - } - if opts.cancelHook != "" { - if err = interruptedChange(ctx, sb, v1, opts.cancelHook, true); err != nil { - return err - } - } - view, err := sb.SetVolumes(ctx, v2) - if err != nil || !slices.Equal(view.Volumes, v2) || !slices.Equal(view.VolumesWant, v2) { - return fmt.Errorf("second dataset did not converge: view=%+v error=%v", view, err) - } - if content, readErr := sb.Exec(ctx, "cat", path.Join(mount, probe)); readErr != nil || content != v2content { - return fmt.Errorf("dataset changed after recovery: content=%q error=%v", content, readErr) - } - if err = requireSettled(view); err != nil { - return err - } - if err = sb.WriteFile(ctx, "/tmp/continuity", []byte("independent file"), nil); err != nil { - return err - } - data, err := sb.ReadFile(ctx, "/tmp/continuity") - if err != nil || string(data) != "independent file" { - return fmt.Errorf("independent file round trip: %q %v", data, err) - } - if _, err = sb.SetVolumes(ctx, []sandbox.Volume{}); err != nil { - return err - } - after, err := identity(ctx, sb, session, pid) - if err != nil || after != before { - return fmt.Errorf("process identity changed: before=%+v after=%+v error=%v", before, after, err) - } - sessions, err := sb.Sessions(ctx) - if err != nil || !slices.Contains(sessions, session.ID) { - return fmt.Errorf("session disappeared: %v %v", sessions, err) - } - state, err := session.Exec(ctx, "printenv", "DATASET_SESSION") - if err != nil || strings.TrimSpace(state) != "same" { - return fmt.Errorf("session state changed: %q %v", state, err) - } - encoded, _ = json.Marshal(after) - fmt.Printf("CONTINUITY PASS after=%s v1=%q v2=%q cwd_409=true bind_409=true foreign_409=true retained_bind=true erofs=true cleared=true\n", encoded, strings.TrimSpace(want), strings.TrimSpace(v2content)) - return nil -} - -func refuseBusy(ctx context.Context, sb *sandbox.Sandbox, target []sandbox.Volume, mount, reference string) error { - view, err := sb.SetVolumes(ctx, target) - api, ok := errors.AsType[*sandbox.APIError](err) - if !ok || api.Status != 409 || !strings.Contains(err.Error(), mount) { - return fmt.Errorf("%s busy removal must return 409 and guest path: view=%+v error=%v", reference, view, err) - } - encoded, _ := json.Marshal(view) - fmt.Printf("BUSY PASS reference=%s view=%s\n", reference, encoded) - return nil -} - -func interruptedChange(ctx context.Context, sb *sandbox.Sandbox, target []sandbox.Volume, hook string, cancelRequest bool) error { - mode := "restart" - if cancelRequest { - mode = "cancel" - } - if err := runHook(ctx, sb.ID, hook, mode, "arm"); err != nil { - return err - } - server, err := sb.Spawn(ctx, sandbox.Cmd{Argv: []string{"/tmp/volume-guestserver", "-addr", "127.0.0.1:18080"}, Cwd: "/tmp"}) - if err != nil { - return err - } - defer func() { _ = sb.Kill(context.Background(), server, 0) }() - for deadline := time.Now().Add(5 * time.Second); ; { - if _, err := harness.HTTPOverPort(ctx, sb, 18080, "GET", "/", nil); err == nil { - break - } - if time.Now().After(deadline) { - return errors.New("independent HTTP server did not start") - } - time.Sleep(20 * time.Millisecond) - } - requestCtx, cancel := context.WithCancel(ctx) - defer cancel() - done := make(chan error, 1) - go func() { _, err := sb.SetVolumes(requestCtx, target); done <- err }() - if err := runHook(ctx, sb.ID, hook, mode, "wait"); err != nil { - return err - } - if _, err := sb.Exec(ctx, "true"); err != nil { - return err - } - if err := sb.WriteFile(ctx, "/tmp/during-change", []byte("still running"), nil); err != nil { - return err - } - if content, readErr := sb.ReadFile(ctx, "/tmp/during-change"); readErr != nil || string(content) != "still running" { - return fmt.Errorf("file read during change: %q %v", content, readErr) - } - if _, err := harness.HTTPOverPort(ctx, sb, 18080, "GET", "/during-change", nil); err != nil { - return err - } - if cancelRequest { - cancel() - } - if err := runHook(ctx, sb.ID, hook, mode, "interrupt"); err != nil { - return err - } - fmt.Println("ACTIVITY PASS exec=true file=true port=true") - requestErr := <-done - fmt.Printf("INTERRUPTION request_error=%v\n", requestErr) - if requestErr == nil { - return errors.New("change hook did not interrupt the request") - } - deadline := time.Now().Add(30 * time.Second) - for { - view, err := sb.GetVolumes(ctx) - if err == nil { - encoded, _ := json.Marshal(view) - fmt.Printf("INTERRUPTION recovered_view=%s\n", encoded) - break - } - if time.Now().After(deadline) { - return fmt.Errorf("daemon did not recover: %w", err) - } - time.Sleep(100 * time.Millisecond) - } - if _, err := sb.SetVolumes(ctx, target); err != nil { - return fmt.Errorf("retry after interruption: %w", err) - } - fmt.Printf("INTERRUPTION PASS sandbox=%s\n", sb.ID) - return nil -} - -func runHook(ctx context.Context, id, hook, mode, stage string) error { - if !filepath.IsAbs(hook) || filepath.Clean(hook) != hook { - return errors.New("fault hook must be a clean absolute Python script path") - } - command := exec.CommandContext(ctx, "python3") - command.Args = append(command.Args, hook, mode) - command.Env = append(os.Environ(), "SANDBOX_ID="+id, "VOLUME_HOOK_STAGE="+stage) - command.Stdout, command.Stderr = os.Stdout, os.Stderr - if err := command.Run(); err != nil { - return fmt.Errorf("change hook %s: %w", stage, err) - } - return nil -} - -func requireSettled(view sandbox.SandboxVolumesResponse) error { - encoded, err := json.Marshal(view) - if err != nil { - return err - } - var progress struct { - Converging *bool `json:"converging"` - } - if err := json.Unmarshal(encoded, &progress); err != nil { - return err - } - if progress.Converging == nil || *progress.Converging { - return fmt.Errorf("settled view must explicitly report converging=false: %s", encoded) - } - return nil -} - -func exerciseMountConflicts(ctx context.Context, sb *sandbox.Sandbox, v1, v2 []sandbox.Volume, probe, want string) (string, error) { - const mount = "/datasets/current" - var err error - if _, err = sb.SetVolumes(ctx, v1); err != nil { - return "", err - } - busy, err := sb.Spawn(ctx, sandbox.Cmd{Argv: []string{"sleep", "3600"}, Cwd: mount}) - if err != nil { - return "", err - } - defer func() { _ = sb.Kill(context.Background(), busy, 0) }() - if err = refuseBusy(ctx, sb, v2, mount, "cwd"); err != nil { - return "", err - } - got, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) - if err != nil || got != want { - return "", fmt.Errorf("busy cwd changed old dataset: content=%q error=%v", got, err) - } - if err = sb.Kill(ctx, busy, 0); err != nil { - return "", err - } - if _, err = sb.SetVolumes(ctx, v2); err != nil { - return "", fmt.Errorf("cwd retry: %w", err) - } - if _, err = sb.SetVolumes(ctx, v1); err != nil { - return "", err - } - if _, err = sb.Exec(ctx, "sh", "-c", "mkdir -p /tmp/volume-bind; mount --bind /datasets/current /tmp/volume-bind"); err != nil { - return "", err - } - defer func() { _, _ = sb.Exec(context.Background(), "umount", "/tmp/volume-bind") }() - fmt.Printf("RETAINED begin=%d\n", time.Now().UnixNano()) - if _, err = sb.SetVolumes(ctx, v1); err != nil { - return "", fmt.Errorf("retained volume with bind was reapplied: %w", err) - } - fmt.Printf("RETAINED end=%d\n", time.Now().UnixNano()) - if err = refuseBusy(ctx, sb, v2, "/tmp/volume-bind", "bind"); err != nil { - return "", err - } - if got, err = sb.Exec(ctx, "cat", path.Join("/tmp/volume-bind", probe)); err != nil || got != want { - return "", fmt.Errorf("busy bind was changed: content=%q error=%v", got, err) - } - if _, err = sb.Exec(ctx, "umount", "/tmp/volume-bind"); err != nil { - return "", err - } - if _, err = sb.SetVolumes(ctx, v2); err != nil { - return "", fmt.Errorf("same-path retry: %w", err) - } - v2content, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) - if err != nil || v2content == "" || v2content == want { - return "", fmt.Errorf("second dataset marker must differ: content=%q error=%v", v2content, err) - } - if _, err = sb.Exec(ctx, "touch", path.Join(mount, ".write-probe")); err == nil { - return "", errors.New("second dataset accepted a write") - } - if exit, ok := errors.AsType[*sandbox.ExitError](err); !ok || !strings.Contains(strings.ToLower(exit.Stderr), "read-only file system") { - return "", fmt.Errorf("second dataset write did not fail with EROFS: %w", err) - } - if _, err = sb.Exec(ctx, "sh", "-c", "awk '$2 == \"/datasets/current\" && $4 ~ /(^|,)ro(,|$)/ { found=1; print } END { exit !found }' /proc/mounts"); err != nil { - return "", fmt.Errorf("guest mount is not read-only: %w", err) - } - if _, err = sb.Exec(ctx, "mount", "-t", "tmpfs", "tmpfs", mount); err != nil { - return "", err - } - if err = refuseBusy(ctx, sb, v1, mount, "foreign"); err != nil { - return "", err - } - if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { - return "", fmt.Errorf("foreign mount was removed: %w", err) - } - if _, err = sb.Exec(ctx, "umount", mount); err != nil { - return "", err - } - if _, err = sb.SetVolumes(ctx, v1); err != nil { - return "", err - } - if _, err = sb.SetVolumes(ctx, v2); err != nil { - return "", err - } - return v2content, nil -} - -func runRetirement(ctx context.Context, client *sandbox.Client, template, volume, hook string) error { - reader, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone), sandbox.WithVolumes(sandbox.Volume{Name: volume, Mount: "/datasets/retiring"})) - if err != nil { - return err - } - defer func() { _ = reader.Close() }() - if err := runHook(ctx, reader.ID, hook, "retire", "arm"); err != nil { - return err - } - fmt.Printf("RETIREMENT release_error=%v\n", reader.Close()) - writerOptions := []sandbox.Option{sandbox.WithNetwork(sandbox.NetNone), sandbox.WithVolumes(sandbox.Volume{Name: volume, Mount: "/datasets/reused", Mode: "rw"})} - refuseWriter := func() error { - writer, err := client.New(ctx, template, writerOptions...) - if err == nil { - _ = writer.Close() - return errors.New("writer admitted while retired VM still owns backing") - } - api, ok := errors.AsType[*sandbox.APIError](err) - if !ok || api.Status != 409 { - return fmt.Errorf("retired backing must return 409: %w", err) - } - fmt.Printf("RETIREMENT writer_refused=%v\n", err) - return nil - } - if err := refuseWriter(); err != nil { - return err - } - if err := runHook(ctx, reader.ID, hook, "retire", "wait"); err != nil { - return err - } - if err := refuseWriter(); err != nil { - return err - } - if err := runHook(ctx, reader.ID, hook, "retire", "interrupt"); err != nil { - return err - } - deadline := time.Now().Add(45 * time.Second) - for { - writer, err := client.New(ctx, template, writerOptions...) - if err == nil { - defer func() { _ = writer.Close() }() - if _, err = writer.Exec(ctx, "touch", "/datasets/reused/.retirement-reuse"); err != nil { - return err - } - if err = writer.Close(); err != nil { - return err - } - fmt.Println("RETIREMENT PASS writer_blocked_before_restart=true writer_blocked_after_restart=true backing_reused=true") - return nil - } - api, ok := errors.AsType[*sandbox.APIError](err) - if !ok || api.Status != 409 || time.Now().After(deadline) { - return fmt.Errorf("backing not reusable after cleanup: %w", err) - } - time.Sleep(200 * time.Millisecond) - } -} - -func uploadListener(ctx context.Context, sb *sandbox.Sandbox, listener string) error { - root, err := os.OpenRoot(filepath.Dir(listener)) - if err != nil { - return err - } - defer func() { _ = root.Close() }() - binary, err := root.ReadFile(filepath.Base(listener)) - if err != nil { - return err - } - return sb.WriteFile(ctx, "/tmp/volume-guestserver", binary, new(uint32(0o755))) -} diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index 2d3834c4..4465fdf6 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -1,5 +1,5 @@ // volumesmoke validates shared read-only dataset mounts, and optionally one -// writable dataset mount, on a live Cloud Hypervisor node. +// writable dataset mount and live volume changes, on a live Cloud Hypervisor node. package main import ( @@ -24,31 +24,24 @@ func main() { template := flag.String("template", "rt:24.04", "template ref") volume := flag.String("volume", "", "catalog volume name") rwVolume := flag.String("rw-volume", "", "writable catalog volume name (adds the writable leg)") - nextVolume := flag.String("next-volume", "", "second catalog dataset for the same-path continuity proof") - changeHook := flag.String("change-hook", "", "absolute Python hook path to interrupt a change; SANDBOX_ID is provided") - cancelHook := flag.String("cancel-hook", "", "absolute Python hook path to gate cancellation; SANDBOX_ID and VOLUME_HOOK_STAGE are provided") - releaseHook := flag.String("release-hook", "", "absolute Python hook path to test failed removal and recovery; uses VOLUME_HOOK_STAGE") - listener := flag.String("listener", "", "path to the existing guestserver binary for activity checks during faults") + live := flag.Bool("live", false, "add the live volume leg (needs cocoon v0.7.2)") probe := flag.String("probe", "volume-e2e.txt", "non-empty file inside the volume") flag.Parse() - if err := run(*addr, *token, *template, *volume, *rwVolume, *probe, liveOptions{nextVolume: *nextVolume, changeHook: *changeHook, cancelHook: *cancelHook, releaseHook: *releaseHook, listener: *listener}); err != nil { + if err := run(*addr, *token, *template, *volume, *rwVolume, *probe, *live); err != nil { fmt.Fprintln(os.Stderr, "volumesmoke:", err) os.Exit(1) } } -func run(addr, token, template, volume, rwVolume, probe string, live liveOptions) error { +func run(addr, token, template, volume, rwVolume, probe string, live bool) error { if volume == "" { return errors.New("volume is required") } - if (live.changeHook != "" || live.cancelHook != "") && live.listener == "" { - return errors.New("listener is required with change-hook or cancel-hook") - } if probe == "" || probe == "." || path.IsAbs(probe) || path.Clean(probe) != probe || probe == ".." || strings.HasPrefix(probe, "../") { return fmt.Errorf("probe %q must be a clean relative file path", probe) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() const ( @@ -126,11 +119,8 @@ func run(addr, token, template, volume, rwVolume, probe string, live liveOptions } released = true fmt.Printf("VOLUME PASS concurrent_read_bytes=%d read_only=true released=true\n", len(outputs[0])) - if err := runLive(ctx, client, template, volume, probe, outputs[0], live); err != nil { - return err - } - if live.releaseHook != "" { - if err := runRetirement(ctx, client, template, volume, live.releaseHook); err != nil { + if live { + if err := runLive(ctx, client, template, volume, probe, outputs[0]); err != nil { return err } } @@ -140,9 +130,7 @@ func run(addr, token, template, volume, rwVolume, probe string, live liveOptions return runWritable(ctx, client, template, rwVolume) } -// runLive starts without volumes, adds a dataset, then removes it without -// replacing the claim. Run this on the target VMM before deploying live mounts. -func runLive(ctx context.Context, client *sandbox.Client, template, volume, probe, want string, live liveOptions) error { +func runLive(ctx context.Context, client *sandbox.Client, template, volume, probe, want string) error { const mount = "/datasets/live" sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone)) if err != nil { @@ -159,7 +147,7 @@ func runLive(ctx context.Context, client *sandbox.Client, template, volume, prob return fmt.Errorf("live get volumes: %w", err) } wantVolumes := []sandbox.Volume{{Name: volume, Mount: mount}} - if !slices.Equal(view.VolumesWant, wantVolumes) || !slices.Equal(view.Volumes, wantVolumes) || len(view.States) != 1 || view.States[0].State != "attached" { + if !slices.Equal(view.VolumesWant, wantVolumes) || !slices.Equal(view.Volumes, wantVolumes) || len(view.States) != 1 || view.States[0].State != sandbox.VolumeAttached { return fmt.Errorf("live volume view not converged: %+v", view) } got, err := sb.Exec(ctx, "cat", path.Join(mount, probe)) @@ -206,11 +194,6 @@ func runLive(ctx context.Context, client *sandbox.Client, template, volume, prob if _, err = sb.Exec(ctx, "test", "!", "-e", path.Join(mount, probe)); err != nil { return fmt.Errorf("cleared volume probe should be absent: %w", err) } - if live.nextVolume != "" { - if err := runContinuity(ctx, sb, volume, probe, want, live); err != nil { - return err - } - } fmt.Printf("LIVE VOLUME PASS read_bytes=%d repeated_put=true repeated_patch=true cleared=true\n", len(got)) return nil } diff --git a/e2e/cmd/volumesmoke/testdata/fault_hook.py b/e2e/cmd/volumesmoke/testdata/fault_hook.py deleted file mode 100755 index b960f7c1..00000000 --- a/e2e/cmd/volumesmoke/testdata/fault_hook.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import signal -import subprocess -import sys -import time -import urllib.request -from pathlib import Path - -root = Path(os.environ["VOLUME_SMOKE_DIR"]) -mode = sys.argv[1] -stage = os.environ["VOLUME_HOOK_STAGE"] -sandbox_id = os.environ["SANDBOX_ID"] -config = json.loads((root / "config.json").read_text()) -address = "http://" + config["listen"] - - -def view(): - request = urllib.request.Request( - f"{address}/v1/sandboxes/{sandbox_id}/volumes", - headers={"Authorization": "Bearer " + config["api_token"]}, - ) - with urllib.request.urlopen(request, timeout=3) as response: - return json.load(response) - - -def restart(): - old = int((root / "daemon.pid").read_text()) - os.kill(old, signal.SIGKILL) - (root / "gate-attach").unlink(missing_ok=True) - with (root / "candidate-daemon.log").open("a") as log: - process = subprocess.Popen( - [str(root / "sandboxd"), "-config", str(root / "config.json")], - stdout=log, - stderr=log, - start_new_session=True, - ) - (root / "daemon.pid").write_text(str(process.pid)) - deadline = time.monotonic() + 30 - while True: - try: - with urllib.request.urlopen(address + "/healthz", timeout=1): - break - except OSError: - if time.monotonic() > deadline: - raise RuntimeError("isolated daemon did not restart") from None - time.sleep(0.05) - print(f"HOOK daemon_restart old={old} new={process.pid}", flush=True) - - -def wait_for(marker): - deadline = time.monotonic() + 10 - while not marker.exists(): - if time.monotonic() > deadline: - raise RuntimeError(f"runtime never reached {marker.name}") - time.sleep(0.01) - print("HOOK runtime_event=" + marker.read_text(), flush=True) - - -def runtime_view(): - data = subprocess.check_output([os.environ["COCOON_REAL_BIN"], "vm", "list", "--format", "json"]) - running = [] - for vm in json.loads(data): - if not vm["config"]["name"].startswith("sbx-"): - continue - pid = vm.get("pid", 0) - running.append( - { - "id": vm["id"], - "name": vm["config"]["name"], - "pid": pid, - "pid_exists": pid > 0 and Path(f"/proc/{pid}").exists(), - "disks": vm.get("storage_configs"), - } - ) - print("HOOK real_vm=" + json.dumps(running), flush=True) - - -if stage == "arm": - marker = "remove-failed" if mode == "retire" else "gate-entered" - (root / marker).unlink(missing_ok=True) - (root / ("fail-remove" if mode == "retire" else "gate-attach")).touch() -elif stage == "wait": - wait_for(root / ("remove-failed" if mode == "retire" else "gate-entered")) - runtime_view() - if mode == "retire": - restart() - runtime_view() - else: - print("HOOK gated_view=" + json.dumps(view()), flush=True) -elif stage == "interrupt": - if mode == "restart": - restart() - elif mode == "cancel": - time.sleep(0.5) - print("HOOK canceled_view=" + json.dumps(view()), flush=True) - (root / "gate-attach").unlink(missing_ok=True) - elif mode == "retire": - (root / "fail-remove").unlink() - else: - raise RuntimeError("unknown hook mode " + mode) -else: - raise RuntimeError("unknown hook stage " + stage) diff --git a/e2e/cmd/volumesmoke/testdata/runtime_observer.py b/e2e/cmd/volumesmoke/testdata/runtime_observer.py deleted file mode 100755 index 4e2ec6c5..00000000 --- a/e2e/cmd/volumesmoke/testdata/runtime_observer.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import subprocess -import sys -import time -from pathlib import Path - -root = Path(os.environ["VOLUME_SMOKE_DIR"]) -args = sys.argv[1:] -event = {"time": time.time(), "pid": os.getpid(), "args": args} -with (root / "commands.jsonl").open("a") as log: - log.write(json.dumps(event) + "\n") -if args[:2] == ["vm", "rm"] and (root / "fail-remove").exists(): - (root / "remove-failed").write_text(json.dumps(event)) - print("volumesmoke injected VM removal failure", file=sys.stderr) - sys.exit(73) -result = subprocess.run([os.environ["COCOON_REAL_BIN"], *args], check=False) -if args[:3] == ["vm", "disk", "attach"] and result.returncode == 0 and (root / "gate-attach").exists(): - (root / "gate-entered").write_text(json.dumps(event)) - while (root / "gate-attach").exists(): - time.sleep(0.02) -sys.exit(result.returncode) diff --git a/scripts/sandboxd-e2e.sh b/scripts/sandboxd-e2e.sh index 1e6f8cb6..2a4ae02b 100755 --- a/scripts/sandboxd-e2e.sh +++ b/scripts/sandboxd-e2e.sh @@ -16,6 +16,8 @@ VOLUME_DIRECTIO=${VOLUME_DIRECTIO:-off} # VOLUME_RW_IMAGE adds the writable leg; its own image is mutated by design, so it must not be VOLUME_IMAGE. VOLUME_RW_IMAGE=${VOLUME_RW_IMAGE:-} VOLUME_RW_NAME=${VOLUME_RW_NAME:-e2e-scratch} +# VOLUME_LIVE=1 adds the live volume leg, which needs cocoon v0.7.2. +VOLUME_LIVE=${VOLUME_LIVE:-} VOLUME_CHECKSUM="" if [[ -n $VOLUME_IMAGE ]]; then @@ -192,7 +194,7 @@ if [[ -n $VOLUME_IMAGE ]]; then done warm_before=$(warm_claims) "$DATA/volumesmoke" -addr "$ADDR" -token "$TOKEN" -template "$TEMPLATE" \ - -volume "$VOLUME_NAME" -probe "$VOLUME_PROBE" ${VOLUME_RW_IMAGE:+-rw-volume "$VOLUME_RW_NAME"} + -volume "$VOLUME_NAME" -probe "$VOLUME_PROBE" ${VOLUME_RW_IMAGE:+-rw-volume "$VOLUME_RW_NAME"} ${VOLUME_LIVE:+-live} warm_after=$(warm_claims) [[ $warm_after -ge $((warm_before + 2)) ]] || { echo "volume claims did not consume the two ready warm VMs: before=$warm_before after=$warm_after" From 935f5aefa8c7f2d3a5d690eaebdfa58e0ba121d3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:53:47 +0800 Subject: [PATCH 27/57] pool: one commit path for live volume runs and no extra lock round trips A failed claims write now keeps memory and recommits, as commitIfLive does, instead of rolling the intent back in one mode and keeping it in the other; after a device change, memory is the only copy that matches the guest. The ownership test, the dropped-hold release and the map setup each live in one place. sourceSnap's captureAllowed repeats its callers' check under the same Transition hold, and liveness reads the holds inside its existing critical section; PatchVolumes leaves the read-only rule to validateLiveTarget. --- sandboxd/pool/liveness.go | 12 ++--- sandboxd/pool/pool.go | 2 + sandboxd/pool/refill.go | 3 -- sandboxd/pool/volume_mutation.go | 76 +++++++-------------------- sandboxd/pool/volume_mutation_test.go | 14 ++--- 5 files changed, 33 insertions(+), 74 deletions(-) diff --git a/sandboxd/pool/liveness.go b/sandboxd/pool/liveness.go index 31219f45..c87398ba 100644 --- a/sandboxd/pool/liveness.go +++ b/sandboxd/pool/liveness.go @@ -132,6 +132,7 @@ func (m *Manager) recoverVMM(ctx context.Context, sb *types.Sandbox) { now := time.Now() m.mu.Lock() live := m.claimed[sb.ID] == sb && runningShaped(sb) && !m.leaseLapsed(sb, now) + volumes := m.hasVolumeHolds(sb) m.mu.Unlock() if !live { return @@ -150,9 +151,7 @@ func (m *Manager) recoverVMM(ctx context.Context, sb *types.Sandbox) { sb.ExitRecorded = true m.recordUsage(ctx, usageEvent{Event: "vmm_exit", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) } - m.mu.Lock() - reason := coldBootBlocker(sb, found, m.hasVolumeHolds(sb)) - m.mu.Unlock() + reason := coldBootBlocker(sb, found, volumes) if reason == "" && m.vmmRestart == types.VMMRestartNone { reason = "vmm exited" } @@ -181,7 +180,7 @@ func (m *Manager) restartFailed(ctx context.Context, sb *types.Sandbox) error { sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - live, failed := m.claimed[sb.ID] == sb, sb.Failed != "" + live, failed, volumes := m.claimed[sb.ID] == sb, sb.Failed != "", m.hasVolumeHolds(sb) m.mu.Unlock() switch { case !live: @@ -193,10 +192,7 @@ func (m *Manager) restartFailed(ctx context.Context, sb *types.Sandbox) error { if err != nil { return fmt.Errorf("wake %s: %w", sb.ID, err) } - m.mu.Lock() - reason := coldBootBlocker(sb, found, m.hasVolumeHolds(sb)) - m.mu.Unlock() - if reason != "" { + if reason := coldBootBlocker(sb, found, volumes); reason != "" { return fmt.Errorf("%w: %s", ErrFailed, reason) } sb.RestartLog = nil diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 007effc3..77f3d649 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -402,6 +402,8 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg store: newClaimStore(cfg.DataDir, cfg.SyncClaims), volumes: make(map[string]catalogVolume, len(cfg.Volumes)), volumeAdmission: map[string]volumeHolders{}, + volumeRuns: map[*types.Sandbox]*volumeRun{}, + volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), pools: make(map[types.PoolKey]*pool, len(cfg.Pools)), claimed: map[string]*types.Sandbox{}, diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index d4d199bc..7214d759 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -357,9 +357,6 @@ func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { // sourceSnap picks the snapshot to export a claimed sandbox from, with its cleanup; the caller holds sb.Transition. func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, func(), error) { - if err := m.captureAllowed(sb); err != nil { - return "", nil, err - } // archive() clears VMName under the same lock, so this is the check that cannot race it if sb.ArchiveCk != "" { return "", nil, ErrArchived diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 8be40d45..07ff5858 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -43,9 +43,6 @@ func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch typ return nil, err } for _, v := range additions { - if volumeIndex(current, v.Name) < 0 && immutableVolume(v) { - return nil, fmt.Errorf("%w: live additions require read-only mounts", ErrBadVolume) - } if slices.Contains(patch.Remove, v.Name) { return nil, fmt.Errorf("%w: volume %q is both added and removed", ErrBadVolume, v.Name) } @@ -189,9 +186,6 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex } runCtx, cancel := context.WithCancel(ctx) run := &volumeRun{cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb)} - if m.volumeRuns == nil { - m.volumeRuns = make(map[*types.Sandbox]*volumeRun) - } m.volumeRuns[sb] = run m.mu.Unlock() return runCtx, run, nil @@ -227,7 +221,7 @@ func (m *Manager) checkVolumeRunLocked(ctx context.Context, sb *types.Sandbox, r if err := ctx.Err(); err != nil { return err } - if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run { + if !m.ownsRunLocked(sb, run) { return ErrUnknownSandbox } switch { @@ -280,7 +274,6 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru m.mu.Unlock() return cleanErr } - previous, applied := sb.VolumesWant, sb.Volumes invalid := volumeDifference(sb.Volumes, sb.VolumesWant) sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), func(v types.Volume) bool { return volumeIndex(invalid, v.Name) >= 0 && volumeIndex(target, v.Name) >= 0 @@ -288,34 +281,7 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru sb.VolumesWant = slices.Clone(target) js := m.store.set(sb) m.mu.Unlock() - err := m.store.commit(js) - if err != nil && m.store.sync { - return fmt.Errorf("persist volume target: %w", err) - } - if err != nil { - m.mu.Lock() - if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { - sb.VolumesWant, sb.Volumes = previous, applied - rb := m.store.set(sb) - m.releaseVolumes(additions) - run.holds = volumeDifference(run.holds, additions) - m.mu.Unlock() - m.recommit(ctx, rb) - } else { - m.mu.Unlock() - } - return fmt.Errorf("persist volume target: %w", err) - } - m.mu.Lock() - if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run || ctx.Err() != nil { - m.mu.Unlock() - return context.Canceled - } - dropped := volumeDifference(run.holds, heldVolumes(sb)) - m.releaseVolumes(dropped) - run.holds = volumeDifference(run.holds, dropped) - m.mu.Unlock() - return nil + return m.commitVolumeRun(ctx, sb, run, js) } func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { @@ -330,29 +296,24 @@ func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *v func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, next []types.Volume) error { m.mu.Lock() - if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run || ctx.Err() != nil { + if !m.ownsRunLocked(sb, run) || ctx.Err() != nil { m.mu.Unlock() return context.Canceled } - previous := sb.Volumes sb.Volumes = next js := m.store.set(sb) m.mu.Unlock() + return m.commitVolumeRun(ctx, sb, run, js) +} + +func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, js claimSnapshot) error { if err := m.store.commit(js); err != nil { - m.mu.Lock() - if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { - sb.Volumes = previous - rb := m.store.set(sb) - m.mu.Unlock() - m.recommit(ctx, rb) - } else { - m.mu.Unlock() - } - return fmt.Errorf("persist volume result: %w", err) + m.recommit(ctx, js) + return fmt.Errorf("persist volumes: %w", err) } m.mu.Lock() defer m.mu.Unlock() - if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run || ctx.Err() != nil { + if !m.ownsRunLocked(sb, run) || ctx.Err() != nil { return context.Canceled } dropped := volumeDifference(run.holds, heldVolumes(sb)) @@ -361,8 +322,13 @@ func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, ru return nil } +func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { + return m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run +} + func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { - return hasVolumes(sb) || m.volumeRuns[sb] != nil && len(m.volumeRuns[sb].holds) > 0 + run := m.volumeRuns[sb] + return hasVolumes(sb) || run != nil && len(run.holds) > 0 } func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { @@ -386,7 +352,7 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name string, err error) { m.mu.Lock() defer m.mu.Unlock() - if m.claimed[sb.ID] != sb || m.volumeRuns[sb] != run { + if !m.ownsRunLocked(sb, run) { return } errs := m.volumeErrors[sb] @@ -409,16 +375,13 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri delete(m.volumeErrors, sb) return } - if m.volumeErrors == nil { - m.volumeErrors = map[*types.Sandbox]map[string]string{} - } m.volumeErrors[sb] = errs } func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name string) { m.mu.Lock() defer m.mu.Unlock() - if m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run { + if m.ownsRunLocked(sb, run) { delete(m.volumeErrors[sb], name) } } @@ -469,6 +432,7 @@ func validateLiveTarget(held, target []types.Volume) error { } func immutableVolume(v types.Volume) bool { return v.RW() || v.AttachOnly || v.Mount == "" } + func volumeIndex(volumes []types.Volume, name string) int { return slices.IndexFunc(volumes, func(v types.Volume) bool { return v.Name == name }) } @@ -490,5 +454,5 @@ func validateDetachNames(names []string) error { } func heldVolumes(sb *types.Sandbox) []types.Volume { - return append(slices.Clone(sb.Volumes), volumeDifference(sb.VolumesWant, sb.Volumes)...) + return slices.Concat(sb.Volumes, volumeDifference(sb.VolumesWant, sb.Volumes)) } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 80f4e635..d623fb86 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -176,14 +176,14 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { } eng.liveVolumeHook = nil waitFor(t, func() bool { return !m.recommitting.Load() }) - if stage == "intent" { - if !slices.Equal(sb.VolumesWant, sb.Volumes) || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { - t.Fatal("failed intent must not reach the engine or retain a hold") - } - return + assertLiveJournal(t, m, sb) + wantApplied := map[string]int{"intent": 0, "attach result": 1, "detach result": 0}[stage] + wantHolders := map[string]int{"intent": 1, "attach result": 1, "detach result": 0}[stage] + if len(sb.Volumes) != wantApplied || volumeHoldersOf(m, "data").readers != wantHolders { + t.Fatalf("failed write lost the physical result or its hold: volumes=%v holders=%+v", sb.Volumes, volumeHoldersOf(m, "data")) } - if slices.Equal(sb.VolumesWant, sb.Volumes) || volumeHoldersOf(m, "data").readers != 1 { - t.Fatal("uncommitted physical result lost its intent or hold") + if stage == "intent" && len(eng.volumeSpecs) != 0 { + t.Fatal("failed intent reached the engine") } if stage == "detach result" { _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) From 0fcf747ac8aaafb345cc5eeab14543e0e63f1e32 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:55:38 +0800 Subject: [PATCH 28/57] review: one top-of-file const and var block, vocabulary types first, message-only mount conflicts asl topdecl: VolumePhase's constants move into api.go's top block. A mount conflict's kind and path were read only by Error() and tests, so it carries its message and the exported enum goes. volumeMount sits ahead of the Engine methods, the mountinfo unescaper is built once, the new wire types get the same one-line godoc as their neighbours, and the two catalog WHY comments the move dropped are back. --- sandboxd/engine/volume.go | 1 - sandboxd/engine/volume_live.go | 45 ++++++++---------------- sandboxd/engine/volume_live_test.go | 16 ++++----- sandboxd/pool/volume.go | 3 +- sandboxd/pool/volume_convergence_test.go | 6 ++-- sandboxd/server/volume_mutation_test.go | 6 ++-- sandboxd/types/api.go | 19 +++++----- 7 files changed, 41 insertions(+), 55 deletions(-) diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index f13e9cd4..b424efb6 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -164,4 +164,3 @@ func isNotFound(err error) bool { respErr, ok := errors.AsType[*wire.ErrorResp](err) return ok && respErr.Kind == wire.KindNotFound } - diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 945e1bd8..97e1445b 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -11,31 +11,14 @@ import ( "github.com/projecteru2/core/log" ) -// VolumeMountConflictKind identifies the guest condition that prevents a volume change. -type VolumeMountConflictKind string - -const ( - VolumeMountForeign VolumeMountConflictKind = "foreign" - VolumeMountBind VolumeMountConflictKind = "bind" - VolumeMountUnmountRefused VolumeMountConflictKind = "unmount-refused" -) +var mountUnescaper = strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) // VolumeMountConflict reports a guest mount that must be resolved before retrying. type VolumeMountConflict struct { - Kind VolumeMountConflictKind - Mount string + Message string } -func (e *VolumeMountConflict) Error() string { - switch e.Kind { - case VolumeMountForeign: - return fmt.Sprintf("volume mount %q has a different device, root, or mode; restore the expected mount before retrying", e.Mount) - case VolumeMountBind: - return fmt.Sprintf("volume device has another mount at %q; remove guest mount references before retrying", e.Mount) - default: - return fmt.Sprintf("cannot unmount %q; close guest references and retry", e.Mount) - } -} +func (e *VolumeMountConflict) Error() string { return e.Message } // LiveVolumeUnsupported reports a runtime that cannot safely change live volumes. type LiveVolumeUnsupported struct { @@ -49,6 +32,10 @@ func (e *LiveVolumeUnsupported) Error() string { return "live volume changes require Cocoon >= " + LiveVolumeCocoon + "; runtime version unavailable" } +type volumeMount struct { + device, root, path, options string +} + // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. func (e *Engine) CheckLiveVolumeSupport(ctx context.Context) error { version, err := e.Version(ctx) @@ -102,7 +89,7 @@ func (e *Engine) unmountLiveVolume(ctx context.Context, sock, device, mount stri if mounted { if err = e.UnmountVolume(ctx, sock, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { - return &VolumeMountConflict{Kind: VolumeMountUnmountRefused, Mount: mount} + return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} } return err } @@ -111,10 +98,6 @@ func (e *Engine) unmountLiveVolume(ctx context.Context, sock, device, mount stri return e.volumeUnmounted(ctx, sock, device) } -type volumeMount struct { - device, root, path, options string -} - func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, []volumeMount, error) { dev, err := e.silkdReadFile(ctx, sock, "/sys/block/"+filepath.Base(device)+"/dev") if err != nil { @@ -130,7 +113,7 @@ func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, if len(fields) < 10 || !slices.Contains(fields[6:], "-") { return "", nil, errors.New("invalid guest mountinfo") } - mounts = append(mounts, volumeMount{fields[2], unescapeMount(fields[3]), unescapeMount(fields[4]), fields[5]}) + mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5]}) } return strings.TrimSpace(string(dev)), mounts, nil } @@ -148,12 +131,12 @@ func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string, for _, entry := range mounts { if entry.path != mount { if entry.device == dev { - return false, &VolumeMountConflict{Kind: VolumeMountBind, Mount: entry.path} + return false, bindConflict(entry.path) } continue } if matched || entry.device != dev || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), mode) { - return false, &VolumeMountConflict{Kind: VolumeMountForeign, Mount: mount} + return false, &VolumeMountConflict{Message: fmt.Sprintf("volume mount %q has a different device, root, or mode; restore the expected mount before retrying", mount)} } matched = true } @@ -167,12 +150,12 @@ func (e *Engine) volumeUnmounted(ctx context.Context, sock, device string) error } for _, mount := range mounts { if mount.device == dev { - return &VolumeMountConflict{Kind: VolumeMountBind, Mount: mount.path} + return bindConflict(mount.path) } } return nil } -func unescapeMount(s string) string { - return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(s) +func bindConflict(path string) *VolumeMountConflict { + return &VolumeMountConflict{Message: fmt.Sprintf("volume device has another mount at %q; remove guest mount references before retrying", path)} } diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 5a8ce13c..48f759fd 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -73,12 +73,12 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { if err == nil { t.Fatal("accepted a foreign mount") } - var conflict *VolumeMountConflict - if errors.As(err, &conflict) != (row != "malformed\n") { + conflict, isConflict := errors.AsType[*VolumeMountConflict](err) + if isConflict != (row != "malformed\n") { t.Fatalf("conflict type: %v", err) } - if conflict != nil && (conflict.Kind != VolumeMountForeign || conflict.Mount != "/data") { - t.Fatalf("conflict: %+v", conflict) + if isConflict && !strings.Contains(conflict.Error(), `"/data" has a different device`) { + t.Fatalf("conflict: %v", conflict) } fake.mu.Lock() defer fake.mu.Unlock() @@ -123,16 +123,16 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { if err == nil { t.Fatal("detach must fail closed") } - var conflict *VolumeMountConflict + conflict, isConflict := errors.AsType[*VolumeMountConflict](err) conflictExpected := mode == "busy" || mode == "second mount" - if errors.As(err, &conflict) != conflictExpected { + if isConflict != conflictExpected { t.Fatalf("conflict=%v error=%v", conflictExpected, err) } if conflictExpected && (!strings.Contains(conflict.Error(), "/data") && !strings.Contains(conflict.Error(), "/other") || strings.Contains(conflict.Error(), "/private")) { t.Fatalf("unsafe or missing guest path: %v", conflict) } - if mode == "busy" && conflict.Kind != VolumeMountUnmountRefused { - t.Fatalf("kind=%s", conflict.Kind) + if mode == "busy" && !strings.HasPrefix(conflict.Error(), "cannot unmount") { + t.Fatalf("busy conflict: %v", conflict) } }) } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index dbc7e296..5a1e8eed 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -160,12 +160,13 @@ func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) return resolvedVolume{}, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) } if _, statErr := os.Stat(entry.disk.Path); statErr != nil { + // the operator's path stays server-side; the refusal matches the unknown one log.WithFunc("pool.resolveCatalogVolume").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return resolvedVolume{}, ErrVolumeUnavailable } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false + volume.AttachOnly = false // past validation the empty mount carries it return resolvedVolume{disk: disk, applied: volume}, nil } diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index a90e94a3..d64f0536 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -168,7 +168,7 @@ func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { if err != nil || view.States[0].Error != "" || view.Converging { t.Fatalf("successful retry retained errors: %+v %v", view, err) } - conflict := &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} + conflict := &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} base.detachVolumeErr = conflict view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}) if !errors.Is(err, conflict) || view.States[0].Error != conflict.Error() { @@ -187,7 +187,7 @@ func TestReleaseDropsVolumeDiagnostics(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - eng.detachVolumeErr = &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} + eng.detachVolumeErr = &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { t.Fatal("expected mount conflict") } @@ -216,7 +216,7 @@ func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { t.Fatal(err) } if priorConflict { - eng.detachVolumeErr = &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/volumes/data"} + eng.detachVolumeErr = &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { t.Fatal("expected mount conflict") } diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 47773b6b..9072ef15 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -92,9 +92,9 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { {"failed", "root", `{"volumes":[]}`, pool.ErrFailed, http.StatusConflict}, {"sandbox token", "sandbox-token", `{"volumes":[]}`, nil, http.StatusUnauthorized}, {"invalid token", "invalid", `{"volumes":[]}`, nil, http.StatusUnauthorized}, - {"foreign mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountForeign, Mount: "/data"}, http.StatusConflict}, - {"bind mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountBind, Mount: "/other"}, http.StatusConflict}, - {"unmount refused", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Kind: engine.VolumeMountUnmountRefused, Mount: "/data"}, http.StatusConflict}, + {"foreign mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Message: `volume mount "/data" has a different device, root, or mode; restore the expected mount before retrying`}, http.StatusConflict}, + {"bind mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Message: `volume device has another mount at "/other"; remove guest mount references before retrying`}, http.StatusConflict}, + {"unmount refused", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Message: `cannot unmount "/data"; close guest references and retry`}, http.StatusConflict}, {"unsupported runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{Version: "v0.7.1"}, http.StatusServiceUnavailable}, {"unknown runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{}, http.StatusServiceUnavailable}, {"timeout", "root", `{"volumes":[]}`, context.DeadlineExceeded, http.StatusGatewayTimeout}, diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index aac4f8ad..a3828440 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -5,6 +5,12 @@ import ( "time" ) +const ( + VolumeAttached VolumePhase = "attached" + VolumePending VolumePhase = "pending" + VolumeDetaching VolumePhase = "detaching" +) + // TTLField is the shared requested-lease field; zero means the server default. type TTLField struct { TTLSeconds int `json:"ttl_seconds,omitzero"` @@ -77,31 +83,28 @@ type VolumeListResponse struct { Volumes []VolumeInfo `json:"volumes"` } -// SetVolumesRequest replaces the desired set of volumes. +// SetVolumesRequest is the wire body of PUT /v1/sandboxes/{id}/volumes. type SetVolumesRequest struct { Volumes []Volume `json:"volumes"` } +// VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { Add []Volume `json:"add,omitempty"` Remove []string `json:"remove,omitempty"` } +// VolumePhase is how far one held volume has converged. type VolumePhase string -const ( - VolumeAttached VolumePhase = "attached" - VolumePending VolumePhase = "pending" - VolumeDetaching VolumePhase = "detaching" -) - +// VolumeState reports one held volume and its last failure. type VolumeState struct { Name string `json:"name"` State VolumePhase `json:"state"` Error string `json:"error,omitempty"` } -// SandboxVolumesResponse retains the desired and applied sets after a failure. +// SandboxVolumesResponse is the wire reply of the sandbox volume routes, failures included. type SandboxVolumesResponse struct { VolumesWant []Volume `json:"volumes_want"` Volumes []Volume `json:"volumes"` From e7454592fe1104ee7e093567f3b5bcb40ee75aa5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 02:56:40 +0800 Subject: [PATCH 29/57] review: Python volume docstrings follow the SDK register and the auth assert runs in the test --- sdk/python/cocoonsandbox/sandbox.py | 6 +++--- sdk/python/cocoonsandbox/volumes.py | 6 ++++++ sdk/python/tests/test_volumes.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index c8dd0ed5..72ef8d6b 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -70,13 +70,13 @@ def __exit__(self, *exc: object) -> None: raise def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse: - """Read desired and applied volumes from the owning node.""" + """Reads the desired and applied volumes from the owning node.""" return self._volumes_request("GET", None, "get volumes", timeout) def set_volumes( self, volumes: Sequence[str | Mapping[str, str] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: - """Replace the desired set and converge. An explicit empty list clears mutable mounts.""" + """Replaces the desired set and converges; an explicit empty list clears mutable mounts.""" if volumes is None: raise TypeError("set volumes requires an explicit list; use [] to clear volumes") return self._volumes_request( @@ -90,7 +90,7 @@ def patch_volumes( remove: Sequence[str] | None = None, timeout: float | None = None, ) -> SandboxVolumesResponse: - """Add or remove mounted read-only entries by name. Omitted arguments leave the set unchanged.""" + """Adds or removes mounted read-only entries by name; omitted arguments leave the set unchanged.""" body: dict[str, Any] = {} if add is not None: body["add"] = [_volume_body(volume, True) for volume in add] diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index 65b9f116..ce156554 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -16,6 +16,8 @@ class _VolumeName(TypedDict): class Volume(_VolumeName, total=False): + """A catalog volume by name, with its guest mount point and an `rw` mode for writable claims.""" + mount: str mode: str @@ -26,6 +28,8 @@ class _VolumeState(TypedDict): class VolumeState(_VolumeState, total=False): + """One held volume's convergence phase and its last failure, if any.""" + error: str @@ -37,6 +41,8 @@ class _SandboxVolumesResponse(TypedDict): class SandboxVolumesResponse(_SandboxVolumesResponse, total=False): + """The desired and applied sets of a sandbox, with per-volume states; error names a refused request.""" + error: str diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 6ece1ea6..4754da0f 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -20,7 +20,6 @@ def test_volume_requests_use_owner_and_api_token(spawn_node, method): def reply(body, path): seen.append(body) - assert FakeNode.last_headers["Authorization"] == "Bearer operator-token" return 200, VIEW addr = spawn_node({(method, PATH): reply}) @@ -35,6 +34,7 @@ def reply(body, path): else: got = sb.patch_volumes(add=["data"], remove=["old"]) assert seen == [{"add": [{"name": "data"}], "remove": ["old"]}] + assert FakeNode.last_headers["Authorization"] == "Bearer operator-token" assert got == VIEW assert sb.volumes == VIEW["volumes"] assert sb.volumes_want == VIEW["volumes_want"] From 2726e7fc18b48e415662e1e523ddb3b94f664395 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:04:08 +0800 Subject: [PATCH 30/57] sdk/python: volume methods take lists so a bare name is not split into characters Sequence[str] accepted set_volumes("data") and sent four one-letter volumes. The volume request reuses the client token fallback, the VolumeError docstring covers every failure it carries, and an import-identity test goes. --- sdk/python/cocoonsandbox/errors.py | 2 +- sdk/python/cocoonsandbox/sandbox.py | 11 +++++------ sdk/python/tests/test_volumes.py | 8 +------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index c2c84bc9..4e1ed4dc 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -28,7 +28,7 @@ def __init__(self, verb: str, status: int, message: str, *, reply: dict[str, Any class VolumeError(APIError): - """Failed convergence with the server's complete view, when returned.""" + """A volume request failed; response holds the server's complete view when one came back.""" def __init__(self, error: APIError, response: SandboxVolumesResponse | None) -> None: super().__init__(error.verb, error.status, error.message, reply=error.reply) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 72ef8d6b..ce4d5586 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -7,7 +7,7 @@ import threading import time from collections import deque -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping from typing import TYPE_CHECKING, Any, cast from .checkpoint import Checkpoint @@ -70,11 +70,11 @@ def __exit__(self, *exc: object) -> None: raise def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse: - """Reads the desired and applied volumes from the owning node.""" + """Reads the desired and applied volumes from the owning node and records them on this handle.""" return self._volumes_request("GET", None, "get volumes", timeout) def set_volumes( - self, volumes: Sequence[str | Mapping[str, str] | Volume], *, timeout: float | None = None + self, volumes: list[str | Mapping[str, str] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: """Replaces the desired set and converges; an explicit empty list clears mutable mounts.""" if volumes is None: @@ -86,8 +86,8 @@ def set_volumes( def patch_volumes( self, *, - add: Sequence[str | Mapping[str, str] | Volume] | None = None, - remove: Sequence[str] | None = None, + add: list[str | Mapping[str, str] | Volume] | None = None, + remove: list[str] | None = None, timeout: float | None = None, ) -> SandboxVolumesResponse: """Adds or removes mounted read-only entries by name; omitted arguments leave the set unchanged.""" @@ -427,7 +427,6 @@ def _volumes_request( f"/v1/sandboxes/{self.id}/volumes", body, verb, - bearer=self._client.api_token, timeout=timeout or 0.0, ) except APIError as exc: diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 4754da0f..6d968e8a 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -124,7 +124,7 @@ def reply(body, path): assert requests[0] == requests[1] -def test_volume_timeout_uses_client_deadline(black_hole): +def test_volume_timeout_bounds_the_call(black_hole): sb = sandbox_at(black_hole, timeout=2) with pytest.raises(APIError) as caught: sb.get_volumes(timeout=0.02) @@ -151,9 +151,3 @@ def test_incomplete_volume_view_preserves_handle(spawn_node, status): if status == 409: assert isinstance(caught.value, VolumeError) assert caught.value.response is None - - -def test_volume_error_export(): - from cocoonsandbox.errors import VolumeError as Error - - assert Error is VolumeError From 3e21909cef0972a63c6d0fccb5b38a8e2b04266e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:05:38 +0800 Subject: [PATCH 31/57] test: live volume tests share their fixtures and prove ordering without wall clocks breakClaimStore and newGatedVolumeEngine replace the copied store-break and gate setup; the concurrent-patch, canceled-writer and attach-deadline tests run in synctest, so the second patch is shown queued behind the first and the deadline is exact. Route tests use doReq and t.Errorf off the test goroutine, errors.AsType replaces errors.As, and the durable old/new legs hold the retrier off claims.json so each leg tests what it names. --- e2e/volume_mutation_test.go | 3 +- sandboxd/pool/volume_convergence_test.go | 10 +- sandboxd/pool/volume_mutation_test.go | 222 ++++++++++++----------- sandboxd/server/volume_mutation_test.go | 40 +--- sdk/go/volume_mutation_test.go | 12 +- 5 files changed, 140 insertions(+), 147 deletions(-) diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 776b9fd8..9fd17e57 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -52,8 +52,7 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { t.Fatal(checkErr) } _, err = sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}, {Name: "dataset"}}) - var refusal *sandbox.VolumeError - if !errors.As(err, &refusal) || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicate volume") { + if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicate volume") { t.Errorf("invalid target explanation: %v", err) } view, err := sb.GetVolumes(t.Context()) diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index d64f0536..bb17f477 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -145,7 +145,7 @@ func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) sb := mustClaim(t, m, testKey) - eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + eng := newGatedVolumeEngine(base) m.eng = eng ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) @@ -340,6 +340,13 @@ func TestUncertainTargetCommitReleasesOnlyConfirmedPrecleanedHolds(t *testing.T) if !slices.Contains(eng.volumeOpsLog(), "detach:data") || len(eng.volumeSpecs) != 1 { t.Fatalf("device calls before uncertain target: attaches=%v detaches=%v", eng.volumeSpecs, eng.volumeOpsLog()) } + if durable == "old" { + m.store.writeMu.Lock() + t.Cleanup(func() { + m.store.writeMu.Unlock() + waitFor(t, func() bool { return !m.recommitting.Load() }) + }) + } if checkErr := os.Remove(m.store.path + ".tmp"); checkErr != nil { t.Fatal(checkErr) } @@ -348,6 +355,7 @@ func TestUncertainTargetCommitReleasesOnlyConfirmedPrecleanedHolds(t *testing.T) if checkErr := m.store.commit(m.store.mark()); checkErr != nil { t.Fatal(checkErr) } + waitFor(t, func() bool { return !m.recommitting.Load() }) } m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) if checkErr := m2.Reconcile(t.Context()); checkErr != nil { diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index d623fb86..a566abc4 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -9,6 +9,7 @@ import ( "strings" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -146,18 +147,11 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { t.Fatal(err) } } - path := m.store.path + ".tmp" - breakStore := func() { - m.store.writeMu.Lock() - defer m.store.writeMu.Unlock() - if err := os.Mkdir(path, 0o700); err != nil { - t.Fatal(err) - } - } + var restore func() if stage == "intent" { - breakStore() + restore = breakClaimStore(t, m) } else { - eng.liveVolumeHook = func(string) error { breakStore(); return nil } + eng.liveVolumeHook = func(string) error { restore = breakClaimStore(t, m); return nil } } var err error if stage == "detach result" { @@ -168,14 +162,8 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { if err == nil { t.Fatal("expected persistence failure") } - m.store.writeMu.Lock() - removeErr := os.Remove(path) - m.store.writeMu.Unlock() - if removeErr != nil { - t.Fatal(removeErr) - } eng.liveVolumeHook = nil - waitFor(t, func() bool { return !m.recommitting.Load() }) + restore() assertLiveJournal(t, m, sb) wantApplied := map[string]int{"intent": 0, "attach result": 1, "detach result": 0}[stage] wantHolders := map[string]int{"intent": 1, "attach result": 1, "detach result": 0}[stage] @@ -496,40 +484,43 @@ func TestSupersedingTargetPrecleansUncertainAddBeforeJournaling(t *testing.T) { } func TestCanceledVolumeWriterOwnsHardwareUntilReturn(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, liveCatalog(t)) - sb := mustClaim(t, m, testKey) - eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} - m.eng = eng - ctx, cancel := context.WithCancel(t.Context()) - first := make(chan error, 1) - go func() { _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); first <- err }() - <-eng.entered - cancel() - waitingCtx, stopWaiting := context.WithTimeout(t.Context(), 25*time.Millisecond) - defer stopWaiting() - _, err := m.PatchVolumes(waitingCtx, sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) - if !errors.Is(err, context.DeadlineExceeded) || eng.calls.Load() != 1 { - t.Fatalf("new writer overtook canceled device call: %v calls=%d", err, eng.calls.Load()) - } - close(eng.resume) - if firstErr := <-first; !errors.Is(firstErr, context.Canceled) { - t.Fatalf("canceled result=%v", firstErr) - } - if len(base.volumeMounts) != 0 { - t.Fatal("canceled writer issued another device call") - } - view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) - if err != nil || len(view.Volumes) != 2 { - t.Fatalf("retry lost prior desired state: %+v %v", view, err) - } + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + ctx, cancel := context.WithCancel(t.Context()) + first := make(chan error, 1) + go func() { _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); first <- err }() + <-eng.entered + cancel() + waitingCtx, stopWaiting := context.WithTimeout(t.Context(), 25*time.Millisecond) + defer stopWaiting() + _, err := m.PatchVolumes(waitingCtx, sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + if !errors.Is(err, context.DeadlineExceeded) || eng.calls.Load() != 1 { + close(eng.resume) + t.Fatalf("new writer overtook canceled device call: %v calls=%d", err, eng.calls.Load()) + } + close(eng.resume) + if firstErr := <-first; !errors.Is(firstErr, context.Canceled) { + t.Fatalf("canceled result=%v", firstErr) + } + if len(base.volumeMounts) != 0 { + t.Fatal("canceled writer issued another device call") + } + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + if err != nil || len(view.Volumes) != 2 { + t.Fatalf("retry lost prior desired state: %+v %v", view, err) + } + }) } func TestReleaseOwnsUncertainHoldsUntilVMRemoval(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) sb := mustClaim(t, m, testKey) - eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + eng := newGatedVolumeEngine(base) m.eng = eng base.removeErrFor = sb.VMName changed := make(chan error, 1) @@ -571,38 +562,45 @@ func TestReleaseOwnsUncertainHoldsUntilVMRemoval(t *testing.T) { } func TestConcurrentVolumePatchesUseLatestDesiredSet(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, liveCatalog(t)) - sb := mustClaim(t, m, testKey) - eng := &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} - m.eng = eng - first := make(chan error, 1) - go func() { - _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) - first <- err - }() - <-eng.entered - second := make(chan error, 1) - go func() { - _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) - second <- err - }() - close(eng.resume) - if err := <-first; err != nil { - t.Fatal(err) - } - if err := <-second; err != nil { - t.Fatal(err) - } - view, err := m.GetVolumes(sb.ID, sb.Tenant) - if err != nil || len(view.Volumes) != 2 || len(view.VolumesWant) != 2 { - t.Fatalf("lost concurrent patch: %+v %v", view, err) - } - for _, name := range []string{"data", "other"} { - if volumeHoldersOf(m, name).readers != 1 { - t.Fatal("duplicate hold for " + name) + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + first := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + first <- err + }() + <-eng.entered + second := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "other"}}}) + second <- err + }() + synctest.Wait() + if len(second) != 0 || eng.calls.Load() != 1 { + close(eng.resume) + t.Fatalf("second patch was not queued behind the first: calls=%d", eng.calls.Load()) } - } + close(eng.resume) + if err := <-first; err != nil { + t.Fatal(err) + } + if err := <-second; err != nil { + t.Fatal(err) + } + view, err := m.GetVolumes(sb.ID, sb.Tenant) + if err != nil || len(view.Volumes) != 2 || len(view.VolumesWant) != 2 { + t.Fatalf("lost concurrent patch: %+v %v", view, err) + } + for _, name := range []string{"data", "other"} { + if volumeHoldersOf(m, name).readers != 1 { + t.Fatal("duplicate hold for " + name) + } + } + }) } func TestSyncIntentFailureRetainsConservativeDesiredHold(t *testing.T) { @@ -610,16 +608,12 @@ func TestSyncIntentFailureRetainsConservativeDesiredHold(t *testing.T) { m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) m.store.sync = true - if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { - t.Fatal(err) - } + restore := breakClaimStore(t, m) view, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) if err == nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || len(eng.volumeSpecs) != 0 || volumeHoldersOf(m, "data").readers != 1 { t.Fatalf("ambiguous write lost recovery state: %+v %v", view, err) } - if err := os.Remove(m.store.path + ".tmp"); err != nil { - t.Fatal(err) - } + restore() if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, nil); err != nil { t.Fatal(err) } @@ -641,9 +635,7 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { sb.Deadline = time.Now().Add(-time.Second) } m.mu.Unlock() - if mkdirErr := os.Mkdir(m.store.path+".tmp", 0o700); mkdirErr != nil { - t.Fatal(mkdirErr) - } + restore := breakClaimStore(t, m) m.store.writeMu.Lock() done := make(chan error, 1) go func() { @@ -675,13 +667,7 @@ func TestFailedReleaseRestoresVolumeHoldOwnership(t *testing.T) { if volumeHoldersOf(m, "data") != (volumeHolders{}) { t.Fatal("rollback orphaned transient admission hold") } - m.store.writeMu.Lock() - err = os.Remove(m.store.path + ".tmp") - m.store.writeMu.Unlock() - if err != nil { - t.Fatal(err) - } - waitFor(t, func() bool { return !m.recommitting.Load() }) + restore() if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } @@ -708,19 +694,20 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { } func TestLiveAttachBoundsWholeDeviceOperation(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, liveCatalog(t)) - sb := mustClaim(t, m, testKey) - eng := &deadlineVolumeEngine{fakeEngine: base, deadlines: make(chan time.Time, 1)} - m.eng = eng - start := time.Now() - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - deadline := <-eng.deadlines - if deadline.IsZero() || deadline.Before(start) || deadline.After(time.Now().Add(30*time.Second)) { - t.Fatalf("live attach deadline=%v", deadline) - } + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := &deadlineVolumeEngine{fakeEngine: base, deadlines: make(chan time.Time, 1)} + m.eng = eng + start := time.Now() + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if deadline := <-eng.deadlines; !deadline.Equal(start.Add(engine.VolumeMutationTimeout)) { + t.Fatalf("live attach deadline=%v, want %v", deadline, start.Add(engine.VolumeMutationTimeout)) + } + }) } func TestLivenessRefusesAdmittedVolumeRun(t *testing.T) { @@ -795,6 +782,10 @@ type gatedVolumeEngine struct { calls atomic.Int32 } +func newGatedVolumeEngine(base *fakeEngine) *gatedVolumeEngine { + return &gatedVolumeEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} +} + func (e *gatedVolumeEngine) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error { if e.calls.Add(1) == 1 { close(e.entered) @@ -830,3 +821,24 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume run.holds = append(run.holds, transient...) return run } + +func breakClaimStore(t *testing.T, m *Manager) (restore func()) { + t.Helper() + tmp := m.store.path + ".tmp" + m.store.writeMu.Lock() + err := os.Mkdir(tmp, 0o700) + m.store.writeMu.Unlock() + if err != nil { + t.Fatal(err) + } + return func() { + t.Helper() + m.store.writeMu.Lock() + err := os.Remove(tmp) + m.store.writeMu.Unlock() + if err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return !m.recommitting.Load() }) + } +} diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 9072ef15..f916d547 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -54,15 +54,7 @@ func TestLiveVolumeRoutesAuthorizeAndDecode(t *testing.T) { if method == http.MethodPatch { body = `{"remove":["data"]}` } - req, err := http.NewRequestWithContext(t.Context(), method, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Authorization", "Bearer "+token) - resp, err := ts.Client().Do(req) - if err != nil { - t.Fatal(err) - } + resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", token, body) defer resp.Body.Close() var got types.SandboxVolumesResponse if err := json.UnmarshalRead(resp.Body, &got); err != nil { @@ -106,17 +98,7 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "data"}}, States: []types.VolumeState{{Name: "data", State: "pending", Error: "volume operation failed; retry after resolving the runtime error"}}}, tt.err }} ts := newTestServer(t, "root", mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(tt.body)) - if err != nil { - t.Fatal(err) - } - if tt.token != "" { - req.Header.Set("Authorization", "Bearer "+tt.token) - } - resp, err := ts.Client().Do(req) - if err != nil { - t.Fatal(err) - } + resp := doReq(t, http.MethodPut, ts.URL+"/v1/sandboxes/sb_1/volumes", tt.token, tt.body) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { @@ -135,9 +117,11 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { t.Run(method, func(t *testing.T) { + called := false check := func(id, tenant string) (types.SandboxVolumesResponse, error) { + called = true if id != "sb_1" || tenant != "other" { - t.Fatalf("scope %s %q", id, tenant) + t.Errorf("scope %s %q", id, tenant) } return types.SandboxVolumesResponse{}, pool.ErrUnknownSandbox } @@ -155,18 +139,10 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { if method == http.MethodPatch { body = `{}` } - req, err := http.NewRequestWithContext(t.Context(), method, ts.URL+"/v1/sandboxes/sb_1/volumes", strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Authorization", "Bearer other-token") - resp, err := ts.Client().Do(req) - if err != nil { - t.Fatal(err) - } + resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", "other-token", body) defer resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status %d", resp.StatusCode) + if resp.StatusCode != http.StatusNotFound || !called { + t.Fatalf("status=%d called=%v", resp.StatusCode, called) } }) } diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 44ec5aad..97e36178 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -76,9 +76,9 @@ func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { case http.MethodPatch: got, err = sb.PatchVolumes(t.Context(), VolumePatch{Remove: []string{"old"}}) } - var failure *VolumeError - var api *APIError - if !errors.As(err, &failure) || !errors.As(err, &api) || api.Status != 500 || failure.Response == nil { + failure, isVolume := errors.AsType[*VolumeError](err) + api, isAPI := errors.AsType[*APIError](err) + if !isVolume || !isAPI || api.Status != 500 || failure.Response == nil { t.Fatalf("%s error lost response or APIError: %v", method, err) } if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" || sb.VolumesWant[0].Name != "new" || sb.Volumes[0].Name != "old" { @@ -100,8 +100,7 @@ func TestLiveVolumeValidationFailurePreservesLastKnownSet(t *testing.T) { c := testClient(t, ts) sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c, Volumes: []Volume{{Name: "old"}}, VolumesWant: []Volume{{Name: "wanted"}}} _, err := sb.SetVolumes(t.Context(), []Volume{{Name: "new"}}) - var failure *VolumeError - if !errors.As(err, &failure) || failure.Response != nil || failure.Status != 400 { + if failure, ok := errors.AsType[*VolumeError](err); !ok || failure.Response != nil || failure.Status != 400 { t.Fatalf("error=%v", err) } if sb.Volumes[0].Name != "old" || sb.VolumesWant[0].Name != "wanted" { @@ -189,8 +188,7 @@ func TestLiveVolumeIncompleteViewPreservesHandle(t *testing.T) { if err == nil || sb.Volumes[0].Name != "old" || sb.VolumesWant[0].Name != "wanted" { t.Fatalf("handle overwritten: %+v, %v", sb, err) } - var failure *VolumeError - if status == http.StatusConflict && (!errors.As(err, &failure) || failure.Response != nil) { + if failure, ok := errors.AsType[*VolumeError](err); status == http.StatusConflict && (!ok || failure.Response != nil) { t.Fatalf("partial view accepted: %v", err) } }) From 6705b86b764828c1ce2d5ba697551f322f237b5e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:06:26 +0800 Subject: [PATCH 32/57] docs: live volume errors, attach-only retention, version caching and timeouts match the code --- README.md | 3 ++- docs/deploy.md | 6 ++++-- docs/sandboxd-api.md | 23 +++++++++++++++-------- docs/sdk-python.md | 3 +++ docs/sdk.md | 3 ++- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7843d205..6d155e34 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,8 @@ TEMPLATE=rt:24.04 scripts/sandboxd-e2e.sh # without a Go toolchain. # VOLUME_IMAGE=/absolute/dataset.img enables the volumes proof (ro sharing; # the image contains volume-e2e.txt); VOLUME_RW_IMAGE=/absolute/scratch.img -# adds the rw claim/release leg. Prebuilt runs also set VOLUME_SMOKE_BIN. +# adds the rw claim/release leg; VOLUME_LIVE=1 adds the live volume leg. +# Prebuilt runs also set VOLUME_SMOKE_BIN. ``` ## CI diff --git a/docs/deploy.md b/docs/deploy.md index 289812d4..4cf64f84 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -166,7 +166,8 @@ with 503 before any intent, hold, or device change; a dev build is assumed current. sandboxd reads the version once, so restart it after upgrading cocoon. GET and release remain available. -The claim journal stores desired `VolumesWant` and confirmed applied `Volumes`. +The claim journal stores confirmed applied `Volumes`, plus desired +`VolumesWant` while it differs. Admission holds cover their union across daemon restart and teardown. Retry an interrupted operation with the same PUT target. Release cancels convergence and transfers holds to teardown. Capture is refused while either set is @@ -709,7 +710,8 @@ using prebuilt binaries, also set `VOLUME_SMOKE_BIN` beside `SANDBOXD_BIN`, Add `VOLUME_RW_IMAGE=/srv/datasets/scratch.img` (a second, writable image) to also run the writable leg: a durable write across release, second-writer -exclusion, and a clean read-only claim afterward. +exclusion, and a clean read-only claim afterward. `VOLUME_LIVE=1` adds the live +volume leg on cocoon v0.7.2 or newer. ## TLS for SDK clients diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 6a059fbe..add40aed 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -299,7 +299,9 @@ array. An explicit empty array removes all mutable entries. Only mounted read-only catalog entries can change live (`mode` omitted or `ro`). A PUT must retain every existing writable or attach-only entry unchanged. The claim-time name, mount, and eight-volume limits apply to the desired set. -An omitted mount defaults to `/volumes/`. To change an existing name's +An omitted mount defaults to `/volumes/`, except that a held attach-only +entry is retained by sending it as the claim returned it, without `mount`. To +change an existing name's mount or mode, first remove that name and wait for successful convergence. An identical target can be retried. Claims with `on_expire: archive` must change that policy before adding a volume. @@ -310,7 +312,7 @@ exits; PUT with an empty target cannot restore its processes. Missing, invalid, or sandbox bearer tokens return 401. Another tenant's sandbox and an unknown id return 404. Invalid, inaccessible, or non-local entries return 400. Removed tenant tokens are rejected at authentication. Root can still repair or remove -held entries, but cannot admit new names for a removed tenant. A conflicting +held entries, but admitting a new name for a removed tenant returns 403. A conflicting writer or dirty image returns 409. ## PATCH /v1/sandboxes/{id}/volumes @@ -335,15 +337,17 @@ timeouts return 504 and cancellation returns 408. PUT and PATCH return the GET response shape on success. A convergence error returns a non-2xx status with the same view, so callers can inspect desired, -applied, and per-volume state. Validation, authentication and unknown-sandbox -errors carry only `error`. A failed request may already have persisted its target and +applied, and per-volume state; semantic validation 400s carry it too. A +malformed body, a missing `volumes` array, authentication and an unknown sandbox +carry only `error`. A failed request may already have persisted its target and completed some device changes. GET reads the last confirmed set, rather than probing the devices. Resolve guest mount references or runtime failures before retrying. Retry the same PUT to converge, or send a new target to abandon unfinished additions. Concurrent requests serialize per sandbox without blocking renewals or new exec, file, and port connections. -Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`. A +Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`, +storing it only while it differs from `Volumes`. A superseding request may first clean up an older desired-only attachment while the old target stays recorded. If that cleanup fails, the response retains the old view; retry the same PUT to finish accepting the new target. Admission @@ -355,14 +359,17 @@ for a blocked device call. Capture operations (hibernate, fork, checkpoint, and promote) are refused while either set is nonempty or a convergence request still owns admitted volume holds. -Device calls honor cancellation and have a 30-second ceiling. Journal writes +Device calls honor cancellation and have a 30-second ceiling per volume; a +caller that disconnects cancels the run, leaving interrupted states to retry. Journal writes are atomic. Host-crash durability follows `sync_claims`. An older daemon ignores `VolumesWant`; finish pending volume changes before downgrading. Live convergence requires Cocoon v0.7.2 or newer for idempotent matching attach and absent detach. PUT and PATCH reject an older release, or a runtime whose version cannot be queried, with 503 before changing intent, holds, or -devices; a dev build is assumed current. GET and release remain available. +devices; a dev build is assumed current. sandboxd caches the first successful +version read, so restart it after upgrading cocoon. GET and release remain +available. Catalog names must keep the same backing image while held, including pending holds. Publish changes under versioned names such as `dataset-v1` and @@ -788,7 +795,7 @@ an SDK caller should set. Auth: node API token. Root sees every live claim; a tenant sees only its own. The index is `{"sandboxes": [{id, key, tenant?, deadline, claimed_at?, hibernated, archived?, from_checkpoint?, claim_ref?, metadata?, on_expire?, cpu_count, mem_total_bytes, -volumes?: [{name, mount, mode?}], volumes_want?: [{name, mount, mode?}], restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the +volumes?: [{name, mount?, mode?}], volumes_want?: [{name, mount?, mode?}], restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the size tier's allocation; `restarts` counts the cold boots that replaced an exited VMM and `restarted_at` stamps the last one (each loses guest memory and the instance-metadata document, so a caller re-establishes guest state when diff --git a/docs/sdk-python.md b/docs/sdk-python.md index a328a8f6..d726d366 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -168,6 +168,9 @@ Use `sb.get_volumes()`, `sb.set_volumes([...])`, and methods use the client's API token on the owning node; a sandbox token cannot manage datasets. `set_volumes([])` clears mutable entries; `None` is rejected. PUT must retain writable and attach-only entries. PATCH retains omitted names. +Each method takes `timeout=` (default: the client timeout, 120 s); a large +replacement can need longer, and a call that times out cancels the server run, +so retry the same target. Responses include `volumes_want`, `volumes`, `states`, and `converging`. `VolumePhase` and `VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` diff --git a/docs/sdk.md b/docs/sdk.md index d776881a..e2ffd912 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -268,7 +268,8 @@ and update both public sets from each complete response, including a partial fai Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. Python raises `VolumeError`, a subclass of `APIError`, with `.response`. A response is absent -if the server returned only an authentication or validation error. In that case +when the reply carries no complete view (a malformed request, authentication, +an unknown sandbox, or a non-JSON proxy reply). In that case the handle retains its last known sets. Retry an interrupted operation with the same full PUT target, or set a new target to recover. `Client.Sandboxes` exposes `VolumesWant` and `Volumes` for inspection with an API token. From b924334404cb92cfa8235ff9cddbfa10ca768fc7 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:07:58 +0800 Subject: [PATCH 33/57] server: answer a store outage with a fixed message, not the backend error ErrTenantStoreDown and ErrPoolStoreDown wrap the pgx error, whose text names the PostgreSQL host and user, and poolErrHTTP fell back to err.Error() for them on every route that maps pool errors. --- sandboxd/server/server.go | 4 ++-- sandboxd/server/volume_mutation_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 32119886..28522715 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -53,8 +53,8 @@ var poolErrHTTP = []struct { {pool.ErrBadConfig, http.StatusBadRequest, ""}, {pool.ErrReloadRefused, http.StatusConflict, ""}, {pool.ErrTenantRemoved, http.StatusForbidden, ""}, - {pool.ErrTenantStoreDown, http.StatusServiceUnavailable, ""}, - {pool.ErrPoolStoreDown, http.StatusServiceUnavailable, ""}, + {pool.ErrTenantStoreDown, http.StatusServiceUnavailable, "tenant store down; retry"}, + {pool.ErrPoolStoreDown, http.StatusServiceUnavailable, "pool set store down; retry"}, {pool.ErrNoEgress, http.StatusConflict, ""}, {pool.ErrNoEgressHibernate, http.StatusConflict, ""}, {pool.ErrNoEgressFork, http.StatusConflict, ""}, diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index f916d547..d8c881f1 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -88,6 +88,7 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { {"bind mount", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Message: `volume device has another mount at "/other"; remove guest mount references before retrying`}, http.StatusConflict}, {"unmount refused", "root", `{"volumes":[]}`, &engine.VolumeMountConflict{Message: `cannot unmount "/data"; close guest references and retry`}, http.StatusConflict}, {"unsupported runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{Version: "v0.7.1"}, http.StatusServiceUnavailable}, + {"tenant store down", "root", `{"volumes":[]}`, fmt.Errorf("%w: connect host /private/pg", pool.ErrTenantStoreDown), http.StatusServiceUnavailable}, {"unknown runtime", "root", `{"volumes":[]}`, &engine.LiveVolumeUnsupported{}, http.StatusServiceUnavailable}, {"timeout", "root", `{"volumes":[]}`, context.DeadlineExceeded, http.StatusGatewayTimeout}, {"cancel", "root", `{"volumes":[]}`, context.Canceled, http.StatusRequestTimeout}, From fe444ad451ef559bc55bdd149bdb25e9f7873113 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:11:01 +0800 Subject: [PATCH 34/57] engine: live mount checks follow a symlinked mount path and read a full mountinfo mountinfo records the kernel-resolved path, so a mount under a symlinked parent read as a bind of its own device: a repeated PUT answered 409 and a live removal stayed refused until release. The check now resolves the requested path with realpath only when the device sits elsewhere. The mountinfo read had the 64 KiB sysfs cap, which a few dozen overlay mounts in the guest exceed; it gets its own 1 MiB cap. In the same functions: silkdOutput carries an exec's stdout, mountMode is the one ro/rw mapping, refuseOtherMounts names what volumeUnmounted did, and the parameter and field names match the rest of the package. --- sandboxd/engine/installca.go | 27 +++++++----- sandboxd/engine/installca_test.go | 5 +++ sandboxd/engine/silkd.go | 7 ++-- sandboxd/engine/volume.go | 15 ++++--- sandboxd/engine/volume_live.go | 65 +++++++++++++++-------------- sandboxd/engine/volume_live_test.go | 61 +++++++++++++++++++++++++++ sandboxd/engine/volume_test.go | 2 +- 7 files changed, 129 insertions(+), 53 deletions(-) diff --git a/sandboxd/engine/installca.go b/sandboxd/engine/installca.go index 761cf2dd..da0d4279 100644 --- a/sandboxd/engine/installca.go +++ b/sandboxd/engine/installca.go @@ -19,12 +19,12 @@ const ( ) type guestExitError struct { - Code int32 - Output string + code int32 + output string } func (e *guestExitError) Error() string { - return fmt.Sprintf("exit code %d: %s", e.Code, e.Output) + return fmt.Sprintf("exit code %d: %s", e.code, e.output) } // InstallCACert makes the guest trust the cluster root without update-ca-certificates. @@ -73,9 +73,14 @@ func (e *Engine) silkdWriteFile(ctx context.Context, vsockSocket, path string, m } func (e *Engine) silkdExec(ctx context.Context, vsockSocket string, argv ...string) error { + _, err := e.silkdOutput(ctx, vsockSocket, argv...) + return err +} + +func (e *Engine) silkdOutput(ctx context.Context, vsockSocket string, argv ...string) ([]byte, error) { s, err := e.dialSilkdSession(ctx, vsockSocket) if err != nil { - return err + return nil, err } defer s.close() sendErr := s.send(wire.Exec{Argv: argv, Env: map[string]string{"PATH": guestExecPATH}}) @@ -83,27 +88,27 @@ func (e *Engine) silkdExec(ctx context.Context, vsockSocket string, argv ...stri // a child exiting without reading stdin races this close; prefer the buffered exit frame. sendErr = s.send(wire.StdinClose{}) } - var out []byte + var out, stdout []byte for { frame, err := s.recv() if err != nil { - return errors.Join(sendErr, err) + return nil, errors.Join(sendErr, err) } switch resp := frame.(type) { case *wire.Started: case *wire.Stdout: - out = appendCapped(out, resp.Data) + out, stdout = appendCapped(out, resp.Data), appendCapped(stdout, resp.Data) case *wire.Stderr: out = appendCapped(out, resp.Data) case *wire.Exit: if resp.Code != 0 { - return &guestExitError{Code: resp.Code, Output: string(bytes.TrimSpace(out))} + return nil, &guestExitError{code: resp.Code, output: string(bytes.TrimSpace(out))} } - return nil + return stdout, nil case *wire.ErrorResp: - return fmt.Errorf("silkd %w", resp) + return nil, fmt.Errorf("silkd %w", resp) default: - return fmt.Errorf("unexpected silkd frame %q", resp.RespType()) + return nil, fmt.Errorf("unexpected silkd frame %q", resp.RespType()) } } } diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 2bbd2bd6..ce7616af 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -79,6 +79,7 @@ type fakeSilkd struct { execCode int32 execFailAt int execHook func([]string) + realpath map[string]string writeErr string writeKind string execErr string @@ -202,6 +203,7 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri if code == 0 && reply == "" && f.execHook != nil { f.execHook(argv) } + resolved := f.realpath[argv[len(argv)-1]] f.mu.Unlock() if disconnect { return @@ -211,6 +213,9 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri return } _, _ = io.WriteString(conn, `{"type":"started","pid":1}`+"\n") + if argv[0] == "realpath" { + writeFakeSilkdResponse(conn, &wire.Stdout{Data: []byte(cmp.Or(resolved, argv[len(argv)-1]) + "\n")}) + } _, _ = fmt.Fprintf(conn, `{"type":"exit","code":%d}`+"\n", code) } diff --git a/sandboxd/engine/silkd.go b/sandboxd/engine/silkd.go index 16fa2134..d7bfab3d 100644 --- a/sandboxd/engine/silkd.go +++ b/sandboxd/engine/silkd.go @@ -13,6 +13,7 @@ import ( const ( silkdListMax = 4096 silkdReadMax = 64 << 10 + mountInfoMax = 1 << 20 ) // silkdSession is a dialed silkd conn bound to a ctx, with wire-typed request/reply helpers. @@ -107,7 +108,7 @@ func (e *Engine) silkdList(ctx context.Context, vsockSocket, path string) ([]wir return entries, nil } -func (e *Engine) silkdReadFile(ctx context.Context, vsockSocket, path string) ([]byte, error) { +func (e *Engine) silkdReadFile(ctx context.Context, vsockSocket, path string, limit int) ([]byte, error) { var data []byte err := e.silkdStream(ctx, vsockSocket, wire.FsRead{Path: path}, func(frame wire.Response) error { resp, ok := frame.(*wire.DataResp) @@ -115,8 +116,8 @@ func (e *Engine) silkdReadFile(ctx context.Context, vsockSocket, path string) ([ return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) } data = append(data, resp.Data...) - if len(data) > silkdReadMax { - return fmt.Errorf("silkd read of %s exceeds %d bytes", path, silkdReadMax) + if len(data) > limit { + return fmt.Errorf("silkd read of %s exceeds %d bytes", path, limit) } return nil }) diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index b424efb6..2cc405c2 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -107,7 +107,7 @@ func (e *Engine) findVolumeDevice(ctx context.Context, vsockSocket, name string) return "", false, err } for _, entry := range entries { - serial, err := e.silkdReadFile(ctx, vsockSocket, "/sys/block/"+entry.Name+"/serial") + serial, err := e.silkdReadFile(ctx, vsockSocket, "/sys/block/"+entry.Name+"/serial", silkdReadMax) if isNotFound(err) { continue } @@ -134,11 +134,7 @@ func (e *Engine) mountDevice(ctx context.Context, vsockSocket, device, name, mou if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil { return fmt.Errorf("create volume mount point %s: %w", mount, err) } - mode := types.VolumeModeRO - if rw { - mode = types.VolumeModeRW - } - if err := e.silkdExec(ctx, vsockSocket, "mount", "-o", mode, "--", device, mount); err != nil { + if err := e.silkdExec(ctx, vsockSocket, "mount", "-o", mountMode(rw), "--", device, mount); err != nil { return fmt.Errorf("mount volume %s: %w", name, err) } return nil @@ -164,3 +160,10 @@ func isNotFound(err error) bool { respErr, ok := errors.AsType[*wire.ErrorResp](err) return ok && respErr.Kind == wire.KindNotFound } + +func mountMode(rw bool) string { + if rw { + return types.VolumeModeRW + } + return types.VolumeModeRO +} diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 97e1445b..594d4150 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -1,6 +1,7 @@ package engine import ( + "cmp" "context" "errors" "fmt" @@ -26,14 +27,11 @@ type LiveVolumeUnsupported struct { } func (e *LiveVolumeUnsupported) Error() string { - if e.Version != "" { - return fmt.Sprintf("live volume changes require Cocoon >= %s; installed version is %s", LiveVolumeCocoon, e.Version) - } - return "live volume changes require Cocoon >= " + LiveVolumeCocoon + "; runtime version unavailable" + return fmt.Sprintf("live volume changes require Cocoon >= %s; installed version is %s", LiveVolumeCocoon, cmp.Or(e.Version, "unavailable")) } type volumeMount struct { - device, root, path, options string + devNum, root, path, options string } // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. @@ -50,30 +48,30 @@ func (e *Engine) CheckLiveVolumeSupport(ctx context.Context) error { } // MountLiveVolume mounts an attached disk unless the same mount already exists. -func (e *Engine) MountLiveVolume(ctx context.Context, sock, name, mount string, rw bool) error { +func (e *Engine) MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error { ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout) defer cancel() - device, err := e.waitForVolumeDevice(ctx, sock, name) + device, err := e.waitForVolumeDevice(ctx, vsockSocket, name) if err != nil { return fmt.Errorf("wait for volume device %s: %w", name, err) } - mounted, err := e.volumeMounted(ctx, sock, device, mount, rw) + mounted, err := e.volumeMounted(ctx, vsockSocket, device, mount, rw) if err != nil || mounted { return err } - return e.mountDevice(ctx, sock, device, name, mount, rw) + return e.mountDevice(ctx, vsockSocket, device, name, mount, rw) } // DetachVolume unmounts a guest volume before confirming host removal. -func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount string) error { +func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) error { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() - device, found, err := e.findVolumeDevice(ctx, sock, name) + device, found, err := e.findVolumeDevice(ctx, vsockSocket, name) if err != nil { return err } if found { - if err = e.unmountLiveVolume(ctx, sock, device, mount); err != nil { + if err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { return err } } @@ -81,29 +79,28 @@ func (e *Engine) DetachVolume(ctx context.Context, vmName, sock, name, mount str return err } -func (e *Engine) unmountLiveVolume(ctx context.Context, sock, device, mount string) error { - mounted, err := e.volumeMounted(ctx, sock, device, mount, false) +func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) error { + mounted, err := e.volumeMounted(ctx, vsockSocket, device, mount, false) if err != nil { return err } if mounted { - if err = e.UnmountVolume(ctx, sock, mount); err != nil { + if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} } return err } } - // Refuse a second/bind mount of the same device before hot-unplug. - return e.volumeUnmounted(ctx, sock, device) + return e.refuseOtherMounts(ctx, vsockSocket, device) } -func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, []volumeMount, error) { - dev, err := e.silkdReadFile(ctx, sock, "/sys/block/"+filepath.Base(device)+"/dev") +func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) (string, []volumeMount, error) { + devNum, err := e.silkdReadFile(ctx, vsockSocket, "/sys/block/"+filepath.Base(device)+"/dev", silkdReadMax) if err != nil { return "", nil, err } - raw, err := e.silkdReadFile(ctx, sock, "/proc/self/mountinfo") + raw, err := e.silkdReadFile(ctx, vsockSocket, "/proc/self/mountinfo", mountInfoMax) if err != nil { return "", nil, err } @@ -115,27 +112,31 @@ func (e *Engine) volumeMounts(ctx context.Context, sock, device string) (string, } mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5]}) } - return strings.TrimSpace(string(dev)), mounts, nil + return strings.TrimSpace(string(devNum)), mounts, nil } -func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string, rw bool) (bool, error) { - dev, mounts, err := e.volumeMounts(ctx, sock, device) +func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount string, rw bool) (bool, error) { + devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { return false, err } - mode := "ro" - if rw { - mode = "rw" + target := mount + if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum == devNum && m.path != mount }) { + resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", "-m", "--", mount) + if err != nil { + return false, fmt.Errorf("resolve volume mount %s: %w", mount, err) + } + target = strings.TrimSpace(string(resolved)) } matched := false for _, entry := range mounts { - if entry.path != mount { - if entry.device == dev { + if entry.path != target { + if entry.devNum == devNum { return false, bindConflict(entry.path) } continue } - if matched || entry.device != dev || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), mode) { + if mode, _, _ := strings.Cut(entry.options, ","); matched || entry.devNum != devNum || entry.root != "/" || mode != mountMode(rw) { return false, &VolumeMountConflict{Message: fmt.Sprintf("volume mount %q has a different device, root, or mode; restore the expected mount before retrying", mount)} } matched = true @@ -143,13 +144,13 @@ func (e *Engine) volumeMounted(ctx context.Context, sock, device, mount string, return matched, nil } -func (e *Engine) volumeUnmounted(ctx context.Context, sock, device string) error { - dev, mounts, err := e.volumeMounts(ctx, sock, device) +func (e *Engine) refuseOtherMounts(ctx context.Context, vsockSocket, device string) error { + devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { return err } for _, mount := range mounts { - if mount.device == dev { + if mount.devNum == devNum { return bindConflict(mount.path) } } diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 48f759fd..8ec24852 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "slices" "strings" "testing" ) @@ -53,6 +54,66 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { } } +func TestLiveMountUnderSymlinkedParentConvergesAndDetaches(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + fake.realpath = map[string]string{"/workspace/data": "/home/u/ws/data"} + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /home/u/ws/data ro - ext4 /dev/vdc ro\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + for range 2 { + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/workspace/data", false); err != nil { + t.Fatal(err) + } + } + if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/workspace/data"); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + unmounted := slices.ContainsFunc(fake.execCalls, func(argv []string) bool { return argv[0] == "umount" }) + if !unmounted || len(slices.DeleteFunc(slices.Clone(fake.execCalls), func(argv []string) bool { return argv[0] != "mount" })) != 1 { + t.Fatalf("want one mount and an unmount, got %v", fake.execCalls) + } +} + +func TestLiveMountReadsMountinfoBeyondTheSysfsCap(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + overlays := strings.Repeat("9 1 0:99 / /var/lib/docker/overlay2/layer/merged rw - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABCDEFGHIJKLMNOPQRSTUVWXYZ\n", 2000) + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + overlays) + fake.mu.Unlock() + if len(rootMountInfo+overlays) <= silkdReadMax { + t.Fatalf("fixture of %d bytes does not exceed the sysfs cap", len(rootMountInfo+overlays)) + } + e := New("cocoon", nil, nil, false, false, "") + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { + t.Fatal(err) + } +} + func TestLiveAttachRejectsForeignMounts(t *testing.T) { for _, row := range []string{ "2 1 8:99 / /data ro - ext4 /dev/vdz ro\n", diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index f9825acc..e41c5c82 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -265,7 +265,7 @@ func TestSilkdListAndReadAreBounded(t *testing.T) { if _, err := e.silkdList(t.Context(), path, "/sys/block"); err == nil || !strings.Contains(err.Error(), "exceeds") { t.Errorf("list of %d entries: %v, want a bound error", silkdListMax+1, err) } - if _, err := e.silkdReadFile(t.Context(), path, "/sys/block/vda/serial"); err == nil || !strings.Contains(err.Error(), "exceeds") { + if _, err := e.silkdReadFile(t.Context(), path, "/sys/block/vda/serial", silkdReadMax); err == nil || !strings.Contains(err.Error(), "exceeds") { t.Errorf("read of %d bytes: %v, want a bound error", silkdReadMax+1, err) } } From 1ae91a7c3c7c9e1fc121c73cabe3984966d032e3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:11:32 +0800 Subject: [PATCH 35/57] server: the volume handler maps pool errors through the shared table and keeps Retry-After The handler hand-rolled writePoolErr's loop and dropped its Retry-After, so a tenant or pool-set store outage on the volume routes answered 503 without the hint every other route sends. --- sandboxd/server/server_http.go | 13 ++++++++++--- sandboxd/server/volume_mutation.go | 14 ++++++-------- sandboxd/server/volume_mutation_test.go | 3 +++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index e239ec35..bc98fb95 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -163,16 +163,23 @@ func metadataFilter(pairs []string) (types.Metadata, error) { } func writePoolErr(w http.ResponseWriter, err error) bool { + code, msg, ok := mapPoolErr(w, err) + if ok { + writeErr(w, code, msg) + } + return ok +} + +func mapPoolErr(w http.ResponseWriter, err error) (code int, msg string, ok bool) { for _, m := range poolErrHTTP { if errors.Is(err, m.err) { if m.code == http.StatusServiceUnavailable { w.Header().Set("Retry-After", retryAfterSeconds) } - writeErr(w, m.code, cmp.Or(m.msg, err.Error())) - return true + return m.code, cmp.Or(m.msg, err.Error()), true } } - return false + return 0, "", false } func writeResult(w http.ResponseWriter, r *http.Request, op, id, failMsg string, err error, ok func()) { diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index d22c162d..fb7eca9e 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -1,7 +1,6 @@ package server import ( - "cmp" "context" "errors" "net/http" @@ -15,8 +14,10 @@ import ( func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { id, tenant := r.PathValue("id"), tenantFrom(r.Context()) - var view types.SandboxVolumesResponse - var err error + var ( + view types.SandboxVolumesResponse + err error + ) switch r.Method { case http.MethodGet: view, err = s.mgr.GetVolumes(id, tenant) @@ -56,11 +57,8 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { case errors.Is(err, context.Canceled): code, view.Error = http.StatusRequestTimeout, "volume change canceled; inspect the volume states before retrying" default: - for _, rule := range poolErrHTTP { - if errors.Is(err, rule.err) { - code, view.Error = rule.code, cmp.Or(rule.msg, err.Error()) - break - } + if mapped, msg, ok := mapPoolErr(w, err); ok { + code, view.Error = mapped, msg } } if code >= http.StatusInternalServerError { diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index d8c881f1..bedce484 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -105,6 +105,9 @@ func TestLiveVolumeRoutesErrors(t *testing.T) { if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { t.Fatalf("status=%d body=%s", resp.StatusCode, body) } + if errors.Is(tt.err, pool.ErrTenantStoreDown) && resp.Header.Get("Retry-After") == "" { + t.Fatal("store outage without Retry-After") + } if tt.err != nil && tt.code != http.StatusNotFound { var got types.SandboxVolumesResponse if err := json.Unmarshal(body, &got); err != nil || len(got.VolumesWant) != 1 || len(got.States) != 1 || got.Error == "" || got.States[0].Error != "volume operation failed; retry after resolving the runtime error" { From 43a426c2e686d4ee5e20de1d8070c4347abaad51 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:12:14 +0800 Subject: [PATCH 36/57] review: Go SDK constants precede their type and volumesmoke shares its read-only probe --- e2e/cmd/volumesmoke/main.go | 27 +++++++++++++++------------ sandboxd/engine/engine.go | 3 ++- sdk/go/sandbox.go | 2 +- sdk/go/volumes.go | 10 +++++----- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index 4465fdf6..7ee5c534 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -105,13 +105,8 @@ func run(addr, token, template, volume, rwVolume, probe string, live bool) error return fmt.Errorf("concurrent reads differ: first_bytes=%d second_bytes=%d", len(outputs[0]), len(outputs[1])) } - _, err = sbA.Exec(ctx, "touch", path.Join(mountA, ".sandboxd-write-probe")) - if err == nil { - return errors.New("write to read-only volume succeeded") - } - exitErr, ok := errors.AsType[*sandbox.ExitError](err) - if !ok || !strings.Contains(strings.ToLower(exitErr.Stderr), "read-only file system") { - return fmt.Errorf("write failed without EROFS: %w", err) + if err := requireReadOnly(ctx, sbA, mountA); err != nil { + return err } if err := errors.Join(sbA.Close(), sbB.Close()); err != nil { @@ -157,11 +152,8 @@ func runLive(ctx context.Context, client *sandbox.Client, template, volume, prob if got != want { return fmt.Errorf("live volume read: bytes=%d want=%d", len(got), len(want)) } - if _, err = sb.Exec(ctx, "touch", path.Join(mount, ".sandboxd-write-probe")); err == nil { - return errors.New("live read-only mount accepted a write") - } - if exitErr, ok := errors.AsType[*sandbox.ExitError](err); !ok || !strings.Contains(strings.ToLower(exitErr.Stderr), "read-only file system") { - return fmt.Errorf("live write probe failed without EROFS: %w", err) + if err = requireReadOnly(ctx, sb, mount); err != nil { + return fmt.Errorf("live volume: %w", err) } for range 2 { if _, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{volume}}); err != nil { @@ -261,3 +253,14 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s volume, float64(claimRW.Microseconds())/1000) return nil } + +func requireReadOnly(ctx context.Context, sb *sandbox.Sandbox, mount string) error { + _, err := sb.Exec(ctx, "touch", path.Join(mount, ".sandboxd-write-probe")) + if err == nil { + return errors.New("write to read-only volume succeeded") + } + if exitErr, ok := errors.AsType[*sandbox.ExitError](err); !ok || !strings.Contains(strings.ToLower(exitErr.Stderr), "read-only file system") { + return fmt.Errorf("write failed without EROFS: %w", err) + } + return nil +} diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index a2a614a8..11dd07ed 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -53,7 +53,7 @@ const ( outputTail = 400 portForwardMax = 4096 // port_forward handshake reply line cap vmNotFound = "vm not found" // cocoon's hypervisor.ErrNotFound text - // at 16 concurrent memory dumps cocoon's sqlite writers already time out on its 5 s lock wait; 8 measured clean + // At 16 concurrent memory dumps cocoon's sqlite writers already time out on its 5 s lock wait; 8 measured clean memoryDumps = 8 ) @@ -144,6 +144,7 @@ func (e *Engine) Start(ctx context.Context, name string) error { return err } +// Stop force-stops a VM and keeps its disk for a later Start. func (e *Engine) Stop(ctx context.Context, name string) error { _, err := e.run(ctx, "vm", "stop", "--force", name) return err diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index a26fcd00..0a3ceb34 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -56,7 +56,7 @@ func (e *ExitError) Error() string { type Sandbox struct { ID string Deadline time.Time - // Volumes lists confirmed applied mounts; only a writable entry echoes a mode. + // Volumes lists confirmed applied volumes; only a writable entry echoes a mode. Volumes []Volume // VolumesWant is the persisted desired set, which may differ after a failed convergence. VolumesWant []Volume diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 7ca24600..69e1dea4 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -11,15 +11,15 @@ import ( "slices" ) -// VolumePhase is the progress of a desired or applied volume. -type VolumePhase string - const ( VolumeAttached VolumePhase = "attached" VolumePending VolumePhase = "pending" VolumeDetaching VolumePhase = "detaching" ) +// VolumePhase is the progress of a desired or applied volume. +type VolumePhase string + // VolumeState reports the progress of one desired or applied volume. type VolumeState struct { Name string `json:"name"` @@ -36,13 +36,13 @@ type SandboxVolumesResponse struct { Error string `json:"error,omitempty"` } -// VolumePatch changes the desired set by name. Omitted slices are unchanged. +// VolumePatch adds and removes volumes by name; a nil slice changes nothing. type VolumePatch struct { Add []Volume `json:"add,omitzero"` Remove []string `json:"remove,omitzero"` } -// VolumeError retains a complete server view after a failed volume request. +// VolumeError reports a failed volume request; Response is nil unless the reply carried the full view. type VolumeError struct { *APIError Response *SandboxVolumesResponse From a6d07b8f119f530f19c77c2111bcabc001e6bde1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:13:41 +0800 Subject: [PATCH 37/57] pool: a rolled-back claim takes over its live run's volume holds rollbackClaim dropped a published claim without cancelling a live volume run and freed heldVolumes(sb), which misses additions the run reserved before its target commit and removals committed while the claim was failing. The leaked reader kept every later writable claim of that volume busy until restart; it now takes the run's holds as Release and reap do. --- sandboxd/pool/claim.go | 5 ++++- sandboxd/pool/volume_mutation_test.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index f3d6dcab..3b49b378 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -477,7 +477,9 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { m.mu.Lock() ids := make([]string, len(sbs)) + holds := make([][]types.Volume, len(sbs)) for i, sb := range sbs { + holds[i] = m.cancelVolumeRun(sb) delete(m.claimed, sb.ID) m.tenantDelta(sb.Tenant, -1) ids[i] = sb.ID @@ -485,8 +487,9 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { rb := m.store.del(ids...) m.mu.Unlock() m.recommit(ctx, rb) - for _, sb := range sbs { + for i, sb := range sbs { td := m.quiesceVolumes(ctx, sb) + td.holds = holds[i] m.out.Disarm(sb.ID, m.removeOrRetry(ctx, sb.VMName, sb.ID, "", td)) } } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index a566abc4..1661a2d6 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -693,6 +693,18 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { } } +func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + run := beginAdmittedVolumeRun(t, m, sb) + m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) + m.finishVolumeRun(sb, run) + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) + } +} + func TestLiveAttachBoundsWholeDeviceOperation(t *testing.T) { synctest.Test(t, func(t *testing.T) { base := newFakeEngine() From ac70e417a173affd150caff0ca9570b2d3ee471d Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:14:16 +0800 Subject: [PATCH 38/57] pool: a live volume run stamps sandbox activity before its device calls The run touched the sandbox only after converging, so an idle sweep could pick a sandbox that was attaching its first volume before prepare reserved any hold, hibernate it, and fail the request with ErrPaused. --- sandboxd/pool/volume_convergence_test.go | 24 ++++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index bb17f477..adf6e580 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -7,6 +7,7 @@ import ( "slices" "strings" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/engine" @@ -202,6 +203,29 @@ func TestReleaseDropsVolumeDiagnostics(t *testing.T) { } } +func TestLiveVolumeRunStampsActivityBeforeDeviceCalls(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + sb.TouchAt(time.Now().Add(-time.Hour)) + eng := newGatedVolumeEngine(base) + m.eng = eng + done := make(chan error, 1) + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) + done <- err + }() + <-eng.entered + idle := time.Since(sb.LastSeen()) + close(eng.resume) + if err := <-done; err != nil { + t.Fatal(err) + } + if idle > time.Minute { + t.Fatalf("a running volume change left the sandbox idle for %v", idle) + } +} + func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { for _, priorConflict := range []bool{false, true} { name := "settled" diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 07ff5858..1fc1d7f5 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -63,6 +63,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if !ok || !tenantOwns(tenant, sb.Tenant) { return view, ErrUnknownSandbox } + sb.Touch() var run *volumeRun if err = m.eng.CheckLiveVolumeSupport(ctx); err == nil { ctx, run, err = m.beginVolumeRun(ctx, sb) @@ -160,7 +161,6 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v } m.clearVolumeError(sb, run, v.Name) } - sb.Touch() return view, nil } From e84f1aa16cf7861d579a4b9c21c2fc971fe728a9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:16:13 +0800 Subject: [PATCH 39/57] pool: live runs write the journal outside Transition and skip an unchanged target stageVolumeTarget holds Transition only for the bookkeeping; the claims write (an fsync under sync_claims) happens after it, so exec and port dials, which take Transition to wake, no longer wait on it. An identical PUT or PATCH reserves nothing, changes nothing and writes no journal. A volume step builds its applied set inside the one locked section that commits it, a run that lost its claim answers ErrUnknownSandbox rather than a cancellation, a failed target commit is not filed under the last precleaned volume, and the two retry hints are constants. The claims DTO carries volumes_want as a plain omitzero slice. --- sandboxd/pool/claims.go | 7 +- sandboxd/pool/volume_convergence_test.go | 5 +- sandboxd/pool/volume_mutation.go | 83 +++++++++++++++--------- 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 21cb4913..6011216c 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -39,7 +39,7 @@ type claimDTO struct { Metadata types.Metadata `json:"metadata,omitempty"` OnExpire types.ExpireAction `json:"on_expire,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` - VolumesWant *[]types.Volume `json:"volumes_want,omitzero"` + VolumesWant []types.Volume `json:"volumes_want,omitzero"` Env types.Env `json:"env,omitempty"` VsockSocket string `json:"vsock_socket,omitempty"` TAP string `json:"tap,omitempty"` @@ -234,10 +234,9 @@ func dtoOf(sb *types.Sandbox) claimDTO { } // volumesWantDTO persists the desired set only when it differs; absent reads back as equal to Volumes. -func volumesWantDTO(sb *types.Sandbox) *[]types.Volume { +func volumesWantDTO(sb *types.Sandbox) []types.Volume { if slices.Equal(sb.VolumesWant, sb.Volumes) { return nil } - want := slices.Clone(sb.VolumesWant) - return &want + return append([]types.Volume{}, sb.VolumesWant...) } diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index adf6e580..2f0ffff0 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -31,11 +31,12 @@ func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { t.Fatalf("retained volume reapplied: %v", eng.volumeSpecs) } before = len(eng.volumeSpecs) + journaled := m.store.mark() if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { t.Fatal(err) } - if len(eng.volumeSpecs) != before { - t.Fatal("identical target reached devices") + if len(eng.volumeSpecs) != before || m.store.mark() != journaled { + t.Fatal("identical target reached devices or the claims journal") } } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 1fc1d7f5..7d01fd9d 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -11,6 +11,11 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +const ( + volumeChangeFailed = "volume change failed; retry the desired target" + volumeChangeInterrupted = "volume change interrupted; retry the desired target" +) + type volumeEdit func([]types.Volume) ([]types.Volume, error) type volumeRun struct { @@ -113,6 +118,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } } + failedName = "" if err = m.prepareVolumeTarget(ctx, sb, run, target, additions, resolved); err != nil { return view, err } @@ -124,10 +130,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.detachVolumeRun(ctx, sb, run, v); err != nil { return view, err } - m.mu.Lock() - next := slices.DeleteFunc(slices.Clone(sb.Volumes), func(a types.Volume) bool { return a.Name == v.Name }) - m.mu.Unlock() - if err = m.commitVolumeApplied(ctx, sb, run, next); err != nil { + if err = m.commitVolumeApplied(ctx, sb, run, v, false); err != nil { return view, err } m.clearVolumeError(sb, run, v.Name) @@ -153,10 +156,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - m.mu.Lock() - next := append(slices.Clone(sb.Volumes), v) - m.mu.Unlock() - if err = m.commitVolumeApplied(ctx, sb, run, next); err != nil { + if err = m.commitVolumeApplied(ctx, sb, run, v, true); err != nil { return view, err } m.clearVolumeError(sb, run, v.Name) @@ -245,43 +245,51 @@ func (m *Manager) cancelVolumeRun(sb *types.Sandbox) []types.Volume { } func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) error { + js, changed, err := m.stageVolumeTarget(ctx, sb, run, target, additions, resolved) + if err != nil || !changed { + return err + } + return m.commitVolumeRun(ctx, sb, run, js) +} + +func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() - return err + return claimSnapshot{}, false, err } if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() - return ErrVolumeCapture + return claimSnapshot{}, false, ErrVolumeCapture } if err := m.reserveVolumes(additions); err != nil { m.mu.Unlock() - return err + return claimSnapshot{}, false, err } run.holds = append(run.holds, additions...) m.mu.Unlock() cleanErr := m.confirmVolumesClean(resolved) m.mu.Lock() + defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { - m.mu.Unlock() - return err + return claimSnapshot{}, false, err } if cleanErr != nil { m.releaseVolumes(additions) run.holds = volumeDifference(run.holds, additions) - m.mu.Unlock() - return cleanErr + return claimSnapshot{}, false, cleanErr } - invalid := volumeDifference(sb.Volumes, sb.VolumesWant) - sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), func(v types.Volume) bool { - return volumeIndex(invalid, v.Name) >= 0 && volumeIndex(target, v.Name) >= 0 - }) + replaced := func(v types.Volume) bool { + return volumeIndex(sb.VolumesWant, v.Name) < 0 && volumeIndex(target, v.Name) >= 0 + } + if len(additions) == 0 && slices.Equal(target, sb.VolumesWant) && !slices.ContainsFunc(sb.Volumes, replaced) { + return claimSnapshot{}, false, nil + } + sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), replaced) sb.VolumesWant = slices.Clone(target) - js := m.store.set(sb) - m.mu.Unlock() - return m.commitVolumeRun(ctx, sb, run, js) + return m.store.set(sb), true, nil } func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { @@ -294,13 +302,16 @@ func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *v return m.checkVolumeRun(ctx, sb, run) } -func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, next []types.Volume) error { +func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume, attached bool) error { m.mu.Lock() - if !m.ownsRunLocked(sb, run) || ctx.Err() != nil { + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() - return context.Canceled + return err + } + sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), func(a types.Volume) bool { return a.Name == v.Name }) + if attached { + sb.Volumes = append(sb.Volumes, v) } - sb.Volumes = next js := m.store.set(sb) m.mu.Unlock() return m.commitVolumeRun(ctx, sb, run, js) @@ -313,8 +324,11 @@ func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *v } m.mu.Lock() defer m.mu.Unlock() - if !m.ownsRunLocked(sb, run) || ctx.Err() != nil { - return context.Canceled + if !m.ownsRunLocked(sb, run) { + return ErrUnknownSandbox + } + if err := ctx.Err(); err != nil { + return err } dropped := volumeDifference(run.holds, heldVolumes(sb)) m.releaseVolumes(dropped) @@ -332,7 +346,12 @@ func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { } func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { - view := types.SandboxVolumesResponse{Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, Converging: m.volumeRuns[sb] != nil} + view := types.SandboxVolumesResponse{ + Volumes: slices.Clone(sb.Volumes), + VolumesWant: slices.Clone(sb.VolumesWant), + States: []types.VolumeState{}, + Converging: m.volumeRuns[sb] != nil, + } for _, v := range heldVolumes(sb) { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { @@ -342,7 +361,7 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { } message := m.volumeErrors[sb][v.Name] if message == "" && state != types.VolumeAttached && !view.Converging { - message = "volume change interrupted; retry the desired target" + message = volumeChangeInterrupted } view.States = append(view.States, types.VolumeState{Name: v.Name, State: state, Error: message}) } @@ -357,9 +376,9 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri } errs := m.volumeErrors[sb] if err != nil && name != "" { - message := "volume change failed; retry the desired target" + message := volumeChangeFailed if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - message = "volume change interrupted; retry the desired target" + message = volumeChangeInterrupted } if conflict, ok := errors.AsType[*engine.VolumeMountConflict](err); ok { message = conflict.Error() From 628e94310e13f8df9b11e99dae1ac3777a108143 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:16:54 +0800 Subject: [PATCH 40/57] review: the exported live volume methods carry one-line godocs like their env siblings --- sandboxd/pool/volume_mutation.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 7d01fd9d..4cfe857c 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -24,6 +24,7 @@ type volumeRun struct { holds []types.Volume } +// GetVolumes reads a claim's desired and applied volumes; a tenant reaches only its own claims. func (m *Manager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { m.mu.Lock() defer m.mu.Unlock() @@ -34,10 +35,12 @@ func (m *Manager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, e return m.volumeView(sb), nil } +// SetVolumes converges a running claim onto the full desired volume set. func (m *Manager) SetVolumes(ctx context.Context, id, tenant string, want []types.Volume) (types.SandboxVolumesResponse, error) { return m.changeVolumes(ctx, id, tenant, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) } +// PatchVolumes adds and removes a running claim's read-only mounted volumes by name. func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { return m.changeVolumes(ctx, id, tenant, func(current []types.Volume) ([]types.Volume, error) { if err := validateDetachNames(patch.Remove); err != nil { From 923612112aeaea41d82b32807bffb72fb33ed43c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:26:29 +0800 Subject: [PATCH 41/57] test: drop dead fakes, share server and pool test builders, and tighten volume test tables Write-only fakeManager fields and the never-set snapSaveErr go. The server tests build every node through serveTestServer and newMeshTestServer, so all of them close their relays, and requests with a bearer token use doReq. The pool tests share waitPoolSettled and newTenantVolumeManager, the restart budget test reuses its assertion helper, and shadowed names, verb-less Errorf calls and helpers split around types are fixed. --- e2e/fakeengine_test.go | 3 +- sandboxd/engine/version_test.go | 12 +- sandboxd/engine/volume_live_test.go | 8 +- sandboxd/pool/claims_test.go | 15 +- sandboxd/pool/egress_test.go | 9 +- sandboxd/pool/liveness_test.go | 26 +- sandboxd/pool/pool_test.go | 177 ++++++----- sandboxd/pool/poolstore_test.go | 61 ++-- sandboxd/pool/reload_test.go | 10 + sandboxd/pool/volume_convergence_test.go | 27 +- sandboxd/pool/volume_mutation_test.go | 144 +++++---- sandboxd/server/server_test.go | 385 ++++++----------------- sandboxd/server/volume_mutation_test.go | 8 +- sdk/go/volume_mutation_test.go | 38 +-- 14 files changed, 347 insertions(+), 576 deletions(-) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index b266b9d7..b4f571fe 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -2,6 +2,7 @@ package e2e import ( "context" + "errors" "fmt" "io" "net" @@ -139,7 +140,7 @@ func (f *fakeEngine) Probe(ctx context.Context, vsockSocket string, timeout time } func (f *fakeEngine) DialGuestPort(context.Context, string, uint16) (net.Conn, error) { - return nil, fmt.Errorf("fakeEngine has no guest port") + return nil, errors.New("fakeEngine has no guest port") } func (f *fakeEngine) InstallCACert(context.Context, string, []byte) error { return nil } diff --git a/sandboxd/engine/version_test.go b/sandboxd/engine/version_test.go index dcfcbaac..0969b28d 100644 --- a/sandboxd/engine/version_test.go +++ b/sandboxd/engine/version_test.go @@ -9,9 +9,9 @@ import ( func TestBelowFloor(t *testing.T) { tests := []struct { - v string - below bool - comparable bool + v string + below bool + known bool }{ {"v0.6.9", false, true}, {"v0.6.10", false, true}, @@ -27,9 +27,9 @@ func TestBelowFloor(t *testing.T) { } for _, tt := range tests { t.Run(tt.v, func(t *testing.T) { - below, comparable := belowFloor(tt.v, RequiredCocoon) - if below != tt.below || comparable != tt.comparable { - t.Errorf("belowFloor(%q) = (%v, %v), want (%v, %v)", tt.v, below, comparable, tt.below, tt.comparable) + below, known := belowFloor(tt.v, RequiredCocoon) + if below != tt.below || known != tt.known { + t.Errorf("belowFloor(%q) = (%v, %v), want (%v, %v)", tt.v, below, known, tt.below, tt.known) } }) } diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 8ec24852..84eed3fc 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" ) @@ -185,12 +186,13 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { t.Fatal("detach must fail closed") } conflict, isConflict := errors.AsType[*VolumeMountConflict](err) - conflictExpected := mode == "busy" || mode == "second mount" + wantPath := map[string]string{"busy": "/data", "second mount": "/other"}[mode] + conflictExpected := wantPath != "" if isConflict != conflictExpected { t.Fatalf("conflict=%v error=%v", conflictExpected, err) } - if conflictExpected && (!strings.Contains(conflict.Error(), "/data") && !strings.Contains(conflict.Error(), "/other") || strings.Contains(conflict.Error(), "/private")) { - t.Fatalf("unsafe or missing guest path: %v", conflict) + if conflictExpected && (!strings.Contains(conflict.Error(), strconv.Quote(wantPath)) || strings.Contains(conflict.Error(), "/private")) { + t.Fatalf("unsafe or missing guest path %s: %v", wantPath, conflict) } if mode == "busy" && !strings.HasPrefix(conflict.Error(), "cannot unmount") { t.Fatalf("busy conflict: %v", conflict) diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 28c0e36f..f9c0009c 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -77,24 +77,25 @@ func TestRecommitDrainsAChangeItWasTooBusyToSee(t *testing.T) { } func TestStoreRoundTrip(t *testing.T) { + dataset := []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}} claims := map[string]*types.Sandbox{ - "sb_a": {ID: "sb_a", VMName: "sbx-1", Key: testKey, Token: "t1", Deadline: time.Now().Add(time.Minute).UTC(), ClaimedAt: time.Now().Add(-time.Minute).UTC(), ClaimRef: "ns/workload", Metadata: types.Metadata{"team": "a"}, OnExpire: types.ExpireArchive, Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VolumesWant: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VsockSocket: "/v/1"}, + "sb_a": {ID: "sb_a", VMName: "sbx-1", Key: testKey, Token: "t1", Deadline: time.Now().Add(time.Minute).UTC(), ClaimedAt: time.Now().Add(-time.Minute).UTC(), ClaimRef: "ns/workload", Metadata: types.Metadata{"team": "a"}, OnExpire: types.ExpireArchive, Volumes: dataset, VolumesWant: dataset, VsockSocket: "/v/1"}, "sb_b": {ID: "sb_b", VMName: "sbx-2", Key: testKey, Token: "t2"}, } - for _, sync := range []bool{false, true} { - s := newClaimStore(t.TempDir(), sync) + for _, fsync := range []bool{false, true} { + s := newClaimStore(t.TempDir(), fsync) if err := s.save(claims); err != nil { - t.Fatalf("save (sync=%v): %v", sync, err) + t.Fatalf("save (sync=%v): %v", fsync, err) } got, err := s.load() if err != nil { - t.Fatalf("load (sync=%v): %v", sync, err) + t.Fatalf("load (sync=%v): %v", fsync, err) } if !reflect.DeepEqual(got, claims) { - t.Errorf("sync=%v: got %+v, want %+v", sync, got, claims) + t.Errorf("sync=%v: got %+v, want %+v", fsync, got, claims) } if _, err := os.Stat(s.path + ".tmp"); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("sync=%v: temp file left behind: %v", sync, err) + t.Errorf("sync=%v: temp file left behind: %v", fsync, err) } } } diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index 6a7559e9..462a8102 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -85,7 +85,7 @@ func TestGoldenStampGatesAdoptionOnLane(t *testing.T) { func TestEgressLaneDoesNotHibernate(t *testing.T) { m := newTestManager(t, newFakeEngine()) - sb := &types.Sandbox{ID: "sb_eg_h", Key: types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall}} + sb := &types.Sandbox{ID: "sb_eg_h", Key: egKey} sb.Transition.Lock() err := m.hibernateLocked(t.Context(), sb) sb.Transition.Unlock() @@ -359,12 +359,7 @@ func TestSetPoolsPreservesEgressPolicy(t *testing.T) { if !policyLive() { t.Fatal("drain + re-add wiped the pool egress policy") } - waitFor(t, func() bool { - m.mu.Lock() - defer m.mu.Unlock() - p := m.pools[egKey] - return p != nil && !p.building && p.refilling == 0 - }) + waitPoolSettled(t, m, egKey) } func TestEgressLaneCannotForkOrCheckpoint(t *testing.T) { diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 682c976f..60a9e5b3 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -129,21 +129,7 @@ func TestLivenessRestartFailureRetriesThenFails(t *testing.T) { killVMM(eng, sb.VMName) eng.startErr = errors.New("boot failed") - for range vmmRestartBudget - 1 { - m.recoverVMM(t.Context(), sb) - if got, _ := m.Sandbox(sb.ID); got.Failed != "" { - t.Fatalf("failed %q after a retryable attempt", got.Failed) - } - } - m.recoverVMM(t.Context(), sb) - - got, _ := m.Sandbox(sb.ID) - if got.Failed != reasonColdBootFailed || got.Restarts != 0 { - t.Errorf("summary %+v, want the fixed reason %q after %d attempts", got, reasonColdBootFailed, vmmRestartBudget) - } - if len(eng.stops) != vmmRestartBudget { - t.Errorf("stops %v, want every failed start stopped", eng.stops) - } + assertRestartFailsUntilBudget(t, m, eng, sb) if got, want := usageEventsOf(t, m, sb.ID), "claim,vmm_exit,vmm_failed"; got != want { t.Errorf("events %s, want one exit across the attempts: %s", got, want) } @@ -524,6 +510,11 @@ func TestReconcileKeepsRunningClaimsTheHostLost(t *testing.T) { } } +type fakeVMEvent struct { + sync []engine.VMStatus + change engine.VMChange +} + func assertRestartFailsUntilBudget(t *testing.T, m *Manager, eng *fakeEngine, sb *types.Sandbox) { t.Helper() for attempt := range vmmRestartBudget { @@ -553,11 +544,6 @@ func runLiveness(t *testing.T, m *Manager) { waitFor(t, func() bool { return !m.livenessSweep.Load() }) } -type fakeVMEvent struct { - sync []engine.VMStatus - change engine.VMChange -} - func killVMM(eng *fakeEngine, name string) { eng.mu.Lock() defer eng.mu.Unlock() diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 676a092e..0d9fc1db 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -153,9 +153,8 @@ func TestClaimCanceledRequestStillDestroys(t *testing.T) { func TestClaimEgressNeedsAttachment(t *testing.T) { m := newTestManager(t, newFakeEngine()) - key := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall} - if _, err := claimAny(t.Context(), m, key, 0); !errors.Is(err, ErrNoEgress) { + if _, err := claimAny(t.Context(), m, egKey, 0); !errors.Is(err, ErrNoEgress) { t.Errorf("got %v, want ErrNoEgress", err) } } @@ -1190,83 +1189,6 @@ func (c *closeWriteConn) CloseWrite() error { return nil } -func newTestManager(t testing.TB, eng *fakeEngine, pools ...config.PoolSpec) *Manager { - t.Helper() - return newTestManagerAt(t, eng, t.TempDir(), pools...) -} - -func claimAny(ctx context.Context, m *Manager, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) { - sb, err := m.ClaimWarm(ctx, key, ClaimOptions{TTL: ttl}) - if errors.Is(err, ErrNoWarm) { - return m.ClaimProvision(ctx, key, ClaimOptions{TTL: ttl}) - } - return sb, err -} - -func mustClaim(t *testing.T, m *Manager, key types.PoolKey) *types.Sandbox { - t.Helper() - sb, err := claimAny(t.Context(), m, key, 0) - if err != nil { - t.Fatalf("Claim: %v", err) - } - return sb -} - -func newTestManagerAt(t testing.TB, eng *fakeEngine, dataDir string, pools ...config.PoolSpec) *Manager { - t.Helper() - m, err := NewManager(t.Context(), &config.Config{DataDir: dataDir, Pools: pools}, eng, testSecrets(t)) - if err != nil { - t.Fatalf("setup manager: %v", err) - } - return m -} - -func testSecrets(t testing.TB, specs ...egress.SecretSpec) *egress.SecretStore { - t.Helper() - s, err := egress.NewSecretStore(specs) - if err != nil { - t.Fatalf("secrets: %v", err) - } - return s -} - -func lockCount(m *Manager) int { - m.recLocksMu.Lock() - defer m.recLocksMu.Unlock() - return len(m.recLocks) -} - -func hasRecLock(m *Manager, id string) bool { - m.recLocksMu.Lock() - defer m.recLocksMu.Unlock() - _, ok := m.recLocks[id] - return ok -} - -func waitFor(t *testing.T, cond func() bool) { - t.Helper() - deadline := time.Now().Add(3 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(5 * time.Millisecond) - } - t.Fatal("condition not met within 3s") -} - -func seedGolden(t *testing.T, m *Manager, imageID string) string { - t.Helper() - g := filepath.Join(m.goldensDir(), testKey.Hash()) - if err := os.MkdirAll(g, 0o750); err != nil { - t.Fatalf("seed golden: %v", err) - } - if err := os.WriteFile(g+goldenStampSuffix, []byte(m.goldenStamp(m.view.Load(), testKey, imageID)), 0o644); err != nil { - t.Fatalf("seed golden stamp: %v", err) - } - return g -} - type fakeEngine struct { mu sync.Mutex vms map[string]string @@ -1317,14 +1239,14 @@ type fakeEngine struct { vsockLateN int sockRoot string - cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapSaveErr, snapListErr, startErr error - removeErrFor string - cloneFailNth int - hibernateErrCompletes bool - tap string - listCount int - inspects int - vmEvents chan fakeVMEvent + cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapListErr, startErr error + removeErrFor string + cloneFailNth int + hibernateErrCompletes bool + tap string + listCount int + inspects int + vmEvents chan fakeVMEvent attachRendezvous *sync.WaitGroup @@ -1445,7 +1367,7 @@ func (f *fakeEngine) SnapshotSave(_ context.Context, _, snapName string) error { f.mu.Lock() defer f.mu.Unlock() f.snapSaves = append(f.snapSaves, snapName) - return f.snapSaveErr + return nil } func (f *fakeEngine) SnapshotExport(_ context.Context, snapName, toDir string) error { @@ -1620,7 +1542,7 @@ func (f *fakeEngine) DialGuestPort(context.Context, string, uint16) (net.Conn, e if f.guestDial != nil { return f.guestDial() } - return nil, fmt.Errorf("fakeEngine has no guest port") + return nil, errors.New("fakeEngine has no guest port") } func (f *fakeEngine) InstallCACert(_ context.Context, vsockSocket string, _ []byte) error { @@ -1899,3 +1821,80 @@ func (f *fakeEngine) snapRemoved(name string) bool { defer f.mu.Unlock() return slices.Contains(f.snapRemoves, name) } + +func newTestManager(t testing.TB, eng *fakeEngine, pools ...config.PoolSpec) *Manager { + t.Helper() + return newTestManagerAt(t, eng, t.TempDir(), pools...) +} + +func claimAny(ctx context.Context, m *Manager, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) { + sb, err := m.ClaimWarm(ctx, key, ClaimOptions{TTL: ttl}) + if errors.Is(err, ErrNoWarm) { + return m.ClaimProvision(ctx, key, ClaimOptions{TTL: ttl}) + } + return sb, err +} + +func mustClaim(t *testing.T, m *Manager, key types.PoolKey) *types.Sandbox { + t.Helper() + sb, err := claimAny(t.Context(), m, key, 0) + if err != nil { + t.Fatalf("Claim: %v", err) + } + return sb +} + +func newTestManagerAt(t testing.TB, eng *fakeEngine, dataDir string, pools ...config.PoolSpec) *Manager { + t.Helper() + m, err := NewManager(t.Context(), &config.Config{DataDir: dataDir, Pools: pools}, eng, testSecrets(t)) + if err != nil { + t.Fatalf("setup manager: %v", err) + } + return m +} + +func testSecrets(t testing.TB, specs ...egress.SecretSpec) *egress.SecretStore { + t.Helper() + s, err := egress.NewSecretStore(specs) + if err != nil { + t.Fatalf("secrets: %v", err) + } + return s +} + +func lockCount(m *Manager) int { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + return len(m.recLocks) +} + +func hasRecLock(m *Manager, id string) bool { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + _, ok := m.recLocks[id] + return ok +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("condition not met within 3s") +} + +func seedGolden(t *testing.T, m *Manager, imageID string) string { + t.Helper() + g := filepath.Join(m.goldensDir(), testKey.Hash()) + if err := os.MkdirAll(g, 0o750); err != nil { + t.Fatalf("seed golden: %v", err) + } + if err := os.WriteFile(g+goldenStampSuffix, []byte(m.goldenStamp(m.view.Load(), testKey, imageID)), 0o644); err != nil { + t.Fatalf("seed golden stamp: %v", err) + } + return g +} diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index 303c4b24..f36e9051 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -217,12 +217,7 @@ func TestAStaleCellVersionIsNotApplied(t *testing.T) { if floor != 3 { t.Errorf("warm floor %d after an older and a repeated version, want 3 from version 5", floor) } - waitFor(t, func() bool { - m.mu.Lock() - defer m.mu.Unlock() - p := m.pools[apiKey] - return p != nil && !p.building && p.refilling == 0 - }) + waitPoolSettled(t, m, apiKey) } func TestAPutTheCellCannotStoreChangesNothing(t *testing.T) { @@ -275,33 +270,6 @@ func TestBootPrefersTheCellSetOverPoolsJSON(t *testing.T) { } } -func metaStoreNode(t *testing.T, cell string) *Manager { - t.Helper() - cfg := &config.Config{ - DataDir: t.TempDir(), - Pools: []config.PoolSpec{{PoolKey: seedKey, Warm: 1}}, - MetaStore: &config.MetaStoreConfig{Kind: "pg", DSNEnv: "SANDBOX_TEST_DSN", Cell: cell}, - } - m, err := NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t)) - if err != nil { - t.Fatalf("setup manager: %v", err) - } - t.Cleanup(func() { _ = m.tenants.Close() }) - if m.poolShared != nil { - t.Cleanup(func() { _ = m.poolShared.Close() }) - go m.followCellPools(t.Context()) - } - return m -} - -func hasAPISet(m *Manager) bool { - m.mu.Lock() - defer m.mu.Unlock() - p, api := m.activePool(apiKey) - _, seed := m.activePool(seedKey) - return api && p.floor == 3 && !seed -} - type fakeShared struct { set *poolset.Set version int64 @@ -334,3 +302,30 @@ func (f *fakeShared) Watch(ctx context.Context, changed func()) { func (f *fakeShared) Close() error { return nil } + +func metaStoreNode(t *testing.T, cell string) *Manager { + t.Helper() + cfg := &config.Config{ + DataDir: t.TempDir(), + Pools: []config.PoolSpec{{PoolKey: seedKey, Warm: 1}}, + MetaStore: &config.MetaStoreConfig{Kind: "pg", DSNEnv: "SANDBOX_TEST_DSN", Cell: cell}, + } + m, err := NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t)) + if err != nil { + t.Fatalf("setup manager: %v", err) + } + t.Cleanup(func() { _ = m.tenants.Close() }) + if m.poolShared != nil { + t.Cleanup(func() { _ = m.poolShared.Close() }) + go m.followCellPools(t.Context()) + } + return m +} + +func hasAPISet(m *Manager) bool { + m.mu.Lock() + defer m.mu.Unlock() + p, api := m.activePool(apiKey) + _, seed := m.activePool(seedKey) + return api && p.floor == 3 && !seed +} diff --git a/sandboxd/pool/reload_test.go b/sandboxd/pool/reload_test.go index 6c09a3fd..e5e38799 100644 --- a/sandboxd/pool/reload_test.go +++ b/sandboxd/pool/reload_test.go @@ -355,6 +355,16 @@ func refilledTo(t *testing.T, m *Manager, key types.PoolKey, n int) bool { return warmLen(m, key) == n } +func waitPoolSettled(t *testing.T, m *Manager, key types.PoolKey) { + t.Helper() + waitFor(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + p := m.pools[key] + return p != nil && !p.building && p.refilling == 0 + }) +} + func warmLen(m *Manager, key types.PoolKey) int { m.mu.Lock() defer m.mu.Unlock() diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 2f0ffff0..32c748f8 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -42,15 +42,7 @@ func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { eng := newFakeEngine() - m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: liveCatalog(t)}, eng, testSecrets(t)) - if err != nil { - t.Fatal(err) - } - putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme"}) - if err != nil { - t.Fatal(err) - } + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { t.Fatal(err) } @@ -170,7 +162,7 @@ func TestVolumeDiagnosticsTrackActualWorkerAndRetry(t *testing.T) { if err != nil || view.States[0].Error != "" || view.Converging { t.Fatalf("successful retry retained errors: %+v %v", view, err) } - conflict := &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} + conflict := unmountConflict() base.detachVolumeErr = conflict view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}) if !errors.Is(err, conflict) || view.States[0].Error != conflict.Error() { @@ -189,7 +181,7 @@ func TestReleaseDropsVolumeDiagnostics(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - eng.detachVolumeErr = &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} + eng.detachVolumeErr = unmountConflict() if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { t.Fatal("expected mount conflict") } @@ -241,7 +233,7 @@ func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { t.Fatal(err) } if priorConflict { - eng.detachVolumeErr = &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} + eng.detachVolumeErr = unmountConflict() if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { t.Fatal("expected mount conflict") } @@ -307,11 +299,12 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { t.Fatal(err) } m.mu.Lock() - if err := m.reserveVolumes(held); err != nil { - t.Fatal(err) - } + err = m.reserveVolumes(held) run.holds = held m.mu.Unlock() + if err != nil { + t.Fatal(err) + } defer m.finishVolumeRun(sb, run) } if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "blocked", ""); !errors.Is(err, ErrVolumeCapture) { @@ -405,3 +398,7 @@ type unsupportedVolumeEngine struct{ *fakeEngine } func (e *unsupportedVolumeEngine) CheckLiveVolumeSupport(context.Context) error { return &engine.LiveVolumeUnsupported{} } + +func unmountConflict() *engine.VolumeMountConflict { + return &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} +} diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 1661a2d6..5af7d9da 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -50,25 +50,27 @@ func TestLiveVolumesCommitAndRetry(t *testing.T) { func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { for _, attachOnly := range []bool{false, true} { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: "rw", AttachOnly: attachOnly}}}) - if err != nil { - t.Fatal(err) - } - original := sb.Volumes[0] - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatal(err) - } - if sb.Volumes[0] != original { - t.Fatalf("changed existing volume: %v -> %v", original, sb.Volumes[0]) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { - t.Fatalf("accepted mode change: %v", err) - } - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { - t.Fatalf("writable detach changed the batch: %v", err) - } + t.Run(fmt.Sprintf("attach_only=%t", attachOnly), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: "rw", AttachOnly: attachOnly}}}) + if err != nil { + t.Fatal(err) + } + original := sb.Volumes[0] + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatal(err) + } + if sb.Volumes[0] != original { + t.Fatalf("changed existing volume: %v -> %v", original, sb.Volumes[0]) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("accepted mode change: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { + t.Fatalf("writable detach changed the batch: %v", err) + } + }) } } @@ -351,15 +353,7 @@ func TestLiveVolumesRespectOwningTenantAndRemoval(t *testing.T) { eng := newFakeEngine() catalog := liveCatalog(t) catalog[1].Tenants = []string{"beta"} - m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: catalog}, eng, testSecrets(t)) - if err != nil { - t.Fatal(err) - } - putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme"}) - if err != nil { - t.Fatal(err) - } + m, sb := newTenantVolumeManager(t, eng, catalog) for _, auth := range []string{sb.Tenant, ""} { if _, err := m.SetVolumes(t.Context(), sb.ID, auth, []types.Volume{{Name: "other"}}); !errors.Is(err, ErrVolumeUnavailable) { t.Fatalf("catalog ACL: %v", err) @@ -382,9 +376,11 @@ func TestLiveVolumesRespectOwningTenantAndRemoval(t *testing.T) { func TestDesiredVolumesJournalMigration(t *testing.T) { for _, tt := range []struct { name, field string - want int + want []types.Volume }{ - {"legacy", "", 1}, {"explicit empty", `,"volumes_want":[]`, 0}, {"desired", `,"volumes_want":[{"name":"other","mount":"/other"}]`, 1}, + {"legacy", "", []types.Volume{{Name: "data", Mount: "/data"}}}, + {"explicit empty", `,"volumes_want":[]`, nil}, + {"desired", `,"volumes_want":[{"name":"other","mount":"/other"}]`, []types.Volume{{Name: "other", Mount: "/other"}}}, } { t.Run(tt.name, func(t *testing.T) { store := newClaimStore(t.TempDir(), false) @@ -396,12 +392,8 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { if err != nil { t.Fatal(err) } - sb := claims["sb_1"] - if len(sb.VolumesWant) != tt.want { - t.Fatalf("want=%v", sb.VolumesWant) - } - if tt.name == "desired" && sb.VolumesWant[0].Name != "other" { - t.Fatalf("target replaced by applied: %v", sb.VolumesWant) + if got := claims["sb_1"].VolumesWant; !slices.Equal(got, tt.want) { + t.Fatalf("loaded target %v, want %v", got, tt.want) } if err = store.save(claims); err != nil { t.Fatal(err) @@ -737,19 +729,11 @@ func TestLivenessRefusesAdmittedVolumeRun(t *testing.T) { func TestRemovedTenantCanRemoveReadOnlyVolumeBesideImmutableVolume(t *testing.T) { eng := newFakeEngine() - m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: liveCatalog(t)}, eng, testSecrets(t)) - if err != nil { + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t), types.Volume{Name: "other", Mode: "rw"}) + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { t.Fatal(err) } - putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "other", Mode: "rw"}}}) - if err != nil { - t.Fatal(err) - } - if _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatal(err) - } - if err = m.DeleteTenant(t.Context(), "acme"); err != nil { + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatal(err) } view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) @@ -761,32 +745,6 @@ func TestRemovedTenantCanRemoveReadOnlyVolumeBesideImmutableVolume(t *testing.T) } } -func liveCatalog(t *testing.T) []config.VolumeSpec { - t.Helper() - return []config.VolumeSpec{ - {Name: "data", Path: writeVolumeImage(t, "data.img", "data"), Writable: true}, - {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, - } -} - -func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { - t.Helper() - claims, err := newClaimStore(m.dataDir, false).load() - if err != nil { - t.Fatal(err) - } - got := claims[sb.ID] - if got == nil || !slices.Equal(got.Volumes, sb.Volumes) { - t.Fatalf("journal volumes: %+v", got) - } - if !slices.Equal(got.VolumesWant, sb.VolumesWant) { - t.Fatalf("journal target: %+v, want %+v", got.VolumesWant, sb.VolumesWant) - } - if _, err := os.Stat(m.store.path); err != nil { - t.Fatal(err) - } -} - type gatedVolumeEngine struct { *fakeEngine entered chan struct{} @@ -818,6 +776,46 @@ func (e *deadlineVolumeEngine) DiskAttach(ctx context.Context, vmName string, sp return e.fakeEngine.DiskAttach(ctx, vmName, spec) } +func liveCatalog(t *testing.T) []config.VolumeSpec { + t.Helper() + return []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "data.img", "data"), Writable: true}, + {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, + } +} + +func newTenantVolumeManager(t *testing.T, eng *fakeEngine, catalog []config.VolumeSpec, volumes ...types.Volume) (*Manager, *types.Sandbox) { + t.Helper() + m, err := NewManager(t.Context(), &config.Config{DataDir: t.TempDir(), APIToken: "root", Volumes: catalog}, eng, testSecrets(t)) + if err != nil { + t.Fatal(err) + } + putTenants(t, m, config.TenantSpec{Name: "acme", Token: "acme-token"}) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: volumes}) + if err != nil { + t.Fatal(err) + } + return m, sb +} + +func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil { + t.Fatal(err) + } + got := claims[sb.ID] + if got == nil || !slices.Equal(got.Volumes, sb.Volumes) { + t.Fatalf("journal volumes: %+v", got) + } + if !slices.Equal(got.VolumesWant, sb.VolumesWant) { + t.Fatalf("journal target: %+v, want %+v", got.VolumesWant, sb.VolumesWant) + } + if _, err := os.Stat(m.store.path); err != nil { + t.Fatal(err) + } +} + func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volumeRun { t.Helper() _, run, err := m.beginVolumeRun(t.Context(), sb) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index e1d560c7..499f192a 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -135,15 +135,7 @@ func TestClaimErrorMapping(t *testing.T) { func TestHibernateVolumeCaptureMapsConflict(t *testing.T) { mgr := &fakeManager{hibernate: func(string, string) error { return pool.ErrVolumeCapture }} ts := newTestServer(t, "", mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/hibernate", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer tok") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("hibernate: %v", err) - } + resp := doReq(t, http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/hibernate", "tok", "") defer resp.Body.Close() if resp.StatusCode != http.StatusConflict { t.Errorf("status %d, want 409", resp.StatusCode) @@ -244,15 +236,7 @@ func TestRenewRejectsWrongToken(t *testing.T) { func TestInfoReportsTheNodeAdvertiseAddr(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/v1/info", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+"/v1/info", "sekret", "") defer resp.Body.Close() var info InfoResponse if err := json.UnmarshalRead(resp.Body, &info); err != nil { @@ -494,18 +478,8 @@ func TestVolumeCatalogReturnsScopedFleetProjection(t *testing.T) { }} placer := &fakePlacer{volumeHolders: map[string]int{"imagenet": 3}} mgr.tenants = []config.TenantSpec{{Name: "acme", Token: "acme-tok"}} - srv := New("root", "node:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(ts.Close) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/v1/volumes", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer acme-tok") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("volumes: %v", err) - } + ts := newMeshTestServer(t, "root", mgr, placer, nil) + resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", "acme-tok", "") defer resp.Body.Close() var got types.VolumeListResponse if err := json.UnmarshalRead(resp.Body, &got); err != nil { @@ -529,15 +503,7 @@ func TestSandboxIndexReturnsScopedAppliedVolumes(t *testing.T) { }} }} ts := newTenantTestServer(t, "root", []config.TenantSpec{{Name: "acme", Token: "acme-tok"}}, mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/v1/sandboxes", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer acme-tok") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("sandboxes: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes", "acme-tok", "") defer resp.Body.Close() var got SandboxListResponse if err := json.UnmarshalRead(resp.Body, &got); err != nil { @@ -555,15 +521,7 @@ func TestSandboxesReportClaimedAt(t *testing.T) { return []pool.SandboxSummary{{ID: "sb_1", ClaimedAt: claimed}} }} ts := newTenantTestServer(t, "root", nil, mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/v1/sandboxes", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer root") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("sandboxes: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes", "root", "") defer resp.Body.Close() var got SandboxListResponse if err := json.UnmarshalRead(resp.Body, &got); err != nil { @@ -623,15 +581,7 @@ func TestSandboxesNarrowToAClaimRef(t *testing.T) { return nil }} ts := newTenantTestServer(t, "root", nil, mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+tt.path, nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer root") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("sandboxes: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+tt.path, "root", "") resp.Body.Close() if resp.StatusCode != http.StatusOK || got != tt.want { t.Errorf("%s: status %d, manager saw claim_ref %q, want %q", tt.path, resp.StatusCode, got, tt.want) @@ -725,16 +675,7 @@ func TestMetricsTenantGauge(t *testing.T) { func TestPutPoolsRejectsUnknownFields(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/pools", - strings.NewReader(`{"pools":[{"template":"rt:24.04","net":"none","size":"small","egres":{"allow":[]}}]}`)) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } + resp := doReq(t, http.MethodPut, ts.URL+"/v1/pools", "sekret", `{"pools":[{"template":"rt:24.04","net":"none","size":"small","egres":{"allow":[]}}]}`) defer resp.Body.Close() if resp.StatusCode != http.StatusBadRequest { t.Fatalf("status %d, want 400", resp.StatusCode) @@ -798,15 +739,7 @@ func TestDrainEndpoints(t *testing.T) { {http.MethodPost, true}, {http.MethodDelete, false}, } { - req, err := http.NewRequestWithContext(t.Context(), tt.method, ts.URL+"/v1/drain", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } + resp := doReq(t, tt.method, ts.URL+"/v1/drain", "sekret", "") var info InfoResponse if err := json.UnmarshalRead(resp.Body, &info); err != nil { t.Fatalf("decode: %v", err) @@ -834,16 +767,7 @@ func TestPutPoolsUpdatesTargets(t *testing.T) { } ts := newTestServer(t, "sekret", mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/pools", - strings.NewReader(`{"pools":[{"template":"rt:24.04","net":"none","size":"small","warm":2}]}`)) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } + resp := doReq(t, http.MethodPut, ts.URL+"/v1/pools", "sekret", `{"pools":[{"template":"rt:24.04","net":"none","size":"small","warm":2}]}`) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("status %d, want 200", resp.StatusCode) @@ -1208,20 +1132,7 @@ func TestCheckpointFlow(t *testing.T) { } ts := newTestServer(t, "api", mgr, &fakeDialer{}) - post := func(path, body string) *http.Response { - req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, ts.URL+path, strings.NewReader(body)) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer api") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("%s: %v", path, err) - } - return resp - } - - resp := post("/v1/sandboxes/sb_1/checkpoint", `{"token":"tok","name":"step-1"}`) + resp := postJSON(t, ts.URL+"/v1/sandboxes/sb_1/checkpoint", "api", `{"token":"tok","name":"step-1"}`) defer resp.Body.Close() var cr types.CheckpointResponse if err := json.UnmarshalRead(resp.Body, &cr); err != nil || cr.Checkpoint.ID != "ck_0011223344556677" { @@ -1231,28 +1142,20 @@ func TestCheckpointFlow(t *testing.T) { t.Errorf("name %q, want step-1", cr.Checkpoint.Name) } - resp2 := post("/v1/checkpoints/"+cr.Checkpoint.ID+"/claim", `{}`) + resp2 := postJSON(t, ts.URL+"/v1/checkpoints/"+cr.Checkpoint.ID+"/claim", "api", `{}`) defer resp2.Body.Close() var claim types.ClaimResponse if err := json.UnmarshalRead(resp2.Body, &claim); err != nil || claim.ID != "sb_branch" { t.Fatalf("claim from checkpoint: status %d, %+v, %v", resp2.StatusCode, claim, err) } - resp3 := post("/v1/checkpoints/ck_ffffffffffffffff/claim", `{}`) + resp3 := postJSON(t, ts.URL+"/v1/checkpoints/ck_ffffffffffffffff/claim", "api", `{}`) defer resp3.Body.Close() if resp3.StatusCode != http.StatusNotFound { t.Errorf("unknown checkpoint claim: status %d, want 404", resp3.StatusCode) } - req, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+"/v1/checkpoints/"+cr.Checkpoint.ID, nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer api") - resp4, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("delete: %v", err) - } + resp4 := doReq(t, http.MethodDelete, ts.URL+"/v1/checkpoints/"+cr.Checkpoint.ID, "api", "") defer resp4.Body.Close() if resp4.StatusCode != http.StatusNoContent { t.Errorf("delete checkpoint: status %d, want 204", resp4.StatusCode) @@ -1264,15 +1167,7 @@ func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { ts := newTestServer(t, "api", mgr, &fakeDialer{}) del := func(path string) int { - req, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+path, nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer api") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("delete: %v", err) - } + resp := doReq(t, http.MethodDelete, ts.URL+path, "api", "") defer resp.Body.Close() return resp.StatusCode } @@ -1297,16 +1192,8 @@ func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { t.Run("original delete", func(t *testing.T) { prober.forgotten = nil mgr := &fakeManager{deleteCheckpoint: func(string) error { return nil }} - ts := newPlacerTestServer(t, "api", mgr, prober) - req, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+"/v1/checkpoints/ck_0011223344556677", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer api") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("delete: %v", err) - } + ts := newProberTestServer(t, "api", mgr, prober) + resp := doReq(t, http.MethodDelete, ts.URL+"/v1/checkpoints/ck_0011223344556677", "api", "") defer resp.Body.Close() if len(prober.forgotten) != 1 || prober.forgotten[0] != "ck_0011223344556677" { t.Errorf("forgotten = %v, want [ck_0011223344556677]", prober.forgotten) @@ -1316,16 +1203,8 @@ func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { t.Run("forwarded delete finding nothing locally", func(t *testing.T) { prober.forgotten = nil mgr := &fakeManager{deleteCheckpoint: func(string) error { return pool.ErrUnknownCheckpoint }} - ts := newPlacerTestServer(t, "api", mgr, prober) - req, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+"/v1/checkpoints/ck_0011223344556677?no_forward=1", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer api") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("delete: %v", err) - } + ts := newProberTestServer(t, "api", mgr, prober) + resp := doReq(t, http.MethodDelete, ts.URL+"/v1/checkpoints/ck_0011223344556677?no_forward=1", "api", "") defer resp.Body.Close() if len(prober.forgotten) != 1 || prober.forgotten[0] != "ck_0011223344556677" { t.Errorf("forgotten = %v, want [ck_0011223344556677] even though nothing was held locally", prober.forgotten) @@ -1339,9 +1218,7 @@ func TestClaimRedirectsOnWarmMiss(t *testing.T) { provisioned = true return &types.Sandbox{ID: "sb_local"}, nil }} - srv := New("", "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: []string{"node-b:7777", "node-c:7777"}}, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, &fakePlacer{addrs: []string{"node-b:7777", "node-c:7777"}}, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader(`{"template":"rt:24.04"}`)) if err != nil { @@ -1381,9 +1258,7 @@ func TestClaimRedirectsToTemplateOwner(t *testing.T) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }, } - srv := New("", "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, &fakePlacer{owners: []string{"node-b:7777"}}, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader(`{"template":"tpl"}`)) if err != nil { @@ -1409,24 +1284,14 @@ func TestATemplateDigestGatesDeleteAndLabelWrites(t *testing.T) { mgr := &fakeManager{deleteGolden: func(types.PoolKey) error { return pool.ErrTemplateReplaced }} ts := newTestServer(t, "sekret", mgr, nil) - req, _ := http.NewRequest(http.MethodDelete, ts.URL+"/v1/templates?template=ns%2Fapp&digest=sha256%3Aold", nil) - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } + resp := doReq(t, http.MethodDelete, ts.URL+"/v1/templates?template=ns%2Fapp&digest=sha256%3Aold", "sekret", "") defer resp.Body.Close() if resp.StatusCode != http.StatusPreconditionFailed || mgr.gotDigest != "sha256:old" { t.Errorf("status=%d digest=%q, want 412 for a replaced generation and the digest passed through", resp.StatusCode, mgr.gotDigest) } mgr.labelErr, mgr.gotDigest = pool.ErrTemplateReplaced, "" - req, _ = http.NewRequest(http.MethodPut, ts.URL+"/v1/templates/labels?template=ns%2Fapp&digest=sha256%3Aold", strings.NewReader(`{"labels":{"v1":"sha256:new"}}`)) - req.Header.Set("Authorization", "Bearer sekret") - resp, err = http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } + resp = doReq(t, http.MethodPut, ts.URL+"/v1/templates/labels?template=ns%2Fapp&digest=sha256%3Aold", "sekret", `{"labels":{"v1":"sha256:new"}}`) defer resp.Body.Close() if resp.StatusCode != http.StatusPreconditionFailed || mgr.gotDigest != "sha256:old" { t.Errorf("labels: status=%d digest=%q, want 412 for a replaced generation and the digest passed through", resp.StatusCode, mgr.gotDigest) @@ -1435,9 +1300,7 @@ func TestATemplateDigestGatesDeleteAndLabelWrites(t *testing.T) { func TestDeleteTemplateRedirectsToOwner(t *testing.T) { mgr := &fakeManager{} - srv := New("", "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, &fakePlacer{owners: []string{"node-b:7777"}}, nil) req, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+"/v1/templates?template=tpl", nil) if err != nil { @@ -1472,9 +1335,7 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { t.Errorf("status %d, want 404 with no_redirect despite known owners", resp2.StatusCode) } - srvNoOwner := New("", "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{}, nil, nil, nil) - ts2 := httptest.NewServer(srvNoOwner.Handler()) - t.Cleanup(func() { ts2.Close(); srvNoOwner.CloseRelays() }) + ts2 := newMeshTestServer(t, "", mgr, &fakePlacer{}, nil) req3, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts2.URL+"/v1/templates?template=tpl", nil) if err != nil { t.Fatalf("request: %v", err) @@ -1493,9 +1354,7 @@ func TestClaimProvisionsWhenNoCandidate(t *testing.T) { mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }} - srv := New("", "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: nil}, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, &fakePlacer{addrs: nil}, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader(`{"template":"rt:24.04","claim_ref":"ns1/w1"}`)) if err != nil { @@ -1547,9 +1406,7 @@ func TestVolumeClaimUsesWarmBeforeRedirectOrProvision(t *testing.T) { placer := &fakePlacer{ addrs: []string{"wrong-general-candidate:7777"}, volumeCandidates: tt.volumeCandidates, } - srv := New("", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, placer, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader(`{"template":"rt:24.04","volumes":[{"name":"imagenet"}]}`)) if err != nil { @@ -1658,9 +1515,7 @@ func TestVolumeClaimUsesVolumeAndTemplateIntersections(t *testing.T) { addrs: []string{"warm-peer:7777"}, owners: tt.templateOwners, volumeOwners: tt.volumeOwners, templateVolumeOwners: tt.templateVolumeOwner, } - srv := New("", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, placer, nil) request := types.ClaimRequest{ Template: "tpl", Volumes: []types.Volume{{Name: "imagenet"}}, @@ -1741,9 +1596,7 @@ func TestVolumeClaimKeepsPoolContentAgainstPeerTemplates(t *testing.T) { } } placer := &fakePlacer{owners: []string{"template:7777"}, templateVolumeOwners: []string{"both:7777"}} - srv := New("", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, placer, nil) body, err := json.Marshal(types.ClaimRequest{ Template: "tpl", Volumes: []types.Volume{{Name: "imagenet"}}, @@ -1802,17 +1655,10 @@ func TestForeignTemplateGossipNeverEscalates(t *testing.T) { types.TemplateGossipHash(hash, types.TemplateRootScope): {"peer:7777"}, }} mgr.tenants = tenants - srv := New("root-tok", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "root-tok", mgr, placer, nil) body, _ := json.Marshal(types.ClaimRequest{Template: "tpl", Volumes: tt.volumes}) - req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/claim", bytes.NewReader(body)) - req.Header.Set("Authorization", "Bearer "+tt.token) - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("claim: %v", err) - } + resp := postJSON(t, ts.URL+"/v1/claim", tt.token, string(body)) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("status=%d, want 200", resp.StatusCode) @@ -1854,9 +1700,7 @@ func TestARequiredPromotedClaimIsNeverSentToAWarmPeer(t *testing.T) { mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return nil, tt.provision }} - srv := New("sekret", "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: []string{"warm:7777"}}, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "sekret", mgr, &fakePlacer{addrs: []string{"warm:7777"}}, nil) body, _ := json.Marshal(types.ClaimRequest{Template: "ns/app", RequirePromoted: tt.require}) resp := postJSON(t, ts.URL+"/v1/claim", "sekret", string(body)) @@ -1920,9 +1764,7 @@ func TestVolumeClaimValidatesKeyBeforePlacement(t *testing.T) { }, } placer := &fakePlacer{volumeOwners: []string{"volume:7777"}} - srv := New("", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, placer, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader( `{"template":"rt:24.04","net":"egress","volumes":[{"name":"imagenet"}]}`)) @@ -1946,9 +1788,7 @@ func TestVolumeClaimQuotaDoesNotRedirect(t *testing.T) { placer := &fakePlacer{ addrs: []string{"wrong-general-candidate:7777"}, volumeCandidates: []string{"wrong-volume-candidate:7777"}, } - srv := New("", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, placer, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader(`{"template":"rt:24.04","volumes":[{"name":"imagenet"}]}`)) if err != nil { @@ -1979,16 +1819,13 @@ func TestVolumeClaimRejectsShapeBeforePlacement(t *testing.T) { } { mgr := &fakeManager{} placer := &fakePlacer{addrs: []string{"warm-peer:7777"}, owners: []string{"owner:7777"}} - srv := New("", "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) + ts := newMeshTestServer(t, "", mgr, placer, nil) resp, err := http.Post(ts.URL+"/v1/claim", "application/json", strings.NewReader(body)) if err != nil { t.Fatalf("claim: %v", err) } resp.Body.Close() - ts.Close() - srv.CloseRelays() if resp.StatusCode != http.StatusBadRequest { t.Errorf("body=%s status=%d, want 400", body, resp.StatusCode) } @@ -2005,16 +1842,9 @@ func TestOwnerEndpoint(t *testing.T) { } return "", pool.ErrUnknownSandbox }} - srv := New("", "node-b:7777", mgr, &fakeDialer{}, nil, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := newMeshTestServer(t, "", mgr, nil, nil) - req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/owner", nil) - req.Header.Set("Authorization", "Bearer good") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("owner: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/owner", "good", "") defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("status %d, want 200", resp.StatusCode) @@ -2023,16 +1853,11 @@ func TestOwnerEndpoint(t *testing.T) { OwnerAddr string `json:"owner_addr"` } _ = json.UnmarshalRead(resp.Body, &body) - if body.OwnerAddr != "node-b:7777" { - t.Errorf("owner %q, want node-b:7777", body.OwnerAddr) + if body.OwnerAddr != "node-a:7777" { + t.Errorf("owner %q, want node-a:7777", body.OwnerAddr) } - req2, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/owner", nil) - req2.Header.Set("Authorization", "Bearer wrong") - resp2, err := http.DefaultClient.Do(req2) - if err != nil { - t.Fatalf("owner: %v", err) - } + resp2 := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/owner", "wrong", "") defer resp2.Body.Close() if resp2.StatusCode != http.StatusNotFound { t.Errorf("status %d, want 404", resp2.StatusCode) @@ -2044,9 +1869,7 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { return time.Time{}, nil }} ps := NewPreviewServer("secret", "node:7777", "node:7777", &fakePreviewMgr{}) - srv := New("", "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := serveTestServer(t, New("", "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps)) resp, err := http.Post(ts.URL+"/v1/sandboxes/sb_1/preview", "application/json", strings.NewReader(`{"token":"tok","port":8080,"ttl_seconds":300}`)) @@ -2077,9 +1900,7 @@ func TestPreviewHandlerRejectsPortZero(t *testing.T) { return time.Now().Add(time.Hour), nil }} ps := NewPreviewServer("secret", "node:7777", "node:7777", &fakePreviewMgr{}) - srv := New("", "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + ts := serveTestServer(t, New("", "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps)) resp, err := http.Post(ts.URL+"/v1/sandboxes/sb_1/preview", "application/json", strings.NewReader(`{"token":"tok","port":0}`)) @@ -2095,7 +1916,7 @@ func TestPreviewHandlerRejectsPortZero(t *testing.T) { func TestCheckpointClaimRedirectsToOwner(t *testing.T) { mgr := &fakeManager{} prober := &fakeProber{owners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, prober) + ts := newProberTestServer(t, "sekret", mgr, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -2117,7 +1938,7 @@ func TestCheckpointClaimRedirectsToOwner(t *testing.T) { func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { prober := &fakeProber{owners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", &fakeManager{}, prober) + ts := newProberTestServer(t, "sekret", &fakeManager{}, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) defer resp.Body.Close() @@ -2129,7 +1950,7 @@ func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { func TestCheckpointClaimNoOwnersIs404(t *testing.T) { mgr := &fakeManager{} - ts := newPlacerTestServer(t, "sekret", mgr, nil) + ts := newProberTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -2145,7 +1966,7 @@ func TestCheckpointClaimNoOwnersIs404(t *testing.T) { func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { mgr := &fakeManager{} prober := &fakeProber{owners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, prober) + ts := newProberTestServer(t, "sekret", mgr, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -2171,7 +1992,7 @@ func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { return &types.Sandbox{ID: "sb_healed", Token: "htok", FromCheckpoint: ckptID}, nil }, } - ts := newPlacerTestServer(t, "sekret", mgr, nil) + ts := newProberTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -2198,7 +2019,7 @@ func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { }, } prober := &fakeProber{owners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, prober) + ts := newProberTestServer(t, "sekret", mgr, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) defer resp.Body.Close() @@ -2227,7 +2048,7 @@ func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { return nil, pool.ErrHealBusy }, } - ts := newPlacerTestServer(t, "sekret", mgr, nil) + ts := newProberTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -2243,16 +2064,7 @@ func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { func TestCheckpointBlobUnknownIs404(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, - ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", "sekret", "") defer resp.Body.Close() if resp.StatusCode != http.StatusNotFound { @@ -2268,16 +2080,7 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { mgr := &fakeManager{ckptDir: src} ts := newTestServer(t, "sekret", mgr, nil) - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, - ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } + resp := doReq(t, http.MethodGet, ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", "sekret", "") defer resp.Body.Close() if resp.StatusCode != http.StatusOK { @@ -2364,15 +2167,11 @@ func TestCheckpointProbeRequiresValidMAC(t *testing.T) { func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { const ckptID = "ck_00000000000000aa" mgrB := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} - srvB := New("sekret", "node-b:7777", mgrB, &fakeDialer{}, nil, nil, nil, nil) - tsB := httptest.NewServer(srvB.Handler()) - t.Cleanup(tsB.Close) + tsB := newTestServer(t, "sekret", mgrB, nil) prober := &peer.HTTPProber{Peers: func() []string { return []string{tsB.URL} }} mgrA := &fakeManager{} - srvA := New("sekret", "node-a:7777", mgrA, &fakeDialer{}, nil, prober, nil, nil) - tsA := httptest.NewServer(srvA.Handler()) - t.Cleanup(tsA.Close) + tsA := newProberTestServer(t, "sekret", mgrA, prober) resp := postJSON(t, tsA.URL+"/v1/checkpoints/"+ckptID+"/claim", "sekret", `{}`) defer resp.Body.Close() @@ -2552,18 +2351,26 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS if fm, ok := mgr.(*fakeManager); ok { fm.tenants = tenants } - srv := New(apiToken, "node:7777", mgr, dialer, nil, nil, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(func() { - ts.Close() - srv.CloseRelays() - }) - return ts + return serveTestServer(t, New(apiToken, "node:7777", mgr, dialer, nil, nil, nil, nil)) } func newProbeAuthTestServer(t *testing.T, mgr Manager, probeKey []byte) *httptest.Server { t.Helper() - srv := New("", "node:7777", mgr, &fakeDialer{}, nil, nil, probeKey, nil) + return serveTestServer(t, New("", "node:7777", mgr, &fakeDialer{}, nil, nil, probeKey, nil)) +} + +func newMeshTestServer(t *testing.T, apiToken string, mgr Manager, placer Placer, prober CheckpointProber) *httptest.Server { + t.Helper() + return serveTestServer(t, New(apiToken, "node-a:7777", mgr, &fakeDialer{}, placer, prober, nil, nil)) +} + +func newProberTestServer(t *testing.T, apiToken string, mgr Manager, prober CheckpointProber) *httptest.Server { + t.Helper() + return newMeshTestServer(t, apiToken, mgr, nil, prober) +} + +func serveTestServer(t *testing.T, srv *Server) *httptest.Server { + t.Helper() ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close() @@ -2586,6 +2393,11 @@ func doReq(t *testing.T, method, url, token, body string) *http.Response { return resp } +func postJSON(t *testing.T, url, token, body string) *http.Response { + t.Helper() + return doReq(t, http.MethodPost, url, token, body) +} + func credToken(cred pool.Cred) string { if cred.Operator { return "" @@ -2593,6 +2405,21 @@ func credToken(cred pool.Cred) string { return cred.Token } +func fakeClaimProvision(ctx context.Context, f *fakeManager, key types.PoolKey, o pool.ClaimOptions) (*types.Sandbox, error) { + f.provisionCalls++ + f.gotTenant = o.Tenant + f.gotClaimRef = o.ClaimRef + f.gotMetadata = o.Metadata + f.gotEnv = o.Env + f.gotOnExpire = o.OnExpire + f.gotRequirePromoted = o.RequirePromoted + f.gotNoEgress = o.NoEgress + if f.claim == nil { + return &types.Sandbox{ID: "sb_1", Token: "tok", Volumes: slices.Clone(o.Volumes)}, nil + } + return f.claim(ctx, key, o.TTL) +} + type claimFunc func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) type sandboxVerbFunc func(string, string) error @@ -2659,7 +2486,6 @@ type fakeManager struct { gotMetadata types.Metadata gotEnv types.Env gotOnExpire types.ExpireAction - gotVolumes []types.Volume gotWarmVolumes []types.Volume gotRequirePromoted bool gotNoEgress bool @@ -2669,8 +2495,6 @@ type fakeManager struct { gotNoForward bool tenantClaims map[string]int volumePlacement func(key types.PoolKey, tenant string, names []string) (bool, error) - placementCalls int - placementNames []string volumeCatalog func(tenant string, holders map[string]int) []types.VolumeInfo sandboxIndex func(tenant, claimRef string) []pool.SandboxSummary sandboxByID func(id string) (pool.SandboxSummary, bool) @@ -2852,9 +2676,7 @@ func (f *fakeManager) DeleteTenant(_ context.Context, name string) error { } func (f *fakeManager) VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) { - f.placementCalls++ f.gotTenant = tenant - f.placementNames = slices.Clone(names) if f.volumePlacement == nil { return true, nil } @@ -3080,32 +2902,3 @@ type fakeProber struct { func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } - -func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, prober CheckpointProber) *httptest.Server { - t.Helper() - srv := New(apiToken, "node:7777", mgr, &fakeDialer{}, nil, prober, nil, nil) - ts := httptest.NewServer(srv.Handler()) - t.Cleanup(ts.Close) - return ts -} - -func postJSON(t *testing.T, url, token, body string) *http.Response { - t.Helper() - return doReq(t, http.MethodPost, url, token, body) -} - -func fakeClaimProvision(ctx context.Context, f *fakeManager, key types.PoolKey, o pool.ClaimOptions) (*types.Sandbox, error) { - f.provisionCalls++ - f.gotTenant = o.Tenant - f.gotClaimRef = o.ClaimRef - f.gotMetadata = o.Metadata - f.gotEnv = o.Env - f.gotOnExpire = o.OnExpire - f.gotVolumes = slices.Clone(o.Volumes) - f.gotRequirePromoted = o.RequirePromoted - f.gotNoEgress = o.NoEgress - if f.claim == nil { - return &types.Sandbox{ID: "sb_1", Token: "tok", Volumes: slices.Clone(o.Volumes)}, nil - } - return f.claim(ctx, key, o.TTL) -} diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index bedce484..0bdfa22c 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -19,12 +19,12 @@ import ( func TestLiveVolumeRoutesAuthorizeAndDecode(t *testing.T) { for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { - for _, token := range []string{"acme-token", "root-token"} { - t.Run(method+"/"+token, func(t *testing.T) { + for _, cred := range []struct{ token, tenant string }{{"acme-token", "acme"}, {"root-token", ""}} { + t.Run(method+"/"+cred.token, func(t *testing.T) { called := false check := func(id, tenant string) { called = true - if id != "sb_1" || tenant != map[string]string{"root-token": "", "acme-token": "acme"}[token] { + if id != "sb_1" || tenant != cred.tenant { t.Errorf("scope %s %q", id, tenant) } } @@ -54,7 +54,7 @@ func TestLiveVolumeRoutesAuthorizeAndDecode(t *testing.T) { if method == http.MethodPatch { body = `{"remove":["data"]}` } - resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", token, body) + resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", cred.token, body) defer resp.Body.Close() var got types.SandboxVolumesResponse if err := json.UnmarshalRead(resp.Body, &got); err != nil { diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 97e36178..7f65427c 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -36,16 +36,7 @@ func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { c := testClient(t, ts, WithAPIToken("operator-token")) sb := &Sandbox{ID: "sb_1", token: "sandbox-token", owner: c.addr, c: c} c.addr = "unused.invalid:7777" - var got SandboxVolumesResponse - var err error - switch method { - case http.MethodGet: - got, err = sb.GetVolumes(t.Context()) - case http.MethodPut: - got, err = sb.SetVolumes(t.Context(), []Volume{{Name: "data"}}) - case http.MethodPatch: - got, err = sb.PatchVolumes(t.Context(), VolumePatch{Add: []Volume{{Name: "data"}}, Remove: []string{"old"}}) - } + got, err := callVolumes(t, sb, method) if err != nil || !slices.Equal(got.Volumes, []Volume{{Name: "data", Mount: "/data"}}) || !slices.Equal(got.VolumesWant, got.Volumes) || got.States[0].State != VolumeAttached || !got.Converging { t.Fatalf("got=%+v err=%v", got, err) } @@ -66,19 +57,10 @@ func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { c := testClient(t, ts) for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c} - var got SandboxVolumesResponse - var err error - switch method { - case http.MethodGet: - got, err = sb.GetVolumes(t.Context()) - case http.MethodPut: - got, err = sb.SetVolumes(t.Context(), []Volume{{Name: "new"}}) - case http.MethodPatch: - got, err = sb.PatchVolumes(t.Context(), VolumePatch{Remove: []string{"old"}}) - } + got, err := callVolumes(t, sb, method) failure, isVolume := errors.AsType[*VolumeError](err) api, isAPI := errors.AsType[*APIError](err) - if !isVolume || !isAPI || api.Status != 500 || failure.Response == nil { + if !isVolume || !isAPI || api.Status != http.StatusInternalServerError || failure.Response == nil { t.Fatalf("%s error lost response or APIError: %v", method, err) } if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" || sb.VolumesWant[0].Name != "new" || sb.Volumes[0].Name != "old" { @@ -100,7 +82,7 @@ func TestLiveVolumeValidationFailurePreservesLastKnownSet(t *testing.T) { c := testClient(t, ts) sb := &Sandbox{ID: "sb_1", token: "tok", owner: c.addr, c: c, Volumes: []Volume{{Name: "old"}}, VolumesWant: []Volume{{Name: "wanted"}}} _, err := sb.SetVolumes(t.Context(), []Volume{{Name: "new"}}) - if failure, ok := errors.AsType[*VolumeError](err); !ok || failure.Response != nil || failure.Status != 400 { + if failure, ok := errors.AsType[*VolumeError](err); !ok || failure.Response != nil || failure.Status != http.StatusBadRequest { t.Fatalf("error=%v", err) } if sb.Volumes[0].Name != "old" || sb.VolumesWant[0].Name != "wanted" { @@ -194,3 +176,15 @@ func TestLiveVolumeIncompleteViewPreservesHandle(t *testing.T) { }) } } + +func callVolumes(t *testing.T, sb *Sandbox, method string) (SandboxVolumesResponse, error) { + t.Helper() + switch method { + case http.MethodPut: + return sb.SetVolumes(t.Context(), []Volume{{Name: "data"}}) + case http.MethodPatch: + return sb.PatchVolumes(t.Context(), VolumePatch{Add: []Volume{{Name: "data"}}, Remove: []string{"old"}}) + default: + return sb.GetVolumes(t.Context()) + } +} From 19ca9c77753a0a591c99b7423b46165b41981345 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:35:31 +0800 Subject: [PATCH 42/57] pool: only a live volume run on a running sandbox stamps activity ac70e41 touched the sandbox before validation, so a rejected or paused request kept refreshing LastSeen: a client retrying PUT on a hibernated claim held off archive_after forever. The stamp now follows the running check and still precedes every device call. --- sandboxd/pool/volume_convergence_test.go | 14 ++++++++++++++ sandboxd/pool/volume_mutation.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 32c748f8..0e4e4b90 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -219,6 +219,20 @@ func TestLiveVolumeRunStampsActivityBeforeDeviceCalls(t *testing.T) { } } +func TestRejectedVolumeRequestLeavesActivityUntouched(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + idleSince := time.Now().Add(-time.Hour) + sb.TouchAt(idleSince) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("duplicate target: %v", err) + } + if !sb.LastSeen().Equal(idleSince) { + t.Fatalf("a rejected request stamped activity: %v", sb.LastSeen()) + } +} + func TestRejectedVolumeTargetPreservesConvergenceDiagnostics(t *testing.T) { for _, priorConflict := range []bool{false, true} { name := "settled" diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 4cfe857c..e836159b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -71,7 +71,6 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if !ok || !tenantOwns(tenant, sb.Tenant) { return view, ErrUnknownSandbox } - sb.Touch() var run *volumeRun if err = m.eng.CheckLiveVolumeSupport(ctx); err == nil { ctx, run, err = m.beginVolumeRun(ctx, sb) @@ -112,6 +111,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } + sb.Touch() m.mu.Lock() uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) m.mu.Unlock() From 4f4ef60026d70c2a181aafdd4fba1f64475d6917 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:35:59 +0800 Subject: [PATCH 43/57] sdk/python: volume methods reject a bare string at run time 2726e7f only narrowed the type hints, so an untyped caller's set_volumes("data") or patch_volumes(remove="old") still sent one volume per character. --- sdk/python/cocoonsandbox/sandbox.py | 4 +++- sdk/python/tests/test_volumes.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index ce4d5586..1a83fa86 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -77,7 +77,7 @@ def set_volumes( self, volumes: list[str | Mapping[str, str] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: """Replaces the desired set and converges; an explicit empty list clears mutable mounts.""" - if volumes is None: + if volumes is None or isinstance(volumes, str): raise TypeError("set volumes requires an explicit list; use [] to clear volumes") return self._volumes_request( "PUT", {"volumes": [_volume_body(volume, True) for volume in volumes]}, "set volumes", timeout @@ -91,6 +91,8 @@ def patch_volumes( timeout: float | None = None, ) -> SandboxVolumesResponse: """Adds or removes mounted read-only entries by name; omitted arguments leave the set unchanged.""" + if isinstance(add, str) or isinstance(remove, str): + raise TypeError("patch volumes takes lists of volumes and names") body: dict[str, Any] = {} if add is not None: body["add"] = [_volume_body(volume, True) for volume in add] diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 6d968e8a..3226b82d 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -151,3 +151,16 @@ def test_incomplete_volume_view_preserves_handle(spawn_node, status): if status == 409: assert isinstance(caught.value, VolumeError) assert caught.value.response is None + + +@pytest.mark.parametrize( + "call", + [ + lambda sb: sb.set_volumes("data"), + lambda sb: sb.patch_volumes(add="data"), + lambda sb: sb.patch_volumes(remove="old"), + ], +) +def test_volume_methods_reject_a_bare_string(black_hole, call): + with pytest.raises(TypeError): + call(sandbox_at(black_hole)) From 37253a68e5edd245aefed93dadf5584d82980dd2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 03:42:13 +0800 Subject: [PATCH 44/57] test: the rollback hold test waits for the claims recommit before its TempDir goes --- sandboxd/pool/volume_mutation_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 5af7d9da..a297ce73 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -692,6 +692,7 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) + waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) } From 05d077caf89dc29fae56c4ac2ce7c75b3fb7bf75 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 09:00:30 +0800 Subject: [PATCH 45/57] pool: a retried target still makes its intent durable before device calls A failed intent write keeps the target in memory for recommit, so a retry of the same PUT matched the unchanged-target shortcut, skipped the commit and attached while the journal still lacked the intent; a restart then lost the attachment's hold. The shortcut now requires a journal in step with memory, and otherwise commits first and fails before any device call. --- sandboxd/pool/volume_mutation.go | 2 +- sandboxd/pool/volume_mutation_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index e836159b..a0fbee7d 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -287,7 +287,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run replaced := func(v types.Volume) bool { return volumeIndex(sb.VolumesWant, v.Name) < 0 && volumeIndex(target, v.Name) >= 0 } - if len(additions) == 0 && slices.Equal(target, sb.VolumesWant) && !slices.ContainsFunc(sb.Volumes, replaced) { + if len(additions) == 0 && slices.Equal(target, sb.VolumesWant) && !slices.ContainsFunc(sb.Volumes, replaced) && m.store.synced() { return claimSnapshot{}, false, nil } sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), replaced) diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index a297ce73..d4e13bf7 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -685,6 +685,27 @@ func TestArchiveRenewRejectsAdmittedUnpublishedVolumes(t *testing.T) { } } +func TestRetriedTargetPersistsIntentBeforeDeviceCalls(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + restore := breakClaimStore(t, m) + request := []types.Volume{{Name: "data"}} + for range 2 { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err == nil { + t.Fatal("expected the intent write to fail") + } + } + if len(eng.volumeSpecs) != 0 { + t.Fatalf("device calls before the intent was durable: %v", eng.volumeSpecs) + } + restore() + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err != nil { + t.Fatal(err) + } + assertLiveJournal(t, m, sb) +} + func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) From 09924f6d79c9740668e56c594bc2658c7f5eaef0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 09:00:30 +0800 Subject: [PATCH 46/57] engine: resolve a live mount path before checking what occupies it The realpath lookup ran only when our device already sat at another path, so a new volume aimed at a symlinked alias of an occupied mount point saw no mount, stacked on top of the other device, and could not be removed later. Every live mount check now resolves the requested path first; it costs one guest exec on the live path only. --- sandboxd/engine/installca_test.go | 3 +++ sandboxd/engine/volume_live.go | 11 ++++----- sandboxd/engine/volume_live_test.go | 35 ++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index ce7616af..32cf99c9 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -197,6 +197,9 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri f.execCalls = append(f.execCalls, argv) f.execEnv = env code, reply, disconnect := f.execCode, f.execErr, f.execDrop + if argv[0] == "realpath" { + code, reply, disconnect = 0, "", false + } if f.execFailAt > 0 && len(f.execCalls) != f.execFailAt { code, reply = 0, "" } diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 594d4150..818ac4e2 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -120,14 +120,11 @@ func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount s if err != nil { return false, err } - target := mount - if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum == devNum && m.path != mount }) { - resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", "-m", "--", mount) - if err != nil { - return false, fmt.Errorf("resolve volume mount %s: %w", mount, err) - } - target = strings.TrimSpace(string(resolved)) + resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", "-m", "--", mount) + if err != nil { + return false, fmt.Errorf("resolve volume mount %s: %w", mount, err) } + target := strings.TrimSpace(string(resolved)) matched := false for _, entry := range mounts { if entry.path != target { diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 84eed3fc..3d12ea99 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -46,7 +46,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if len(fake.execCalls) != 3 || fake.execCalls[2][0] != "umount" { + if changes := guestChanges(fake.execCalls); len(changes) != 3 || changes[2][0] != "umount" { t.Fatalf("expected one mkdir/mount/umount, got %v", fake.execCalls) } args, err := os.ReadFile(os.Getenv("COCOON_TEST_ATTACH_ARGS")) @@ -93,6 +93,31 @@ func TestLiveMountUnderSymlinkedParentConvergesAndDetaches(t *testing.T) { } } +func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:48 / /home/u/ws/data ro - ext4 /dev/vdd ro\n") + fake.realpath = map[string]string{"/workspace/data": "/home/u/ws/data"} + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/workspace/data", false) + if _, ok := errors.AsType[*VolumeMountConflict](err); !ok { + t.Fatalf("mounted over another device behind a symlink: %v", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if slices.ContainsFunc(fake.execCalls, func(argv []string) bool { return argv[0] == "mount" }) { + t.Fatalf("mount ran over an occupied path: %v", fake.execCalls) + } +} + func TestLiveMountReadsMountinfoBeyondTheSysfsCap(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -144,8 +169,8 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if len(fake.execCalls) != 0 { - t.Fatalf("modified foreign mount: %v", fake.execCalls) + if changes := guestChanges(fake.execCalls); len(changes) != 0 { + t.Fatalf("modified foreign mount: %v", changes) } }) } @@ -244,3 +269,7 @@ exit 2 `) return VolumeSpec{Name: "imagenet", Path: image} } + +func guestChanges(calls [][]string) [][]string { + return slices.DeleteFunc(slices.Clone(calls), func(argv []string) bool { return argv[0] == "realpath" }) +} From 609072b231c65bdd5be059ea9fbf3696cc6ac81e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 09:27:36 +0800 Subject: [PATCH 47/57] pool: a converged live run clears the sandbox's volume diagnostics A journal write that failed after a device change filed "volume change failed" on that volume; the retry that converged it neither found it pending nor cleared it, so GET kept showing the stale error on an attached volume. --- sandboxd/pool/volume_mutation.go | 6 +++++- sandboxd/pool/volume_mutation_test.go | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index a0fbee7d..79104100 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -377,8 +377,12 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri if !m.ownsRunLocked(sb, run) { return } + if err == nil { + delete(m.volumeErrors, sb) + return + } errs := m.volumeErrors[sb] - if err != nil && name != "" { + if name != "" { message := volumeChangeFailed if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { message = volumeChangeInterrupted diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index d4e13bf7..caddb5a0 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -175,14 +175,18 @@ func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { if stage == "intent" && len(eng.volumeSpecs) != 0 { t.Fatal("failed intent reached the engine") } + var view types.SandboxVolumesResponse if stage == "detach result" { - _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) + view, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) } else { - _, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) + view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) } if err != nil { t.Fatal(err) } + if slices.ContainsFunc(view.States, func(s types.VolumeState) bool { return s.Error != "" }) { + t.Fatalf("converged view kept a stale error: %+v", view.States) + } assertLiveJournal(t, m, sb) }) } From 1ff41c0b3c9dde8bd649f1e1bd9e5ce8ae9998dc Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:06:43 +0800 Subject: [PATCH 48/57] engine: a live mount check resolves only an unlisted path and survives a guest without realpath -m 09924f6 ran `realpath -m` on every check and trimmed all whitespace: a mount path ending in a space read back as a bind of its own device, and a busybox or toybox-without-PATH guest failed every live add and removal. A path that mountinfo already lists is canonical and needs no lookup, only the trailing newline is trimmed, and a failed lookup checks the path as given, as before 09924f6, with a warning. --- sandboxd/engine/installca_test.go | 6 ++- sandboxd/engine/volume_live.go | 18 +++++--- sandboxd/engine/volume_live_test.go | 66 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 32cf99c9..09deafe1 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -80,6 +80,7 @@ type fakeSilkd struct { execFailAt int execHook func([]string) realpath map[string]string + noRealpath bool writeErr string writeKind string execErr string @@ -199,6 +200,9 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri code, reply, disconnect := f.execCode, f.execErr, f.execDrop if argv[0] == "realpath" { code, reply, disconnect = 0, "", false + if f.noRealpath { + code = 1 + } } if f.execFailAt > 0 && len(f.execCalls) != f.execFailAt { code, reply = 0, "" @@ -216,7 +220,7 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri return } _, _ = io.WriteString(conn, `{"type":"started","pid":1}`+"\n") - if argv[0] == "realpath" { + if argv[0] == "realpath" && code == 0 { writeFakeSilkdResponse(conn, &wire.Stdout{Data: []byte(cmp.Or(resolved, argv[len(argv)-1]) + "\n")}) } _, _ = fmt.Fprintf(conn, `{"type":"exit","code":%d}`+"\n", code) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 818ac4e2..64590d27 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -120,11 +120,7 @@ func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount s if err != nil { return false, err } - resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", "-m", "--", mount) - if err != nil { - return false, fmt.Errorf("resolve volume mount %s: %w", mount, err) - } - target := strings.TrimSpace(string(resolved)) + target := e.resolveMount(ctx, vsockSocket, mount, mounts) matched := false for _, entry := range mounts { if entry.path != target { @@ -141,6 +137,18 @@ func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount s return matched, nil } +func (e *Engine) resolveMount(ctx context.Context, vsockSocket, mount string, mounts []volumeMount) string { + if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.path == mount }) { + return mount + } + resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", "-m", "--", mount) + if err != nil { + log.WithFunc("engine.resolveMount").Warnf(ctx, "resolve volume mount %s, checking it as given: %v", mount, err) + return mount + } + return strings.TrimSuffix(string(resolved), "\n") +} + func (e *Engine) refuseOtherMounts(ctx context.Context, vsockSocket, device string) error { devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 3d12ea99..8141238a 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -118,6 +118,72 @@ func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { } } +func TestLiveMountKeepsATrailingSpaceInItsPath(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data\\040 ro - ext4 /dev/vdc ro\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + for range 2 { + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data ", false); err != nil { + t.Fatal(err) + } + } + if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data "); err != nil { + t.Fatal(err) + } +} + +func TestLiveMountWorksWithoutAGuestRealpath(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + fake.mu.Lock() + fake.noRealpath = true + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { + t.Fatal(err) + } + if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if changes := guestChanges(fake.execCalls); len(changes) != 3 || changes[2][0] != "umount" { + t.Fatalf("expected one mkdir/mount/umount, got %v", fake.execCalls) + } +} + func TestLiveMountReadsMountinfoBeyondTheSysfsCap(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) From 5434756aa162b1a677a99be1cf491a6ef78988b6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:06:43 +0800 Subject: [PATCH 49/57] test: a refused volume request on a hibernated claim leaves its activity stamp alone --- sandboxd/pool/volume_convergence_test.go | 37 +++++++++++++++++------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 0e4e4b90..19927ec8 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -220,16 +220,33 @@ func TestLiveVolumeRunStampsActivityBeforeDeviceCalls(t *testing.T) { } func TestRejectedVolumeRequestLeavesActivityUntouched(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - sb := mustClaim(t, m, testKey) - idleSince := time.Now().Add(-time.Hour) - sb.TouchAt(idleSince) - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "data"}}); !errors.Is(err, ErrBadVolume) { - t.Fatalf("duplicate target: %v", err) - } - if !sb.LastSeen().Equal(idleSince) { - t.Fatalf("a rejected request stamped activity: %v", sb.LastSeen()) + for _, tt := range []struct { + name string + request []types.Volume + paused bool + want error + }{ + {"invalid target", []types.Volume{{Name: "data"}, {Name: "data"}}, false, ErrBadVolume}, + {"hibernated claim", []types.Volume{{Name: "data"}}, true, ErrPaused}, + } { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if tt.paused { + m.mu.Lock() + sb.HibernateSnap = "snap" + m.mu.Unlock() + } + idleSince := time.Now().Add(-time.Hour) + sb.TouchAt(idleSince) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, tt.request); !errors.Is(err, tt.want) { + t.Fatalf("got %v, want %v", err, tt.want) + } + if !sb.LastSeen().Equal(idleSince) { + t.Fatalf("a refused request stamped activity: %v", sb.LastSeen()) + } + }) } } From 3d0afccf28b483400e28946a7580b8fd2c612298 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:15:35 +0800 Subject: [PATCH 50/57] engine: call guest realpath without flags Mount targets are validated as absolute paths, so -- guards nothing, and a missing target already falls back to the literal path; plain realpath runs on every guest toolchain. --- sandboxd/engine/volume_live.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 64590d27..da60151f 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -141,7 +141,7 @@ func (e *Engine) resolveMount(ctx context.Context, vsockSocket, mount string, mo if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.path == mount }) { return mount } - resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", "-m", "--", mount) + resolved, err := e.silkdOutput(ctx, vsockSocket, "realpath", mount) if err != nil { log.WithFunc("engine.resolveMount").Warnf(ctx, "resolve volume mount %s, checking it as given: %v", mount, err) return mount From 329639b2b665fa9cebf48d99519ff18de80269d3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:15:35 +0800 Subject: [PATCH 51/57] test: fold the live mount convergence cases into one table that counts realpath lookups The trailing-space case now sits behind a symlink so TrimSpace fails it, and every case pins its realpath call count so dropping the listed-path shortcut fails too. --- sandboxd/engine/volume_live_test.go | 199 ++++++++-------------------- 1 file changed, 57 insertions(+), 142 deletions(-) diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 8141238a..53f07f14 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -13,83 +13,64 @@ import ( const rootMountInfo = "1 0 8:0 / / rw - ext4 /dev/vda rw\n" func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { - sock := sockPath(t) - fake := serveFakeSilkd(t, sock) - configureVolumeDevices(fake) - spec := liveDiskCLI(t, "") - if err := os.Remove(os.Getenv("COCOON_TEST_ATTACHED")); err != nil { - t.Fatal(err) - } - fake.mu.Lock() - fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - fake.execHook = func(argv []string) { - switch argv[0] { - case "mount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") - case "umount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - } - } - fake.mu.Unlock() - e := New("cocoon", nil, nil, false, false, "") - for range 2 { - if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { - t.Fatal(err) - } - if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { - t.Fatal(err) - } - } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { - t.Fatal(err) - } - fake.mu.Lock() - defer fake.mu.Unlock() - if changes := guestChanges(fake.execCalls); len(changes) != 3 || changes[2][0] != "umount" { - t.Fatalf("expected one mkdir/mount/umount, got %v", fake.execCalls) - } - args, err := os.ReadFile(os.Getenv("COCOON_TEST_ATTACH_ARGS")) - if err != nil || !strings.Contains(string(args), "--readonly --directio off") { - t.Fatalf("attach args=%s error=%v", args, err) - } -} - -func TestLiveMountUnderSymlinkedParentConvergesAndDetaches(t *testing.T) { - sock := sockPath(t) - fake := serveFakeSilkd(t, sock) - configureVolumeDevices(fake) - spec := liveDiskCLI(t, "") - fake.mu.Lock() - fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - fake.realpath = map[string]string{"/workspace/data": "/home/u/ws/data"} - fake.execHook = func(argv []string) { - switch argv[0] { - case "mount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /home/u/ws/data ro - ext4 /dev/vdc ro\n") - case "umount": + for _, tt := range []struct { + name, mount, kernelPath string + realpath map[string]string + noRealpath bool + lookups int + }{ + {name: "canonical path", mount: "/data", kernelPath: "/data", lookups: 1}, + {name: "symlinked parent", mount: "/workspace/data", kernelPath: "/home/u/ws/data", realpath: map[string]string{"/workspace/data": "/home/u/ws/data"}, lookups: 3}, + {name: "trailing space behind a symlink", mount: "/ws/data ", kernelPath: `/home/u/ws/data\040`, realpath: map[string]string{"/ws/data ": "/home/u/ws/data "}, lookups: 3}, + {name: "guest without realpath", mount: "/data", kernelPath: "/data", noRealpath: true, lookups: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + if err := os.Remove(os.Getenv("COCOON_TEST_ATTACHED")); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + fake.realpath, fake.noRealpath = tt.realpath, tt.noRealpath + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - } - } - fake.mu.Unlock() - e := New("cocoon", nil, nil, false, false, "") - if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { - t.Fatal(err) - } - for range 2 { - if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/workspace/data", false); err != nil { - t.Fatal(err) - } - } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/workspace/data"); err != nil { - t.Fatal(err) - } - fake.mu.Lock() - defer fake.mu.Unlock() - unmounted := slices.ContainsFunc(fake.execCalls, func(argv []string) bool { return argv[0] == "umount" }) - if !unmounted || len(slices.DeleteFunc(slices.Clone(fake.execCalls), func(argv []string) bool { return argv[0] != "mount" })) != 1 { - t.Fatalf("want one mount and an unmount, got %v", fake.execCalls) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / " + tt.kernelPath + " ro - ext4 /dev/vdc ro\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + for range 2 { + if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { + t.Fatal(err) + } + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, tt.mount, false); err != nil { + t.Fatal(err) + } + } + if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + changes := guestChanges(fake.execCalls) + if len(changes) != 3 || changes[2][0] != "umount" { + t.Fatalf("expected one mkdir/mount/umount, got %v", fake.execCalls) + } + if lookups := len(fake.execCalls) - len(changes); lookups != tt.lookups { + t.Fatalf("realpath lookups = %d, want %d: %v", lookups, tt.lookups, fake.execCalls) + } + args, err := os.ReadFile(os.Getenv("COCOON_TEST_ATTACH_ARGS")) + if err != nil || !strings.Contains(string(args), "--readonly --directio off") { + t.Fatalf("attach args=%s error=%v", args, err) + } + }) } } @@ -118,72 +99,6 @@ func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { } } -func TestLiveMountKeepsATrailingSpaceInItsPath(t *testing.T) { - sock := sockPath(t) - fake := serveFakeSilkd(t, sock) - configureVolumeDevices(fake) - spec := liveDiskCLI(t, "") - fake.mu.Lock() - fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - fake.execHook = func(argv []string) { - switch argv[0] { - case "mount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data\\040 ro - ext4 /dev/vdc ro\n") - case "umount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - } - } - fake.mu.Unlock() - e := New("cocoon", nil, nil, false, false, "") - if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { - t.Fatal(err) - } - for range 2 { - if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data ", false); err != nil { - t.Fatal(err) - } - } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data "); err != nil { - t.Fatal(err) - } -} - -func TestLiveMountWorksWithoutAGuestRealpath(t *testing.T) { - sock := sockPath(t) - fake := serveFakeSilkd(t, sock) - configureVolumeDevices(fake) - spec := liveDiskCLI(t, "") - fake.mu.Lock() - fake.noRealpath = true - fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - fake.execHook = func(argv []string) { - switch argv[0] { - case "mount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") - case "umount": - fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) - } - } - fake.mu.Unlock() - e := New("cocoon", nil, nil, false, false, "") - if err := e.DiskAttach(t.Context(), "sbx-1", spec); err != nil { - t.Fatal(err) - } - if err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/data", false); err != nil { - t.Fatal(err) - } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { - t.Fatal(err) - } - fake.mu.Lock() - defer fake.mu.Unlock() - if changes := guestChanges(fake.execCalls); len(changes) != 3 || changes[2][0] != "umount" { - t.Fatalf("expected one mkdir/mount/umount, got %v", fake.execCalls) - } -} - func TestLiveMountReadsMountinfoBeyondTheSysfsCap(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) From 0020f54ee472e3a2a7718fb80a06888ea7b797ab Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:24:37 +0800 Subject: [PATCH 52/57] engine: name the resolved guest path in a live mount conflict --- sandboxd/engine/volume_live.go | 2 +- sandboxd/engine/volume_live_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index da60151f..a485fac3 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -130,7 +130,7 @@ func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount s continue } if mode, _, _ := strings.Cut(entry.options, ","); matched || entry.devNum != devNum || entry.root != "/" || mode != mountMode(rw) { - return false, &VolumeMountConflict{Message: fmt.Sprintf("volume mount %q has a different device, root, or mode; restore the expected mount before retrying", mount)} + return false, &VolumeMountConflict{Message: fmt.Sprintf("volume mount %q has a different device, root, or mode; restore the expected mount before retrying", target)} } matched = true } diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 53f07f14..4ce7eb10 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -89,8 +89,8 @@ func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { t.Fatal(err) } err := e.MountLiveVolume(t.Context(), sock, spec.Name, "/workspace/data", false) - if _, ok := errors.AsType[*VolumeMountConflict](err); !ok { - t.Fatalf("mounted over another device behind a symlink: %v", err) + if _, ok := errors.AsType[*VolumeMountConflict](err); !ok || !strings.Contains(err.Error(), `"/home/u/ws/data" has a different device`) { + t.Fatalf("mounted over another device behind a symlink or hid its real path: %v", err) } fake.mu.Lock() defer fake.mu.Unlock() From 55593be9f203648ad92799cf88698ed6d3f8abea Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:27:31 +0800 Subject: [PATCH 53/57] test: name the realpath fallback case after what it covers Guest images are Ubuntu with coreutils, so realpath fails only on a path with a missing component, and no mount can sit behind a path that does not exist. --- sandboxd/engine/volume_live_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 4ce7eb10..6c16faf9 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -22,7 +22,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { {name: "canonical path", mount: "/data", kernelPath: "/data", lookups: 1}, {name: "symlinked parent", mount: "/workspace/data", kernelPath: "/home/u/ws/data", realpath: map[string]string{"/workspace/data": "/home/u/ws/data"}, lookups: 3}, {name: "trailing space behind a symlink", mount: "/ws/data ", kernelPath: `/home/u/ws/data\040`, realpath: map[string]string{"/ws/data ": "/home/u/ws/data "}, lookups: 3}, - {name: "guest without realpath", mount: "/data", kernelPath: "/data", noRealpath: true, lookups: 1}, + {name: "unresolvable path", mount: "/data", kernelPath: "/data", noRealpath: true, lookups: 1}, } { t.Run(tt.name, func(t *testing.T) { sock := sockPath(t) From cc453afd491439e04662324747ab66174b7b47e7 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:31:25 +0800 Subject: [PATCH 54/57] engine: a live detach ignores another device mounted at the volume's path After a mount refused because another device occupies the path, removing the pending volume hit the same conflict, so it could not be dropped until the guest unmounted the other device. Detach now unmounts only its own device at the path. --- sandboxd/engine/volume_live.go | 5 +++-- sandboxd/engine/volume_live_test.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index a485fac3..ee7341ac 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -80,11 +80,12 @@ func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mo } func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) error { - mounted, err := e.volumeMounted(ctx, vsockSocket, device, mount, false) + devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { return err } - if mounted { + target := e.resolveMount(ctx, vsockSocket, mount, mounts) + if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum == devNum && m.path == target }) { if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 6c16faf9..15e14c29 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -222,6 +222,25 @@ func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { } } +func TestLiveDetachIgnoresAnotherDeviceAtThePath(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:48 / /data ro - ext4 /dev/vdd ro\n") + fake.mu.Unlock() + if err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { + t.Fatalf("detach of an unmounted device blocked by another device's mount: %v", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if changes := guestChanges(fake.execCalls); len(changes) != 0 { + t.Fatalf("touched another device's mount: %v", changes) + } +} + func liveDiskCLI(t *testing.T, detachErr string) VolumeSpec { t.Helper() dir := t.TempDir() From 3a8e114c3a20ea29b81d698bcfe839beb11f98d9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:44:04 +0800 Subject: [PATCH 55/57] engine: a live detach leaves a guest mount stacked on the volume alone umount removes the top mount at a path, so with a guest mount stacked on the volume it removed the guest's mount and then refused. Detach now refuses before unmounting when another mount shares the volume's path. --- sandboxd/engine/volume_live.go | 3 ++- sandboxd/engine/volume_live_test.go | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index ee7341ac..0242c798 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -85,7 +85,8 @@ func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mou return err } target := e.resolveMount(ctx, vsockSocket, mount, mounts) - if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum == devNum && m.path == target }) { + if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum == devNum && m.path == target }) && + !slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum != devNum && m.path == target }) { if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 15e14c29..cf32e33b 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -158,7 +158,7 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { } func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { - for _, mode := range []string{"busy", "second mount", "probe error", "guest error", "transport disconnect", "host error"} { + for _, mode := range []string{"busy", "second mount", "probe error", "guest error", "transport disconnect", "host error", "stacked mount"} { t.Run(mode, func(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -185,6 +185,8 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { fake.execErr = "guest connection failed /private/runtime" case "probe error": fake.readErr["/proc/self/mountinfo"] = "read failed" + case "stacked mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n3 2 0:39 / /data rw - tmpfs guesttmp rw\n") } fake.mu.Unlock() err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") @@ -192,7 +194,7 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { t.Fatal("detach must fail closed") } conflict, isConflict := errors.AsType[*VolumeMountConflict](err) - wantPath := map[string]string{"busy": "/data", "second mount": "/other"}[mode] + wantPath := map[string]string{"busy": "/data", "second mount": "/other", "stacked mount": "/data"}[mode] conflictExpected := wantPath != "" if isConflict != conflictExpected { t.Fatalf("conflict=%v error=%v", conflictExpected, err) @@ -203,6 +205,11 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { if mode == "busy" && !strings.HasPrefix(conflict.Error(), "cannot unmount") { t.Fatalf("busy conflict: %v", conflict) } + fake.mu.Lock() + defer fake.mu.Unlock() + if mode == "stacked mount" && len(guestChanges(fake.execCalls)) != 0 { + t.Fatalf("unmounted the guest's mount stacked on the volume: %v", fake.execCalls) + } }) } } From 5ab6662848bb2d45f7863e5d19814611ed9ffbbe Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:52:48 +0800 Subject: [PATCH 56/57] engine: a live detach unmounts only when its own mount is alone at the path A guest bind of the volume's own subdirectory stacked on the volume shares its device, so the device-only check still unmounted the top mount, which was the guest's. Detach now unmounts only when the path holds exactly one mount and it is the volume's root. --- sandboxd/engine/volume_live.go | 4 ++-- sandboxd/engine/volume_live_test.go | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 0242c798..e1033a84 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -85,8 +85,8 @@ func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mou return err } target := e.resolveMount(ctx, vsockSocket, mount, mounts) - if slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum == devNum && m.path == target }) && - !slices.ContainsFunc(mounts, func(m volumeMount) bool { return m.devNum != devNum && m.path == target }) { + atTarget := slices.DeleteFunc(mounts, func(m volumeMount) bool { return m.path != target }) + if len(atTarget) == 1 && atTarget[0].devNum == devNum && atTarget[0].root == "/" { if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index cf32e33b..19f13782 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -158,7 +158,7 @@ func TestLiveAttachRejectsForeignMounts(t *testing.T) { } func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { - for _, mode := range []string{"busy", "second mount", "probe error", "guest error", "transport disconnect", "host error", "stacked mount"} { + for _, mode := range []string{"busy", "second mount", "probe error", "guest error", "transport disconnect", "host error", "stacked mount", "stacked bind"} { t.Run(mode, func(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -187,6 +187,8 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { fake.readErr["/proc/self/mountinfo"] = "read failed" case "stacked mount": fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n3 2 0:39 / /data rw - tmpfs guesttmp rw\n") + case "stacked bind": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n3 2 8:32 /sub /data ro - ext4 /dev/vdc ro\n") } fake.mu.Unlock() err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") @@ -194,7 +196,7 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { t.Fatal("detach must fail closed") } conflict, isConflict := errors.AsType[*VolumeMountConflict](err) - wantPath := map[string]string{"busy": "/data", "second mount": "/other", "stacked mount": "/data"}[mode] + wantPath := map[string]string{"busy": "/data", "second mount": "/other", "stacked mount": "/data", "stacked bind": "/data"}[mode] conflictExpected := wantPath != "" if isConflict != conflictExpected { t.Fatalf("conflict=%v error=%v", conflictExpected, err) @@ -207,8 +209,8 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if mode == "stacked mount" && len(guestChanges(fake.execCalls)) != 0 { - t.Fatalf("unmounted the guest's mount stacked on the volume: %v", fake.execCalls) + if !slices.Contains([]string{"busy", "guest error", "transport disconnect"}, mode) && len(guestChanges(fake.execCalls)) != 0 { + t.Fatalf("changed guest mounts: %v", fake.execCalls) } }) } From 4c8d53bb2fe1c813bc64f0113d453fdc152bbb9b Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 10:52:49 +0800 Subject: [PATCH 57/57] test: give the unresolvable realpath case a path with a missing parent --- sandboxd/engine/volume_live_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index 19f13782..a731f2e1 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -22,7 +22,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { {name: "canonical path", mount: "/data", kernelPath: "/data", lookups: 1}, {name: "symlinked parent", mount: "/workspace/data", kernelPath: "/home/u/ws/data", realpath: map[string]string{"/workspace/data": "/home/u/ws/data"}, lookups: 3}, {name: "trailing space behind a symlink", mount: "/ws/data ", kernelPath: `/home/u/ws/data\040`, realpath: map[string]string{"/ws/data ": "/home/u/ws/data "}, lookups: 3}, - {name: "unresolvable path", mount: "/data", kernelPath: "/data", noRealpath: true, lookups: 1}, + {name: "unresolvable path", mount: "/missing/data", kernelPath: "/missing/data", noRealpath: true, lookups: 1}, } { t.Run(tt.name, func(t *testing.T) { sock := sockPath(t)