From c7cb4de262f31fe402ab27b541767017a2b73363 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:30:35 +0000 Subject: [PATCH 1/7] Avoid blocking auto-standby holds on persistence --- lib/autostandby/controller.go | 212 +++++++++++++++++++---------- lib/autostandby/controller_test.go | 79 +++++++++-- 2 files changed, 206 insertions(+), 85 deletions(-) diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 3c4407f5f..f42ddcc40 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -118,6 +118,15 @@ type controllerState struct { standbyExecuting bool } +// runtimePersistence preserves controller mutation order while metadata writes +// run without holding the controller mutex. +type runtimePersistence struct { + id string + runtime *Runtime + generation uint64 + bestEffort bool +} + // Controller decides when eligible instances should transition to standby. type Controller struct { store InstanceStore @@ -136,11 +145,15 @@ type Controller struct { standbySlots chan struct{} standbyWG sync.WaitGroup - mu sync.RWMutex - states map[string]*controllerState - standbyInFlight int - observerConnected bool - lastObserverErr error + mu sync.RWMutex + states map[string]*controllerState + runtimeGenerations map[string]uint64 + nextRuntimeGeneration uint64 + standbyInFlight int + observerConnected bool + lastObserverErr error + + runtimePersistMu sync.Mutex } // NewController creates a new event-driven auto-standby controller. @@ -184,6 +197,7 @@ func NewController(store InstanceStore, source ConnectionSource, opts Controller streamReady: make(chan ConnectionStream, 4), standbySlots: make(chan struct{}, maxConcurrentStandbys), states: make(map[string]*controllerState), + runtimeGenerations: make(map[string]uint64), } c.metrics = newMetrics(opts.Meter, opts.Tracer, c) return c @@ -537,16 +551,19 @@ func (c *Controller) periodicSnapshotSync(ctx context.Context) error { func (c *Controller) seedInstanceState(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { c.mu.Lock() - defer c.mu.Unlock() - - return c.refreshInstanceLocked(ctx, inst, conns, now) + persistence, err := c.refreshInstanceLocked(inst, conns, now) + c.mu.Unlock() + if err != nil { + return err + } + return c.persistRuntime(ctx, persistence) } func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEvent) error { if event.Action == InstanceEventDelete { c.mu.Lock() - defer c.mu.Unlock() c.removeStateLocked(event.InstanceID) + c.mu.Unlock() return nil } if event.Instance == nil { @@ -559,11 +576,15 @@ func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEven } c.mu.Lock() - defer c.mu.Unlock() - return c.refreshInstanceLocked(ctx, *event.Instance, conns, c.now().UTC()) + persistence, err := c.refreshInstanceLocked(*event.Instance, conns, c.now().UTC()) + c.mu.Unlock() + if err != nil { + return err + } + return c.persistRuntime(ctx, persistence) } -func (c *Controller) refreshInstanceLocked(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { +func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, now time.Time) (runtimePersistence, error) { state := c.ensureStateLocked(inst.ID) state.instance = cloneInstance(inst) @@ -571,21 +592,21 @@ func (c *Controller) refreshInstanceLocked(ctx context.Context, inst Instance, c hadRuntime := inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil c.clearStateLocked(state) if hadRuntime { - return c.persistRuntime(ctx, inst.ID, nil) + return c.prepareRuntimePersistenceLocked(inst.ID, nil, false), nil } - return nil + return runtimePersistence{}, nil } compiled, err := compilePolicy(inst.AutoStandby) if err != nil { - return err + return runtimePersistence{}, err } state.compiledPolicy = compiled state.idleTimeout = compiled.idleTimeout activeSet, err := matchingConnections(inst, compiled, conns) if err != nil { - return err + return runtimePersistence{}, err } // Cancel any queued standby attempt only once the refresh is guaranteed to // re-establish a countdown or reconcile below; an erroring refresh above @@ -603,11 +624,12 @@ func (c *Controller) refreshInstanceLocked(ctx context.Context, inst Instance, c } c.cancelTimerLocked(state) c.armReconcileLocked(inst.ID, state) - return c.persistRuntime(ctx, inst.ID, &Runtime{ + return c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }) + }, false), nil } + var persistence runtimePersistence if runtime != nil && runtime.IdleSince != nil { state.idleSince = cloneTimePtr(runtime.IdleSince) state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) @@ -618,19 +640,13 @@ func (c *Controller) refreshInstanceLocked(ctx context.Context, inst Instance, c } else { state.lastInboundAt = nil } - runtime = &Runtime{ + persistence = c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - } - // Persist failures must not strand the instance without a countdown; - // the runtime only matters for recovery across controller restarts. - if err := c.persistRuntime(ctx, inst.ID, runtime); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime during refresh", "instance_id", inst.ID, "error", err) - } + }, true) } c.armTimerLocked(inst.ID, state, now) - return nil + return persistence, nil } func (c *Controller) handleConnectionEvent(ctx context.Context, event ConnectionEvent) { @@ -649,8 +665,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.recordConntrackEvent(string(event.Type), "received") c.mu.Lock() - defer c.mu.Unlock() - + persistences := make([]runtimePersistence, 0, 1) for id, state := range c.states { if state.compiledPolicy == nil { continue @@ -676,13 +691,10 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, idleSince) - if err := c.persistRuntime(ctx, id, &Runtime{ + persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime when idle countdown started", "instance_id", id, "error", err) - } + }, false)) c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) continue } @@ -695,12 +707,9 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelTimerLocked(state) c.armReconcileLocked(id, state) - if err := c.persistRuntime(ctx, id, &Runtime{ + persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after inbound activity", "instance_id", id, "error", err) - } + }, false)) c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) case ConnectionEventDestroy: if _, ok := state.activeInbound[key]; !ok { @@ -716,16 +725,21 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, idleSince) - if err := c.persistRuntime(ctx, id, &Runtime{ + persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime when idle countdown started", "instance_id", id, "error", err) - } + }, false)) c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) } } + c.mu.Unlock() + + for _, persistence := range persistences { + if err := c.persistRuntime(ctx, persistence); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after connection event", "instance_id", persistence.id, "error", err) + } + } } // confirmIdleBeforeStandby re-reads the conntrack table and reports whether the @@ -738,12 +752,11 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo conns, listErr := c.source.ListConnections(ctx) c.mu.Lock() - defer c.mu.Unlock() - state := c.states[id] // Activity can land between the timer firing and this check, and it already // owns idleSince and the reconcile loop; the paths below must not clobber it. if state == nil || state.compiledPolicy == nil || len(state.activeInbound) > 0 { + c.mu.Unlock() return false } @@ -756,10 +769,13 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo idleSince := c.now().UTC() state.idleSince = &idleSince c.armTimerLocked(id, state, idleSince) - if persistErr := c.persistRuntime(ctx, id, &Runtime{ + persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); persistErr != nil { + }, false) + c.mu.Unlock() + + if persistErr := c.persistRuntime(ctx, persistence); persistErr != nil { c.recordControllerError("persist_runtime") c.log.Warn("auto-standby failed to persist runtime after unconfirmed standby", "instance_id", id, "error", persistErr) } @@ -768,6 +784,7 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo return false } if len(activeSet) == 0 { + c.mu.Unlock() return true } @@ -777,9 +794,12 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo state.lastInboundAt = &now c.cancelTimerLocked(state) c.armReconcileLocked(id, state) - if err := c.persistRuntime(ctx, id, &Runtime{ + persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); err != nil { + }, false) + c.mu.Unlock() + + if err := c.persistRuntime(ctx, persistence); err != nil { c.recordControllerError("persist_runtime") c.log.Warn("auto-standby failed to persist runtime after standby confirmation found connections", "instance_id", id, "error", err) } @@ -902,32 +922,34 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName c.recordControllerError("standby") c.mu.Lock() - defer c.mu.Unlock() if errors.Is(err, ErrInstanceNotFound) { c.log.Info("auto-standby target instance no longer exists, dropping state", "instance_id", id, "instance_name", instanceName) c.removeStateLocked(id) + c.mu.Unlock() return } c.log.Warn("auto-standby standby attempt failed", "instance_id", id, "instance_name", instanceName, "error", err) + var persistence runtimePersistence if state := c.states[id]; state != nil { state.standbyRequested = false // Inbound activity that arrived during the attempt owns the state // now; the reconcile/destroy flow restarts the countdown once the // connections drain. - if len(state.activeInbound) > 0 { - return - } - idleSince := c.now().UTC() - state.idleSince = &idleSince - c.armTimerLocked(id, state, idleSince) - if persistErr := c.persistRuntime(ctx, id, &Runtime{ - IdleSince: cloneTimePtr(state.idleSince), - LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); persistErr != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after standby failure", "instance_id", id, "error", persistErr) + if len(state.activeInbound) == 0 { + idleSince := c.now().UTC() + state.idleSince = &idleSince + c.armTimerLocked(id, state, idleSince) + persistence = c.prepareRuntimePersistenceLocked(id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }, false) } } + c.mu.Unlock() + if persistErr := c.persistRuntime(ctx, persistence); persistErr != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after standby failure", "instance_id", id, "error", persistErr) + } return } @@ -935,13 +957,15 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) c.mu.Lock() - defer c.mu.Unlock() + var persistence runtimePersistence if state := c.states[id]; state != nil { c.clearStateLocked(state) - if err := c.persistRuntime(ctx, id, nil); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) - } + persistence = c.prepareRuntimePersistenceLocked(id, nil, false) + } + c.mu.Unlock() + if err := c.persistRuntime(ctx, persistence); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) } } @@ -962,10 +986,9 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { now := c.now().UTC() c.mu.Lock() - defer c.mu.Unlock() - state := c.states[id] if state == nil || state.compiledPolicy == nil { + c.mu.Unlock() return } @@ -976,6 +999,7 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { if len(state.activeInbound) > 0 { c.armReconcileLocked(id, state) } + c.mu.Unlock() return } @@ -983,6 +1007,7 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { if len(activeSet) > 0 { state.standbyRequested = false c.armReconcileLocked(id, state) + c.mu.Unlock() return } @@ -990,14 +1015,18 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, now) - if err := c.persistRuntime(ctx, id, &Runtime{ + persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }); err != nil { + }, false) + idleTimeout := state.idleTimeout + c.mu.Unlock() + + if err := c.persistRuntime(ctx, persistence); err != nil { c.recordControllerError("persist_runtime") c.log.Warn("auto-standby failed to persist runtime after active connection reconcile drained", "instance_id", id, "error", err) } - c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", state.idleTimeout) + c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", idleTimeout) } func (c *Controller) reconnectStream(ctx context.Context) { @@ -1048,6 +1077,7 @@ func (c *Controller) removeStateLocked(id string) { c.cancelReconcileLocked(state) } delete(c.states, id) + delete(c.runtimeGenerations, id) } func (c *Controller) clearStateLocked(state *controllerState) { @@ -1129,8 +1159,40 @@ func (c *Controller) stopAllTimers() { } } -func (c *Controller) persistRuntime(ctx context.Context, id string, runtime *Runtime) error { - return c.store.SetRuntime(ctx, id, runtime) +func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, bestEffort bool) runtimePersistence { + c.nextRuntimeGeneration++ + generation := c.nextRuntimeGeneration + c.runtimeGenerations[id] = generation + return runtimePersistence{ + id: id, + runtime: cloneRuntime(runtime), + generation: generation, + bestEffort: bestEffort, + } +} + +func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePersistence) error { + if persistence.generation == 0 { + return nil + } + + c.runtimePersistMu.Lock() + defer c.runtimePersistMu.Unlock() + + c.mu.RLock() + generation := c.runtimeGenerations[persistence.id] + c.mu.RUnlock() + if generation != persistence.generation { + return nil + } + + err := c.store.SetRuntime(ctx, persistence.id, persistence.runtime) + if err != nil && persistence.bestEffort { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime", "instance_id", persistence.id, "error", err) + return nil + } + return err } func (c *Controller) setObserverConnected(connected bool) { diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index 10d8b7d7d..05444238e 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -14,16 +14,18 @@ import ( ) type fakeInstanceStore struct { - mu sync.Mutex - instances []Instance - standbyIDs []string - persistedRuntime map[string]*Runtime - events chan InstanceEvent - standbyErr error - listErr error - setRuntimeErr error - standbyStarted chan string - standbyRelease chan struct{} + mu sync.Mutex + instances []Instance + standbyIDs []string + persistedRuntime map[string]*Runtime + events chan InstanceEvent + standbyErr error + listErr error + setRuntimeErr error + setRuntimeStarted chan string + setRuntimeRelease chan struct{} + standbyStarted chan string + standbyRelease chan struct{} } func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { @@ -67,6 +69,13 @@ func (f *fakeInstanceStore) standbyCalls() []string { } func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { + if f.setRuntimeStarted != nil { + f.setRuntimeStarted <- id + } + if f.setRuntimeRelease != nil { + <-f.setRuntimeRelease + } + f.mu.Lock() defer f.mu.Unlock() if f.setRuntimeErr != nil { @@ -1348,6 +1357,56 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller.mu.RUnlock() } +func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + inst := Instance{ + ID: "inst-hold-during-persist", + Name: "inst-hold-during-persist", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.100", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + } + conn := Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50010, + OriginalDestinationIP: mustAddr(inst.IP), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + } + store := newFakeInstanceStore([]Instance{inst}) + store.setRuntimeStarted = make(chan string, 1) + store.setRuntimeRelease = make(chan struct{}) + controller := NewController(store, &fakeConnectionSource{connections: []Connection{conn}}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + resyncDone := make(chan error, 1) + go func() { + resyncDone <- controller.startupResync(context.Background()) + }() + require.Equal(t, inst.ID, <-store.setRuntimeStarted) + + holdDone := make(chan error, 1) + go func() { + _, err := controller.HoldStandby(context.Background(), inst) + holdDone <- err + }() + + select { + case err := <-holdDone: + require.NoError(t, err) + case <-time.After(time.Second): + close(store.setRuntimeRelease) + require.FailNow(t, "hold waited for runtime persistence") + } + + close(store.setRuntimeRelease) + require.NoError(t, <-resyncDone) +} + func TestHoldStandbyExtendsArmedCountdown(t *testing.T) { t.Parallel() From 118f11c520bd94ad0a71448239fa75eec525ac68 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:53:12 +0000 Subject: [PATCH 2/7] Refine runtime persistence concurrency --- lib/autostandby/controller.go | 105 +++++++++++++--------- lib/autostandby/controller_test.go | 139 ++++++++++++++++++++++++++--- 2 files changed, 186 insertions(+), 58 deletions(-) diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index f42ddcc40..dff84cc2e 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -118,13 +118,27 @@ type controllerState struct { standbyExecuting bool } +type runtimePersistenceErrorMode uint8 + +const ( + runtimePersistencePropagate runtimePersistenceErrorMode = iota + runtimePersistenceBestEffort +) + +type runtimePersistenceLock struct { + mu sync.Mutex + refs int +} + // runtimePersistence preserves controller mutation order while metadata writes // run without holding the controller mutex. type runtimePersistence struct { id string runtime *Runtime generation uint64 - bestEffort bool + errorMode runtimePersistenceErrorMode + operation string + lock *runtimePersistenceLock } // Controller decides when eligible instances should transition to standby. @@ -148,12 +162,11 @@ type Controller struct { mu sync.RWMutex states map[string]*controllerState runtimeGenerations map[string]uint64 + runtimePersistLocks map[string]*runtimePersistenceLock nextRuntimeGeneration uint64 standbyInFlight int observerConnected bool lastObserverErr error - - runtimePersistMu sync.Mutex } // NewController creates a new event-driven auto-standby controller. @@ -198,6 +211,7 @@ func NewController(store InstanceStore, source ConnectionSource, opts Controller standbySlots: make(chan struct{}, maxConcurrentStandbys), states: make(map[string]*controllerState), runtimeGenerations: make(map[string]uint64), + runtimePersistLocks: make(map[string]*runtimePersistenceLock), } c.metrics = newMetrics(opts.Meter, opts.Tracer, c) return c @@ -592,7 +606,7 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no hadRuntime := inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil c.clearStateLocked(state) if hadRuntime { - return c.prepareRuntimePersistenceLocked(inst.ID, nil, false), nil + return c.prepareRuntimePersistenceLocked(inst.ID, nil, runtimePersistencePropagate, "refresh disabled instance"), nil } return runtimePersistence{}, nil } @@ -626,7 +640,7 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no c.armReconcileLocked(inst.ID, state) return c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false), nil + }, runtimePersistencePropagate, "refresh active instance"), nil } var persistence runtimePersistence @@ -643,7 +657,7 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no persistence = c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, true) + }, runtimePersistenceBestEffort, "refresh idle instance") } c.armTimerLocked(inst.ID, state, now) return persistence, nil @@ -694,7 +708,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false)) + }, runtimePersistenceBestEffort, "start idle countdown")) c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) continue } @@ -709,7 +723,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.armReconcileLocked(id, state) persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false)) + }, runtimePersistenceBestEffort, "record inbound activity")) c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) case ConnectionEventDestroy: if _, ok := state.activeInbound[key]; !ok { @@ -728,17 +742,14 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false)) + }, runtimePersistenceBestEffort, "restart idle countdown")) c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) } } c.mu.Unlock() for _, persistence := range persistences { - if err := c.persistRuntime(ctx, persistence); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after connection event", "instance_id", persistence.id, "error", err) - } + _ = c.persistRuntime(ctx, persistence) } } @@ -772,13 +783,10 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false) + }, runtimePersistenceBestEffort, "handle unconfirmed standby") c.mu.Unlock() - if persistErr := c.persistRuntime(ctx, persistence); persistErr != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after unconfirmed standby", "instance_id", id, "error", persistErr) - } + _ = c.persistRuntime(ctx, persistence) c.recordControllerError("standby_confirm") c.log.Warn("auto-standby could not confirm idle before standby", "instance_id", id, "error", err) return false @@ -796,13 +804,10 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo c.armReconcileLocked(id, state) persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false) + }, runtimePersistenceBestEffort, "record standby confirmation activity") c.mu.Unlock() - if err := c.persistRuntime(ctx, persistence); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after standby confirmation found connections", "instance_id", id, "error", err) - } + _ = c.persistRuntime(ctx, persistence) c.log.Info("auto-standby skipped standby, conntrack still reports inbound connections", "instance_id", id, "active_inbound_connections", len(activeSet)) return false } @@ -942,14 +947,11 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName persistence = c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false) + }, runtimePersistenceBestEffort, "recover from standby failure") } } c.mu.Unlock() - if persistErr := c.persistRuntime(ctx, persistence); persistErr != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after standby failure", "instance_id", id, "error", persistErr) - } + _ = c.persistRuntime(ctx, persistence) return } @@ -960,13 +962,10 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName var persistence runtimePersistence if state := c.states[id]; state != nil { c.clearStateLocked(state) - persistence = c.prepareRuntimePersistenceLocked(id, nil, false) + persistence = c.prepareRuntimePersistenceLocked(id, nil, runtimePersistenceBestEffort, "clear runtime after standby") } c.mu.Unlock() - if err := c.persistRuntime(ctx, persistence); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) - } + _ = c.persistRuntime(ctx, persistence) } func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { @@ -1018,14 +1017,11 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, false) + }, runtimePersistenceBestEffort, "finish active connection reconcile") idleTimeout := state.idleTimeout c.mu.Unlock() - if err := c.persistRuntime(ctx, persistence); err != nil { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime after active connection reconcile drained", "instance_id", id, "error", err) - } + _ = c.persistRuntime(ctx, persistence) c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", idleTimeout) } @@ -1159,15 +1155,25 @@ func (c *Controller) stopAllTimers() { } } -func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, bestEffort bool) runtimePersistence { +func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, errorMode runtimePersistenceErrorMode, operation string) runtimePersistence { c.nextRuntimeGeneration++ generation := c.nextRuntimeGeneration c.runtimeGenerations[id] = generation + + lock := c.runtimePersistLocks[id] + if lock == nil { + lock = &runtimePersistenceLock{} + c.runtimePersistLocks[id] = lock + } + lock.refs++ + return runtimePersistence{ id: id, runtime: cloneRuntime(runtime), generation: generation, - bestEffort: bestEffort, + errorMode: errorMode, + operation: operation, + lock: lock, } } @@ -1176,8 +1182,9 @@ func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePers return nil } - c.runtimePersistMu.Lock() - defer c.runtimePersistMu.Unlock() + persistence.lock.mu.Lock() + defer c.releaseRuntimePersistenceLock(persistence) + defer persistence.lock.mu.Unlock() c.mu.RLock() generation := c.runtimeGenerations[persistence.id] @@ -1187,14 +1194,24 @@ func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePers } err := c.store.SetRuntime(ctx, persistence.id, persistence.runtime) - if err != nil && persistence.bestEffort { + if err != nil && persistence.errorMode == runtimePersistenceBestEffort { c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime", "instance_id", persistence.id, "error", err) + c.log.Warn("auto-standby failed to persist runtime", "instance_id", persistence.id, "operation", persistence.operation, "error", err) return nil } return err } +func (c *Controller) releaseRuntimePersistenceLock(persistence runtimePersistence) { + c.mu.Lock() + defer c.mu.Unlock() + + persistence.lock.refs-- + if persistence.lock.refs == 0 && c.runtimePersistLocks[persistence.id] == persistence.lock { + delete(c.runtimePersistLocks, persistence.id) + } +} + func (c *Controller) setObserverConnected(connected bool) { c.mu.Lock() defer c.mu.Unlock() diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index 05444238e..b8ead05c9 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -14,18 +14,19 @@ import ( ) type fakeInstanceStore struct { - mu sync.Mutex - instances []Instance - standbyIDs []string - persistedRuntime map[string]*Runtime - events chan InstanceEvent - standbyErr error - listErr error - setRuntimeErr error - setRuntimeStarted chan string - setRuntimeRelease chan struct{} - standbyStarted chan string - standbyRelease chan struct{} + mu sync.Mutex + instances []Instance + standbyIDs []string + persistedRuntime map[string]*Runtime + events chan InstanceEvent + standbyErr error + listErr error + setRuntimeErr error + setRuntimeStarted chan string + setRuntimeRelease chan struct{} + setRuntimeReleaseByID map[string]chan struct{} + standbyStarted chan string + standbyRelease chan struct{} } func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { @@ -72,8 +73,12 @@ func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Ru if f.setRuntimeStarted != nil { f.setRuntimeStarted <- id } - if f.setRuntimeRelease != nil { - <-f.setRuntimeRelease + release := f.setRuntimeRelease + if f.setRuntimeReleaseByID != nil { + release = f.setRuntimeReleaseByID[id] + } + if release != nil { + <-release } f.mu.Lock() @@ -1357,6 +1362,112 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller.mu.RUnlock() } +func TestRuntimePersistenceDoesNotBlockOtherInstances(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + first := Instance{ + ID: "inst-persist-first", + Name: "inst-persist-first", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.110", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + } + second := first + second.ID = "inst-persist-second" + second.Name = second.ID + second.IP = "192.168.100.111" + conns := []Connection{ + { + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50010, + OriginalDestinationIP: mustAddr(first.IP), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50011, + OriginalDestinationIP: mustAddr(second.IP), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + } + firstRelease := make(chan struct{}) + store := newFakeInstanceStore([]Instance{first, second}) + store.setRuntimeStarted = make(chan string, 2) + store.setRuntimeReleaseByID = map[string]chan struct{}{first.ID: firstRelease} + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + firstDone := make(chan error, 1) + go func() { + firstDone <- controller.seedInstanceState(context.Background(), first, conns, now) + }() + require.Equal(t, first.ID, <-store.setRuntimeStarted) + + secondDone := make(chan error, 1) + go func() { + secondDone <- controller.seedInstanceState(context.Background(), second, conns, now) + }() + + select { + case id := <-store.setRuntimeStarted: + require.Equal(t, second.ID, id) + case <-time.After(time.Second): + close(firstRelease) + require.FailNow(t, "runtime persistence waited on another instance") + } + select { + case err := <-secondDone: + require.NoError(t, err) + case <-time.After(time.Second): + close(firstRelease) + require.FailNow(t, "runtime persistence did not complete for another instance") + } + + close(firstRelease) + require.NoError(t, <-firstDone) +} + +func TestRuntimePersistenceKeepsLatestGeneration(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore(nil) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) + firstTime := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + secondTime := firstTime.Add(time.Second) + + controller.mu.Lock() + first := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &firstTime}, runtimePersistencePropagate, "test first generation") + second := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &secondTime}, runtimePersistencePropagate, "test second generation") + controller.mu.Unlock() + + require.NoError(t, controller.persistRuntime(context.Background(), first)) + assert.NotContains(t, store.persistedRuntime, "inst-persist-order") + require.NoError(t, controller.persistRuntime(context.Background(), second)) + require.NotNil(t, store.persistedRuntime["inst-persist-order"]) + assert.Equal(t, secondTime, *store.persistedRuntime["inst-persist-order"].IdleSince) +} + +func TestRuntimePersistenceSkipsDeletedInstance(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore(nil) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) + now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + + controller.mu.Lock() + persistence := controller.prepareRuntimePersistenceLocked("inst-persist-deleted", &Runtime{IdleSince: &now}, runtimePersistencePropagate, "test deleted instance") + controller.removeStateLocked("inst-persist-deleted") + controller.mu.Unlock() + + require.NoError(t, controller.persistRuntime(context.Background(), persistence)) + assert.NotContains(t, store.persistedRuntime, "inst-persist-deleted") +} + func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { t.Parallel() From 0654fb3fb28a13a8ed42f3c36c096e14ee899e80 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:04:32 +0000 Subject: [PATCH 3/7] Simplify runtime persistence ordering --- lib/autostandby/controller.go | 112 ++++++++--------------- lib/autostandby/controller_test.go | 139 +++++++---------------------- 2 files changed, 65 insertions(+), 186 deletions(-) diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index dff84cc2e..24f264af1 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -118,27 +118,15 @@ type controllerState struct { standbyExecuting bool } -type runtimePersistenceErrorMode uint8 - -const ( - runtimePersistencePropagate runtimePersistenceErrorMode = iota - runtimePersistenceBestEffort -) - -type runtimePersistenceLock struct { - mu sync.Mutex - refs int -} - -// runtimePersistence preserves controller mutation order while metadata writes -// run without holding the controller mutex. +// runtimePersistence preserves mutation order while metadata writes run +// without holding the controller mutex. type runtimePersistence struct { id string runtime *Runtime - generation uint64 - errorMode runtimePersistenceErrorMode + previous <-chan struct{} + done chan struct{} + bestEffort bool operation string - lock *runtimePersistenceLock } // Controller decides when eligible instances should transition to standby. @@ -159,14 +147,12 @@ type Controller struct { standbySlots chan struct{} standbyWG sync.WaitGroup - mu sync.RWMutex - states map[string]*controllerState - runtimeGenerations map[string]uint64 - runtimePersistLocks map[string]*runtimePersistenceLock - nextRuntimeGeneration uint64 - standbyInFlight int - observerConnected bool - lastObserverErr error + mu sync.RWMutex + states map[string]*controllerState + runtimePersistenceTail chan struct{} + standbyInFlight int + observerConnected bool + lastObserverErr error } // NewController creates a new event-driven auto-standby controller. @@ -210,8 +196,6 @@ func NewController(store InstanceStore, source ConnectionSource, opts Controller streamReady: make(chan ConnectionStream, 4), standbySlots: make(chan struct{}, maxConcurrentStandbys), states: make(map[string]*controllerState), - runtimeGenerations: make(map[string]uint64), - runtimePersistLocks: make(map[string]*runtimePersistenceLock), } c.metrics = newMetrics(opts.Meter, opts.Tracer, c) return c @@ -606,7 +590,7 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no hadRuntime := inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil c.clearStateLocked(state) if hadRuntime { - return c.prepareRuntimePersistenceLocked(inst.ID, nil, runtimePersistencePropagate, "refresh disabled instance"), nil + return c.prepareRuntimePersistenceLocked(inst.ID, nil, false, "refresh disabled instance"), nil } return runtimePersistence{}, nil } @@ -640,7 +624,7 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no c.armReconcileLocked(inst.ID, state) return c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistencePropagate, "refresh active instance"), nil + }, false, "refresh active instance"), nil } var persistence runtimePersistence @@ -657,7 +641,7 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no persistence = c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "refresh idle instance") + }, true, "refresh idle instance") } c.armTimerLocked(inst.ID, state, now) return persistence, nil @@ -708,7 +692,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "start idle countdown")) + }, true, "start idle countdown")) c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) continue } @@ -723,7 +707,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.armReconcileLocked(id, state) persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "record inbound activity")) + }, true, "record inbound activity")) c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) case ConnectionEventDestroy: if _, ok := state.activeInbound[key]; !ok { @@ -742,7 +726,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "restart idle countdown")) + }, true, "restart idle countdown")) c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) } } @@ -783,7 +767,7 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "handle unconfirmed standby") + }, true, "handle unconfirmed standby") c.mu.Unlock() _ = c.persistRuntime(ctx, persistence) @@ -804,7 +788,7 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo c.armReconcileLocked(id, state) persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "record standby confirmation activity") + }, true, "record standby confirmation activity") c.mu.Unlock() _ = c.persistRuntime(ctx, persistence) @@ -947,7 +931,7 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName persistence = c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "recover from standby failure") + }, true, "recover from standby failure") } } c.mu.Unlock() @@ -962,7 +946,7 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName var persistence runtimePersistence if state := c.states[id]; state != nil { c.clearStateLocked(state) - persistence = c.prepareRuntimePersistenceLocked(id, nil, runtimePersistenceBestEffort, "clear runtime after standby") + persistence = c.prepareRuntimePersistenceLocked(id, nil, true, "clear runtime after standby") } c.mu.Unlock() _ = c.persistRuntime(ctx, persistence) @@ -1017,7 +1001,7 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), - }, runtimePersistenceBestEffort, "finish active connection reconcile") + }, true, "finish active connection reconcile") idleTimeout := state.idleTimeout c.mu.Unlock() @@ -1073,7 +1057,6 @@ func (c *Controller) removeStateLocked(id string) { c.cancelReconcileLocked(state) } delete(c.states, id) - delete(c.runtimeGenerations, id) } func (c *Controller) clearStateLocked(state *controllerState) { @@ -1155,46 +1138,31 @@ func (c *Controller) stopAllTimers() { } } -func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, errorMode runtimePersistenceErrorMode, operation string) runtimePersistence { - c.nextRuntimeGeneration++ - generation := c.nextRuntimeGeneration - c.runtimeGenerations[id] = generation - - lock := c.runtimePersistLocks[id] - if lock == nil { - lock = &runtimePersistenceLock{} - c.runtimePersistLocks[id] = lock - } - lock.refs++ - - return runtimePersistence{ +func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, bestEffort bool, operation string) runtimePersistence { + done := make(chan struct{}) + persistence := runtimePersistence{ id: id, runtime: cloneRuntime(runtime), - generation: generation, - errorMode: errorMode, + previous: c.runtimePersistenceTail, + done: done, + bestEffort: bestEffort, operation: operation, - lock: lock, } + c.runtimePersistenceTail = done + return persistence } func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePersistence) error { - if persistence.generation == 0 { + if persistence.done == nil { return nil } - - persistence.lock.mu.Lock() - defer c.releaseRuntimePersistenceLock(persistence) - defer persistence.lock.mu.Unlock() - - c.mu.RLock() - generation := c.runtimeGenerations[persistence.id] - c.mu.RUnlock() - if generation != persistence.generation { - return nil + if persistence.previous != nil { + <-persistence.previous } + defer close(persistence.done) err := c.store.SetRuntime(ctx, persistence.id, persistence.runtime) - if err != nil && persistence.errorMode == runtimePersistenceBestEffort { + if err != nil && persistence.bestEffort { c.recordControllerError("persist_runtime") c.log.Warn("auto-standby failed to persist runtime", "instance_id", persistence.id, "operation", persistence.operation, "error", err) return nil @@ -1202,16 +1170,6 @@ func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePers return err } -func (c *Controller) releaseRuntimePersistenceLock(persistence runtimePersistence) { - c.mu.Lock() - defer c.mu.Unlock() - - persistence.lock.refs-- - if persistence.lock.refs == 0 && c.runtimePersistLocks[persistence.id] == persistence.lock { - delete(c.runtimePersistLocks, persistence.id) - } -} - func (c *Controller) setObserverConnected(connected bool) { c.mu.Lock() defer c.mu.Unlock() diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index b8ead05c9..75e45651e 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -14,19 +14,18 @@ import ( ) type fakeInstanceStore struct { - mu sync.Mutex - instances []Instance - standbyIDs []string - persistedRuntime map[string]*Runtime - events chan InstanceEvent - standbyErr error - listErr error - setRuntimeErr error - setRuntimeStarted chan string - setRuntimeRelease chan struct{} - setRuntimeReleaseByID map[string]chan struct{} - standbyStarted chan string - standbyRelease chan struct{} + mu sync.Mutex + instances []Instance + standbyIDs []string + persistedRuntime map[string]*Runtime + events chan InstanceEvent + standbyErr error + listErr error + setRuntimeErr error + setRuntimeStarted chan string + setRuntimeRelease chan struct{} + standbyStarted chan string + standbyRelease chan struct{} } func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { @@ -73,12 +72,8 @@ func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Ru if f.setRuntimeStarted != nil { f.setRuntimeStarted <- id } - release := f.setRuntimeRelease - if f.setRuntimeReleaseByID != nil { - release = f.setRuntimeReleaseByID[id] - } - if release != nil { - <-release + if f.setRuntimeRelease != nil { + <-f.setRuntimeRelease } f.mu.Lock() @@ -1362,112 +1357,38 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller.mu.RUnlock() } -func TestRuntimePersistenceDoesNotBlockOtherInstances(t *testing.T) { +func TestRuntimePersistencePreservesInstanceOrder(t *testing.T) { t.Parallel() - now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) - first := Instance{ - ID: "inst-persist-first", - Name: "inst-persist-first", - State: StateRunning, - NetworkEnabled: true, - IP: "192.168.100.110", - AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - } - second := first - second.ID = "inst-persist-second" - second.Name = second.ID - second.IP = "192.168.100.111" - conns := []Connection{ - { - OriginalSourceIP: mustAddr("1.2.3.4"), - OriginalSourcePort: 50010, - OriginalDestinationIP: mustAddr(first.IP), - OriginalDestinationPort: 8080, - TCPState: TCPStateEstablished, - }, - { - OriginalSourceIP: mustAddr("1.2.3.4"), - OriginalSourcePort: 50011, - OriginalDestinationIP: mustAddr(second.IP), - OriginalDestinationPort: 8080, - TCPState: TCPStateEstablished, - }, - } - firstRelease := make(chan struct{}) - store := newFakeInstanceStore([]Instance{first, second}) + store := newFakeInstanceStore(nil) store.setRuntimeStarted = make(chan string, 2) - store.setRuntimeReleaseByID = map[string]chan struct{}{first.ID: firstRelease} - controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ - Now: func() time.Time { return now }, - }) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) + firstTime := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + secondTime := firstTime.Add(time.Second) - firstDone := make(chan error, 1) - go func() { - firstDone <- controller.seedInstanceState(context.Background(), first, conns, now) - }() - require.Equal(t, first.ID, <-store.setRuntimeStarted) + controller.mu.Lock() + first := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &firstTime}, false, "test first write") + second := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &secondTime}, false, "test second write") + controller.mu.Unlock() secondDone := make(chan error, 1) go func() { - secondDone <- controller.seedInstanceState(context.Background(), second, conns, now) + secondDone <- controller.persistRuntime(context.Background(), second) }() - select { - case id := <-store.setRuntimeStarted: - require.Equal(t, second.ID, id) - case <-time.After(time.Second): - close(firstRelease) - require.FailNow(t, "runtime persistence waited on another instance") - } - select { - case err := <-secondDone: - require.NoError(t, err) - case <-time.After(time.Second): - close(firstRelease) - require.FailNow(t, "runtime persistence did not complete for another instance") + case <-store.setRuntimeStarted: + require.FailNow(t, "second runtime write started before first") + case <-time.After(50 * time.Millisecond): } - close(firstRelease) - require.NoError(t, <-firstDone) -} - -func TestRuntimePersistenceKeepsLatestGeneration(t *testing.T) { - t.Parallel() - - store := newFakeInstanceStore(nil) - controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) - firstTime := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) - secondTime := firstTime.Add(time.Second) - - controller.mu.Lock() - first := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &firstTime}, runtimePersistencePropagate, "test first generation") - second := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &secondTime}, runtimePersistencePropagate, "test second generation") - controller.mu.Unlock() - require.NoError(t, controller.persistRuntime(context.Background(), first)) - assert.NotContains(t, store.persistedRuntime, "inst-persist-order") - require.NoError(t, controller.persistRuntime(context.Background(), second)) + require.Equal(t, "inst-persist-order", <-store.setRuntimeStarted) + require.NoError(t, <-secondDone) + require.Equal(t, "inst-persist-order", <-store.setRuntimeStarted) require.NotNil(t, store.persistedRuntime["inst-persist-order"]) assert.Equal(t, secondTime, *store.persistedRuntime["inst-persist-order"].IdleSince) } -func TestRuntimePersistenceSkipsDeletedInstance(t *testing.T) { - t.Parallel() - - store := newFakeInstanceStore(nil) - controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) - now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) - - controller.mu.Lock() - persistence := controller.prepareRuntimePersistenceLocked("inst-persist-deleted", &Runtime{IdleSince: &now}, runtimePersistencePropagate, "test deleted instance") - controller.removeStateLocked("inst-persist-deleted") - controller.mu.Unlock() - - require.NoError(t, controller.persistRuntime(context.Background(), persistence)) - assert.NotContains(t, store.persistedRuntime, "inst-persist-deleted") -} - func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { t.Parallel() From 629dca4c86c4c08c8020fc0f428795a0f6b0dada Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:29:16 +0000 Subject: [PATCH 4/7] Document persistence chain invariants --- lib/autostandby/controller.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 24f264af1..5cea9391e 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -119,7 +119,8 @@ type controllerState struct { } // runtimePersistence preserves mutation order while metadata writes run -// without holding the controller mutex. +// without holding the controller mutex. Every prepared value must be passed to +// persistRuntime exactly once so its successor can proceed. type runtimePersistence struct { id string runtime *Runtime @@ -1138,6 +1139,9 @@ func (c *Controller) stopAllTimers() { } } +// prepareRuntimePersistenceLocked reserves this write's place in the global +// persistence order. The caller must pass the result to persistRuntime exactly +// once after releasing c.mu. func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, bestEffort bool, operation string) runtimePersistence { done := make(chan struct{}) persistence := runtimePersistence{ @@ -1157,6 +1161,8 @@ func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePers return nil } if persistence.previous != nil { + // Do not abandon this wait on context cancellation: a successor must not + // start until the predecessor has stopped writing. <-persistence.previous } defer close(persistence.done) From 6d8ab724a32a0a182a650782499a3525c198cd0f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:45:45 +0000 Subject: [PATCH 5/7] Rename auto-standby state identifiers --- cmd/api/api/auto_standby_hold_test.go | 14 +- cmd/api/api/auto_standby_status_test.go | 14 +- lib/autostandby/controller.go | 152 ++++++++-------- lib/autostandby/controller_test.go | 164 +++++++++--------- lib/autostandby/types.go | 20 +-- lib/instances/auto_standby.go | 14 +- .../auto_standby_integration_linux_test.go | 10 +- lib/instances/auto_standby_runtime.go | 34 ---- lib/instances/auto_standby_state.go | 34 ++++ lib/instances/metadata_clone.go | 2 +- lib/instances/storage.go | 5 +- lib/providers/auto_standby_linux.go | 42 ++--- lib/providers/auto_standby_linux_test.go | 36 ++-- 13 files changed, 271 insertions(+), 270 deletions(-) delete mode 100644 lib/instances/auto_standby_runtime.go create mode 100644 lib/instances/auto_standby_state.go diff --git a/cmd/api/api/auto_standby_hold_test.go b/cmd/api/api/auto_standby_hold_test.go index fe347b42f..a1f9259b5 100644 --- a/cmd/api/api/auto_standby_hold_test.go +++ b/cmd/api/api/auto_standby_hold_test.go @@ -31,13 +31,13 @@ func TestHoldAutoStandbyExtendsCountdown(t *testing.T) { idleSince := now.Add(-4 * time.Minute) store := &statusStore{ instances: []autostandby.Instance{{ - ID: "inst-hold", - Name: "inst-hold", - State: autostandby.StateRunning, - NetworkEnabled: true, - IP: "192.168.100.30", - AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, - Runtime: &autostandby.Runtime{IdleSince: &idleSince}, + ID: "inst-hold", + Name: "inst-hold", + State: autostandby.StateRunning, + NetworkEnabled: true, + IP: "192.168.100.30", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + AutoStandbyState: &autostandby.AutoStandbyState{IdleSince: &idleSince}, }}, } controller := autostandby.NewController(store, &statusConnectionSource{}, autostandby.ControllerOptions{ diff --git a/cmd/api/api/auto_standby_status_test.go b/cmd/api/api/auto_standby_status_test.go index b4aecfd1d..22b830830 100644 --- a/cmd/api/api/auto_standby_status_test.go +++ b/cmd/api/api/auto_standby_status_test.go @@ -27,9 +27,9 @@ func (m *captureStatusManager) GetInstance(context.Context, string) (*instances. } type statusStore struct { - instances []autostandby.Instance - runtime map[string]*autostandby.Runtime - events chan autostandby.InstanceEvent + instances []autostandby.Instance + autoStandbyState map[string]*autostandby.AutoStandbyState + events chan autostandby.InstanceEvent } func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, error) { @@ -38,11 +38,11 @@ func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, er func (s *statusStore) StandbyInstance(context.Context, string) error { return nil } -func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { - if s.runtime == nil { - s.runtime = make(map[string]*autostandby.Runtime) +func (s *statusStore) SetAutoStandbyState(_ context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { + if s.autoStandbyState == nil { + s.autoStandbyState = make(map[string]*autostandby.AutoStandbyState) } - s.runtime[id] = runtime + s.autoStandbyState[id] = autoStandbyState return nil } diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 5cea9391e..0064b798d 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -53,11 +53,11 @@ type InstanceEvent struct { } // InstanceStore supplies the controller with instance state, lifecycle events, -// runtime persistence, and standby actions. +// auto-standby state persistence, and standby actions. type InstanceStore interface { ListInstances(ctx context.Context) ([]Instance, error) StandbyInstance(ctx context.Context, id string) error - SetRuntime(ctx context.Context, id string, runtime *Runtime) error + SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *AutoStandbyState) error SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) } @@ -118,16 +118,16 @@ type controllerState struct { standbyExecuting bool } -// runtimePersistence preserves mutation order while metadata writes run +// autoStandbyStatePersistence preserves mutation order while metadata writes run // without holding the controller mutex. Every prepared value must be passed to -// persistRuntime exactly once so its successor can proceed. -type runtimePersistence struct { - id string - runtime *Runtime - previous <-chan struct{} - done chan struct{} - bestEffort bool - operation string +// persistAutoStandbyState exactly once so its successor can proceed. +type autoStandbyStatePersistence struct { + id string + autoStandbyState *AutoStandbyState + previous <-chan struct{} + done chan struct{} + bestEffort bool + operation string } // Controller decides when eligible instances should transition to standby. @@ -148,12 +148,12 @@ type Controller struct { standbySlots chan struct{} standbyWG sync.WaitGroup - mu sync.RWMutex - states map[string]*controllerState - runtimePersistenceTail chan struct{} - standbyInFlight int - observerConnected bool - lastObserverErr error + mu sync.RWMutex + states map[string]*controllerState + autoStandbyStatePersistenceTail chan struct{} + standbyInFlight int + observerConnected bool + lastObserverErr error } // NewController creates a new event-driven auto-standby controller. @@ -555,7 +555,7 @@ func (c *Controller) seedInstanceState(ctx context.Context, inst Instance, conns if err != nil { return err } - return c.persistRuntime(ctx, persistence) + return c.persistAutoStandbyState(ctx, persistence) } func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEvent) error { @@ -580,32 +580,32 @@ func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEven if err != nil { return err } - return c.persistRuntime(ctx, persistence) + return c.persistAutoStandbyState(ctx, persistence) } -func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, now time.Time) (runtimePersistence, error) { +func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, now time.Time) (autoStandbyStatePersistence, error) { state := c.ensureStateLocked(inst.ID) state.instance = cloneInstance(inst) if !eligible(inst) { - hadRuntime := inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil + hadAutoStandbyState := inst.AutoStandbyState != nil || state.idleSince != nil || state.lastInboundAt != nil c.clearStateLocked(state) - if hadRuntime { - return c.prepareRuntimePersistenceLocked(inst.ID, nil, false, "refresh disabled instance"), nil + if hadAutoStandbyState { + return c.prepareAutoStandbyStatePersistenceLocked(inst.ID, nil, false, "refresh disabled instance"), nil } - return runtimePersistence{}, nil + return autoStandbyStatePersistence{}, nil } compiled, err := compilePolicy(inst.AutoStandby) if err != nil { - return runtimePersistence{}, err + return autoStandbyStatePersistence{}, err } state.compiledPolicy = compiled state.idleTimeout = compiled.idleTimeout activeSet, err := matchingConnections(inst, compiled, conns) if err != nil { - return runtimePersistence{}, err + return autoStandbyStatePersistence{}, err } // Cancel any queued standby attempt only once the refresh is guaranteed to // re-establish a countdown or reconcile below; an erroring refresh above @@ -613,33 +613,33 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no state.standbyRequested = false state.activeInbound = activeSet - runtime := cloneRuntime(inst.Runtime) + autoStandbyState := cloneAutoStandbyState(inst.AutoStandbyState) if len(activeSet) > 0 { state.idleSince = nil - if runtime != nil && runtime.LastInboundActivityAt != nil { - state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + if autoStandbyState != nil && autoStandbyState.LastInboundActivityAt != nil { + state.lastInboundAt = cloneTimePtr(autoStandbyState.LastInboundActivityAt) } else { state.lastInboundAt = &now } c.cancelTimerLocked(state) c.armReconcileLocked(inst.ID, state) - return c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ + return c.prepareAutoStandbyStatePersistenceLocked(inst.ID, &AutoStandbyState{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, false, "refresh active instance"), nil } - var persistence runtimePersistence - if runtime != nil && runtime.IdleSince != nil { - state.idleSince = cloneTimePtr(runtime.IdleSince) - state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + var persistence autoStandbyStatePersistence + if autoStandbyState != nil && autoStandbyState.IdleSince != nil { + state.idleSince = cloneTimePtr(autoStandbyState.IdleSince) + state.lastInboundAt = cloneTimePtr(autoStandbyState.LastInboundActivityAt) } else { state.idleSince = &now - if runtime != nil { - state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + if autoStandbyState != nil { + state.lastInboundAt = cloneTimePtr(autoStandbyState.LastInboundActivityAt) } else { state.lastInboundAt = nil } - persistence = c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ + persistence = c.prepareAutoStandbyStatePersistenceLocked(inst.ID, &AutoStandbyState{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "refresh idle instance") @@ -664,7 +664,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.recordConntrackEvent(string(event.Type), "received") c.mu.Lock() - persistences := make([]runtimePersistence, 0, 1) + persistences := make([]autoStandbyStatePersistence, 0, 1) for id, state := range c.states { if state.compiledPolicy == nil { continue @@ -690,7 +690,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, idleSince) - persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistences = append(persistences, c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "start idle countdown")) @@ -706,7 +706,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelTimerLocked(state) c.armReconcileLocked(id, state) - persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistences = append(persistences, c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "record inbound activity")) c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) @@ -724,7 +724,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, idleSince) - persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistences = append(persistences, c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "restart idle countdown")) @@ -734,7 +734,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.mu.Unlock() for _, persistence := range persistences { - _ = c.persistRuntime(ctx, persistence) + _ = c.persistAutoStandbyState(ctx, persistence) } } @@ -765,13 +765,13 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo idleSince := c.now().UTC() state.idleSince = &idleSince c.armTimerLocked(id, state, idleSince) - persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistence := c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "handle unconfirmed standby") c.mu.Unlock() - _ = c.persistRuntime(ctx, persistence) + _ = c.persistAutoStandbyState(ctx, persistence) c.recordControllerError("standby_confirm") c.log.Warn("auto-standby could not confirm idle before standby", "instance_id", id, "error", err) return false @@ -787,12 +787,12 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo state.lastInboundAt = &now c.cancelTimerLocked(state) c.armReconcileLocked(id, state) - persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistence := c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "record standby confirmation activity") c.mu.Unlock() - _ = c.persistRuntime(ctx, persistence) + _ = c.persistAutoStandbyState(ctx, persistence) c.log.Info("auto-standby skipped standby, conntrack still reports inbound connections", "instance_id", id, "active_inbound_connections", len(activeSet)) return false } @@ -919,7 +919,7 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName return } c.log.Warn("auto-standby standby attempt failed", "instance_id", id, "instance_name", instanceName, "error", err) - var persistence runtimePersistence + var persistence autoStandbyStatePersistence if state := c.states[id]; state != nil { state.standbyRequested = false // Inbound activity that arrived during the attempt owns the state @@ -929,14 +929,14 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName idleSince := c.now().UTC() state.idleSince = &idleSince c.armTimerLocked(id, state, idleSince) - persistence = c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistence = c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "recover from standby failure") } } c.mu.Unlock() - _ = c.persistRuntime(ctx, persistence) + _ = c.persistAutoStandbyState(ctx, persistence) return } @@ -944,13 +944,13 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) c.mu.Lock() - var persistence runtimePersistence + var persistence autoStandbyStatePersistence if state := c.states[id]; state != nil { c.clearStateLocked(state) - persistence = c.prepareRuntimePersistenceLocked(id, nil, true, "clear runtime after standby") + persistence = c.prepareAutoStandbyStatePersistenceLocked(id, nil, true, "clear auto-standby state after standby") } c.mu.Unlock() - _ = c.persistRuntime(ctx, persistence) + _ = c.persistAutoStandbyState(ctx, persistence) } func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { @@ -999,14 +999,14 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, now) - persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ + persistence := c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "finish active connection reconcile") idleTimeout := state.idleTimeout c.mu.Unlock() - _ = c.persistRuntime(ctx, persistence) + _ = c.persistAutoStandbyState(ctx, persistence) c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", idleTimeout) } @@ -1139,24 +1139,24 @@ func (c *Controller) stopAllTimers() { } } -// prepareRuntimePersistenceLocked reserves this write's place in the global -// persistence order. The caller must pass the result to persistRuntime exactly +// prepareAutoStandbyStatePersistenceLocked reserves this write's place in the global +// persistence order. The caller must pass the result to persistAutoStandbyState exactly // once after releasing c.mu. -func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, bestEffort bool, operation string) runtimePersistence { +func (c *Controller) prepareAutoStandbyStatePersistenceLocked(id string, autoStandbyState *AutoStandbyState, bestEffort bool, operation string) autoStandbyStatePersistence { done := make(chan struct{}) - persistence := runtimePersistence{ - id: id, - runtime: cloneRuntime(runtime), - previous: c.runtimePersistenceTail, - done: done, - bestEffort: bestEffort, - operation: operation, - } - c.runtimePersistenceTail = done + persistence := autoStandbyStatePersistence{ + id: id, + autoStandbyState: cloneAutoStandbyState(autoStandbyState), + previous: c.autoStandbyStatePersistenceTail, + done: done, + bestEffort: bestEffort, + operation: operation, + } + c.autoStandbyStatePersistenceTail = done return persistence } -func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePersistence) error { +func (c *Controller) persistAutoStandbyState(ctx context.Context, persistence autoStandbyStatePersistence) error { if persistence.done == nil { return nil } @@ -1167,10 +1167,10 @@ func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePers } defer close(persistence.done) - err := c.store.SetRuntime(ctx, persistence.id, persistence.runtime) + err := c.store.SetAutoStandbyState(ctx, persistence.id, persistence.autoStandbyState) if err != nil && persistence.bestEffort { - c.recordControllerError("persist_runtime") - c.log.Warn("auto-standby failed to persist runtime", "instance_id", persistence.id, "operation", persistence.operation, "error", err) + c.recordControllerError("persist_auto_standby_state") + c.log.Warn("auto-standby failed to persist state", "instance_id", persistence.id, "operation", persistence.operation, "error", err) return nil } return err @@ -1245,20 +1245,20 @@ func connectionKey(conn Connection) ConnectionKey { } } -func cloneRuntime(runtime *Runtime) *Runtime { - if runtime == nil { +func cloneAutoStandbyState(autoStandbyState *AutoStandbyState) *AutoStandbyState { + if autoStandbyState == nil { return nil } - return &Runtime{ - IdleSince: cloneTimePtr(runtime.IdleSince), - LastInboundActivityAt: cloneTimePtr(runtime.LastInboundActivityAt), + return &AutoStandbyState{ + IdleSince: cloneTimePtr(autoStandbyState.IdleSince), + LastInboundActivityAt: cloneTimePtr(autoStandbyState.LastInboundActivityAt), } } func cloneInstance(inst Instance) Instance { cloned := inst cloned.AutoStandby = clonePolicy(inst.AutoStandby) - cloned.Runtime = cloneRuntime(inst.Runtime) + cloned.AutoStandbyState = cloneAutoStandbyState(inst.AutoStandbyState) return cloned } diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index 75e45651e..fe63d0c6c 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -14,25 +14,25 @@ import ( ) type fakeInstanceStore struct { - mu sync.Mutex - instances []Instance - standbyIDs []string - persistedRuntime map[string]*Runtime - events chan InstanceEvent - standbyErr error - listErr error - setRuntimeErr error - setRuntimeStarted chan string - setRuntimeRelease chan struct{} - standbyStarted chan string - standbyRelease chan struct{} + mu sync.Mutex + instances []Instance + standbyIDs []string + persistedAutoStandbyState map[string]*AutoStandbyState + events chan InstanceEvent + standbyErr error + listErr error + setAutoStandbyStateErr error + setAutoStandbyStateStarted chan string + setAutoStandbyStateRelease chan struct{} + standbyStarted chan string + standbyRelease chan struct{} } func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { return &fakeInstanceStore{ - instances: append([]Instance(nil), instances...), - persistedRuntime: make(map[string]*Runtime), - events: make(chan InstanceEvent, 16), + instances: append([]Instance(nil), instances...), + persistedAutoStandbyState: make(map[string]*AutoStandbyState), + events: make(chan InstanceEvent, 16), } } @@ -68,23 +68,23 @@ func (f *fakeInstanceStore) standbyCalls() []string { return append([]string(nil), f.standbyIDs...) } -func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { - if f.setRuntimeStarted != nil { - f.setRuntimeStarted <- id +func (f *fakeInstanceStore) SetAutoStandbyState(_ context.Context, id string, autoStandbyState *AutoStandbyState) error { + if f.setAutoStandbyStateStarted != nil { + f.setAutoStandbyStateStarted <- id } - if f.setRuntimeRelease != nil { - <-f.setRuntimeRelease + if f.setAutoStandbyStateRelease != nil { + <-f.setAutoStandbyStateRelease } f.mu.Lock() defer f.mu.Unlock() - if f.setRuntimeErr != nil { - return f.setRuntimeErr + if f.setAutoStandbyStateErr != nil { + return f.setAutoStandbyStateErr } - f.persistedRuntime[id] = cloneRuntime(runtime) + f.persistedAutoStandbyState[id] = cloneAutoStandbyState(autoStandbyState) for i := range f.instances { if f.instances[i].ID == id { - f.instances[i].Runtime = cloneRuntime(runtime) + f.instances[i].AutoStandbyState = cloneAutoStandbyState(autoStandbyState) } } return nil @@ -146,7 +146,7 @@ func TestStartupResyncClearsPersistedIdleWhenCurrentConnectionsExist(t *testing. NetworkEnabled: true, IP: "192.168.100.10", AutoStandby: &Policy{Enabled: true, IdleTimeout: "5m"}, - Runtime: &Runtime{ + AutoStandbyState: &AutoStandbyState{ IdleSince: &idleSince, LastInboundActivityAt: &lastInbound, }, @@ -168,8 +168,8 @@ func TestStartupResyncClearsPersistedIdleWhenCurrentConnectionsExist(t *testing. status := controller.Describe(store.instances[0]) require.Equal(t, StatusActive, status.Status) require.Nil(t, status.IdleSince) - require.NotNil(t, store.persistedRuntime["inst-active"]) - require.Nil(t, store.persistedRuntime["inst-active"].IdleSince) + require.NotNil(t, store.persistedAutoStandbyState["inst-active"]) + require.Nil(t, store.persistedAutoStandbyState["inst-active"].IdleSince) } func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { @@ -183,7 +183,7 @@ func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { NetworkEnabled: true, IP: "192.168.100.20", AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, - Runtime: &Runtime{ + AutoStandbyState: &AutoStandbyState{ IdleSince: &idleSince, }, }}) @@ -238,7 +238,7 @@ func TestPeriodicSnapshotSyncRefreshesTrackedState(t *testing.T) { require.Equal(t, 1, status.ActiveInboundCount) } -func TestInstanceEventClearsPersistedRuntimeWhenInstanceBecomesIneligible(t *testing.T) { +func TestInstanceEventClearsPersistedAutoStandbyStateWhenInstanceBecomesIneligible(t *testing.T) { t.Parallel() idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) @@ -250,7 +250,7 @@ func TestInstanceEventClearsPersistedRuntimeWhenInstanceBecomesIneligible(t *tes NetworkEnabled: true, IP: "192.168.100.22", AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, - Runtime: &Runtime{ + AutoStandbyState: &AutoStandbyState{ IdleSince: &idleSince, LastInboundActivityAt: &lastInbound, }, @@ -274,9 +274,9 @@ func TestInstanceEventClearsPersistedRuntimeWhenInstanceBecomesIneligible(t *tes }, })) - runtime, ok := store.persistedRuntime["inst-ineligible"] + autoStandbyState, ok := store.persistedAutoStandbyState["inst-ineligible"] require.True(t, ok) - require.Nil(t, runtime) + require.Nil(t, autoStandbyState) } func TestConnectionEventsClearIdleAndStartCountdown(t *testing.T) { @@ -566,7 +566,7 @@ func TestHandleStandbyTimerCallsStandbyAndClearsState(t *testing.T) { NetworkEnabled: true, IP: "192.168.100.61", AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - Runtime: &Runtime{ + AutoStandbyState: &AutoStandbyState{ IdleSince: &idleSince, }, }}) @@ -580,7 +580,7 @@ func TestHandleStandbyTimerCallsStandbyAndClearsState(t *testing.T) { controller.standbyWG.Wait() require.Equal(t, []string{"inst-standby"}, store.standbyCalls()) - require.Nil(t, store.persistedRuntime["inst-standby"]) + require.Nil(t, store.persistedAutoStandbyState["inst-standby"]) controller.mu.RLock() state := controller.states["inst-standby"] @@ -606,7 +606,7 @@ func TestHandleStandbyTimerFailureRearmsIdleCountdown(t *testing.T) { NetworkEnabled: true, IP: "192.168.100.62", AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - Runtime: &Runtime{ + AutoStandbyState: &AutoStandbyState{ IdleSince: &idleSince, }, }}) @@ -621,9 +621,9 @@ func TestHandleStandbyTimerFailureRearmsIdleCountdown(t *testing.T) { controller.standbyWG.Wait() require.Equal(t, []string{"inst-standby-fail"}, store.standbyCalls()) - require.NotNil(t, store.persistedRuntime["inst-standby-fail"]) - require.NotNil(t, store.persistedRuntime["inst-standby-fail"].IdleSince) - assert.Equal(t, now, *store.persistedRuntime["inst-standby-fail"].IdleSince) + require.NotNil(t, store.persistedAutoStandbyState["inst-standby-fail"]) + require.NotNil(t, store.persistedAutoStandbyState["inst-standby-fail"].IdleSince) + assert.Equal(t, now, *store.persistedAutoStandbyState["inst-standby-fail"].IdleSince) controller.mu.RLock() state := controller.states["inst-standby-fail"] @@ -664,8 +664,8 @@ func TestHandleStandbyTimerSkipsStandbyWhenConntrackReportsConnection(t *testing controller.standbyWG.Wait() assert.Empty(t, store.standbyCalls()) - require.NotNil(t, store.persistedRuntime["inst-missed-event"]) - assert.Nil(t, store.persistedRuntime["inst-missed-event"].IdleSince) + require.NotNil(t, store.persistedAutoStandbyState["inst-missed-event"]) + assert.Nil(t, store.persistedAutoStandbyState["inst-missed-event"].IdleSince) controller.mu.RLock() state := controller.states["inst-missed-event"] @@ -698,9 +698,9 @@ func TestHandleStandbyTimerRearmsCountdownWhenConfirmationFails(t *testing.T) { controller.standbyWG.Wait() assert.Empty(t, store.standbyCalls()) - require.NotNil(t, store.persistedRuntime["inst-confirm-fail"]) - require.NotNil(t, store.persistedRuntime["inst-confirm-fail"].IdleSince) - assert.Equal(t, now, *store.persistedRuntime["inst-confirm-fail"].IdleSince) + require.NotNil(t, store.persistedAutoStandbyState["inst-confirm-fail"]) + require.NotNil(t, store.persistedAutoStandbyState["inst-confirm-fail"].IdleSince) + assert.Equal(t, now, *store.persistedAutoStandbyState["inst-confirm-fail"].IdleSince) controller.mu.RLock() state := controller.states["inst-confirm-fail"] @@ -745,8 +745,8 @@ func TestHandleStandbyTimerKeepsActiveStateWhenConfirmationFails(t *testing.T) { controller.standbyWG.Wait() assert.Empty(t, store.standbyCalls()) - require.NotNil(t, store.persistedRuntime["inst-confirm-active"]) - assert.Nil(t, store.persistedRuntime["inst-confirm-active"].IdleSince) + require.NotNil(t, store.persistedAutoStandbyState["inst-confirm-active"]) + assert.Nil(t, store.persistedAutoStandbyState["inst-confirm-active"].IdleSince) controller.mu.RLock() state := controller.states["inst-confirm-active"] @@ -984,13 +984,13 @@ func (s *restoreConnectionSource) OpenStream(context.Context) (ConnectionStream, func idleTestInstance(id, ip string, idleSince time.Time) Instance { since := idleSince return Instance{ - ID: id, - Name: id, - State: StateRunning, - NetworkEnabled: true, - IP: ip, - AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - Runtime: &Runtime{IdleSince: &since}, + ID: id, + Name: id, + State: StateRunning, + NetworkEnabled: true, + IP: ip, + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + AutoStandbyState: &AutoStandbyState{IdleSince: &since}, } } @@ -1245,7 +1245,7 @@ func TestStandbyFailureWithMidFlightActivityDoesNotRearmIdle(t *testing.T) { controller.mu.RUnlock() store.mu.Lock() - persisted := cloneRuntime(store.persistedRuntime["inst-fail-busy"]) + persisted := cloneAutoStandbyState(store.persistedAutoStandbyState["inst-fail-busy"]) store.mu.Unlock() require.NotNil(t, persisted) assert.Nil(t, persisted.IdleSince, "failure must not persist a false idle window while connections are active") @@ -1338,7 +1338,7 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ Now: func() time.Time { return idleSince }, }) - store.setRuntimeErr = errors.New("metadata write failed") + store.setAutoStandbyStateErr = errors.New("metadata write failed") // Fresh idle state forces the persist that fails; the countdown must be // armed regardless. @@ -1357,39 +1357,39 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller.mu.RUnlock() } -func TestRuntimePersistencePreservesInstanceOrder(t *testing.T) { +func TestAutoStandbyStatePersistencePreservesInstanceOrder(t *testing.T) { t.Parallel() store := newFakeInstanceStore(nil) - store.setRuntimeStarted = make(chan string, 2) + store.setAutoStandbyStateStarted = make(chan string, 2) controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) firstTime := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) secondTime := firstTime.Add(time.Second) controller.mu.Lock() - first := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &firstTime}, false, "test first write") - second := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &secondTime}, false, "test second write") + first := controller.prepareAutoStandbyStatePersistenceLocked("inst-persist-order", &AutoStandbyState{IdleSince: &firstTime}, false, "test first write") + second := controller.prepareAutoStandbyStatePersistenceLocked("inst-persist-order", &AutoStandbyState{IdleSince: &secondTime}, false, "test second write") controller.mu.Unlock() secondDone := make(chan error, 1) go func() { - secondDone <- controller.persistRuntime(context.Background(), second) + secondDone <- controller.persistAutoStandbyState(context.Background(), second) }() select { - case <-store.setRuntimeStarted: - require.FailNow(t, "second runtime write started before first") + case <-store.setAutoStandbyStateStarted: + require.FailNow(t, "second auto-standby state write started before first") case <-time.After(50 * time.Millisecond): } - require.NoError(t, controller.persistRuntime(context.Background(), first)) - require.Equal(t, "inst-persist-order", <-store.setRuntimeStarted) + require.NoError(t, controller.persistAutoStandbyState(context.Background(), first)) + require.Equal(t, "inst-persist-order", <-store.setAutoStandbyStateStarted) require.NoError(t, <-secondDone) - require.Equal(t, "inst-persist-order", <-store.setRuntimeStarted) - require.NotNil(t, store.persistedRuntime["inst-persist-order"]) - assert.Equal(t, secondTime, *store.persistedRuntime["inst-persist-order"].IdleSince) + require.Equal(t, "inst-persist-order", <-store.setAutoStandbyStateStarted) + require.NotNil(t, store.persistedAutoStandbyState["inst-persist-order"]) + assert.Equal(t, secondTime, *store.persistedAutoStandbyState["inst-persist-order"].IdleSince) } -func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { +func TestHoldStandbyDoesNotWaitForAutoStandbyStatePersistence(t *testing.T) { t.Parallel() now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) @@ -1409,8 +1409,8 @@ func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { TCPState: TCPStateEstablished, } store := newFakeInstanceStore([]Instance{inst}) - store.setRuntimeStarted = make(chan string, 1) - store.setRuntimeRelease = make(chan struct{}) + store.setAutoStandbyStateStarted = make(chan string, 1) + store.setAutoStandbyStateRelease = make(chan struct{}) controller := NewController(store, &fakeConnectionSource{connections: []Connection{conn}}, ControllerOptions{ Now: func() time.Time { return now }, }) @@ -1419,7 +1419,7 @@ func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { go func() { resyncDone <- controller.startupResync(context.Background()) }() - require.Equal(t, inst.ID, <-store.setRuntimeStarted) + require.Equal(t, inst.ID, <-store.setAutoStandbyStateStarted) holdDone := make(chan error, 1) go func() { @@ -1431,11 +1431,11 @@ func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { case err := <-holdDone: require.NoError(t, err) case <-time.After(time.Second): - close(store.setRuntimeRelease) - require.FailNow(t, "hold waited for runtime persistence") + close(store.setAutoStandbyStateRelease) + require.FailNow(t, "hold waited for auto-standby state persistence") } - close(store.setRuntimeRelease) + close(store.setAutoStandbyStateRelease) require.NoError(t, <-resyncDone) } @@ -1452,7 +1452,7 @@ func TestHoldStandbyExtendsArmedCountdown(t *testing.T) { }) require.NoError(t, controller.startupResync(context.Background())) - persistedBefore := cloneRuntime(store.persistedRuntime["inst-hold"]) + persistedBefore := cloneAutoStandbyState(store.persistedAutoStandbyState["inst-hold"]) snapshot, err := controller.HoldStandby(context.Background(), store.instances[0]) require.NoError(t, err) @@ -1463,7 +1463,7 @@ func TestHoldStandbyExtendsArmedCountdown(t *testing.T) { assert.Equal(t, now.Add(time.Minute), *snapshot.NextStandbyAt) // Holds are in-memory only: the persisted countdown is untouched. - assert.Equal(t, persistedBefore, store.persistedRuntime["inst-hold"]) + assert.Equal(t, persistedBefore, store.persistedAutoStandbyState["inst-hold"]) // A resync must keep the held deadline. require.NoError(t, controller.startupResync(context.Background())) @@ -1480,13 +1480,13 @@ func TestHoldStandbyReplacesLongerHold(t *testing.T) { idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) longPolicy := Instance{ - ID: "inst-reset-hold", - Name: "inst-reset-hold", - State: StateRunning, - NetworkEnabled: true, - IP: "192.168.100.110", - AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, - Runtime: &Runtime{IdleSince: &idleSince}, + ID: "inst-reset-hold", + Name: "inst-reset-hold", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.110", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, + AutoStandbyState: &AutoStandbyState{IdleSince: &idleSince}, } store := newFakeInstanceStore([]Instance{longPolicy}) now := idleSince diff --git a/lib/autostandby/types.go b/lib/autostandby/types.go index 0dfa319b8..7bd881fd5 100644 --- a/lib/autostandby/types.go +++ b/lib/autostandby/types.go @@ -19,14 +19,14 @@ type Policy struct { // Instance is the minimal instance view needed by the auto-standby controller. type Instance struct { - ID string - Name string - State string - NetworkEnabled bool - IP string - HasVGPU bool - AutoStandby *Policy - Runtime *Runtime + ID string + Name string + State string + NetworkEnabled bool + IP string + HasVGPU bool + AutoStandby *Policy + AutoStandbyState *AutoStandbyState } // Connection is the normalized network view used by activity classification. @@ -38,8 +38,8 @@ type Connection struct { TCPState TCPState } -// Runtime stores persisted and in-memory idle-tracking timestamps. -type Runtime struct { +// AutoStandbyState stores persisted and in-memory idle-tracking timestamps. +type AutoStandbyState struct { IdleSince *time.Time `json:"idle_since,omitempty"` LastInboundActivityAt *time.Time `json:"last_inbound_activity_at,omitempty"` } diff --git a/lib/instances/auto_standby.go b/lib/instances/auto_standby.go index de061dbfb..0f0ba8b2e 100644 --- a/lib/instances/auto_standby.go +++ b/lib/instances/auto_standby.go @@ -24,18 +24,18 @@ func cloneAutoStandbyPolicy(policy *autostandby.Policy) *autostandby.Policy { return cloned } -func cloneAutoStandbyRuntime(runtime *autostandby.Runtime) *autostandby.Runtime { - if runtime == nil { +func cloneAutoStandbyState(autoStandbyState *autostandby.AutoStandbyState) *autostandby.AutoStandbyState { + if autoStandbyState == nil { return nil } - cloned := &autostandby.Runtime{} - if runtime.IdleSince != nil { - idleSince := runtime.IdleSince.UTC() + cloned := &autostandby.AutoStandbyState{} + if autoStandbyState.IdleSince != nil { + idleSince := autoStandbyState.IdleSince.UTC() cloned.IdleSince = &idleSince } - if runtime.LastInboundActivityAt != nil { - lastInboundActivityAt := runtime.LastInboundActivityAt.UTC() + if autoStandbyState.LastInboundActivityAt != nil { + lastInboundActivityAt := autoStandbyState.LastInboundActivityAt.UTC() cloned.LastInboundActivityAt = &lastInboundActivityAt } return cloned diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go index d4e25b2d9..7c9cff544 100644 --- a/lib/instances/auto_standby_integration_linux_test.go +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -58,8 +58,8 @@ func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id str return err } -func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { - return s.manager.SetAutoStandbyRuntime(ctx, id, runtime) +func (s integrationAutoStandbyStore) SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { + return s.manager.SetAutoStandbyState(ctx, id, autoStandbyState) } func (s integrationAutoStandbyStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { @@ -375,7 +375,7 @@ func TestAutoStandbyCloudHypervisorHoldStandby(t *testing.T) { require.NoError(t, err) require.Equal(t, StateRunning, current.State) - runtimeBefore, err := mgr.GetAutoStandbyRuntime(ctx, instanceID) + stateBefore, err := mgr.GetAutoStandbyState(ctx, instanceID) require.NoError(t, err) snapshot, err := controller.HoldStandby(ctx, toAutoStandby(current)) @@ -386,9 +386,9 @@ func TestAutoStandbyCloudHypervisorHoldStandby(t *testing.T) { "hold_until %s must be after the countdown deadline %s", snapshot.HoldUntil, countdownDeadline) // Holds are in-memory only: the persisted countdown is untouched. - runtimeAfter, err := mgr.GetAutoStandbyRuntime(ctx, instanceID) + stateAfter, err := mgr.GetAutoStandbyState(ctx, instanceID) require.NoError(t, err) - require.Equal(t, runtimeBefore, runtimeAfter) + require.Equal(t, stateBefore, stateAfter) // Just past the countdown deadline the instance must still be running; // without the hold the standby would have fired here. diff --git a/lib/instances/auto_standby_runtime.go b/lib/instances/auto_standby_runtime.go deleted file mode 100644 index a625bc632..000000000 --- a/lib/instances/auto_standby_runtime.go +++ /dev/null @@ -1,34 +0,0 @@ -package instances - -import ( - "context" - - "github.com/kernel/hypeman/lib/autostandby" -) - -// GetAutoStandbyRuntime returns the persisted auto-standby runtime metadata for an instance. -func (m *manager) GetAutoStandbyRuntime(_ context.Context, id string) (*autostandby.Runtime, error) { - lock := m.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - - meta, err := m.loadMetadata(id) - if err != nil { - return nil, err - } - return cloneAutoStandbyRuntime(meta.AutoStandbyRuntime), nil -} - -// SetAutoStandbyRuntime persists auto-standby runtime metadata for an instance. -func (m *manager) SetAutoStandbyRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { - lock := m.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - - meta, err := m.loadMetadata(id) - if err != nil { - return err - } - meta.AutoStandbyRuntime = cloneAutoStandbyRuntime(runtime) - return m.saveMetadata(meta) -} diff --git a/lib/instances/auto_standby_state.go b/lib/instances/auto_standby_state.go new file mode 100644 index 000000000..3808eca2a --- /dev/null +++ b/lib/instances/auto_standby_state.go @@ -0,0 +1,34 @@ +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/autostandby" +) + +// GetAutoStandbyState returns the persisted auto-standby state for an instance. +func (m *manager) GetAutoStandbyState(_ context.Context, id string) (*autostandby.AutoStandbyState, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + return cloneAutoStandbyState(meta.AutoStandbyState), nil +} + +// SetAutoStandbyState persists auto-standby state for an instance. +func (m *manager) SetAutoStandbyState(_ context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.AutoStandbyState = cloneAutoStandbyState(autoStandbyState) + return m.saveMetadata(meta) +} diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go index 70742edc4..478771e7f 100644 --- a/lib/instances/metadata_clone.go +++ b/lib/instances/metadata_clone.go @@ -11,7 +11,7 @@ func deepCopyMetadata(src *metadata) *metadata { return &metadata{ StoredMetadata: cloneStoredMetadata(src.StoredMetadata), - AutoStandbyRuntime: cloneAutoStandbyRuntime(src.AutoStandbyRuntime), + AutoStandbyState: cloneAutoStandbyState(src.AutoStandbyState), HealthCheckRuntime: healthcheck.CloneRuntime(src.HealthCheckRuntime), } } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e1..80b5fde1b 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -40,8 +40,9 @@ const ( // metadata wraps StoredMetadata for JSON serialization type metadata struct { StoredMetadata - AutoStandbyRuntime *autostandby.Runtime `json:"auto_standby_runtime,omitempty"` - HealthCheckRuntime *healthcheck.Runtime `json:"health_check_runtime,omitempty"` + // Keep the legacy JSON key so existing instance metadata remains readable. + AutoStandbyState *autostandby.AutoStandbyState `json:"auto_standby_runtime,omitempty"` + HealthCheckRuntime *healthcheck.Runtime `json:"health_check_runtime,omitempty"` } // ensureDirectories creates the instance directory structure diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go index d54d7aeaa..eb1f9b950 100644 --- a/lib/providers/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -14,15 +14,15 @@ import ( "go.opentelemetry.io/otel" ) -type autoStandbyRuntimeManager interface { - GetAutoStandbyRuntime(ctx context.Context, id string) (*autostandby.Runtime, error) - SetAutoStandbyRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error +type autoStandbyStateManager interface { + GetAutoStandbyState(ctx context.Context, id string) (*autostandby.AutoStandbyState, error) + SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error SubscribeLifecycleEvents(consumer instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) } type autoStandbyInstanceStore struct { - manager instances.Manager - runtimeManager autoStandbyRuntimeManager + manager instances.Manager + stateManager autoStandbyStateManager } func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { @@ -33,20 +33,20 @@ func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostan out := make([]autostandby.Instance, 0, len(insts)) for _, inst := range insts { - runtime, err := s.runtimeManager.GetAutoStandbyRuntime(ctx, inst.Id) + autoStandbyState, err := s.stateManager.GetAutoStandbyState(ctx, inst.Id) if err != nil { return nil, err } out = append(out, autostandby.Instance{ - ID: inst.Id, - Name: inst.Name, - State: string(inst.State), - NetworkEnabled: inst.NetworkEnabled, - IP: inst.IP, - HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", - AutoStandby: inst.AutoStandby, - Runtime: runtime, + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + AutoStandbyState: autoStandbyState, }) } return out, nil @@ -60,21 +60,21 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string return err } -func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { - return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime) +func (s autoStandbyInstanceStore) SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { + return s.stateManager.SetAutoStandbyState(ctx, id, autoStandbyState) } func (s autoStandbyInstanceStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { - src, unsub := s.runtimeManager.SubscribeLifecycleEvents(instances.LifecycleEventConsumerAutoStandby) + src, unsub := s.stateManager.SubscribeLifecycleEvents(instances.LifecycleEventConsumerAutoStandby) dst := make(chan autostandby.InstanceEvent, 32) go func() { defer close(dst) for event := range src { inst := toAutoStandbyInstance(event.Instance) if inst != nil { - runtime, err := s.runtimeManager.GetAutoStandbyRuntime(context.Background(), inst.ID) + autoStandbyState, err := s.stateManager.GetAutoStandbyState(context.Background(), inst.ID) if err == nil { - inst.Runtime = runtime + inst.AutoStandbyState = autoStandbyState } } dst <- autostandby.InstanceEvent{ @@ -108,13 +108,13 @@ func ProvideAutoStandbyController(instanceManager instances.Manager, cfg *config return nil } - runtimeManager, ok := instanceManager.(autoStandbyRuntimeManager) + stateManager, ok := instanceManager.(autoStandbyStateManager) if !ok { return nil } return autostandby.NewController( - autoStandbyInstanceStore{manager: instanceManager, runtimeManager: runtimeManager}, + autoStandbyInstanceStore{manager: instanceManager, stateManager: stateManager}, autostandby.NewConntrackSource(), autostandby.ControllerOptions{ Log: log.With("controller", "auto_standby"), diff --git a/lib/providers/auto_standby_linux_test.go b/lib/providers/auto_standby_linux_test.go index b1a407668..67e593bee 100644 --- a/lib/providers/auto_standby_linux_test.go +++ b/lib/providers/auto_standby_linux_test.go @@ -12,34 +12,34 @@ import ( "github.com/stretchr/testify/require" ) -type autoStandbyRuntimeManagerStub struct { - runtimeByID map[string]*autostandby.Runtime - events chan instances.LifecycleEvent +type autoStandbyStateManagerStub struct { + stateByID map[string]*autostandby.AutoStandbyState + events chan instances.LifecycleEvent } -func (s *autoStandbyRuntimeManagerStub) GetAutoStandbyRuntime(_ context.Context, id string) (*autostandby.Runtime, error) { - if runtime, ok := s.runtimeByID[id]; ok { - cloned := *runtime +func (s *autoStandbyStateManagerStub) GetAutoStandbyState(_ context.Context, id string) (*autostandby.AutoStandbyState, error) { + if autoStandbyState, ok := s.stateByID[id]; ok { + cloned := *autoStandbyState return &cloned, nil } return nil, nil } -func (s *autoStandbyRuntimeManagerStub) SetAutoStandbyRuntime(context.Context, string, *autostandby.Runtime) error { +func (s *autoStandbyStateManagerStub) SetAutoStandbyState(context.Context, string, *autostandby.AutoStandbyState) error { return nil } -func (s *autoStandbyRuntimeManagerStub) SubscribeLifecycleEvents(instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) { +func (s *autoStandbyStateManagerStub) SubscribeLifecycleEvents(instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) { return s.events, func() {} } -func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesRuntime(t *testing.T) { +func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesState(t *testing.T) { t.Parallel() idleSince := time.Date(2026, 4, 7, 12, 0, 0, 0, time.UTC) lastInbound := idleSince.Add(-time.Minute) - runtimeManager := &autoStandbyRuntimeManagerStub{ - runtimeByID: map[string]*autostandby.Runtime{ + stateManager := &autoStandbyStateManagerStub{ + stateByID: map[string]*autostandby.AutoStandbyState{ "inst-1": { IdleSince: &idleSince, LastInboundActivityAt: &lastInbound, @@ -48,13 +48,13 @@ func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesRuntime(t *testi events: make(chan instances.LifecycleEvent, 1), } store := autoStandbyInstanceStore{ - runtimeManager: runtimeManager, + stateManager: stateManager, } eventCh, _, err := store.SubscribeInstanceEvents() require.NoError(t, err) - runtimeManager.events <- instances.LifecycleEvent{ + stateManager.events <- instances.LifecycleEvent{ Action: instances.LifecycleEventUpdate, InstanceID: "inst-1", Instance: &instances.Instance{ @@ -71,9 +71,9 @@ func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesRuntime(t *testi event := <-eventCh require.NotNil(t, event.Instance) - require.NotNil(t, event.Instance.Runtime) - require.NotNil(t, event.Instance.Runtime.IdleSince) - require.Equal(t, idleSince, *event.Instance.Runtime.IdleSince) - require.NotNil(t, event.Instance.Runtime.LastInboundActivityAt) - require.Equal(t, lastInbound, *event.Instance.Runtime.LastInboundActivityAt) + require.NotNil(t, event.Instance.AutoStandbyState) + require.NotNil(t, event.Instance.AutoStandbyState.IdleSince) + require.Equal(t, idleSince, *event.Instance.AutoStandbyState.IdleSince) + require.NotNil(t, event.Instance.AutoStandbyState.LastInboundActivityAt) + require.Equal(t, lastInbound, *event.Instance.AutoStandbyState.LastInboundActivityAt) } From 374c1851f359ee289b618d31f545712eb96f5c42 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:46:15 +0000 Subject: [PATCH 6/7] Update auto-standby state documentation --- lib/instances/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/README.md b/lib/instances/README.md index 13ada0565..a9f2ac888 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -52,7 +52,7 @@ Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEM memory-ranges # Memory state ``` -`metadata.json` also carries controller-owned auto-standby runtime timestamps when that feature is enabled, so idle countdown state can survive Hypeman restarts. +`metadata.json` also carries controller-owned `AutoStandbyState` timestamps when that feature is enabled, so idle countdown state can survive Hypeman restarts. **Benefits:** - Content-addressable IDs (ULID = time-ordered) From 4b70809f2dcc70f63c06db0214e981787e5c08e5 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:05:54 +0000 Subject: [PATCH 7/7] Revert auto-standby state rename --- cmd/api/api/auto_standby_hold_test.go | 14 +- cmd/api/api/auto_standby_status_test.go | 14 +- lib/autostandby/controller.go | 152 ++++++++-------- lib/autostandby/controller_test.go | 164 +++++++++--------- lib/autostandby/types.go | 20 +-- lib/instances/README.md | 2 +- lib/instances/auto_standby.go | 14 +- .../auto_standby_integration_linux_test.go | 10 +- lib/instances/auto_standby_runtime.go | 34 ++++ lib/instances/auto_standby_state.go | 34 ---- lib/instances/metadata_clone.go | 2 +- lib/instances/storage.go | 5 +- lib/providers/auto_standby_linux.go | 42 ++--- lib/providers/auto_standby_linux_test.go | 36 ++-- 14 files changed, 271 insertions(+), 272 deletions(-) create mode 100644 lib/instances/auto_standby_runtime.go delete mode 100644 lib/instances/auto_standby_state.go diff --git a/cmd/api/api/auto_standby_hold_test.go b/cmd/api/api/auto_standby_hold_test.go index a1f9259b5..fe347b42f 100644 --- a/cmd/api/api/auto_standby_hold_test.go +++ b/cmd/api/api/auto_standby_hold_test.go @@ -31,13 +31,13 @@ func TestHoldAutoStandbyExtendsCountdown(t *testing.T) { idleSince := now.Add(-4 * time.Minute) store := &statusStore{ instances: []autostandby.Instance{{ - ID: "inst-hold", - Name: "inst-hold", - State: autostandby.StateRunning, - NetworkEnabled: true, - IP: "192.168.100.30", - AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, - AutoStandbyState: &autostandby.AutoStandbyState{IdleSince: &idleSince}, + ID: "inst-hold", + Name: "inst-hold", + State: autostandby.StateRunning, + NetworkEnabled: true, + IP: "192.168.100.30", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + Runtime: &autostandby.Runtime{IdleSince: &idleSince}, }}, } controller := autostandby.NewController(store, &statusConnectionSource{}, autostandby.ControllerOptions{ diff --git a/cmd/api/api/auto_standby_status_test.go b/cmd/api/api/auto_standby_status_test.go index 22b830830..b4aecfd1d 100644 --- a/cmd/api/api/auto_standby_status_test.go +++ b/cmd/api/api/auto_standby_status_test.go @@ -27,9 +27,9 @@ func (m *captureStatusManager) GetInstance(context.Context, string) (*instances. } type statusStore struct { - instances []autostandby.Instance - autoStandbyState map[string]*autostandby.AutoStandbyState - events chan autostandby.InstanceEvent + instances []autostandby.Instance + runtime map[string]*autostandby.Runtime + events chan autostandby.InstanceEvent } func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, error) { @@ -38,11 +38,11 @@ func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, er func (s *statusStore) StandbyInstance(context.Context, string) error { return nil } -func (s *statusStore) SetAutoStandbyState(_ context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { - if s.autoStandbyState == nil { - s.autoStandbyState = make(map[string]*autostandby.AutoStandbyState) +func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { + if s.runtime == nil { + s.runtime = make(map[string]*autostandby.Runtime) } - s.autoStandbyState[id] = autoStandbyState + s.runtime[id] = runtime return nil } diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 0064b798d..5cea9391e 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -53,11 +53,11 @@ type InstanceEvent struct { } // InstanceStore supplies the controller with instance state, lifecycle events, -// auto-standby state persistence, and standby actions. +// runtime persistence, and standby actions. type InstanceStore interface { ListInstances(ctx context.Context) ([]Instance, error) StandbyInstance(ctx context.Context, id string) error - SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *AutoStandbyState) error + SetRuntime(ctx context.Context, id string, runtime *Runtime) error SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) } @@ -118,16 +118,16 @@ type controllerState struct { standbyExecuting bool } -// autoStandbyStatePersistence preserves mutation order while metadata writes run +// runtimePersistence preserves mutation order while metadata writes run // without holding the controller mutex. Every prepared value must be passed to -// persistAutoStandbyState exactly once so its successor can proceed. -type autoStandbyStatePersistence struct { - id string - autoStandbyState *AutoStandbyState - previous <-chan struct{} - done chan struct{} - bestEffort bool - operation string +// persistRuntime exactly once so its successor can proceed. +type runtimePersistence struct { + id string + runtime *Runtime + previous <-chan struct{} + done chan struct{} + bestEffort bool + operation string } // Controller decides when eligible instances should transition to standby. @@ -148,12 +148,12 @@ type Controller struct { standbySlots chan struct{} standbyWG sync.WaitGroup - mu sync.RWMutex - states map[string]*controllerState - autoStandbyStatePersistenceTail chan struct{} - standbyInFlight int - observerConnected bool - lastObserverErr error + mu sync.RWMutex + states map[string]*controllerState + runtimePersistenceTail chan struct{} + standbyInFlight int + observerConnected bool + lastObserverErr error } // NewController creates a new event-driven auto-standby controller. @@ -555,7 +555,7 @@ func (c *Controller) seedInstanceState(ctx context.Context, inst Instance, conns if err != nil { return err } - return c.persistAutoStandbyState(ctx, persistence) + return c.persistRuntime(ctx, persistence) } func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEvent) error { @@ -580,32 +580,32 @@ func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEven if err != nil { return err } - return c.persistAutoStandbyState(ctx, persistence) + return c.persistRuntime(ctx, persistence) } -func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, now time.Time) (autoStandbyStatePersistence, error) { +func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, now time.Time) (runtimePersistence, error) { state := c.ensureStateLocked(inst.ID) state.instance = cloneInstance(inst) if !eligible(inst) { - hadAutoStandbyState := inst.AutoStandbyState != nil || state.idleSince != nil || state.lastInboundAt != nil + hadRuntime := inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil c.clearStateLocked(state) - if hadAutoStandbyState { - return c.prepareAutoStandbyStatePersistenceLocked(inst.ID, nil, false, "refresh disabled instance"), nil + if hadRuntime { + return c.prepareRuntimePersistenceLocked(inst.ID, nil, false, "refresh disabled instance"), nil } - return autoStandbyStatePersistence{}, nil + return runtimePersistence{}, nil } compiled, err := compilePolicy(inst.AutoStandby) if err != nil { - return autoStandbyStatePersistence{}, err + return runtimePersistence{}, err } state.compiledPolicy = compiled state.idleTimeout = compiled.idleTimeout activeSet, err := matchingConnections(inst, compiled, conns) if err != nil { - return autoStandbyStatePersistence{}, err + return runtimePersistence{}, err } // Cancel any queued standby attempt only once the refresh is guaranteed to // re-establish a countdown or reconcile below; an erroring refresh above @@ -613,33 +613,33 @@ func (c *Controller) refreshInstanceLocked(inst Instance, conns []Connection, no state.standbyRequested = false state.activeInbound = activeSet - autoStandbyState := cloneAutoStandbyState(inst.AutoStandbyState) + runtime := cloneRuntime(inst.Runtime) if len(activeSet) > 0 { state.idleSince = nil - if autoStandbyState != nil && autoStandbyState.LastInboundActivityAt != nil { - state.lastInboundAt = cloneTimePtr(autoStandbyState.LastInboundActivityAt) + if runtime != nil && runtime.LastInboundActivityAt != nil { + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) } else { state.lastInboundAt = &now } c.cancelTimerLocked(state) c.armReconcileLocked(inst.ID, state) - return c.prepareAutoStandbyStatePersistenceLocked(inst.ID, &AutoStandbyState{ + return c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, false, "refresh active instance"), nil } - var persistence autoStandbyStatePersistence - if autoStandbyState != nil && autoStandbyState.IdleSince != nil { - state.idleSince = cloneTimePtr(autoStandbyState.IdleSince) - state.lastInboundAt = cloneTimePtr(autoStandbyState.LastInboundActivityAt) + var persistence runtimePersistence + if runtime != nil && runtime.IdleSince != nil { + state.idleSince = cloneTimePtr(runtime.IdleSince) + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) } else { state.idleSince = &now - if autoStandbyState != nil { - state.lastInboundAt = cloneTimePtr(autoStandbyState.LastInboundActivityAt) + if runtime != nil { + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) } else { state.lastInboundAt = nil } - persistence = c.prepareAutoStandbyStatePersistenceLocked(inst.ID, &AutoStandbyState{ + persistence = c.prepareRuntimePersistenceLocked(inst.ID, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "refresh idle instance") @@ -664,7 +664,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.recordConntrackEvent(string(event.Type), "received") c.mu.Lock() - persistences := make([]autoStandbyStatePersistence, 0, 1) + persistences := make([]runtimePersistence, 0, 1) for id, state := range c.states { if state.compiledPolicy == nil { continue @@ -690,7 +690,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, idleSince) - persistences = append(persistences, c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "start idle countdown")) @@ -706,7 +706,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelTimerLocked(state) c.armReconcileLocked(id, state) - persistences = append(persistences, c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "record inbound activity")) c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) @@ -724,7 +724,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, idleSince) - persistences = append(persistences, c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistences = append(persistences, c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "restart idle countdown")) @@ -734,7 +734,7 @@ func (c *Controller) handleConnectionEvent(ctx context.Context, event Connection c.mu.Unlock() for _, persistence := range persistences { - _ = c.persistAutoStandbyState(ctx, persistence) + _ = c.persistRuntime(ctx, persistence) } } @@ -765,13 +765,13 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo idleSince := c.now().UTC() state.idleSince = &idleSince c.armTimerLocked(id, state, idleSince) - persistence := c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "handle unconfirmed standby") c.mu.Unlock() - _ = c.persistAutoStandbyState(ctx, persistence) + _ = c.persistRuntime(ctx, persistence) c.recordControllerError("standby_confirm") c.log.Warn("auto-standby could not confirm idle before standby", "instance_id", id, "error", err) return false @@ -787,12 +787,12 @@ func (c *Controller) confirmIdleBeforeStandby(ctx context.Context, id string) bo state.lastInboundAt = &now c.cancelTimerLocked(state) c.armReconcileLocked(id, state) - persistence := c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "record standby confirmation activity") c.mu.Unlock() - _ = c.persistAutoStandbyState(ctx, persistence) + _ = c.persistRuntime(ctx, persistence) c.log.Info("auto-standby skipped standby, conntrack still reports inbound connections", "instance_id", id, "active_inbound_connections", len(activeSet)) return false } @@ -919,7 +919,7 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName return } c.log.Warn("auto-standby standby attempt failed", "instance_id", id, "instance_name", instanceName, "error", err) - var persistence autoStandbyStatePersistence + var persistence runtimePersistence if state := c.states[id]; state != nil { state.standbyRequested = false // Inbound activity that arrived during the attempt owns the state @@ -929,14 +929,14 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName idleSince := c.now().UTC() state.idleSince = &idleSince c.armTimerLocked(id, state, idleSince) - persistence = c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistence = c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "recover from standby failure") } } c.mu.Unlock() - _ = c.persistAutoStandbyState(ctx, persistence) + _ = c.persistRuntime(ctx, persistence) return } @@ -944,13 +944,13 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) c.mu.Lock() - var persistence autoStandbyStatePersistence + var persistence runtimePersistence if state := c.states[id]; state != nil { c.clearStateLocked(state) - persistence = c.prepareAutoStandbyStatePersistenceLocked(id, nil, true, "clear auto-standby state after standby") + persistence = c.prepareRuntimePersistenceLocked(id, nil, true, "clear runtime after standby") } c.mu.Unlock() - _ = c.persistAutoStandbyState(ctx, persistence) + _ = c.persistRuntime(ctx, persistence) } func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { @@ -999,14 +999,14 @@ func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { state.standbyRequested = false c.cancelReconcileLocked(state) c.armTimerLocked(id, state, now) - persistence := c.prepareAutoStandbyStatePersistenceLocked(id, &AutoStandbyState{ + persistence := c.prepareRuntimePersistenceLocked(id, &Runtime{ IdleSince: cloneTimePtr(state.idleSince), LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), }, true, "finish active connection reconcile") idleTimeout := state.idleTimeout c.mu.Unlock() - _ = c.persistAutoStandbyState(ctx, persistence) + _ = c.persistRuntime(ctx, persistence) c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", idleTimeout) } @@ -1139,24 +1139,24 @@ func (c *Controller) stopAllTimers() { } } -// prepareAutoStandbyStatePersistenceLocked reserves this write's place in the global -// persistence order. The caller must pass the result to persistAutoStandbyState exactly +// prepareRuntimePersistenceLocked reserves this write's place in the global +// persistence order. The caller must pass the result to persistRuntime exactly // once after releasing c.mu. -func (c *Controller) prepareAutoStandbyStatePersistenceLocked(id string, autoStandbyState *AutoStandbyState, bestEffort bool, operation string) autoStandbyStatePersistence { +func (c *Controller) prepareRuntimePersistenceLocked(id string, runtime *Runtime, bestEffort bool, operation string) runtimePersistence { done := make(chan struct{}) - persistence := autoStandbyStatePersistence{ - id: id, - autoStandbyState: cloneAutoStandbyState(autoStandbyState), - previous: c.autoStandbyStatePersistenceTail, - done: done, - bestEffort: bestEffort, - operation: operation, - } - c.autoStandbyStatePersistenceTail = done + persistence := runtimePersistence{ + id: id, + runtime: cloneRuntime(runtime), + previous: c.runtimePersistenceTail, + done: done, + bestEffort: bestEffort, + operation: operation, + } + c.runtimePersistenceTail = done return persistence } -func (c *Controller) persistAutoStandbyState(ctx context.Context, persistence autoStandbyStatePersistence) error { +func (c *Controller) persistRuntime(ctx context.Context, persistence runtimePersistence) error { if persistence.done == nil { return nil } @@ -1167,10 +1167,10 @@ func (c *Controller) persistAutoStandbyState(ctx context.Context, persistence au } defer close(persistence.done) - err := c.store.SetAutoStandbyState(ctx, persistence.id, persistence.autoStandbyState) + err := c.store.SetRuntime(ctx, persistence.id, persistence.runtime) if err != nil && persistence.bestEffort { - c.recordControllerError("persist_auto_standby_state") - c.log.Warn("auto-standby failed to persist state", "instance_id", persistence.id, "operation", persistence.operation, "error", err) + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime", "instance_id", persistence.id, "operation", persistence.operation, "error", err) return nil } return err @@ -1245,20 +1245,20 @@ func connectionKey(conn Connection) ConnectionKey { } } -func cloneAutoStandbyState(autoStandbyState *AutoStandbyState) *AutoStandbyState { - if autoStandbyState == nil { +func cloneRuntime(runtime *Runtime) *Runtime { + if runtime == nil { return nil } - return &AutoStandbyState{ - IdleSince: cloneTimePtr(autoStandbyState.IdleSince), - LastInboundActivityAt: cloneTimePtr(autoStandbyState.LastInboundActivityAt), + return &Runtime{ + IdleSince: cloneTimePtr(runtime.IdleSince), + LastInboundActivityAt: cloneTimePtr(runtime.LastInboundActivityAt), } } func cloneInstance(inst Instance) Instance { cloned := inst cloned.AutoStandby = clonePolicy(inst.AutoStandby) - cloned.AutoStandbyState = cloneAutoStandbyState(inst.AutoStandbyState) + cloned.Runtime = cloneRuntime(inst.Runtime) return cloned } diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index fe63d0c6c..75e45651e 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -14,25 +14,25 @@ import ( ) type fakeInstanceStore struct { - mu sync.Mutex - instances []Instance - standbyIDs []string - persistedAutoStandbyState map[string]*AutoStandbyState - events chan InstanceEvent - standbyErr error - listErr error - setAutoStandbyStateErr error - setAutoStandbyStateStarted chan string - setAutoStandbyStateRelease chan struct{} - standbyStarted chan string - standbyRelease chan struct{} + mu sync.Mutex + instances []Instance + standbyIDs []string + persistedRuntime map[string]*Runtime + events chan InstanceEvent + standbyErr error + listErr error + setRuntimeErr error + setRuntimeStarted chan string + setRuntimeRelease chan struct{} + standbyStarted chan string + standbyRelease chan struct{} } func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { return &fakeInstanceStore{ - instances: append([]Instance(nil), instances...), - persistedAutoStandbyState: make(map[string]*AutoStandbyState), - events: make(chan InstanceEvent, 16), + instances: append([]Instance(nil), instances...), + persistedRuntime: make(map[string]*Runtime), + events: make(chan InstanceEvent, 16), } } @@ -68,23 +68,23 @@ func (f *fakeInstanceStore) standbyCalls() []string { return append([]string(nil), f.standbyIDs...) } -func (f *fakeInstanceStore) SetAutoStandbyState(_ context.Context, id string, autoStandbyState *AutoStandbyState) error { - if f.setAutoStandbyStateStarted != nil { - f.setAutoStandbyStateStarted <- id +func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { + if f.setRuntimeStarted != nil { + f.setRuntimeStarted <- id } - if f.setAutoStandbyStateRelease != nil { - <-f.setAutoStandbyStateRelease + if f.setRuntimeRelease != nil { + <-f.setRuntimeRelease } f.mu.Lock() defer f.mu.Unlock() - if f.setAutoStandbyStateErr != nil { - return f.setAutoStandbyStateErr + if f.setRuntimeErr != nil { + return f.setRuntimeErr } - f.persistedAutoStandbyState[id] = cloneAutoStandbyState(autoStandbyState) + f.persistedRuntime[id] = cloneRuntime(runtime) for i := range f.instances { if f.instances[i].ID == id { - f.instances[i].AutoStandbyState = cloneAutoStandbyState(autoStandbyState) + f.instances[i].Runtime = cloneRuntime(runtime) } } return nil @@ -146,7 +146,7 @@ func TestStartupResyncClearsPersistedIdleWhenCurrentConnectionsExist(t *testing. NetworkEnabled: true, IP: "192.168.100.10", AutoStandby: &Policy{Enabled: true, IdleTimeout: "5m"}, - AutoStandbyState: &AutoStandbyState{ + Runtime: &Runtime{ IdleSince: &idleSince, LastInboundActivityAt: &lastInbound, }, @@ -168,8 +168,8 @@ func TestStartupResyncClearsPersistedIdleWhenCurrentConnectionsExist(t *testing. status := controller.Describe(store.instances[0]) require.Equal(t, StatusActive, status.Status) require.Nil(t, status.IdleSince) - require.NotNil(t, store.persistedAutoStandbyState["inst-active"]) - require.Nil(t, store.persistedAutoStandbyState["inst-active"].IdleSince) + require.NotNil(t, store.persistedRuntime["inst-active"]) + require.Nil(t, store.persistedRuntime["inst-active"].IdleSince) } func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { @@ -183,7 +183,7 @@ func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { NetworkEnabled: true, IP: "192.168.100.20", AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, - AutoStandbyState: &AutoStandbyState{ + Runtime: &Runtime{ IdleSince: &idleSince, }, }}) @@ -238,7 +238,7 @@ func TestPeriodicSnapshotSyncRefreshesTrackedState(t *testing.T) { require.Equal(t, 1, status.ActiveInboundCount) } -func TestInstanceEventClearsPersistedAutoStandbyStateWhenInstanceBecomesIneligible(t *testing.T) { +func TestInstanceEventClearsPersistedRuntimeWhenInstanceBecomesIneligible(t *testing.T) { t.Parallel() idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) @@ -250,7 +250,7 @@ func TestInstanceEventClearsPersistedAutoStandbyStateWhenInstanceBecomesIneligib NetworkEnabled: true, IP: "192.168.100.22", AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, - AutoStandbyState: &AutoStandbyState{ + Runtime: &Runtime{ IdleSince: &idleSince, LastInboundActivityAt: &lastInbound, }, @@ -274,9 +274,9 @@ func TestInstanceEventClearsPersistedAutoStandbyStateWhenInstanceBecomesIneligib }, })) - autoStandbyState, ok := store.persistedAutoStandbyState["inst-ineligible"] + runtime, ok := store.persistedRuntime["inst-ineligible"] require.True(t, ok) - require.Nil(t, autoStandbyState) + require.Nil(t, runtime) } func TestConnectionEventsClearIdleAndStartCountdown(t *testing.T) { @@ -566,7 +566,7 @@ func TestHandleStandbyTimerCallsStandbyAndClearsState(t *testing.T) { NetworkEnabled: true, IP: "192.168.100.61", AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - AutoStandbyState: &AutoStandbyState{ + Runtime: &Runtime{ IdleSince: &idleSince, }, }}) @@ -580,7 +580,7 @@ func TestHandleStandbyTimerCallsStandbyAndClearsState(t *testing.T) { controller.standbyWG.Wait() require.Equal(t, []string{"inst-standby"}, store.standbyCalls()) - require.Nil(t, store.persistedAutoStandbyState["inst-standby"]) + require.Nil(t, store.persistedRuntime["inst-standby"]) controller.mu.RLock() state := controller.states["inst-standby"] @@ -606,7 +606,7 @@ func TestHandleStandbyTimerFailureRearmsIdleCountdown(t *testing.T) { NetworkEnabled: true, IP: "192.168.100.62", AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - AutoStandbyState: &AutoStandbyState{ + Runtime: &Runtime{ IdleSince: &idleSince, }, }}) @@ -621,9 +621,9 @@ func TestHandleStandbyTimerFailureRearmsIdleCountdown(t *testing.T) { controller.standbyWG.Wait() require.Equal(t, []string{"inst-standby-fail"}, store.standbyCalls()) - require.NotNil(t, store.persistedAutoStandbyState["inst-standby-fail"]) - require.NotNil(t, store.persistedAutoStandbyState["inst-standby-fail"].IdleSince) - assert.Equal(t, now, *store.persistedAutoStandbyState["inst-standby-fail"].IdleSince) + require.NotNil(t, store.persistedRuntime["inst-standby-fail"]) + require.NotNil(t, store.persistedRuntime["inst-standby-fail"].IdleSince) + assert.Equal(t, now, *store.persistedRuntime["inst-standby-fail"].IdleSince) controller.mu.RLock() state := controller.states["inst-standby-fail"] @@ -664,8 +664,8 @@ func TestHandleStandbyTimerSkipsStandbyWhenConntrackReportsConnection(t *testing controller.standbyWG.Wait() assert.Empty(t, store.standbyCalls()) - require.NotNil(t, store.persistedAutoStandbyState["inst-missed-event"]) - assert.Nil(t, store.persistedAutoStandbyState["inst-missed-event"].IdleSince) + require.NotNil(t, store.persistedRuntime["inst-missed-event"]) + assert.Nil(t, store.persistedRuntime["inst-missed-event"].IdleSince) controller.mu.RLock() state := controller.states["inst-missed-event"] @@ -698,9 +698,9 @@ func TestHandleStandbyTimerRearmsCountdownWhenConfirmationFails(t *testing.T) { controller.standbyWG.Wait() assert.Empty(t, store.standbyCalls()) - require.NotNil(t, store.persistedAutoStandbyState["inst-confirm-fail"]) - require.NotNil(t, store.persistedAutoStandbyState["inst-confirm-fail"].IdleSince) - assert.Equal(t, now, *store.persistedAutoStandbyState["inst-confirm-fail"].IdleSince) + require.NotNil(t, store.persistedRuntime["inst-confirm-fail"]) + require.NotNil(t, store.persistedRuntime["inst-confirm-fail"].IdleSince) + assert.Equal(t, now, *store.persistedRuntime["inst-confirm-fail"].IdleSince) controller.mu.RLock() state := controller.states["inst-confirm-fail"] @@ -745,8 +745,8 @@ func TestHandleStandbyTimerKeepsActiveStateWhenConfirmationFails(t *testing.T) { controller.standbyWG.Wait() assert.Empty(t, store.standbyCalls()) - require.NotNil(t, store.persistedAutoStandbyState["inst-confirm-active"]) - assert.Nil(t, store.persistedAutoStandbyState["inst-confirm-active"].IdleSince) + require.NotNil(t, store.persistedRuntime["inst-confirm-active"]) + assert.Nil(t, store.persistedRuntime["inst-confirm-active"].IdleSince) controller.mu.RLock() state := controller.states["inst-confirm-active"] @@ -984,13 +984,13 @@ func (s *restoreConnectionSource) OpenStream(context.Context) (ConnectionStream, func idleTestInstance(id, ip string, idleSince time.Time) Instance { since := idleSince return Instance{ - ID: id, - Name: id, - State: StateRunning, - NetworkEnabled: true, - IP: ip, - AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, - AutoStandbyState: &AutoStandbyState{IdleSince: &since}, + ID: id, + Name: id, + State: StateRunning, + NetworkEnabled: true, + IP: ip, + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + Runtime: &Runtime{IdleSince: &since}, } } @@ -1245,7 +1245,7 @@ func TestStandbyFailureWithMidFlightActivityDoesNotRearmIdle(t *testing.T) { controller.mu.RUnlock() store.mu.Lock() - persisted := cloneAutoStandbyState(store.persistedAutoStandbyState["inst-fail-busy"]) + persisted := cloneRuntime(store.persistedRuntime["inst-fail-busy"]) store.mu.Unlock() require.NotNil(t, persisted) assert.Nil(t, persisted.IdleSince, "failure must not persist a false idle window while connections are active") @@ -1338,7 +1338,7 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ Now: func() time.Time { return idleSince }, }) - store.setAutoStandbyStateErr = errors.New("metadata write failed") + store.setRuntimeErr = errors.New("metadata write failed") // Fresh idle state forces the persist that fails; the countdown must be // armed regardless. @@ -1357,39 +1357,39 @@ func TestRefreshPersistFailureStillArmsIdleTimer(t *testing.T) { controller.mu.RUnlock() } -func TestAutoStandbyStatePersistencePreservesInstanceOrder(t *testing.T) { +func TestRuntimePersistencePreservesInstanceOrder(t *testing.T) { t.Parallel() store := newFakeInstanceStore(nil) - store.setAutoStandbyStateStarted = make(chan string, 2) + store.setRuntimeStarted = make(chan string, 2) controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) firstTime := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) secondTime := firstTime.Add(time.Second) controller.mu.Lock() - first := controller.prepareAutoStandbyStatePersistenceLocked("inst-persist-order", &AutoStandbyState{IdleSince: &firstTime}, false, "test first write") - second := controller.prepareAutoStandbyStatePersistenceLocked("inst-persist-order", &AutoStandbyState{IdleSince: &secondTime}, false, "test second write") + first := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &firstTime}, false, "test first write") + second := controller.prepareRuntimePersistenceLocked("inst-persist-order", &Runtime{IdleSince: &secondTime}, false, "test second write") controller.mu.Unlock() secondDone := make(chan error, 1) go func() { - secondDone <- controller.persistAutoStandbyState(context.Background(), second) + secondDone <- controller.persistRuntime(context.Background(), second) }() select { - case <-store.setAutoStandbyStateStarted: - require.FailNow(t, "second auto-standby state write started before first") + case <-store.setRuntimeStarted: + require.FailNow(t, "second runtime write started before first") case <-time.After(50 * time.Millisecond): } - require.NoError(t, controller.persistAutoStandbyState(context.Background(), first)) - require.Equal(t, "inst-persist-order", <-store.setAutoStandbyStateStarted) + require.NoError(t, controller.persistRuntime(context.Background(), first)) + require.Equal(t, "inst-persist-order", <-store.setRuntimeStarted) require.NoError(t, <-secondDone) - require.Equal(t, "inst-persist-order", <-store.setAutoStandbyStateStarted) - require.NotNil(t, store.persistedAutoStandbyState["inst-persist-order"]) - assert.Equal(t, secondTime, *store.persistedAutoStandbyState["inst-persist-order"].IdleSince) + require.Equal(t, "inst-persist-order", <-store.setRuntimeStarted) + require.NotNil(t, store.persistedRuntime["inst-persist-order"]) + assert.Equal(t, secondTime, *store.persistedRuntime["inst-persist-order"].IdleSince) } -func TestHoldStandbyDoesNotWaitForAutoStandbyStatePersistence(t *testing.T) { +func TestHoldStandbyDoesNotWaitForRuntimePersistence(t *testing.T) { t.Parallel() now := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) @@ -1409,8 +1409,8 @@ func TestHoldStandbyDoesNotWaitForAutoStandbyStatePersistence(t *testing.T) { TCPState: TCPStateEstablished, } store := newFakeInstanceStore([]Instance{inst}) - store.setAutoStandbyStateStarted = make(chan string, 1) - store.setAutoStandbyStateRelease = make(chan struct{}) + store.setRuntimeStarted = make(chan string, 1) + store.setRuntimeRelease = make(chan struct{}) controller := NewController(store, &fakeConnectionSource{connections: []Connection{conn}}, ControllerOptions{ Now: func() time.Time { return now }, }) @@ -1419,7 +1419,7 @@ func TestHoldStandbyDoesNotWaitForAutoStandbyStatePersistence(t *testing.T) { go func() { resyncDone <- controller.startupResync(context.Background()) }() - require.Equal(t, inst.ID, <-store.setAutoStandbyStateStarted) + require.Equal(t, inst.ID, <-store.setRuntimeStarted) holdDone := make(chan error, 1) go func() { @@ -1431,11 +1431,11 @@ func TestHoldStandbyDoesNotWaitForAutoStandbyStatePersistence(t *testing.T) { case err := <-holdDone: require.NoError(t, err) case <-time.After(time.Second): - close(store.setAutoStandbyStateRelease) - require.FailNow(t, "hold waited for auto-standby state persistence") + close(store.setRuntimeRelease) + require.FailNow(t, "hold waited for runtime persistence") } - close(store.setAutoStandbyStateRelease) + close(store.setRuntimeRelease) require.NoError(t, <-resyncDone) } @@ -1452,7 +1452,7 @@ func TestHoldStandbyExtendsArmedCountdown(t *testing.T) { }) require.NoError(t, controller.startupResync(context.Background())) - persistedBefore := cloneAutoStandbyState(store.persistedAutoStandbyState["inst-hold"]) + persistedBefore := cloneRuntime(store.persistedRuntime["inst-hold"]) snapshot, err := controller.HoldStandby(context.Background(), store.instances[0]) require.NoError(t, err) @@ -1463,7 +1463,7 @@ func TestHoldStandbyExtendsArmedCountdown(t *testing.T) { assert.Equal(t, now.Add(time.Minute), *snapshot.NextStandbyAt) // Holds are in-memory only: the persisted countdown is untouched. - assert.Equal(t, persistedBefore, store.persistedAutoStandbyState["inst-hold"]) + assert.Equal(t, persistedBefore, store.persistedRuntime["inst-hold"]) // A resync must keep the held deadline. require.NoError(t, controller.startupResync(context.Background())) @@ -1480,13 +1480,13 @@ func TestHoldStandbyReplacesLongerHold(t *testing.T) { idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) longPolicy := Instance{ - ID: "inst-reset-hold", - Name: "inst-reset-hold", - State: StateRunning, - NetworkEnabled: true, - IP: "192.168.100.110", - AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, - AutoStandbyState: &AutoStandbyState{IdleSince: &idleSince}, + ID: "inst-reset-hold", + Name: "inst-reset-hold", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.110", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, + Runtime: &Runtime{IdleSince: &idleSince}, } store := newFakeInstanceStore([]Instance{longPolicy}) now := idleSince diff --git a/lib/autostandby/types.go b/lib/autostandby/types.go index 7bd881fd5..0dfa319b8 100644 --- a/lib/autostandby/types.go +++ b/lib/autostandby/types.go @@ -19,14 +19,14 @@ type Policy struct { // Instance is the minimal instance view needed by the auto-standby controller. type Instance struct { - ID string - Name string - State string - NetworkEnabled bool - IP string - HasVGPU bool - AutoStandby *Policy - AutoStandbyState *AutoStandbyState + ID string + Name string + State string + NetworkEnabled bool + IP string + HasVGPU bool + AutoStandby *Policy + Runtime *Runtime } // Connection is the normalized network view used by activity classification. @@ -38,8 +38,8 @@ type Connection struct { TCPState TCPState } -// AutoStandbyState stores persisted and in-memory idle-tracking timestamps. -type AutoStandbyState struct { +// Runtime stores persisted and in-memory idle-tracking timestamps. +type Runtime struct { IdleSince *time.Time `json:"idle_since,omitempty"` LastInboundActivityAt *time.Time `json:"last_inbound_activity_at,omitempty"` } diff --git a/lib/instances/README.md b/lib/instances/README.md index a9f2ac888..13ada0565 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -52,7 +52,7 @@ Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEM memory-ranges # Memory state ``` -`metadata.json` also carries controller-owned `AutoStandbyState` timestamps when that feature is enabled, so idle countdown state can survive Hypeman restarts. +`metadata.json` also carries controller-owned auto-standby runtime timestamps when that feature is enabled, so idle countdown state can survive Hypeman restarts. **Benefits:** - Content-addressable IDs (ULID = time-ordered) diff --git a/lib/instances/auto_standby.go b/lib/instances/auto_standby.go index 0f0ba8b2e..de061dbfb 100644 --- a/lib/instances/auto_standby.go +++ b/lib/instances/auto_standby.go @@ -24,18 +24,18 @@ func cloneAutoStandbyPolicy(policy *autostandby.Policy) *autostandby.Policy { return cloned } -func cloneAutoStandbyState(autoStandbyState *autostandby.AutoStandbyState) *autostandby.AutoStandbyState { - if autoStandbyState == nil { +func cloneAutoStandbyRuntime(runtime *autostandby.Runtime) *autostandby.Runtime { + if runtime == nil { return nil } - cloned := &autostandby.AutoStandbyState{} - if autoStandbyState.IdleSince != nil { - idleSince := autoStandbyState.IdleSince.UTC() + cloned := &autostandby.Runtime{} + if runtime.IdleSince != nil { + idleSince := runtime.IdleSince.UTC() cloned.IdleSince = &idleSince } - if autoStandbyState.LastInboundActivityAt != nil { - lastInboundActivityAt := autoStandbyState.LastInboundActivityAt.UTC() + if runtime.LastInboundActivityAt != nil { + lastInboundActivityAt := runtime.LastInboundActivityAt.UTC() cloned.LastInboundActivityAt = &lastInboundActivityAt } return cloned diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go index 7c9cff544..d4e25b2d9 100644 --- a/lib/instances/auto_standby_integration_linux_test.go +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -58,8 +58,8 @@ func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id str return err } -func (s integrationAutoStandbyStore) SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { - return s.manager.SetAutoStandbyState(ctx, id, autoStandbyState) +func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { + return s.manager.SetAutoStandbyRuntime(ctx, id, runtime) } func (s integrationAutoStandbyStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { @@ -375,7 +375,7 @@ func TestAutoStandbyCloudHypervisorHoldStandby(t *testing.T) { require.NoError(t, err) require.Equal(t, StateRunning, current.State) - stateBefore, err := mgr.GetAutoStandbyState(ctx, instanceID) + runtimeBefore, err := mgr.GetAutoStandbyRuntime(ctx, instanceID) require.NoError(t, err) snapshot, err := controller.HoldStandby(ctx, toAutoStandby(current)) @@ -386,9 +386,9 @@ func TestAutoStandbyCloudHypervisorHoldStandby(t *testing.T) { "hold_until %s must be after the countdown deadline %s", snapshot.HoldUntil, countdownDeadline) // Holds are in-memory only: the persisted countdown is untouched. - stateAfter, err := mgr.GetAutoStandbyState(ctx, instanceID) + runtimeAfter, err := mgr.GetAutoStandbyRuntime(ctx, instanceID) require.NoError(t, err) - require.Equal(t, stateBefore, stateAfter) + require.Equal(t, runtimeBefore, runtimeAfter) // Just past the countdown deadline the instance must still be running; // without the hold the standby would have fired here. diff --git a/lib/instances/auto_standby_runtime.go b/lib/instances/auto_standby_runtime.go new file mode 100644 index 000000000..a625bc632 --- /dev/null +++ b/lib/instances/auto_standby_runtime.go @@ -0,0 +1,34 @@ +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/autostandby" +) + +// GetAutoStandbyRuntime returns the persisted auto-standby runtime metadata for an instance. +func (m *manager) GetAutoStandbyRuntime(_ context.Context, id string) (*autostandby.Runtime, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + return cloneAutoStandbyRuntime(meta.AutoStandbyRuntime), nil +} + +// SetAutoStandbyRuntime persists auto-standby runtime metadata for an instance. +func (m *manager) SetAutoStandbyRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.AutoStandbyRuntime = cloneAutoStandbyRuntime(runtime) + return m.saveMetadata(meta) +} diff --git a/lib/instances/auto_standby_state.go b/lib/instances/auto_standby_state.go deleted file mode 100644 index 3808eca2a..000000000 --- a/lib/instances/auto_standby_state.go +++ /dev/null @@ -1,34 +0,0 @@ -package instances - -import ( - "context" - - "github.com/kernel/hypeman/lib/autostandby" -) - -// GetAutoStandbyState returns the persisted auto-standby state for an instance. -func (m *manager) GetAutoStandbyState(_ context.Context, id string) (*autostandby.AutoStandbyState, error) { - lock := m.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - - meta, err := m.loadMetadata(id) - if err != nil { - return nil, err - } - return cloneAutoStandbyState(meta.AutoStandbyState), nil -} - -// SetAutoStandbyState persists auto-standby state for an instance. -func (m *manager) SetAutoStandbyState(_ context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { - lock := m.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - - meta, err := m.loadMetadata(id) - if err != nil { - return err - } - meta.AutoStandbyState = cloneAutoStandbyState(autoStandbyState) - return m.saveMetadata(meta) -} diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go index 478771e7f..70742edc4 100644 --- a/lib/instances/metadata_clone.go +++ b/lib/instances/metadata_clone.go @@ -11,7 +11,7 @@ func deepCopyMetadata(src *metadata) *metadata { return &metadata{ StoredMetadata: cloneStoredMetadata(src.StoredMetadata), - AutoStandbyState: cloneAutoStandbyState(src.AutoStandbyState), + AutoStandbyRuntime: cloneAutoStandbyRuntime(src.AutoStandbyRuntime), HealthCheckRuntime: healthcheck.CloneRuntime(src.HealthCheckRuntime), } } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 80b5fde1b..a293fc6e1 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -40,9 +40,8 @@ const ( // metadata wraps StoredMetadata for JSON serialization type metadata struct { StoredMetadata - // Keep the legacy JSON key so existing instance metadata remains readable. - AutoStandbyState *autostandby.AutoStandbyState `json:"auto_standby_runtime,omitempty"` - HealthCheckRuntime *healthcheck.Runtime `json:"health_check_runtime,omitempty"` + AutoStandbyRuntime *autostandby.Runtime `json:"auto_standby_runtime,omitempty"` + HealthCheckRuntime *healthcheck.Runtime `json:"health_check_runtime,omitempty"` } // ensureDirectories creates the instance directory structure diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go index eb1f9b950..d54d7aeaa 100644 --- a/lib/providers/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -14,15 +14,15 @@ import ( "go.opentelemetry.io/otel" ) -type autoStandbyStateManager interface { - GetAutoStandbyState(ctx context.Context, id string) (*autostandby.AutoStandbyState, error) - SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error +type autoStandbyRuntimeManager interface { + GetAutoStandbyRuntime(ctx context.Context, id string) (*autostandby.Runtime, error) + SetAutoStandbyRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error SubscribeLifecycleEvents(consumer instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) } type autoStandbyInstanceStore struct { - manager instances.Manager - stateManager autoStandbyStateManager + manager instances.Manager + runtimeManager autoStandbyRuntimeManager } func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { @@ -33,20 +33,20 @@ func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostan out := make([]autostandby.Instance, 0, len(insts)) for _, inst := range insts { - autoStandbyState, err := s.stateManager.GetAutoStandbyState(ctx, inst.Id) + runtime, err := s.runtimeManager.GetAutoStandbyRuntime(ctx, inst.Id) if err != nil { return nil, err } out = append(out, autostandby.Instance{ - ID: inst.Id, - Name: inst.Name, - State: string(inst.State), - NetworkEnabled: inst.NetworkEnabled, - IP: inst.IP, - HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", - AutoStandby: inst.AutoStandby, - AutoStandbyState: autoStandbyState, + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + Runtime: runtime, }) } return out, nil @@ -60,21 +60,21 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string return err } -func (s autoStandbyInstanceStore) SetAutoStandbyState(ctx context.Context, id string, autoStandbyState *autostandby.AutoStandbyState) error { - return s.stateManager.SetAutoStandbyState(ctx, id, autoStandbyState) +func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { + return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime) } func (s autoStandbyInstanceStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { - src, unsub := s.stateManager.SubscribeLifecycleEvents(instances.LifecycleEventConsumerAutoStandby) + src, unsub := s.runtimeManager.SubscribeLifecycleEvents(instances.LifecycleEventConsumerAutoStandby) dst := make(chan autostandby.InstanceEvent, 32) go func() { defer close(dst) for event := range src { inst := toAutoStandbyInstance(event.Instance) if inst != nil { - autoStandbyState, err := s.stateManager.GetAutoStandbyState(context.Background(), inst.ID) + runtime, err := s.runtimeManager.GetAutoStandbyRuntime(context.Background(), inst.ID) if err == nil { - inst.AutoStandbyState = autoStandbyState + inst.Runtime = runtime } } dst <- autostandby.InstanceEvent{ @@ -108,13 +108,13 @@ func ProvideAutoStandbyController(instanceManager instances.Manager, cfg *config return nil } - stateManager, ok := instanceManager.(autoStandbyStateManager) + runtimeManager, ok := instanceManager.(autoStandbyRuntimeManager) if !ok { return nil } return autostandby.NewController( - autoStandbyInstanceStore{manager: instanceManager, stateManager: stateManager}, + autoStandbyInstanceStore{manager: instanceManager, runtimeManager: runtimeManager}, autostandby.NewConntrackSource(), autostandby.ControllerOptions{ Log: log.With("controller", "auto_standby"), diff --git a/lib/providers/auto_standby_linux_test.go b/lib/providers/auto_standby_linux_test.go index 67e593bee..b1a407668 100644 --- a/lib/providers/auto_standby_linux_test.go +++ b/lib/providers/auto_standby_linux_test.go @@ -12,34 +12,34 @@ import ( "github.com/stretchr/testify/require" ) -type autoStandbyStateManagerStub struct { - stateByID map[string]*autostandby.AutoStandbyState - events chan instances.LifecycleEvent +type autoStandbyRuntimeManagerStub struct { + runtimeByID map[string]*autostandby.Runtime + events chan instances.LifecycleEvent } -func (s *autoStandbyStateManagerStub) GetAutoStandbyState(_ context.Context, id string) (*autostandby.AutoStandbyState, error) { - if autoStandbyState, ok := s.stateByID[id]; ok { - cloned := *autoStandbyState +func (s *autoStandbyRuntimeManagerStub) GetAutoStandbyRuntime(_ context.Context, id string) (*autostandby.Runtime, error) { + if runtime, ok := s.runtimeByID[id]; ok { + cloned := *runtime return &cloned, nil } return nil, nil } -func (s *autoStandbyStateManagerStub) SetAutoStandbyState(context.Context, string, *autostandby.AutoStandbyState) error { +func (s *autoStandbyRuntimeManagerStub) SetAutoStandbyRuntime(context.Context, string, *autostandby.Runtime) error { return nil } -func (s *autoStandbyStateManagerStub) SubscribeLifecycleEvents(instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) { +func (s *autoStandbyRuntimeManagerStub) SubscribeLifecycleEvents(instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) { return s.events, func() {} } -func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesState(t *testing.T) { +func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesRuntime(t *testing.T) { t.Parallel() idleSince := time.Date(2026, 4, 7, 12, 0, 0, 0, time.UTC) lastInbound := idleSince.Add(-time.Minute) - stateManager := &autoStandbyStateManagerStub{ - stateByID: map[string]*autostandby.AutoStandbyState{ + runtimeManager := &autoStandbyRuntimeManagerStub{ + runtimeByID: map[string]*autostandby.Runtime{ "inst-1": { IdleSince: &idleSince, LastInboundActivityAt: &lastInbound, @@ -48,13 +48,13 @@ func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesState(t *testing events: make(chan instances.LifecycleEvent, 1), } store := autoStandbyInstanceStore{ - stateManager: stateManager, + runtimeManager: runtimeManager, } eventCh, _, err := store.SubscribeInstanceEvents() require.NoError(t, err) - stateManager.events <- instances.LifecycleEvent{ + runtimeManager.events <- instances.LifecycleEvent{ Action: instances.LifecycleEventUpdate, InstanceID: "inst-1", Instance: &instances.Instance{ @@ -71,9 +71,9 @@ func TestAutoStandbyInstanceStoreSubscribeInstanceEventsIncludesState(t *testing event := <-eventCh require.NotNil(t, event.Instance) - require.NotNil(t, event.Instance.AutoStandbyState) - require.NotNil(t, event.Instance.AutoStandbyState.IdleSince) - require.Equal(t, idleSince, *event.Instance.AutoStandbyState.IdleSince) - require.NotNil(t, event.Instance.AutoStandbyState.LastInboundActivityAt) - require.Equal(t, lastInbound, *event.Instance.AutoStandbyState.LastInboundActivityAt) + require.NotNil(t, event.Instance.Runtime) + require.NotNil(t, event.Instance.Runtime.IdleSince) + require.Equal(t, idleSince, *event.Instance.Runtime.IdleSince) + require.NotNil(t, event.Instance.Runtime.LastInboundActivityAt) + require.Equal(t, lastInbound, *event.Instance.Runtime.LastInboundActivityAt) }