From d30f6df6fbebfcdb65b910eea976dd784613dd18 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 16 May 2026 02:35:51 +0000 Subject: [PATCH 01/13] Add instance restart policy --- cmd/api/api/instances.go | 25 +- cmd/api/api/instances_test.go | 147 ++++++ cmd/api/api/restart_policy.go | 75 +++ cmd/api/main.go | 8 + lib/instances/create.go | 6 + lib/instances/fork.go | 5 + lib/instances/lifecycle_events.go | 8 +- lib/instances/manager.go | 15 +- lib/instances/restart_policy.go | 149 ++++++ lib/instances/restart_policy_test.go | 48 ++ lib/instances/snapshot.go | 2 + lib/instances/types.go | 14 +- lib/instances/update.go | 15 +- lib/oapi/oapi.go | 645 ++++++++++++++------------ lib/restart-policy/README.md | 77 +++ lib/restart-policy/controller.go | 213 +++++++++ lib/restart-policy/controller_test.go | 116 +++++ lib/restart-policy/policy.go | 142 ++++++ lib/restart-policy/policy_test.go | 57 +++ openapi.yaml | 66 +++ stainless.yaml | 2 + 21 files changed, 1538 insertions(+), 297 deletions(-) create mode 100644 cmd/api/api/restart_policy.go create mode 100644 lib/instances/restart_policy.go create mode 100644 lib/instances/restart_policy_test.go create mode 100644 lib/restart-policy/README.md create mode 100644 lib/restart-policy/controller.go create mode 100644 lib/restart-policy/controller_test.go create mode 100644 lib/restart-policy/policy.go create mode 100644 lib/restart-policy/policy_test.go diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index be399f86..2c85284b 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -293,6 +293,13 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Message: err.Error(), }, nil } + restartPolicy, err := toDomainRestartPolicy(request.Body.RestartPolicy) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_restart_policy", + Message: err.Error(), + }, nil + } domainReq := instances.CreateInstanceRequest{ Name: request.Body.Name, @@ -319,6 +326,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, AutoStandby: autoStandby, HealthCheck: healthCheck, + RestartPolicy: restartPolicy, } if request.Body.SnapshotPolicy != nil { snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) @@ -970,11 +978,20 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst Message: err.Error(), }, nil } + restartPolicy, err := toDomainRestartPolicy(request.Body.RestartPolicy) + if err != nil { + return oapi.UpdateInstance400JSONResponse{ + Code: "invalid_restart_policy", + Message: err.Error(), + }, nil + } result, err := s.InstanceManager.UpdateInstance(ctx, inst.Id, instances.UpdateInstanceRequest{ - Env: env, - AutoStandby: autoStandby, - HealthCheck: healthCheck, + Env: env, + AutoStandby: autoStandby, + HealthCheck: healthCheck, + RestartPolicy: restartPolicy, + RestartPolicySet: request.Body.RestartPolicy != nil, }) if err != nil { switch { @@ -1108,6 +1125,8 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { oapiInst.AutoStandby = toOAPIAutoStandbyPolicy(inst.AutoStandby) oapiInst.HealthCheck = toOAPIHealthCheck(inst.HealthCheck) oapiInst.HealthStatus = toOAPIHealthStatus(healthcheck.Snapshot(inst.HealthCheck, string(inst.State), inst.HealthCheckRuntime)) + oapiInst.RestartPolicy = toOAPIRestartPolicy(inst.RestartPolicy) + oapiInst.RestartStatus = toOAPIRestartStatus(inst.RestartStatus) // Convert volume attachments if len(inst.Volumes) > 0 { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b28fbc58..4fadadee 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -16,6 +16,7 @@ import ( mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/kernel/hypeman/lib/system" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -281,6 +282,7 @@ func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, re Env: req.Env, AutoStandby: req.AutoStandby, HealthCheck: req.HealthCheck, + RestartPolicy: req.RestartPolicy, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -304,6 +306,7 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances Vcpus: req.Vcpus, AutoStandby: req.AutoStandby, HealthCheck: req.HealthCheck, + RestartPolicy: req.RestartPolicy, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -705,6 +708,48 @@ func TestCreateInstance_MapsHealthCheckPolicy(t *testing.T) { assert.Equal(t, oapi.InstanceHealthStatusStatusStarting, instance.HealthStatus.Status) } +func TestCreateInstance_MapsRestartPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + policy := oapi.OnFailure + backoff := "7s" + stableAfter := "2m" + maxAttempts := 4 + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-restart-policy", + Image: "docker.io/library/alpine:latest", + RestartPolicy: &oapi.RestartPolicy{ + Policy: &policy, + Backoff: &backoff, + StableAfter: &stableAfter, + MaxAttempts: &maxAttempts, + }, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.RestartPolicy) + assert.Equal(t, restartpolicy.PolicyOnFailure, mockMgr.lastReq.RestartPolicy.Policy) + assert.Equal(t, "7s", mockMgr.lastReq.RestartPolicy.Backoff) + assert.Equal(t, "2m", mockMgr.lastReq.RestartPolicy.StableAfter) + assert.Equal(t, 4, mockMgr.lastReq.RestartPolicy.MaxAttempts) + + instance := oapi.Instance(created) + require.NotNil(t, instance.RestartPolicy) + require.NotNil(t, instance.RestartPolicy.Policy) + assert.Equal(t, oapi.OnFailure, *instance.RestartPolicy.Policy) +} + func TestUpdateInstance_MapsEnvPatch(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -883,6 +928,108 @@ func TestUpdateInstance_MapsHealthCheckPatch(t *testing.T) { assert.Equal(t, oapi.InstanceHealthStatusStatusUnknown, instance.HealthStatus.Status) } +func TestUpdateInstance_MapsRestartPolicyPatch(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{ + Manager: origMgr, + result: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-restart-policy", + Name: "inst-update-restart-policy", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartPolicy: &restartpolicy.Policy{ + Policy: restartpolicy.PolicyAlways, + Backoff: "5s", + StableAfter: "10m0s", + }, + RestartStatus: restartpolicy.Status{ + BlockedReason: restartpolicy.BlockedReasonManualStop, + }, + }, + State: instances.StateStopped, + }, + } + svc.InstanceManager = mockMgr + + policy := oapi.Always + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-restart-policy", + Name: "inst-update-restart-policy", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + RestartPolicy: &oapi.RestartPolicy{Policy: &policy}, + }, + }) + require.NoError(t, err) + updated, ok := resp.(oapi.UpdateInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + assert.True(t, mockMgr.lastReq.RestartPolicySet) + require.NotNil(t, mockMgr.lastReq.RestartPolicy) + assert.Equal(t, restartpolicy.PolicyAlways, mockMgr.lastReq.RestartPolicy.Policy) + + instance := oapi.Instance(updated) + require.NotNil(t, instance.RestartPolicy) + require.NotNil(t, instance.RestartStatus) + require.NotNil(t, instance.RestartStatus.BlockedReason) + assert.Equal(t, oapi.ManualStop, *instance.RestartStatus.BlockedReason) +} + +func TestUpdateInstance_RejectsInvalidRestartPolicy(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureUpdateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + now := time.Now() + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-restart-policy", + Name: "inst-update-restart-policy", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + policy := oapi.OnFailure + backoff := "0s" + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + RestartPolicy: &oapi.RestartPolicy{ + Policy: &policy, + Backoff: &backoff, + }, + }, + }) + require.NoError(t, err) + + badReq, ok := resp.(oapi.UpdateInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_restart_policy", badReq.Code) + assert.Nil(t, mockMgr.lastReq) +} + func TestUpdateInstance_RejectsZeroAutoStandbyIgnoreDestinationPort(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/cmd/api/api/restart_policy.go b/cmd/api/api/restart_policy.go new file mode 100644 index 00000000..46c908f2 --- /dev/null +++ b/cmd/api/api/restart_policy.go @@ -0,0 +1,75 @@ +package api + +import ( + "github.com/kernel/hypeman/lib/oapi" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + "github.com/samber/lo" +) + +func toDomainRestartPolicy(policy *oapi.RestartPolicy) (*restartpolicy.Policy, error) { + if policy == nil { + return nil, nil + } + + out := &restartpolicy.Policy{} + if policy.Policy != nil { + out.Policy = restartpolicy.PolicyMode(*policy.Policy) + } + if policy.Backoff != nil { + out.Backoff = *policy.Backoff + } + if policy.MaxAttempts != nil { + out.MaxAttempts = *policy.MaxAttempts + } + if policy.StableAfter != nil { + out.StableAfter = *policy.StableAfter + } + if _, err := restartpolicy.NormalizePolicy(out); err != nil { + return nil, err + } + return out, nil +} + +func toOAPIRestartPolicy(policy *restartpolicy.Policy) *oapi.RestartPolicy { + if policy == nil { + return nil + } + + mode := oapi.RestartPolicyPolicy(policy.Policy) + out := &oapi.RestartPolicy{ + Policy: &mode, + } + if policy.Backoff != "" { + out.Backoff = lo.ToPtr(policy.Backoff) + } + if policy.MaxAttempts > 0 { + out.MaxAttempts = lo.ToPtr(policy.MaxAttempts) + } + if policy.StableAfter != "" { + out.StableAfter = lo.ToPtr(policy.StableAfter) + } + return out +} + +func toOAPIRestartStatus(status restartpolicy.Status) *oapi.RestartStatus { + if status.IsZero() { + return nil + } + + out := &oapi.RestartStatus{ + Attempts: lo.ToPtr(status.Attempts), + } + if status.BlockedReason != "" { + reason := oapi.RestartStatusBlockedReason(status.BlockedReason) + out.BlockedReason = &reason + } + if status.LastAttemptAt != nil { + lastAttemptAt := status.LastAttemptAt.UTC() + out.LastAttemptAt = &lastAttemptAt + } + if status.NextAttemptAt != nil { + nextAttemptAt := status.NextAttemptAt.UTC() + out.NextAttemptAt = &nextAttemptAt + } + return out +} diff --git a/cmd/api/main.go b/cmd/api/main.go index f7e24c4e..da2a2326 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -571,6 +571,14 @@ func run() error { return app.HealthCheckController.Run(gctx) }) } + if restartController, ok := app.InstanceManager.(interface { + StartRestartPolicyController(context.Context) error + }); ok { + grp.Go(func() error { + logger.Info("starting restart policy controller") + return restartController.StartRestartPolicyController(gctx) + }) + } // Run the server grp.Go(func() error { diff --git a/lib/instances/create.go b/lib/instances/create.go index dc2f57b0..df991674 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -375,6 +375,7 @@ func (m *manager) createInstance( SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), + RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } // 12. Ensure directories @@ -638,6 +639,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { if err := validateHealthCheckCompatibility(req.HealthCheck, req.NetworkEnabled, req.SkipGuestAgent); err != nil { return err } + normalizedRestartPolicy, err := normalizeRestartPolicy(req.RestartPolicy) + if err != nil { + return err + } + req.RestartPolicy = normalizedRestartPolicy // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 7c0b25db..c29d7189 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -16,6 +16,7 @@ import ( "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "gvisor.dev/gvisor/pkg/cleanup" @@ -281,6 +282,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.VsockSocket = m.paths.InstanceSocket(forkID, hypervisor.VsockSocketNameForType(forkMeta.HypervisorType)) forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + forkMeta.RestartStatus = restartpolicy.Status{} // Forks are new instances; phase accounting must not inherit the source's // cumulative durations. The first transition into the fork's runtime // phase (Standby for snapshot forks, Stopped for stopped forks) will be @@ -481,6 +483,9 @@ func cloneStoredMetadata(src StoredMetadata) StoredMetadata { if src.HealthCheck != nil { dst.HealthCheck = cloneHealthCheckPolicy(src.HealthCheck) } + if src.RestartPolicy != nil { + dst.RestartPolicy = cloneRestartPolicy(src.RestartPolicy) + } if src.SnapshotPolicy != nil { dst.SnapshotPolicy = cloneSnapshotPolicy(src.SnapshotPolicy) } diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go index 4bba4851..9eb15b75 100644 --- a/lib/instances/lifecycle_events.go +++ b/lib/instances/lifecycle_events.go @@ -12,15 +12,17 @@ const defaultLifecycleEventBufferSize = 256 type LifecycleEventConsumer string const ( - LifecycleEventConsumerWaitForState LifecycleEventConsumer = "wait_for_state" - LifecycleEventConsumerAutoStandby LifecycleEventConsumer = "auto_standby" - LifecycleEventConsumerHealthCheck LifecycleEventConsumer = "health_check" + LifecycleEventConsumerWaitForState LifecycleEventConsumer = "wait_for_state" + LifecycleEventConsumerAutoStandby LifecycleEventConsumer = "auto_standby" + LifecycleEventConsumerHealthCheck LifecycleEventConsumer = "health_check" + LifecycleEventConsumerRestartPolicy LifecycleEventConsumer = "restart_policy" ) var allLifecycleEventConsumers = []LifecycleEventConsumer{ LifecycleEventConsumerWaitForState, LifecycleEventConsumerAutoStandby, LifecycleEventConsumerHealthCheck, + LifecycleEventConsumerRestartPolicy, } // LifecycleEventAction identifies which instance lifecycle action occurred. diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 36e7a627..e8e26655 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -475,7 +475,17 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error return nil, err } if current.State == StateStopped { - return current, nil + if err := m.markRestartManualStopLocked(ctx, id); err != nil { + return nil, err + } + updated, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return nil, err + } + return updated, nil + } + if err := m.markRestartManualStopLocked(ctx, id); err != nil { + return nil, err } inst, err := m.stopInstance(ctx, id) if err == nil { @@ -498,6 +508,9 @@ func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanc return current, nil } } + if err := m.clearRestartStatusLocked(ctx, id); err != nil { + return nil, err + } inst, err := m.startInstance(ctx, id, req) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventStart, inst) diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go new file mode 100644 index 00000000..9f1ea4bd --- /dev/null +++ b/lib/instances/restart_policy.go @@ -0,0 +1,149 @@ +package instances + +import ( + "context" + "fmt" + + "github.com/kernel/hypeman/lib/logger" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" +) + +type restartPolicyStore struct { + manager *manager +} + +func cloneRestartPolicy(policy *restartpolicy.Policy) *restartpolicy.Policy { + if policy == nil { + return nil + } + return &restartpolicy.Policy{ + Policy: policy.Policy, + Backoff: policy.Backoff, + MaxAttempts: policy.MaxAttempts, + StableAfter: policy.StableAfter, + } +} + +func normalizeRestartPolicy(policy *restartpolicy.Policy) (*restartpolicy.Policy, error) { + normalized, err := restartpolicy.NormalizePolicy(policy) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return normalized, nil +} + +func restartStatusAfterPolicyUpdate(status restartpolicy.Status) restartpolicy.Status { + if status.BlockedReason == restartpolicy.BlockedReasonManualStop { + return restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop} + } + return restartpolicy.Status{} +} + +func (m *manager) markRestartManualStopLocked(ctx context.Context, id string) error { + if err := m.updateRestartStatusLocked(id, restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop}); err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to mark restart policy manual stop", "instance_id", id, "error", err) + return err + } + return nil +} + +func (m *manager) clearRestartStatusLocked(ctx context.Context, id string) error { + if err := m.updateRestartStatusLocked(id, restartpolicy.Status{}); err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to clear restart policy status", "instance_id", id, "error", err) + return err + } + return nil +} + +func (m *manager) updateRestartStatusLocked(id string, status restartpolicy.Status) error { + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.RestartStatus = status + return m.saveMetadata(meta) +} + +func (m *manager) RestartInstance(ctx context.Context, id string) (*Instance, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + inst, err := m.startInstance(ctx, id, StartInstanceRequest{}) + if err == nil { + m.notifyLifecycleEvent(ctx, LifecycleEventStart, inst) + } + return inst, err +} + +func (m *manager) StartRestartPolicyController(ctx context.Context) error { + controller := restartpolicy.NewController( + restartPolicyStore{manager: m}, + restartpolicy.ControllerOptions{ + Log: logger.FromContext(ctx).With("controller", "restart_policy"), + }, + ) + return controller.Run(ctx) +} + +func (s restartPolicyStore) ListInstances(ctx context.Context) ([]restartpolicy.Instance, error) { + insts, err := s.manager.ListInstances(ctx, nil) + if err != nil { + return nil, err + } + out := make([]restartpolicy.Instance, 0, len(insts)) + for _, inst := range insts { + out = append(out, *toRestartPolicyInstance(&inst)) + } + return out, nil +} + +func (s restartPolicyStore) RestartInstance(ctx context.Context, id string) error { + _, err := s.manager.RestartInstance(ctx, id) + return err +} + +func (s restartPolicyStore) SetRestartStatus(ctx context.Context, id string, status restartpolicy.Status) error { + lock := s.manager.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + return s.manager.updateRestartStatusLocked(id, status) +} + +func (s restartPolicyStore) SubscribeInstanceEvents() (<-chan restartpolicy.InstanceEvent, func(), error) { + src, unsub := s.manager.SubscribeLifecycleEvents(LifecycleEventConsumerRestartPolicy) + dst := make(chan restartpolicy.InstanceEvent, 32) + go func() { + defer close(dst) + for event := range src { + dst <- restartpolicy.InstanceEvent{ + Action: restartpolicy.InstanceEventAction(event.Action), + InstanceID: event.InstanceID, + Instance: toRestartPolicyInstance(event.Instance), + } + } + }() + return dst, unsub, nil +} + +func toRestartPolicyInstance(inst *Instance) *restartpolicy.Instance { + if inst == nil { + return nil + } + return &restartpolicy.Instance{ + ID: inst.Id, + State: string(inst.State), + StartedAt: inst.StartedAt, + ExitCode: inst.ExitCode, + RestartPolicy: inst.RestartPolicy, + RestartStatus: inst.RestartStatus, + } +} + +var _ restartpolicy.Store = restartPolicyStore{} +var _ interface { + StartRestartPolicyController(context.Context) error +} = (*manager)(nil) +var _ interface { + RestartInstance(context.Context, string) (*Instance, error) +} = (*manager)(nil) diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go new file mode 100644 index 00000000..d500f578 --- /dev/null +++ b/lib/instances/restart_policy_test.go @@ -0,0 +1,48 @@ +package instances + +import ( + "errors" + "testing" + + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateUpdateInstanceRequestAllowsRestartPolicyOnly(t *testing.T) { + err := validateUpdateInstanceRequest(&metadata{}, UpdateInstanceRequest{ + RestartPolicy: &restartpolicy.Policy{Policy: restartpolicy.PolicyAlways}, + RestartPolicySet: true, + }) + + require.NoError(t, err) +} + +func TestNormalizeRestartPolicyWrapsInvalidRequest(t *testing.T) { + _, err := normalizeRestartPolicy(&restartpolicy.Policy{ + Policy: restartpolicy.PolicyOnFailure, + Backoff: "0s", + }) + + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestRestartStatusAfterPolicyUpdatePreservesManualStop(t *testing.T) { + status := restartStatusAfterPolicyUpdate(restartpolicy.Status{ + Attempts: 3, + BlockedReason: restartpolicy.BlockedReasonManualStop, + }) + + assert.Equal(t, restartpolicy.BlockedReasonManualStop, status.BlockedReason) + assert.Zero(t, status.Attempts) +} + +func TestRestartStatusAfterPolicyUpdateClearsRetryState(t *testing.T) { + status := restartStatusAfterPolicyUpdate(restartpolicy.Status{ + Attempts: 3, + BlockedReason: restartpolicy.BlockedReasonMaxAttemptsExceeded, + }) + + assert.True(t, status.IsZero()) +} diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 707d071c..05d8084e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/tags" "github.com/nrednav/cuid2" @@ -448,6 +449,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS forkMeta.VsockSocket = m.paths.InstanceSocket(forkID, hypervisor.VsockSocketNameForType(targetHypervisor)) forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + forkMeta.RestartStatus = restartpolicy.Status{} if rec.Snapshot.Kind == SnapshotKindStandby { forkMeta.VsockCID = rec.StoredMetadata.VsockCID } else { diff --git a/lib/instances/types.go b/lib/instances/types.go index 892191cb..8c031fc9 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -7,6 +7,7 @@ import ( "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances/phasetracking" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/tags" ) @@ -152,6 +153,10 @@ type StoredMetadata struct { // Workload health check policy. Health is reported separately from lifecycle state. HealthCheck *healthcheck.Policy + // Whole-instance restart supervision policy and runtime status. + RestartPolicy *restartpolicy.Policy + RestartStatus restartpolicy.Status + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -243,6 +248,7 @@ type CreateInstanceRequest struct { SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance AutoStandby *autostandby.Policy // Optional automatic standby policy HealthCheck *healthcheck.Policy // Optional workload health check policy + RestartPolicy *restartpolicy.Policy // Optional whole-instance restart policy } // StartInstanceRequest is the domain request for starting a stopped instance @@ -253,9 +259,11 @@ type StartInstanceRequest struct { // UpdateInstanceRequest is the domain request for updating mutable instance properties. type UpdateInstanceRequest struct { - Env map[string]string // Updated environment variables (merged with existing) - AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil - HealthCheck *healthcheck.Policy // Replaces the persisted health check policy when non-nil + Env map[string]string // Updated environment variables (merged with existing) + AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil + HealthCheck *healthcheck.Policy // Replaces the persisted health check policy when non-nil + RestartPolicy *restartpolicy.Policy // Replaces the persisted restart policy when non-nil + RestartPolicySet bool // True when restart policy was present in the update request } // ForkInstanceRequest is the domain request for forking an instance. diff --git a/lib/instances/update.go b/lib/instances/update.go index d8f97833..cdea9e4c 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -41,6 +41,13 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta return nil, err } req.HealthCheck = normalizedHealthCheck + if req.RestartPolicySet { + normalizedRestartPolicy, err := normalizeRestartPolicy(req.RestartPolicy) + if err != nil { + return nil, err + } + req.RestartPolicy = normalizedRestartPolicy + } if err := validateUpdateInstanceRequest(meta, req); err != nil { return nil, err @@ -56,6 +63,10 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta nextMeta.HealthCheck = cloneHealthCheckPolicy(req.HealthCheck) nextMeta.HealthCheckRuntime = nil } + if req.RestartPolicySet { + nextMeta.RestartPolicy = cloneRestartPolicy(req.RestartPolicy) + nextMeta.RestartStatus = restartStatusAfterPolicyUpdate(nextMeta.RestartStatus) + } if len(req.Env) == 0 { if err := m.saveMetadata(nextMeta); err != nil { return nil, fmt.Errorf("save metadata: %w", err) @@ -103,8 +114,8 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta } func validateUpdateInstanceRequest(meta *metadata, req UpdateInstanceRequest) error { - if len(req.Env) == 0 && req.AutoStandby == nil && req.HealthCheck == nil { - return fmt.Errorf("%w: request must include env, auto_standby, and/or health_check", ErrInvalidRequest) + if len(req.Env) == 0 && req.AutoStandby == nil && req.HealthCheck == nil && !req.RestartPolicySet { + return fmt.Errorf("%w: request must include env, auto_standby, health_check, and/or restart_policy", ErrInvalidRequest) } if req.HealthCheck != nil { if meta == nil { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 607c5a4a..7ef14c6e 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -185,6 +185,19 @@ const ( MemoryReclaimResponseHostPressureStatePressure MemoryReclaimResponseHostPressureState = "pressure" ) +// Defines values for RestartPolicyPolicy. +const ( + Always RestartPolicyPolicy = "always" + Never RestartPolicyPolicy = "never" + OnFailure RestartPolicyPolicy = "on_failure" +) + +// Defines values for RestartStatusBlockedReason. +const ( + ManualStop RestartStatusBlockedReason = "manual_stop" + MaxAttemptsExceeded RestartStatusBlockedReason = "max_attempts_exceeded" +) + // Defines values for RestoreSnapshotRequestTargetHypervisor. const ( RestoreSnapshotRequestTargetHypervisorCloudHypervisor RestoreSnapshotRequestTargetHypervisor = "cloud-hypervisor" @@ -506,6 +519,9 @@ type CreateInstanceRequest struct { // OverlaySize Writable overlay disk size (human-readable format like "10GB", "50G") OverlaySize *string `json:"overlay_size,omitempty"` + // RestartPolicy Whole-instance restart supervision policy. + RestartPolicy *RestartPolicy `json:"restart_policy,omitempty"` + // Size Base memory size (human-readable format like "1GB", "512MB", "2G") Size *string `json:"size,omitempty"` @@ -1025,6 +1041,12 @@ type Instance struct { // billable. PhaseDurationsMs *map[string]int64 `json:"phase_durations_ms,omitempty"` + // RestartPolicy Whole-instance restart supervision policy. + RestartPolicy *RestartPolicy `json:"restart_policy,omitempty"` + + // RestartStatus Runtime status for restart policy decisions. + RestartStatus *RestartStatus `json:"restart_status,omitempty"` + // Size Base memory size (human-readable) Size *string `json:"size,omitempty"` SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` @@ -1296,6 +1318,48 @@ type Resources struct { Network ResourceStatus `json:"network"` } +// RestartPolicy Whole-instance restart supervision policy. +type RestartPolicy struct { + // Backoff Delay before each restart attempt, expressed as a Go duration like "5s" or "1m". + Backoff *string `json:"backoff,omitempty"` + + // MaxAttempts Consecutive automatic restart attempts before blocking retries. 0 means unlimited. + MaxAttempts *int `json:"max_attempts,omitempty"` + + // Policy Restart behavior when the guest program exits: + // - never: do not automatically restart + // - always: restart after any guest exit + // - on_failure: restart only for nonzero, signaled, OOM, or unknown exits + Policy *RestartPolicyPolicy `json:"policy,omitempty"` + + // StableAfter Running this long resets the consecutive restart attempt count. + StableAfter *string `json:"stable_after,omitempty"` +} + +// RestartPolicyPolicy Restart behavior when the guest program exits: +// - never: do not automatically restart +// - always: restart after any guest exit +// - on_failure: restart only for nonzero, signaled, OOM, or unknown exits +type RestartPolicyPolicy string + +// RestartStatus Runtime status for restart policy decisions. +type RestartStatus struct { + // Attempts Consecutive automatic restart attempts in the current failure window. + Attempts *int `json:"attempts,omitempty"` + + // BlockedReason Reason automatic restarts are currently blocked. + BlockedReason *RestartStatusBlockedReason `json:"blocked_reason"` + + // LastAttemptAt Last time Hypeman attempted an automatic restart. + LastAttemptAt *time.Time `json:"last_attempt_at"` + + // NextAttemptAt Next scheduled automatic restart attempt after backoff. + NextAttemptAt *time.Time `json:"next_attempt_at"` +} + +// RestartStatusBlockedReason Reason automatic restarts are currently blocked. +type RestartStatusBlockedReason string + // RestoreSnapshotRequest defines model for RestoreSnapshotRequest. type RestoreSnapshotRequest struct { // TargetHypervisor Optional hypervisor override. Allowed only when restoring from a Stopped snapshot. @@ -1472,6 +1536,9 @@ type UpdateInstanceRequest struct { // HealthCheck Workload health check policy. Health is reported separately from instance lifecycle state. HealthCheck *HealthCheck `json:"health_check,omitempty"` + + // RestartPolicy Whole-instance restart supervision policy. + RestartPolicy *RestartPolicy `json:"restart_policy,omitempty"` } // Volume defines model for Volume. @@ -15789,291 +15856,299 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZI/+ioIntkYaYakSN1sa6Pjf9SS7dZ2y9axZM/ZbfpQYBVIYlQFVAMoSrTD", - "X/cB9hH3SU4gAdQVRZZkSbamHbsxLbOqcEkkEpmJzF9+7gQ8TjgjTMnOweeODOYkxvDnoVI4mH/gURqT", - "d+SPlEilf04ET4hQlMBLMU+ZGidYzfW/QiIDQRNFOescdM6wmqPrOREELaAVJOc8jUI0IQi+I2Gn2yE3", - "OE4i0jnobMVMbYVY4U63o5aJ/kkqQdms86XbEQSHnEVL080Up5HqHExxJEm30u2pbhphifQnPfgma2/C", - "eUQw63yBFv9IqSBh5+D34jQ+Zi/zyT9JoHTnh6ni5wqzcLI84xENlvXJ/kZZegO9IZwqHmNFAyTNNyiB", - "j9AESxIizhAOFF0QRNmEpyxEF0dnKOCMkUA3JkeMTyQRCxKiqeAxUnOC5lwqeEcJHFwhhScR6Y9Yp1tZ", - "D8L0k3A9lf4xJ2pOhGewVCLbCppygdScSkSZfhqQfnHBlEhJnbLdDg0jMlY0JjxVdUL9wq9RxNkMpuXa", - "RXEqFZrjBUGfiODojxRHdLqkbNZMpAmZckHQL8uExJihJMIBkYgqRJnibjaGRjmP7cU+5qIzxgUZh0Qq", - "yrBuf5xwYXZEefRv4Q8cocK7MDR4H6k5Vo7LGVfoipCkPFF8ja/KZPx9e7v7YjAYfOx2qCKx2Vb4hsZp", - "3DnY39vb2et2YsrMv4fZ6ClTZEaEHr79BQuBl4XpSJ6KgIwDGopVMwkiSphCRyfH7+44gc5w0If/23re", - "6XaGL7b7w/3n8O/hfqc4rRrhyyP/snrrnSusUlmXQWY3jS2jjAtMUp/1mzSeEIH4FAWpEISpaIlgS5Gw", - "BdOVpj3wLUXA2ZTOUuG2oG/Llcg5xxJhZoRGryIv8sZa7btAC7GQX7OxIDGmTNO4Noh37hHSOxTZTaSH", - "FHCmBI8iLRSUInGipNtFXS3GGcJJEtEARE9pU+3GA9npdlgaRfphZYT5apOIzii80Io0VBYWyX2LFEeE", - "KSKyHd6GNCWx2NRxTm7vauRysb0UlJQF/umyKs1jLeEFCcx0sxOgRJEJCXhMkG66vALbg+393mC3N9i/", - "GD47GOweDPb+q9PtTLmIseocdEKsSE8veJtlWi2/j3Iq6ReRfTE/qjy061dkcDt2ibBU2a6GTU7Vcow9", - "Y7qgMZEKx4ne2HoMBWI2bWvXYHUdHOVXEnj4VQRm5EaNLYW88/HxB7lJSKCPGO62Z3Zi6/a6iE4RRpkM", - "0OxqBOPKibz4qokIgqUesNY79On0eydlMk30WUjCcRJhpdvVSgqwwTimUupPsx9CKs3G7HYck48ZV2OR", - "MmZeZERdc3FVfNO2MqZJp9uZYzlezJK00111DpSZGrogEU4ktGdXXIyJEFx0jK65HE+5cIukD7GchCua", - "qlFIZmeWh0KdbqdEgEw+urm4cWer6h0c9AK8JIyabvRqmEx94MW26sPNhrZaUhqxbLRSt8zIfizLEiCk", - "eMa4VDSQreQmnMZ6eWMeekTncdYcoiFhik4pEVZRJUikDI411wjSjSDKUCor+yDTpcdkoY2f8WJ3rIKk", - "TpSKpVBcvMJhnx8xhWMuW/5sp6xh0vLcvZbIAlPYk8dkQc3RUlaG7NKMQ0EXRHjEd3aiGlFo3kMbeq9r", - "EcI4I5slSrEFDSluIw5CGNOYerjn7OgEmcfo5BhtzMlNuZPtZ5PnneYmGY49vPBLGmPW0xtCD8u1D+8W", - "2/5t16vz8zhOxzPB06Te8snb09P3CB4iBipjscXn2z7VLwnoGIehIFL65+8eFsc2GAwGB3j7YDDoD3yj", - "XBAWctFIUvPYT9LhICQrmmxFUtt+jaRvPpwcnxyiIy4SLsAIWrtxiuQpzqvINuVV8fH/zymNwjrXT/TP", - "RIyzQ8RHsBOnRp0cOz3Bfoc+nKINLUNCMklnM8pmm234PeCaHPqo8x3iMFRk39FmonJayp3P20AQvKY7", - "/UarzupbLTUrOY5lU+vuFS1RYxpFVJKAs1AW+6BM7e82T6awYcwJVevqpf4ZxURKPCNoA1wqYH4YYaoV", - "mymmEQk32ymzTZP5J58UjpASewNb9PAkGG7veGVHjGdkHNKZ9YlVjyj9u2Yx3Y5C8LZ/InCYt5sHdCnI", - "tN7fKxDd0IkgUyKI5vGv7C4RfEEYttbLX6Dfzv+1lTsLt6yncAuIeZa//qXb+SMlKRknXFIzwprksk80", - "GwGpEXzhHzM8WrXWBY6SCovV+wPeuIedmOt1a2lj3RZatcGztZ9c6HeqshNEY6ZLFKRAo4h8qZUaj3bA", - "mbIPKu5LPkMRZcbi0KqdWQvQq5YJ+SniIBLviQ4Z+eubX4/7DsLL/NDQmn7WzRTwiM+K1JwTLNSElIjZ", - "cITZhvLRNZL/rLR9KmcVlmS8WoKcUcZICP5iu7HNm1qN9ZoZsIuuqBoviJDePQfD+pUqZN9obCriwdWU", - "RmQ8x3JuHWxhSI2z8Kw0E4+2VnLEY7DHXYOgRYD9ev7L4fbePrIdeGhoPZf6hfpMCl/r5s27SGExwVHk", - "5Y1mdrv9GV3nED8H5M7KprMn40DHmEbSdexqWjs5lXPzF8huPSo4+7QY0OwV6b8/eiZ9BELCWAmNtzd+", - "HTDzDM8irmm6RCmjf6QlBbuPTqbgINYHBQ1J2EUYHoDfQdt/M8KI0HIq9wwVlGC0QfqzfheNtF7Y01pw", - "D2/3BoPeYNQpq7HRbs+Y9wlWigg9wP/vd9z7dNj7r0Hvxcf8z3G/9/Hvf/ExQFvN3GmFdp4bbu93kRts", - "UV2vDnSdKn9n6V8cvk/imKU+0XLitit9dFJXHMxcQx5cEdGnfCuiE4HFcovNKLs5iLAiUpVnvvrde6UF", - "zGMFEdhMk+mWZKgYPcDGGxG/JiLQEjgimvFkVwthqmQXYW03g/BC+pT8dxRgpveCUS64QISF6JqqOcLw", - "Xpla8bKHE9qjZqidbifGN78RNlPzzsH+To3PNZNv2D96H//mftr8P15WF2lEPEz+jqeKshmCx8VrPTeG", - "7Ipm1Yo46qYRqHkxZSfms2H9DurrVthNZNVKG2Oucam1EMpcZGsGUr/f1cZW7DEd3i6IEDR0x/LR6THa", - "iOgVsfsFiZShUToY7ATwAvxJ7C8Bj2PMQvPbZh+9janSx2Gan/LmyrZyu0aCOQdFJYr4ba7TQFMEAwdH", - "K8/xVaTxUvsoa7d+6v/CperFmOEZAXPUvogmgl8RPVBzJ0CJRFdkqbWcJZrpRnsLKuGGh7AFWmDjdeiP", - "2MWcS2JecY8k+PbpgqCYB1fm6nfOwZJf4Cglsouu51rlAJ8gwZH9GZmLsRGb60HKgCck1EaIeQ2mhi4J", - "W1yiGCewzbEgsMdRjBURFEf0k7nCh1sGElJ9wo0YgY2BEqz3fBBwEcING0cEB/MCFf4q0aVRWC6h+UvK", - "NFtfmo1Zuaz+3Hn7/uLnt+/fHI/fnr18c3gy/vXlf+qfzUedg98/d0yoRqap/EywIAL95TPM94tRb0Mi", - "Ogedw1TNuaCfjLfmS7ejaSA1f+GE9nlCGKb9gMedbudvxX9+/PLRKWTGjb3Q28AzsC9eZcicpR6RdOy8", - "gRJZD5O729Ak0yLq9dn7LX06J1hKNRc8nc3LG8OqBrfaEiGVV2PKx5PENyYqr9DJ1lukFRcUUb1BM0Vl", - "OBic/rwlRx39jz33j80+Oja7FoavZRAXVn+Sc80+WdTH0dl7hKOIB9aHMm264HVd+QQ8YUosE059RlxF", - "OOWv1mVUr5c/vYUo2ppQtiX1MvSC29Ed+ObOpsRLtqCCs1ibcwssqD6nZXmvvHl7/HL88s2HzoE+CMI0", - "sF7Js7fvLjoHnZ3BYNDxMajmoDUy8PXZe3PrCduG4EjNx8GcBFfrPvwF3j2CV2HHqSRKZ2NJP3m0kMOM", - "NCgmMRfG+rbfoI15WUkxWx7Buo46O69/Nnw5fA0s6dbTXi9lrZiGKzeCr3/2Mdp8mRCxoNLnovsle+aY", - "ph4pVNoW5oIt43fYAP2C6RNEPA17hS67nSkVJIDIDP2vP0isbYDFp/KNluc7v+esle67RqnFUUIZWaHV", - "fifa5TUXVxHHYW94z8qlvYv1RNWYB+X1zS7lHEvUgtUmmIXXNFTzccivmR6yRyTbJyh7OZPLN3omOPrf", - "//6fD6e5iTZ8PUmskB5u732lkK6IZd201/2STSRN/NN4n/gn8eH0f//7f9xMvu0kjA5zJ33Qrv9L00I1", - "3saGIRpPasOlcnbwZ7EuiltbHD5HjvfW3iD7ZDxfEBHhZUHw2jF1hgOQfpVRCQoBlsh+p8XoFdIfrxHD", - "ujWnH7yu+ge2B35B6xmUZ0w/a1lhz4U2I8kGMtw+tX9u14fUMKIrmoxB4R7jWeYuXhVNen5FE6vFwxdm", - "GaPICIIwBb1/wrnqj5gJbtFrBwtMbkgAMk8qrNDh2YlE1zSKwLkEQqV+tGiboBAVBa9Lpf9XpKyLJqnS", - "ij5XBFmTCzpJYSzw8oSglGF3lV5Ru+0E65EJQJYrIhiJxkatli0pYz5C9qNG4sBUp1ja6Dah0qRMr+Nf", - "T8/RxvGS4ZgG6FfT6ikP04igcxOYsFmmXnfEEgERDroTvZ+p7ZdPEU9Vj097ShDihhhDY5l7zt7zLl6f", - "vbeRAnKzP2LviCYsYaGNEXYnjo0fDTn7q96xJCw3W+y/QvSmaBDJcCLnXI2TLO56lXQ6t6/nVnx7P0S3", - "swiStLyk293G+NEFFSrFkZa1JU3UGxtg4t89FocJry9aPlbu5fG2qnyp29ZZY1qGYHhvpK3H52I0pdY+", - "l4IXoOZ9cSbq53aDXdP+CXMDWelzyq3Ur+jr3DRSi/sxP3fdzO5ApZOMJhVP1f2Q51AWrPpWcesmfMto", - "hBJtXOKE9i0fa9P/sosu/1b6Qe99Z1po/eIaGWqAPGH6p2L7VX/GWk/DrSLFi4uD5d3X41A2BkmhxRAp", - "gZk04W1znJA++gWEOFIkTrQkYzNEJcqiwhDj1/+OuFFq3KcjpocmTYiJJUfmb5J0xiibbWo1Xx9MOAyN", - "U2qaqlTo9xZU5tQss45z/NQCYs3oiJHHkFxBWRClIUGXzjl0WdYL666juklofUk1C8eQBCwbMPbUVpwq", - "3b2ecIxVMNd04qkyMWd26uV4wIqDat1drB1Ldkt3h/U/z8RFNYdm4TFx9OTs/RB4FAuuzSYPolVU/N7N", - "K7KEJXeeTFzzZRadmH5XoyCSRwtij92iG3QCWULcKE65B9T4Mq37Um//an6Mz7G3bik0vVqTv2wqeLKD", - "pOq5yeYcY7V/F07upJCenOmvqw1jSYD4YHocIFDHLrvGViLggUBMM0uEQipIoGrNUzYbMQg/ubS/9G1r", - "l3qTax3lXnKuIIUBlPbi0qLCyjq1D5rRU+MxVYqE3bJucEVIItdPSqvX1uftccwLci2oE2QuHrmlekbY", - "lIuAxNZI+DrD8WWhMa8Zd7sm6tEghr6FMbvUDkhsIaEJPTLrAR7aUsZHNfExrFhtJvqg3OUljqJLtGFf", - "2kSC/BOC+O1aMc5yZr84OnMskN2Yfzjtao7UUuByrlQy1v8jx3oXX1Ybs9+6HZ4npT0fgH21u7tjV9U6", - "3cyAK82W/WveiIrmpXHqd+OlnOYLPUobotJGlT/KP8mdsFeUhW0b+FW/2+idyxQjZ2k8tIMuEaSXJjOB", - "ITr3Pt1zd75yBWo2S/A1KcC+CMs8uTCVisfFUP2NSnQILceRlIm14FEvxAqDK7Olv9UMtx6zHC9NU8YW", - "a/LEjGcTT8gR/QRZBDM6w5OlKl89DL2JgF97/+3G4luWpth/Y0GScKz46uhnOkXu3TbBjiZVQfHxYkr5", - "6swQGzpTSt0zx5G1a3UTvSSg1p0AOk4wN8GphgigNH44LV779UesB8fvATrOOsiazZrEoFvi0NycbHBR", - "GITJAUGT5SbC6MNpH11ko/2rRNpgWRCXDDHHEk0IYSgF1zOchj1zFhcHkEo4NFX1c+s7MXkTm3C7ye2z", - "fpauDF6aLPkaoqwmtDIfk3QJC2WvkzEresFaea1WxYy/IzMqlahEjKONd6+OdnZ2XlT9l9t7vcGwN9y7", - "GA4OBvr//6t9cPn9p4b42josyxYbt1aUPkfvT463rbO03I/6tItfPL+5werFPr2WLz7FEzH75w5+lOQR", - "vyg7zgPu0EYqieg5Mam5yhdmV4hmawiju3N03AMFu+Wxu6veNZS40G8+RFaML97aRvvePm+lKjDXRmwX", - "Jle35JcJ2J35LilocDYwMqDeENBjKq9+FgRfQbZf/dyO8YzIsTnP/KEQqTTxOeTGejcE52oqzb1p2es5", - "3H22+3xnf/f5YOBJBqkzPA/oONAnUKsBvD06QRFeEoHgG7QBF14hmkR8Umb0vZ39588GL4bbbcdhrnja", - "0SEzvNxXaMNS5O8O4sQ9KQ1qe/vZ/s7OzmB/f3u31aisv7jVoJxvuaSSPNt5tjt8vr3bigo+hf6lS86p", - "KvC+pMxDAwyg/9WTCQnolAYI0nuQ/gBtxHCEkey2qrwnJzh0qav+s0NhGsmVEROmM/umcbTFaaRoEhHz", - "DBaklS8aZn4MLXnBNRjLUoVv15JNaVobIeDmkr2CSqlpJdKdmlzogvJESRQemB26Vs7BauYD+9jEB3YO", - "LbnhN2069SKyIFGRCczRZZJyBUEZn5hFK82KsgWOaDimLEm9LNFIylepAF3UNIrwhKfKXDPa3O68EwiY", - "BttjqsV1Ozv3FRdXa0NP9UmcpbCv9QodgiN9al01cIpjZL922Q0FpS+7DjSXpva5RO/MF8ZDlP+cpGVA", - "nC70ZD1JDAkiFQdJah2Gtpm22qVfbwFnqQv/MP3lsvORYl96UxMucL8WtpgRgG5QazUWzSkX8P45vN46", - "kl1/uNaR0oLujFw/BtEh1L+n2bYnGU4ehuKrgtEyX0P+EpzCgoakj2B3QVSMSy2s7LRzxZOEhJn/pz9i", - "NhQ8+0maGxT9oaGDmhMqEBd0Rssdlx1sDxnVdhtWdNx0Z3YsfljXUOEhhG80b3o8VQam4cplW5Fi6pNd", - "hE63c56BWlhJVCbNuwwYpEaRPEqzNsTXZ+9vG5uWCD6lPqgiiIWwT61l5qK2ftsdnPeG/4+JwNT8Bioa", - "ZSZ+IuZhBYPCvt/u5Hl99v6saUwZKgQqjq42pyziZRUulqOIvVSyt5LWgnHsrw+WrJNc937h02WnAsdk", - "kk6nRIxjj3PtlX6OzAsmtIkydPpzWZ/VenNbq/mstDhgNk9xYJP621Hf45CrTKNboOZH/3K9I+YYbkoF", - "1Esl7Ds2G7CP3mQ4HOj12XuJ8iglj6euvLyNofZn86WkAY5Miyazl7Kigw2Ys7WGfJZ/aF2RHj3ZD9/i", - "NgLaWMySFLbh+bveydsPW3FIFt3SmCCyaM4jose9WZAWC5cQmOcFlITEosnTYRhDtt1ABVplO7g1kQr7", - "1UMdxRWOxjLivmCNC/0QwUO08eGVSdjSI+iipLSU+vcCFUr8ve/dMVoiNXV7Dh1WXaalDe61HctAmsa9", - "UpheqVPfVjFR8nUdp46dxK/KC82v1uP1mEaa+z1ygfwVp7ZVJJGJ90cQ7+/un5H51HitrWtEkgQLrEi0", - "NJpFdvRFdEqCZRCZPU7qV4nkhgS3yCR4qV//YhKBU0HGai6InPOofAG9063jtkkIglwQi59h5lRwvCuO", - "Yiyu4GB0ijRKmaFAOWtgZx0m5Vyp5BaT+uXi4sxY14qIhYkoK0bpytrV6jGJ8BJNiLomhLmpYIkwes0z", - "oJJqWo1swDAQapwQQXmZhp0dT7/nJjATzQQOCDJfORhFuyQSTs22pLS9eLC4goBI2bC+w1Xraz+dplG7", - "NfYNa7gWdTS4zQJfHJ25XPwMVtCRebtO5TMiembLOXzB1Uu7LVejQriuGGek3pngEwIwETZzppjj4wJL", - "AAZDf17KqykIB1lMaLH9wC4wpOqaff6xlbJX3e6+i/QYs9AHz2gink3y4iyN9YLoIYsU7o5oaKJOTD6l", - "UcuLgduC4JAyImUl8ytIRdTpdnpTO6uDra2IBziac6kOdneGz7dWx++tDNy0cSrjkK6y71w0i4l3cAlS", - "BiYRJl1miS2cJC08YIaOa84HEE/1QDGAZNRnW0HDc5HDg0Etie0GB8oh24BLrHTliYvbNtEsWZoPNJjh", - "Au+9eFHcnwPvHXQOy+3Yf6vG+3pmJphM84hxN1ToaJj8k1ej4sIHX8KFsikqE+KizbLz0MVy2UuVUmfP", - "B8+Ls2yFfgzCprLN7b7zTNW8XU5bIwVy2/1rG4C4mLLO4bb0arAzTZc1PKUlYo2lVhOUJ4Tdip57uzvb", - "t6Nn24mcuKy6ilzyJc0fnR4bnSjgTGHKiEAxUdhCvReEDPiStJTRBn6ISQx5CtN/Xy1aGuIXilnwjTfg", - "RzW8tAe5/W7A+Xln4jdDFGNGp1og2zeLPcs53t7bPzBoZCGZ7u7t9/v92+YGv8yTgVstxZbJfyykCffl", - "/OvW4QFSgNvM5XPn7PDiFy3IUinMobUlJ5QdFP6d/TN/AH+Yf04o86YOtwKwo9MacF05Hkwb/Ob3gwLW", - "uNN7WuEJ+53BEBYKeAVenBeFZ1q3MRz3tYAud4Z8y3FHVQHqrZhN0wL2jX5afY3svErwju0zZYpGOSJe", - "/QL5TpiGciXsUw3yKSEsA3qKIvNXwNlC7wof6lPpJHLPvir4YqXq9Y+6xrV+vznFaw3b+r1smfxri3Zn", - "MWk8J9E3l/p3CVAq9/529h9//L/y7Nk/h3/89uHDfy5e/8fxG/qfH6Kzt1+Vfr4ajuibYgrdG4wQROWU", - "sITastIpVoHHG6UNnQYK2yfGtlbBvI+OwGt+MGI99BtVRODoAI06lfyqUQdtELAJ4Cut2OmmbJropv74", - "zNyd6Y8/O4XvS7WN0OaDCrsgWRq4TCchjzFlmyM2YrYt5CYiQQPWf4UowIkCxwVlSFt6SzQRUE7F3m3k", - "nXfRZ5wkXzZHDK4HyI0SegYJFirDT3M9AFPYUZmAS/s6CR0gj7leGLHsXMrweMwFVz9TcyGwoZqu4ifK", - "akvF2gjPBz7kIgiZ1wsZUamMsp1xtmajLJYfPR9s1i2XNdp0xkMr2A92Qr3OkmPKFnvJMDB0bQT32Dnj", - "1gQiaNlk9ggCW0lx+O85cg3ltMiW2HjITQKFNBesKpKF1InNjhflHVa35YTMDSN8FrVIuX5pcmsufjtH", - "iojYZTtuBJqcUxro+UHsJJUy1axIMTo8On252W9RKApom41/xTpeZDOsZsbaG8emi9TcsMMx6aKTY8ht", - "sjs0V+AgJvkVFygyAibf1wfovSQVGxGy1yAk0qxktMyvLc0JMOpsuhaTqqQ4QO8yvRFnQykVpyrfhOb7", - "Epq1USsmYLrWerdW9kU4u8iKNgiPxirLsNMnbrMoaO+osBSHPV8xq2+9t4s3yY1Gc2Ht7xut7v7VnZ3b", - "qTuu8kAyx9LH3fPiVQi8tKLcD634d0XzvX+p37UVd7LuoIiQPvqKn/tqpez1hsOL4e7tbf7bAo2VAT0K", - "YDAZ1lh7kLCHANuq2783VI0bI0KRfmzjP52V9+EUzbFkf1XwsGLrDXeetYKN1722jaUsRlHyqRlSJqUc", - "OkgWA2hwUq5oFJnQWklnDEfoBdo4P3n968lvv22iHnr79rS6FKu+8K1PC8wxJypen72H6zQsxy4cqTkD", - "B+dZbOSGSiXr4Cmtovq+BuPMfNoOF9xN0rSRA4SvBkr7pQRm5kXD2bxHhDMXilkj42Ngl33LHJfvDzdt", - "JdLZ18KVWePlgdDKGoW7D+mrLOfNz/eLO/Ygw1lbZa941rsExDsDfXU71JN8dSi1CCYhOjnLscZzJ6Nr", - "vjInW7JyOBj0h4M2LtcYByv6Pj08at/5YNuoFgd4chCEB2T6FS5fy9hGGcfRNV5KNHLm0qhj7LOCYVbY", - "ttakanU9XcdTuxt8WlWh8ctpo9i5e39p69M0KTctsmuqYC9xGpkEzmJNm3qZTpkYZDMD6pvpsiMGA+xa", - "hJWstCYOApHm/gxXM81ovmli+X7EBJEJZ9JUUuyjX8lSopjCHULWPUQOSZQFcYcjtiFcwH8W2Z/gVJJQ", - "/wDRtF0XtamHRhWAF+sPRkzOUygBt9lHR5zJNCbCunrQhIIfehPJ1Bh3MF6gBhQklTQkYsT0ax7stM+Z", - "on6wPxgMBllpus7Bjv73wMdNfi5aB3jXDsnua+HDVlWUOS/XkmltU31FwcVW4d5Oq7KB3var8W1uqwgK", - "eBqFWlGfaNFs/CgktO4eSVRepgek+Xt2xTRDlaZuw8wUR3+kRCzRh9PT0hWXIFNbhaTFxIGlG9aBJ7da", - "hu01pu3a0dwRVO4xgOSqx2pBnbl32Liij90lnBkObeFrz80bb5AwZWZpNJ+smFPFSxqSxThNfVqzfuTS", - "zN+/PzkuMQfG+8Png+cves8nw/3ebjgY9vBwZ7+3vYcH053g2U5DHbD2SQJ3j/v32ki+wk4u+m/sohC9", - "tXsbYkArh5SNa7umLOTXrapeZ73bIKZ13ddDFFsPwRvYDMWCoaUGKXFaKAlsAvsqZX8afD/7F4PhGt9P", - "u0rGDfL3QqQsMPhPIIkzr2qxhnFxsep1TG8vTmFALoB4HbWKnbcn2uBg78XB3tcSzQXBrhtjlZ0ecXGb", - "rvwdiGAlytZlehQ8CIUiwKBvGEenDcrtdDtZ3DD8DQdtJSYte9wqGL5pw3b9YmSV/G7ICTspKa5wD2qw", - "hMIDrQVk6USTVKEs1VCrF0cRT0NU8L4YaBW4mjgpKLG6GbgpsM4Zg5Flglq1sgugjAAJTJkWxHAloxux", - "CWQH6DW8C49wbPR7OwjMwsptBA6X5jZW7y/XtdG2Vw/53Cra8I3WuhHU5tbT1mSwTrrVTRjN5wC94fBN", - "pvYzXvX2mddB366/XvUMbtjMLpcBDJ1ZNe4AvcpUt0z5s8rehiT2z7EVWHni/WYp/dGueEdzS75yhcy+", - "bsdQtNPtOEJBBmA9F/B9zvW1/VdkRV+YAsER7OU81ypVNLI4kzATCtWpbRCuXtwm/cJiqpNwbAyTpqAj", - "k8BjjZfsI6e+fDhFG4Ao9XdkXZn6X5tZgFLprNt+sfti/9n2i/1WuBH5ANernUeQXlYf3FodNEjSsSv8", - "2jD1o7P3xggOjHkJvnE790KabiK4Fj165nkl2bzzF/0XRbiMkKemqrYdksXW+VKoHb+y7G9DlM0fNFrQ", - "6ZT98Sm42v6noPHwZl9uT7wuxbxIvdf/clK8b605K8mkZ+pg+BENgKGEbAT9eEckzACdE4WAf3oIB2A6", - "ZFlhluUcNIiluJexdnd2dp4/29tuxVd2dIWNMwZvjOdQtiMobDF4E228Oz9HWwWGM226VFmA6GTWrPTv", - "M2SraQ3KCml/ONjxcUnDwZ1zjW17ETeS/IM1zeykLNEhuS0z22q73EvtnZ3Bs92953vttrH1E47FzWoJ", - "40K/DXkskmxx5TdAm7w4PEOQWDXFQdlvMtze2d3bf/b8VqNStxoVoCAb9NJbDOz5s/293Z3tYTv0Gt89", - "vMVlKm3YsuzybDoPU3hWw0OKuujtNp0WPnXKMNg7EkSYxoeBi6GtnD4GpXQszGv5IrQ5GKxrunZwtfi2", - "leOoUrXZqAZcoJRl2Nj99Zdwd7tTaxbT5jxYL8brln2EmSaXhVkwxTDuQLtEkAXlqbyHhrgyyUbTiHNx", - "q2+bLJR3RKaRMhdfVKIPp38FIaKZC0lFkrLRZNlvBRjFHSd3qw1c4gk/VzcRq9VqtFn6VRPuNmzT7qpM", - "5NL2b8R8CbWoStn6ALgjHAUpwL/jbD31rAC9AXIpkyRamlDRKOKcoWCO2YxAJT5TbILNEEZzHoV9b/ie", - "fjKeei/O+TWKuEGrvCIkscjoZhD6M62z0AVBG4VcTmRYqVLhaC82UsViX5e5cc8b5SkIlr70gyyHUNMT", - "K14AUjSflHyMEZ9JsAIVBMH2q/i9CRYmthUzg/S/iI3xWAa/2danvWeIFentO0LN0cmn1qK1OgZk6BlK", - "4kBwKRGJ6AxQ5T+cVhK/VmQxZOlf64PayoNtwbrmKstzdsGZJlsXBPEdiJ7w8K85EoGHIQtkRbiY80bG", - "mKWAlV5gZHKTUGHYo11I2JxLNc4APW45WKnGgIOdCpKj/mTpipkDyL3jPRedaLsLuWzs5Z2+rnGVv6mm", - "ATbLVC9F/dTqZjzoY+M6pMlKFJUclqWKwXEb0J0cOJlKaJUW8F7QBuOqJJYK4L+bbcIj/Daq7qdmnto6", - "T7/tDs7b4uGshr85w2p+wqbckzR9i2tI53q2EXsJETEFJHgUEkZJ6IzH7D7S+rYg0y6SBIUpsZQzCqnA", - "luDYbG9IfGbOKUbZrCLrqx228QebMayGyYZ+7YttAl2kPz/rQqRAKxOaJhHOM7VaxflROfbfX9UbFmSW", - "RligKubTiiHLZRxRdtWmdbmMJzyiAdIfVC+ZpzyK+PVYP5I/wVw2W81OfzDOEw0ql8ZmcDbNxCxIpd98", - "Cj/pWW5WktzA9bJlvt+CVPs2cUPeaNlXNCIWFuk9ozcFRi/jyO5uD5ryHxsaLWU+1iG1biu5Lcv6drxD", - "uzrM6k56bilN3GvlrrTsiFx70weB1auyPeuuGLThQpEcTm+ZrgW83FaekHax3dWgOzeaLUmCcu+7z/ee", - "7bcELP4qX6dJIr9vz+YiXuHRbFip0zZus+d7z1+82Nnde7F9KweVi89sWJ+mGM3i+lTKy1acZnsQyzS4", - "1aBMhKZ/SA1RmuUBlUrF3nlAX1Zs3abggnxvNl1yRsWVdPcsZQ9oOx/jCm3psKRyFQqpb5DplIBROTZ0", - "6+WDqeTotRpDgBMcULX0OEzwtSmal71SAVxr400rD9ZDUtu2BefRkkumkzyNYcN1jv5mXOsVXnjeGvdc", - "ppMmN/7baq/GiZ/7gIpXRC1uaPLSjHV3QTafayxLsWb67wDiF/NC+dWoVfPGavCnKuQGXAJaeP9CJIUP", - "NLBy/tmPistfWc6C27ekJFcpvuoIbd6Ct7KhPSeyx4QO1ueiVOSDPQDv9tV4UqxIsLLkQ6l8QX7q3r7f", - "diX+69+ZE+z2/RUSD27zYRWaCvjRjsGSPG+7W2KJBm5SXKyvyfUAEMsmpuBOIMs2HOFRcJbtzw+CrVxb", - "jnOi3Lvn2qJPoxUltUooiCWzxUVxuFfKflQjibvIuvjQMN6sRD7tzv26msXxaBnUqrXAcSLIlN6s4Bbz", - "gjmuy0lL0lIgLJddk2gjxjdo9xkK5ljIytgZnc1VtCw7WXc9OYNfhTguiNKqc/sCdflqug/rNxp2OYut", - "+7bseSHDz184j4TjVWg3R9lrzmec4CXolo2G4LOd3cFgZ3twJ7ib+6rnV2inKU6y8J115pSuHostZFHp", - "9aIP14KasvCOTFIJguMDiKZKcEBQRKaQDJ4V21lr09e6Xj14e0lq080y/ncLZdfN+VnKeJJZVxYpyE2j", - "425pyxmGxef1Ya/IGM/ETFBLHfcEUu70BvsXw52Dvf2D4fAh8HEyIjWF8Dz7NLx+Fm3j6W70fPnsj+H8", - "2Ww73vEmqT1A6chyYkq1kqSdQ0JEtZpHtQqOJBFlpCezsLf1sacrZIG5SVq7/2/nfTAzWKksnJcnWdQZ", - "sMqJUy1q/xjptHb0K10o1eGfHK8e9p3iyKoD8TNYdSjAT+0GA3htw68FB0tZy3PnfeHF1ifPytjGdWeP", - "LxUFtrZ3lRso7uPnkmAs7bBVJ3b9VPOYcDMuqJrHq4+H7LUMagguwz9JFZbTN/voZMagdk/x5+zuoxhg", - "rj/udDvRp93ynrG/t0/ktdA6GQPapS6qAS3uBqA01GoqwCu5aSFMeAIWBAjx07A3fAE39NGn3Z8GvRd9", - "9I9CpEDXUKtIvqF7u/TroA0Ni4DYDkh1+OJW1+iOnqs46Ffqg3POD2ILumN5PC+c4s4KFzZdWuD8cW2N", - "K7mOD1aq2Z5m46KWFJIIL33lLg2kO2Cpy4p9WGQyNCEzymQXkRsndKrw4CiiVwSNOjsDOeogLtCosxeP", - "On10aKGqwFrN62KVmoeKSAU+obGt0m1rIjWHpWzH7bK6qsbD7fAL3Vce9azv189erE90WRdFt+6Y7H9F", - "VPVXmbvtTNxVOVi/YakymxSQ0uHFLhRKZhW4fluczqZDQHgrJKIfuLzrnGWtDJC54uc8IV004wrliRDt", - "sqFEyrz8UB4/uYEUnhWZT4Yhtu8lrS1LsaarxNfJMUoED9MgjwKOYNB53pZIK5hPK7T69fesD+nQgPD6", - "KRdovUOjyYOxHuCB3DSv9xtyowpdaoZtXurhYP1SP4gXpNtJk3C9DDMvtZNgt4IkWxNX6vHJlMle0QQL", - "k/nYQqK/K1KwbuRC2W8UaJUoTWxdGeCpOifJesmYGN+Mveg8xyQi+piqN4J4FOahL1TmUnS9SB3uP/e7", - "DfHNOIBU8dpAfiUk0bYKZLFCfzFmS+/AqrXY0MbAFdiQCJrvGUhTS63y4J6t1cQal6p9WbuKV9tkvRWr", - "CGbYXPdb085+ubbi6EP44b6lkvbWXi5UUFgcRlCGaub6N2G9gOLEosp5vbsn2+lkF9YybkKWqYb2Ft3M", - "h73/Mm5lNO4fbP309/+79/Fvf/EXMy7ZzZKIXkimcN95RZY9gMVF2kbvl3FVANJPK9O2BLAiOAanUXBF", - "jJMqxjfF8e4NMqGxfIPj2hTgojimLPv32gn9/S/N16wFMr4HObmWZb8a8fIhygko7o6jjZiImSt46KID", - "N/sjBgGHV2QpUQGz2Ko0jlH/KrNPtIoOTkscoUujBvYJW1yiCQXodzli2qrFQUASbU1Y7FZqavZwkD6C", - "4KjYjsVOdtH8Wi6ZrHGbXVUD5Xn7/uLnt+/fHI/fnr18c3gy/vXlf2pthFz3TA9hT/Pe7t6+rdRTpOTQ", - "s8R3xgf08YtBHPEwCIS+Qv0ij8ZLJSSuuPjswstog8SJWrqiAi6CdvN2CCiHWYPeS/N7RlsdvLgPcPn3", - "K9HkFzzqaZW4ASzP64E0tPAGfUFTJpiu0+SZnnnKfZ5bd+CMzrDHGe2tNHYfIPBuQGuhaWrr3wjh7A/B", - "O64iCZrtbEhVQb6rGJZS9Zoj9GKtCY3zmk5lKIyU2SBWC6MyS6t1ZLZiprZssQZf4kyoj87VYcv5LnO4", - "CD34aH007kpdvDCzwkia1+bUqZwVpXgFgc40aa7nRJDCQsAHOULbLUlmQ0pbpGMZSPWEiF7GEi4eFWqc", - "Cgoxqpm3wJEgCzuuu1BXI/Cd4pusB3C/Y1m7pIJ55Fi4w9c/Q93id64kGZ26JmAYFYPAD7dW5qJVNHFc", - "VV+MIlfV523e9248K6tWSL+mvVVhzryPEmv6+PEfmKpXXIAJ0Zz89OCobWCehERA9ncVk60VoBmNSTjO", - "CjM27X9Xi9FkPmVlL/NyDc5cwsDEWsith9R36Tn5GOqU1uQgQSqoWp5DATeDyEqwIOIwNRve1YGzP+cd", - "Q/WDL1/A0Tj1xDq+JowIGqDDsxPYjzFmoGWjD6cFxHIDXl9DagH98O3RiTVRHdgPmBxUAevpt2PMdPud", - "bmdBhDHTOoP+dn8AmzkhDCe0c9DZ6Q/7g46p3AdT3IJiSfCnTWPITJ2T0OpBP5tX9FcCx0QRITsHv3vS", - "ARQRpviSBIUVzwomR4KpsDZHEkGSgmEVqr8F+D53lB6Y89gW3mvtYZNqaUM2SfLWLutHzQlm18AUtwcD", - "C2am7MELAacmym3rnza1M++3lT4H5PFg2dVMAqdTWpJ/6XZ2B8NbjWfVMGDH+rp9z3Cq5lzQTwSGuXdL", - "Ityp0xNm4siRQSSxkTLFfQYsVNxhv3/U6yXTOMZi6ciV0yrhskkZJtrsZ+TaVv76J5/0kb0/AHh4Oedp", - "pKUJMkHyzlOgsOjPPiEsgjldkBGz53ScRoomWIAfIEb6fDYWT3lrmK7N6mfpiz/zcFmhbtbclm6u57zG", - "OYGrMNCSjAHtcNxU1i/3F1OmDXv9icXMzupb1cNxtLgcy4D7opYvCMNM9WRCAjqlAYKX9e61Lmlvg60A", - "gLTAg2UhAnA5nItle9Of+QLo3v6ksePsGbLkLasTDO5xgigNc53LBWNjMcFR5EWImEV8gqOxoc8V8aio", - "r+ENS5QiELpTbhgPiQG1TpZqzpn5O52kTKXm74ng15IIrQLZYiWW1iQ05UkM614DolgMBUNMKTTd55YZ", - "4tbnK7L80h+xwzB2Ze6k+QRHkutT0xYXAoPARe8a3vXDrzcEhhylUvHYslRWaSkfJk9Vkip7KS6JshVW", - "4HUqUZLKOQlHTHH0WZAZlUosv2x9znv8ArYLwaHmk8IrZkpbn2n4pWnUcoz17Mfwqsf6I0CAUUefLqOO", - "/nsmsLZdUjkHL4gEz8esuKQbWda+1gs3qxQOMEMJTwziATDVHGuWK7UB1apwFCEFW8l9q7VNWMmG+dgk", - "pnjSmMFkUk4q24gydPpzYTMNdp/795MkgSA+B8d/nL99g+Co0mtgXss9TuZWmulTFIUpaPLQe3/EXuJg", - "jozeBKh2ow4NR53Mugg3YayptCHWvR6ouD9BeWvTTZeGP/X7uimjPR+g3z+bVg70XkriseJXhI06X7qo", - "8GBG1TydZM8++gnalAhyXhIEaMPI/k1XaxAAKfJj0JwbmIWIW1kbLRFGuQQq+lEmlGGxslCih/SWgtqU", - "xzNZJMbnEThfR52DkXO/jjrdUYewBfxmfbSjzhc/BawS3QyhZmpFOl07Y6L9wWBzfYampa9HhS69qLff", - "l5r2tX1viodVuuqKh5mcw3/UK2iqfhp16xE0n59x6OpI/VDx1qh41nNRUN7g++I5YNg3IsbArWhg2p6N", - "nAa20joxbAGwzGBxuHxqY3BQp8HlzFs0P6rmfN2s2G3aZQEMMXL8t/sI/Af9ZiWHTL8vHqtfHAGaqYPH", - "fWLsCIvlGLHrt4hfE/U9cNzgsUSphV79lvz7VPjnNbF6X060ijTbIgt33+RHjYBsEWlbMS9rW/UcxtQ7", - "J0yhl/Br3/7XWTyAgXwZ8dnlATIkjPgMRZTZi7zCbZE+FC0t4SOTMJJ9Z/NHHGTXhjk///e//wcGRdns", - "f//7f7Q2bf6C7b5lcFQA4vdyTrBQE4LV5QH6lZCkhyO6IG4yAMJJFkQs0c4A1MxEwCNPbXI5YiP2jqhU", - "sMKFp0HPkrZBMD0YzIeylEibcKNfpFML7WEczB4T3u1lQ8pH3dFdD3g9zKAwAX0qOh6AXG1bZsXaXx2/", - "98zMueQ/q/rKax7T9fJFkRtluLdnBnhLAQMk9u07eGAnjTbOz19u9hHYGIYrAL4FNOa8Gas893/IpPUy", - "yUiUskABKhvZZLLQVvt/j+077RzAtsU/kwfYInrdwgVsXB5QH9StwA9boYU72E835xr2+WePXZpls4P2", - "7vMtduECkVoZwve3zo736jQ3Twok+xYmMNpw4eyuRPTZ0YmrXbf5zZj+UU4NPVNbESg7OhA3hakfzSw7", - "4mwa0UChnhsLlH+ISWaqlRnkqYiDd3bUCLt5VYESi+fbVgn3p/GkyyCA8iPv4U+PSqe3OUZyMMec136c", - "JOtY55jKgOtvC9zSC3BiCzob9SXbp0UuWueQMtHx2ZGzUl2y4vnk2G3Ix3NN2a5TVj0bHkEoHlcE4jcU", - "hJUisQX406fEze+zVXSQEis8V98Xaw4eTwt6bC+Wj82fkhsrrJBNS0ETld14gL4mysRidx5woW0Pnomf", - "E+F2tUOrhlln0zKfmvJtZkJwIb3a9j0xr7QzfU17fybLF8hzG43FkvyHitLC2M1ptcrAPbGFUR/OvoUe", - "bmXe3t89r2UwD5Eh2GTiPNamuh+WSxZs/qmueh/lNDPEfpKH2VkaRe7GY0GEQm+PTszOKp4BW58hLGm9", - "bu9228rj4P2733qEBRzi0LIYKr8SZZ/cs4ZvFsxM5QebtLEJTV4zdedZk4bzFetvC+CbCMc+5f+2/Sqi", - "E4HF8t+2X+EooYz8285hhBWRavPBmGXwWKL5sTXuJ8x8WuGmZaKBaGJQVXadhpq91VJJde//qfRUM+lb", - "aaoZXX8oq22U1SK5VuqrdikeVGM1fXyjK5mM2XzUhkcuPvFPpqk+rpfPcqRDeqayfO1hS/lwAX5eeEQZ", - "SiV5ggGUNOO44rHR0l2db8iVx4dj3ZPjLhCyq0kHsEs2QeSRnNduHI+u3Np+H99zfRhP6CzlqSzmnsRY", - "BXMibbJSRMoC+Kmp3fnx3Kh4f8dcOnjMo+PR9eoffP9AGn91QY3wNjdQ63R+91Zbnd++r3V+k0Jtc9cs", - "NlTX4QZuNgQVuiTqtmxcyjWvBzv6xuWzRdB7bajk5gICC+JgxP6Ptj9+VwTHH39ySTLpYLC9D78Ttvj4", - "k8uTYaeOVQhTglqY18M3x3DtN4Psc0CCzVPyquMwpSOA9Rz2zb+cgZTffLa3kBwX/rCQWllIBXKttpDs", - "WjysiVTGz3p0G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF5ZrRYkZgszPsHcMmbvhwrBHaWDtrWVlG3KNQpo", - "juv/6IE9JzmQ4WMbR66EwNOMkeeJxeS25kh+GDbbI98bPwweVzg/vh3ylFnMKPx10iVap/QV7ASQyDhV", - "EJSYI4RA1CcSRmvPWuyjvE6mTJOECyUN0CQowAaKfq4VYB8oZRln0gcsCfDBlMjuiEGpAf3Y5PJvXZGl", - "gZGknGWIkdlMLXSkL/eqDOP5TbfR/etYfozSVjrWI29jizr97XSsbyY6HkXTOimB+W9kGwMMygnJdjLP", - "kvvoJ8pmm08qAtUIq2xuBTwjj6q1BaX6LDDvlsxKAjcdtAVsXlvI8l/wxK1P0qe1OyzaAgFRSPGMcalo", - "4BJ3q3DnP07o1if0asp6uXlqC7H6DfpXXFy1PeI8hcGewElXnOF36EvQwwM0sG/vUgBj25wGmmke/RSs", - "VXv7likYtHouBlEa6oPQHYhOlZwKHo/tjwavVu8KiwYKLorAtvqthY3u/REcRm+4QjROIqK1eBKinuEm", - "vZpW9Xd48VQWaiPeThjqbVNMiDFgdNLVFrIiEi7X3IJtwD17fbm8UjPis/UgGFnnDvHBg4IxYgbPnjjw", - "+0uUCVmovkUiEih0PafBHBAxoCIXlGQFsAqcJJcZBNbmAXoNO7WIBAadb0gitCEUcCZ5RAzQxSKOLw/q", - "iK0fTk/hIwOGYbBZLw+QQ2nNDgip3yoiXGRFi95Y3I4NzUmCR5FZ0UttNRbmt2mxL3KIshHz4WAwcm0b", - "pFN0WYDEuGzAxHAC9Tc++2baVrcZWNLMRXEkgHCGNwkLO00XMTTyo2EMB96CLi2ROcwwHhiYozaY3/gs", - "A7UssTJOkrbsa4cJXLyI4xU8jDYK1VWlCnmq/i5VSISAjy13NzE32sCB+YfCV5pRbWWgrD4tsJ/3utGg", - "zHlJpYVqoQyO+dcijjvdjh1PAZ3uFtr7GoSTaoP1azG9MgUYkx96920ASsrCvoBQUjk5bP3+ZpX7nXnh", - "T++ftYQK/wxelvJ9Vj4KyvIKTgJKsLsyWk8K6QAWsqaLmcpGvj3iZtmTheqf7a63anVDvwOjdd2tV1YE", - "MqtQ+djXX/URPOUkGFmbzZSLanr8unux756R7m9JalNtwyE/ePP27rlWjJmkK4qBQi1TCX4+KJAJuM7B", - "nHNZYPsJmeMF5cIisFuva8aZ4LIw1qONnrvUrHpp/beXVj0/sL4mhIuPbB99+NzG3Pm/cI/yL14VrO1M", - "4nedSg0okBJhNBGUTFGCU0m0tpTGBJkKIxbIm+Bg7sp990fsYk6QLXBZcCBk9ZCpRJfD+LKLJqlCERYz", - "sHbMQxNJJ0jA45iw0BStHbE5wQuqTTWBIqwIC5Y9SaCI8YLkBUy06W5vKE2t7KxMahe56rrgYLgs1M69", - "RIkgwETGXGalQrUjJlL27wa5Ujd76QZ6iYhUeBJROc9qRQQ4JCzwwkKef99i7P6duOdE1cvLfpM7yzvJ", - "0m95iVn0ZWYFvr+L+80nFqjFhSuN2ULMr1B6ZbNpWI58PM9L6v4LbmkzVzfHb3Qzk5F41S7+Pq5kSjX1", - "f1zLKLslw9R0R8p15/+0dy15IeiUla5brE/2rhcuWSWEjMy3knlbn92fJ3fwkX0nkrDbaNg3YW7nk/4e", - "RK6l6p1k7jdyDlpfUsEr9g1FsB3Ut1OfuChIue9CDJsNl0njosxRAoNNxdkPYVwVxjY84K7C2Hlcaxfg", - "BfFMWS+JcJNczsvO+wWwdQj8i0a/VmZXEITfXPDlNwKPJuxOMvFmBF6ClxHHf/Z7mYALYRI6bTnipwMo", - "VvAFFi6YNsDj1s0kRNdlk3w4Pd1skhJCrZQRQj1hCVEuaxrEnmqNbxdECBq60pFHp8c2epVKJFLWR29j", - "CvUcrwhJoFAM5alEkJnb1/Nzqa31InilHNZuhzAllgmnTK0dRf7qwwzmy51K5z2ynLSQin/6y2Pwwj89", - "IQWyQ6srdgKrrUiFVWMwngtOo8zUu9TaFp7wVLeuJYsrtDuDs21KIyKXUpHYROZN0wg2EYDu2ppM9juT", - "UdpFVEmk90MXMvASImIqJeVMjpgt/54QofvWn0Px3zzIyOu8VziTmmdG9H0fAWx6MCZmC6smqgG0ANQB", - "7Rx0tnCSbEG5aH+QlB3eVwzpFUSkIbmMJzyiAYoou5JoI6JXxuhAC4ki/cfmypC2MXx33xWn7r6zNKVP", - "2JR7i3IYns2Y+c+RhFQWa+4S8cmJtdekuFmc/IGF9os1uVauCYKjnqIxyZLfUapoRD8ZUacboVLRwOTV", - "5KmXUITZZl+O2ClRQr+DBUEBjyISKOdc2UoED7ZG6WCwEyQUUEp2CAwOBF7z4xh6PDp7D++ZQtHdEdP/", - "gIYvDs/MTewUWx9BYaCMqGsurtDJ1ts1Qb7nQKZ/4Sg5M8GVOZDeBf9xfXf7zObGPSQbtihPVhlAPPnT", - "h3FaDe6Ht+BpegsAWiKbzcZM4ACUYjlPVcivmd8zsOBRGut/mD9O1gGUKBzMP8Cr3422a4azths3wSex", - "Ke2cQmKKBn2TCwpDsKcaX6oJ56YASkwpcs97ChyqPyN3379TvkjH7/Bq0lLUFeT6bvbWY598dgwOd6tI", - "j6eyzQ2nuZkovtr7dI1ps/fp54gHVxKlTNGoBGqg7TbAAdU/5riN9uIP1ATIjnSlxBG5SagABJsKPAIi", - "esYSYaSIiCnD0RbM2TQCCJTOi4UXnEKSchBRSBOjIUEJjyJA2bmeE4b0bMBR5Roo3NNKWwGi+E7xilFx", - "NCEBj4lD5dz0mW7/wFS94qIMsfm9yMWLAv31fPRU9TzXoIo29/hVKKOn+AbCmsPUXhO7EW285vmPxhXU", - "RbA2o87OQI46XTTqbMejjl6BIwwuVKzQHoopSxWRfXRs/FuQhro/QJIEnIXSgYM6D97OQDYlpRq2bMhw", - "3IfvHlPtsVwFpHxnO/GJB/0e0t9Dgg3aKG44uyfDLmy6EPFUQQC321f2rZAocI9sPvoNbGGP/LDt20jy", - "f9jtW5JRsMpaXBaW3kj2DD5yrdfNJVXMucxRJ1GAExxQtewiHEU8yL0HqcxuB3rZUCaC4CttQ/VH7F0G", - "XGkTIdDR2fuuc5qhkMor04L1i/XR2wURMp1kg0MgDYwHDxaDhCOmOApwFKSR5lsynZIAchgiGlMlG/xq", - "2VAesgxi3oln4d3DDLbmaTmT/DwBq5ezhaxw3JZZ6i1BggjTuOhUqhIHVF+40gW370Q3yvUxPI3s9VYg", - "uJTINtUjEZ3RSWQva2QfXWiVA8dkxJIIM0YESqWJO9JD7yWCSJmaxBjdANSZNRzVRTnQSSK4sm7iiHMh", - "jWdXc/iHUyQVSVaw2TvT8inM+YFggk3jtqdvZDBUxtB8LNlXkF4QwymG4JqP9DH9DYJ9zIC+NZzwU9n4", - "F4LOZkToXYGNkDVXo2ZbO3KaTV/K9GjEyD/P3mqHkZ+1WojmLkQ6rwSqGLsXx6BA3+YG1tP5FW3EMrGP", - "bpd98av+qGXf5Sh//yDso6+c5Z+l9Nh5Ibi6LbJ+zuFPDeS+MPLSVi0lKKyHI2idkfCQGQKtcQe+GdzA", - "U0YZwKW0gyY4ge+PEQaPmx332DDbT5u3SigBpcI6DalS6+E7vwsOfBjczm+cHXoH3M7vKl8JcBe/Xd7o", - "d5WpVPIDuuIhf3pkzodKUDLwnABj0ZSgZKSeDSRYaSh9sO+0M5Nsi38mDd7ePd9Cf3dk/2H1tzAZCsTy", - "u+xMbrTDbSFxopbucpFPKxeAkn6CZAwf8EMWQ/BweAt3uF6/P/ZwfNp4uf6jntaj3d/nRYdPjp9+Ea3i", - "nisdLFv61OlhEczpgjQ73cs72JIoEaSX8AQuV0JDMEsPd5YpLPqzT8g2b7Gq7L8QdRDHJEQhFSRQ0RJR", - "pjhIBNPHXyUSXFsC8JyLpc+ZXty5rwSPD+1s1pyHdk9ZZ1h+5xsveyFWuLdw0maFC+0rbtrd3bYWeIgy", - "9PpntEFulDCIu2iqLR9EpxlJyU1ASCiBJzeLAx4OGjyb9BMZzyZtRrkCO/mtxaZGQSoVj93anxyjDSi2", - "MCNMr4VW9aegySaCL2hoCpHmRF3wyFB12EDQ2/pdtVKRVcpwxoUZ3DfRYdocSLNPNCmLBRO60DnoTCjD", - "MLi1KMXlPWUSqnR/mEJaQ753HOd0fhxh1vLbcMaO5kRt5DgiKs4NNN7mj2PuKR9zxcBUd6aVTrt2pSLb", - "xaq2DCF9CMDcLI75cd3WH76f8Eoqn2RkpXWdLzKDtMlt/n2x4ODxzofHdpd/eMLh+K+JM74LrnJoQLfo", - "Y5jfeIAjFJIFiXgCVSTNu51uJxVR56AzVyo52NqK9HtzLtXB88HzQefLxy//fwAAAP//QhsHTwiGAQA=", + "H4sIAAAAAAAC/+y963IbOZI/+ioIntkYaYakSN1sa6Pjf9SS7dZ2y9axbM/ZbfpQYBVIYlQFVAMoSrTD", + "X/cB9hH3SU4gAdQVRZautqYduzEts6pwSSQSmYnMX37pBDxOOCNMyc7Bl44M5iTG8OehUjiYf+RRGpN3", + "5I+USKV/TgRPiFCUwEsxT5kaJ1jN9b9CIgNBE0U56xx0zrCao6s5EQQtoBUk5zyNQjQhCL4jYafbIdc4", + "TiLSOehsxUxthVjhTrejlon+SSpB2azztdsRBIecRUvTzRSnkeocTHEkSbfS7aluGmGJ9Cc9+CZrb8J5", + "RDDrfIUW/0ipIGHn4PfiND5lL/PJP0mgdOeHqeLnCrNwsjzjEQ2W9cn+Rll6Db0hnCoeY0UDJM03KIGP", + "0ARLEiLOEA4UXRBE2YSnLETvj85QwBkjgW5MjhifSCIWJERTwWOk5gTNuVTwjhI4uEQKTyLSH7FOt7Ie", + "hOkn4Xoq/WNO1JwIz2CpRLYVNOUCqTmViDL9NCD94oIpkZI6ZbsdGkZkrGhMeKrqhPqFX6GIsxlMy7WL", + "4lQqNMcLgj4TwdEfKY7odEnZrJlIEzLlgqBflgmJMUNJhAMiEVWIMsXdbAyNch7bi33MRWeMCzIOiVSU", + "Yd3+OOHC7Ijy6N/CHzhChXdhaPA+UnOsHJczrtAlIUl5ovgKX5bJ+Pv2dvfFYDD41O1QRWKzrfA1jdO4", + "c7C/t7ez1+3ElJl/D7PRU6bIjAg9fPsLFgIvC9ORPBUBGQc0FKtmEkSUMIWOTo7f3XICneGgD/+39bzT", + "7QxfbPeH+8/h38P9TnFaNcKXR/519dY7V1ilsi6DzG4aW0YZF5ikPus3aTwhAvEpClIhCFPREsGWImEL", + "pitNe+BbioCzKZ2lwm1B35YrkXOOJcLMCI1eRV7kjbXad4EWYiG/YmNBYkyZpnFtEO/cI6R3KLKbSA8p", + "4EwJHkVaKChF4kRJt4u6WowzhJMkogGIntKm2o0HstPtsDSK9MPKCPPVJhGdUXihFWmoLCyS+xYpjghT", + "RGQ7vA1pSmKxqeOc3N7VyOVieykoKQv802VVmsdawgsSmOlmJ0CJIhMS8Jgg3XR5BbYH2/u9wW5vsP9+", + "+OxgsHsw2PuvTrcz5SLGqnPQCbEiPb3gbZZptfw+yqmkX0T2xfyo8tCuX5HB7dglwlJluxo2OVXLMfaM", + "6T2NiVQ4TvTG1mMoELNpW7sGq+vgKL+SwMM7EZiRazW2FPLOx8cf5DohgT5iuNue2Ymt2+siOkUYZTJA", + "s6sRjCsn8uJOExEESz1grXfo0+n3TspkmuizkITjJMJKt6uVFGCDcUyl1J9mP4RUmo3Z7TgmHzOuxiJl", + "zLzIiLri4rL4pm1lTJNOtzPHcryYJWmnu+ocKDM1dEEinEhoz664GBMhuOgYXXM5nnLhFkkfYjkJVzRV", + "o5DMziwPhTrdTokAmXx0c3HjzlbVOzjoBXhJGDXd6NUwmfrAi23Vh5sNbbWkNGLZaKVumZH9WJYlQEjx", + "jHGpaCBbyU04jfXyxjz0iM7jrDlEQ8IUnVIirKJKkEgZHGuuEaQbQZShVFb2QaZLj8lCGz/jxe5YBUmd", + "KBVLobh4hcM+P2IKx1y2/NlOWcOk5bl7LZEFprAnj8mCmqOlrAzZpRmHgi6I8Ijv7EQ1otC8hzb0Xtci", + "hHFGNkuUYgsaUtxGHIQwpjH1cM/Z0Qkyj9HJMdqYk+tyJ9vPJs87zU0yHHt44Zc0xqynN4Qelmsf3i22", + "/duuV+fncZyOZ4KnSb3lk7enpx8QPEQMVMZii8+3fapfEtAxDkNBpPTP3z0sjm0wGAwO8PbBYNAf+Ea5", + "ICzkopGk5rGfpMNBSFY02Yqktv0aSd98PDk+OURHXCRcgBG0duMUyVOcV5Ftyqvi4/+fUxqFda6f6J+J", + "GGeHiI9gJ06NOjl2eoL9Dn08RRtahoRkks5mlM022/B7wDU59FHnO8RhqMi+o81E5bSUW5+3gSB4TXf6", + "jVad1bdaalZyHMum1t0rWqLGNIqoJAFnoSz2QZna322eTGHDmBOq1tVL/TOKiZR4RtAGuFTA/DDCVCs2", + "U0wjEm62U2abJvNPPikcISX2Brbo4Ukw3N7xyo4Yz8g4pDPrE6seUfp3zWK6HYXgbf9E4DBvNw/oUpBp", + "vb9XILqhE0GmRBDN43fsLhF8QRi21stfoN/O/7WVOwu3rKdwC4h5lr/+tdv5IyUpGSdcUjPCmuSyTzQb", + "AakRfOEfMzxatdYFjpIKi9X7A964h52Y63VraWPdFlq1wbO1n7zX71RlJ4jGTJcoSIFGEflSKzUe7YAz", + "ZR9U3Jd8hiLKjMWhVTuzFqBXLRPyU8RBJN4THTLy1ze/HvcthJf5oaE1/aybKeARnxWpOSdYqAkpEbPh", + "CLMN5aNrJP9ZaftUziosyXi1BDmjjJEQ/MV2Y5s3tRrrNTNgF11SNV4QIb17Dob1K1XIvtHYVMSDyymN", + "yHiO5dw62MKQGmfhWWkmHm2t5IjHYI+7BkGLAPv1/JfD7b19ZDvw0NB6LvUL9ZkUvtbNm3eRwmKCo8jL", + "G83sdvMzus4hfg7InZVNZ0/GgY4xjaTr2NW0dnIq5+YvkN16VHD2aTGg2SvSf3/yTPoIhISxEhpvb/w6", + "YOYZnkVc03SJUkb/SEsKdh+dTMFBrA8KGpKwizA8AL+Dtv9mhBGh5VTuGSoowWiD9Gf9LhppvbCnteAe", + "3u4NBr3BqFNWY6PdnjHvE6wUEXqA/9/vuPf5sPdfg96LT/mf437v09//4mOAtpq50wrtPDfc3u8iN9ii", + "ul4d6DpV/tbSvzh8n8QxS32i5cRNV/ropK44mLmGPLgkok/5VkQnAovlFptRdn0QYUWkKs989bv3SguY", + "xwoisJkm0w3JUDF6gI03In5FRKAlcEQ048muFsJUyS7C2m4G4YX0KfnvKMBM7wWjXHCBCAvRFVVzhOG9", + "MrXiZQ8ntEfNUDvdToyvfyNspuadg/2dGp9rJt+wf/Q+/c39tPl/vKwu0oh4mPwdTxVlMwSPi9d6bgzZ", + "Fc2qFXHUTSNQ82LKTsxnw/od1N1W2E1k1UobY65xqbUQylxkawZSv9/VxlbsMR3eLogQNHTH8tHpMdqI", + "6CWx+wWJlKFROhjsBPAC/EnsLwGPY8xC89tmH72NqdLHYZqf8ubKtnK7RoI5B0UlivhNrtNAUwQDB0cr", + "z/FVpPFS+yhrt37q/8Kl6sWY4RkBc9S+iCaCXxI9UHMnQIlEl2SptZwlmulGewsq4YaHsAVaYON16I/Y", + "+zmXxLziHknw7dMFQTEPLs3V75yDJb/AUUpkF13NtcoBPkGCI/szMhdjIzbXg5QBT0iojRDzGkwNXRC2", + "uEAxTmCbY0Fgj6MYKyIojuhnc4UPtwwkpPqEGzECGwMlWO/5IOAihBs2jggO5gUq/FWiC6OwXEDzF5Rp", + "tr4wG7NyWf2l8/bD+5/ffnhzPH579vLN4cn415f/qX82H3UOfv/SMaEamabyM8GCCPSXLzDfr0a9DYno", + "HHQOUzXngn423pqv3Y6mgdT8hRPa5wlhmPYDHne6nb8V//np6yenkBk39kJvA8/AvnqVIXOWekTSsfMG", + "SmQ9TO5uQ5NMi6jXZx+29OmcYCnVXPB0Ni9vDKsa3GhLhFRejikfTxLfmKi8RCdbb5FWXFBE9QbNFJXh", + "YHD685YcdfQ/9tw/Nvvo2OxaGL6WQVxY/UnONftkUR9HZx8QjiIeWB/KtOmC13XlE/CEKbFMOPUZcRXh", + "lL9al1G9Xv70BqJoa0LZltTL0AtuRnfgm1ubEi/ZggrOYm3OLbCg+pyW5b3y5u3xy/HLNx87B/ogCNPA", + "eiXP3r573zno7AwGg46PQTUHrZGBr88+mFtP2DYER2o+DuYkuFz34S/w7hG8CjtOJVE6G0v62aOFHGak", + "QTGJuTDWt/0GbczLSorZ8gjWddTZef2z4cvha2BJt572eilrxTRcuRF8/bOP0ebLhIgFlT4X3S/ZM8c0", + "9Uih0rYwF2wZv8MG6BdMnyDiadgrdNntTKkgAURm6H/9QWJtAyw+l2+0PN/5PWetdN81Si2OEsrICq32", + "O9Eur7i4jDgOe8N7Vi7tXawnqsY8KK9vdinnWKIWrDbBLLyioZqPQ37F9JA9Itk+QdnLmVy+1jPB0f/+", + "9/98PM1NtOHrSWKF9HB7745CuiKWddNe90s2kTTxT+ND4p/Ex9P//e//cTP5tpMwOsyt9EG7/i9NC9V4", + "GxuGaDypDZfK2cGfxboobm1x+Bw53lt7g+yT8XxBRISXBcFrx9QZDkD6VUYlKARYIvudFqOXSH+8Rgzr", + "1px+8LrqH9ge+AWtILCzx0kWWbqK/u/M27mZ4pmTZ0o/a1Fjj5U2E8nmMdw+tX9u12fkn5C8pMkY9PUx", + "nmXe5lXBqOeXNLFGAHxhuCCKjBwJUzAbJpyr/oiZ2Bi99MAf5JoEIDKlwgodnp1IdEWjCHxTIJPqJ5M2", + "KQpBVfC6VPp/Rcq6aJIqbSdwRZC12KCTFMYCL08IShl2N/EVrd1OsB7YAGS5JIKRaGy0ctmSMuYjZD9q", + "JA5MdYqlDY4TKk3K9Dr+9fQcbRwvGY5pgH41rZ7yMI0IOjdxDZtl6nVHLBEQIKE70exIbb98iniqenza", + "U4IQN8QYGsu8e/aaePH67IMNNJCb/RF7RzRhCQttiLE7sGz4acjZX/WGJ2G52WL/FaI3BZNIhhM55203", + "17l9Pd9d7d0Y3c4iSNLykm53G8NPF1SoFEdaVJcUWW9ogQmf9xgsJjq/aDhZsZmH66rynXBbX49pGWLp", + "vYG6HpeNUbRau2wKToSa88ZZuF/aDXZN+yfMDWSlyyo3cu/Q17lppBY2ZH7uupndgkonGU0qjq77Ic+h", + "LDgFWoW9m+gvo1BKtHGBE9q3fNwPeHzRRRd/K/2g976zTLR6coUMNUCeMP1Tsf2qO2Sto+JGgebFxcHy", + "9utxKBtjrNBiiJTATJrouDlOSB/9AkIcKRInWpKxGaISZUFliPGrf0fc6ETu0xHTQ5MmQsWSI3NXSTpj", + "lM02tZWgDyYchsanNU1VKvR7CypzapZZx/mNavG0ZnTEyGPIzaAsiNKQoAvnW7ooq5V1z1PdorSuqJqB", + "ZEgChhHYimorTpXuXk84xiqYazrxVJmQNTv1cjhhxb+17irXjiW75LvF+p9n4qKagrPwWEh6cvZ6CRyS", + "Bc9okwPSKip+5+glWcKSO0corrlCiz5Qv6dSEMmjBbHHbtGLOoEkI24Up9yBalyh1vupt381vcbnF1y3", + "FJperclftjQ8yUVS9dxkc46xxoOLRndSSE/O9NfVdrUkQHywXA4QqGMXXWNqEXBgIKaZJUIhFSRQteYp", + "m40YRK9c2F/6trULvcm1jnIvKVuQAQFKe3FpUWFlndoHzeip8ZgqRcJuWTe4JCSR6yel1WvrMvf49QW5", + "EtQJMhfO3FI9I2zKRUBiayTcze58WWjMawXerIl6MImhb2HMLjME8mJIaCKXzHqAg7eUMFLNmwwrVpsJ", + "Xih3eYGj6AJt2Jc2kSD/hBwAu1aMs5zZ3x+dORbILtw/nnY1R2opcDFXKhnr/5FjvYsvqo3Zb90Oz3Pa", + "ng/Avtrd3bGran12ZsCVZsvuOW9ARvPSOPW78U5P84UepY1waaPKH+Wf5D7cS8rCtg38qt9tdO5lipGz", + "NB7av5cI0kuTmcAQ3Huf3r1b39gCNZsl+JoMYl+AZp6bmErF42Kk/0YluISWw1DKxFrwqBdihcET2tJd", + "a4ZbD3mOl6YpY4t5/R70MxnPJp6IJfoZkhBmdIYnS1W+uRh68wjven3uxuJblqbUAWNBknCs+OrgaTpF", + "7t02sZIm00Hx8WJK+erEEht5U8r8M8eRtWt1E70koNadADpOMDexrYYIoDR+PC3eGvZHrAfH7wE6zjrI", + "ms2axKBb4tBcvGxwURiESSFBk+UmwujjaR+9z0b7V4m0wbIgLpdijiWaEMJQCp5rOA175iwuDiCVcGiq", + "6ufWd2LSLjbhcpTbZ/0s2xm8NFnuNgRpTWhlPiZnExbK3kZjVvSCtfJarQo5f0dmVCpRCThHG+9eHe3s", + "7Lyouj+393qDYW+49344OBjo//+v9rHp959Z4mvrsCxbbNhbUfocfTg53rbO0nI/6vMufvH8+hqrF/v0", + "Sr74HE/E7J87+FFyT/yi7DiP10MbqSSi58Sk5ipflF4hGK4hCu/WwXUPFCuXh/6uetdQ4r1+8yGSanzh", + "2jZY+OZpL1WBuTbguzC5uiW/TMDuzHdJQYOzcZUB9UaQHlN5+bMg+BKSBevndoxnRI7NeeaPpEilCe8h", + "19a7IThXU2muXctez+Hus93nO/u7zwcDTy5JneF5QMeBPoFaDeDt0QmK8JIIBN+gDbgvC9Ek4pMyo+/t", + "7D9/Nngx3G47DnND1I4OmeHlvkIbliJ/dwgp7klpUNvbz/Z3dnYG+/vbu61GZf3FrQblfMslleTZzrPd", + "4fPt3VZU8Cn0L11uT1WB9+V0HhpcAf2vnkxIQKc0QJAdhPQHaCOGI4xkt1XlPTnBoct89Z8dCtNIrgy4", + "MJ3ZN42jLU4jRZOImGewIK180TDzY2jJi83BWJZpfLOWbEbU2gADN5fsFVTKbCuR7tSkUheUJ0qi8MDs", + "0LVyDlYzH9inJj6wc2jJDb9p06kXkQWJikxgji6T0ysIyvjELFppVpQtcETDMWVJ6mWJRlK+SgXooqZR", + "hCc8Veaa0aaG551AvDXYHlMtrtvZua+4uFwbuapP4iwDfq1X6BAc6VPrqoFTHCP7tUuOKCh92XWguTS1", + "zyV6Z74wHqL85yQt4+l0oSfrSWJIEKk4SFLrMLTNtNUu/XoLOEtd9IjpL5edjxQ605uaaIP7tbDFjADy", + "g1qrsWhOeQ/vn8PrrQPh9YdrHSkt6M7I1WMQHTIFeppte5Lh5GEoviqWLfM15C/BKSxoSPoIdhcE1bjM", + "xMpOO1c8SUiY+X/6I2YjybOfpLlB0R8aOqg5oQJxQWe03HHZwfaQQXE3YUXHTbdmx+KHdQ0VHkL4RvOm", + "x1NlUB4uXbIWKWZO2UXodDvnGSaGlURl0rzLcEVqFMmDPGtDfH324aahbYngU+pDOoJYCPvUWmYu6Ou3", + "3cF5b/j/mABOzW+golFm4idiHlYgLOz77U6e12cfzprGlIFKoOLoanPKIl5WwWo5ithLJXsraS0Yx/76", + "YMk6yXXvFz5ddipwTCbpdErEOPY4117p58i8YEKbKEOnP5f1Wa03t7Waz0qLA2bzFAcWE6Ad9T0Ouco0", + "ugVqfvIv1ztijuGmTEK9VMK+Y5MJ++hNBuOBXp99kCiPUvJ46srL2xipfzZfShrgyLRoEoMpKzrYgDlb", + "a8hn+YfWFenRk/3oL24joI3FLElhG56/6528/bgVh2TRLY0JIovmPCJ63JsFabFw+YR5WkFJSCyaPB2G", + "MWTbDVSgVbaDWxOpsF891FFc4WgsI+4L1nivHyJ4iDY+vjL5XnoEXZSUllL/XqBCib/3vTtGS6Smbs+h", + "w6rLtLTBvbZjGYfTuFcK0yt16tsqJsi+ruPUoZf4ZXmh+eV6uB/TSHO/Ry4PoOLUtookMukCCNIF3P0z", + "Mp8ar7V1jUiSYIEViZZGs8iOvohOSbAMIrPHSf0qkVyT4AaJCC/1619NHnEqyFjNBZFzHpUvoHe6ddg3", + "CUGQC2LhN8ycCo53xVGMxSUcjE6RRikzFCgnHeysg7ScK5XcYFK/vH9/ZqxrRcTCRJQVg3xl7Wr1mER4", + "iSZEXRHC3FSwRBi95hnOSTUrRzZAIAg1ToigvEzDzo6n33MTmIlmAgcEma8cCqNdEgmnZltS2l48UF5B", + "QKRsWN/hqvW1n07TqN0a+4Y1XAtaGtxkgd8fnblU/gyV0JF5u07lMyJ6Zss5eMLVS7stV4NKuK4YZ6Te", + "meATAigTNvGmmCLkAksARUN/XkrLKQgHWcyHsf3ALjCk6pp9/qmVslfd7r6L9Biz0IfuaCKeTe7jLI31", + "gughixTujmhook5MOqZRy4uB24LgkDIiZSVxLEhF1Ol2elM7q4OtrYgHOJpzqQ52d4bPt1bH760M3LRx", + "KuOQrrLvXDSLiXdw+VUGZREmXWaJLZwkLTxgho5rzgcQT/VAMUB01GdbQcNzkcODQS0H7hoHygHjgEus", + "dOWJi9s20SxZmg80mMEK7714UdyfA+8ddI7q7dh/q8b7emYmmEzziHE3VOhomPyzV6Piwod+woWyGS4T", + "4qLNsvPQxXLZS5VSZ88Hz4uzbAWeDMKmss3tvvNM1bxdznojBXLb/WsbgLiYss7htvRqrDRNlzU8pSVi", + "jaVWE5QnhN2Innu7O9s3o2fbiZy4pLyKXPLl3B+dHhudKOBMYcqIQDFR2CLFF4QM+JK0lNEGfohJDHkK", + "039fLVoa4heKSfSNN+BHNbi1B7n9boAJemfiN0MUY0anWiDbN4s9yzne3ts/MGBmIZnu7u33+/2bpha/", + "zHOJWy3FlkmfLGQZ9+X8buvwABnEbebypXN2+P4XLchSKcyhtSUnlB0U/p39M38Af5h/TijzZh63wr+j", + "0xruXTkeTBv85veDAlS503tawRH7ncEQFgpwB16YGIVnWrcxHHdXPJhbI8blsKWqgBRXzKZpgRpHP6++", + "RnZeJXjH9pkyRaMcUK9+gXwrSES5EjWqhhiVEJbhREWR+SvgbKF3hQ80qnQSuWd3Cr5YqXr9o65xrd9v", + "TvFaw7Z+L1sm/9qC5VlIG89J9M2l/m0ClMq9v539xx//rzx79s/hH799/Pifi9f/cfyG/ufH6OztnbLX", + "V6MZfVNIontDIYKonBIUUVtWOsUq8HijtKHTQGH7xNjWKpj30RF4zQ9GrId+o4oIHB2gUaeSXzXqoA0C", + "NgF8pRU73ZRNE93UH5+ZuzP98Ren8H2tthHafFBhFyTLIpfpJOQxpmxzxEbMtoXcRCRowPqvEAU4UeC4", + "oAxpS2+JJgKqsdi7jbzzLvqCk+Tr5ojB9QC5VkLPIMFCZfBrrgdgCjsqE3BpXyehw/Mx1wsjlp1LGZyP", + "ueDqZ2ouBDZU01X8RFltqVgb4fnAB3wEIfN6ISMqlVG2M87WbJTF8qPng8265bJGm854aAX7wU6ol2ly", + "TNliLxkGhq6N4B47Z9yaQAQtm8weQWArKQ7/PUeuoZwW2RIbD7lJoJDmglVFspA6sdnxgsTD6rackLlh", + "hM+iFinXL01uzfvfzpEiInbZjhuBJueUBnp+EDtJpUw1K1KMDo9OX272W9SZAtpm41+xju+zGVYzY+2N", + "Y9NFam7Y4Zh00ckx5DbZHZorcBCT/IoLFBkBk+/rA/RBkoqNCNlrEBJpVjJa5teW5gQYdTZdi0lVUhyg", + "d5neiLOhlGpblW9C830JzdqoFRMwXWu9W6saI5xdZEUbhEdjlWXY6RO3WRS0d1RYisOer5jVN97bxZvk", + "RqO5sPb3DXZ3/+rOzs3UHVe4IJlj6ePuefEqBF5aUS2IVvy7ovnev9Tv2oI9WXdQg0gffcXPfaVW9nrD", + "4fvh7s1t/pvilJUBPQpYMhlUWXuMsYfA6qrbv9dUjRsjQpF+bOM/nZX38RTNsWR/VfCwYusNd561Qp3X", + "vbaNpSxGUfKpGVImpRw6SBYDaHBSLmkUmdBaSWcMR+gF2jg/ef3ryW+/baIeevv2tLoUq77wrU8LyDIn", + "Kl6ffYDrNCzHLhypOQMH51ls5JpKJevgKa2i+u4CkWY+bQcr7iZp2sjxxVfjrP1SwkLzouFs3iNAmgvF", + "rJHxMaDPvmWOy/cHu7YSKO2uaGfWeHkgsLNG4e4DCivLefPz/cKWPchw1hbpK571LgHx1jhh3Q71JF8d", + "Si2CSYhOznKo8tzJ6JqvzMlWvBwOBv3hoI3LNcbBir5PD4/adz7YNqrFAZ4cBOEBmd7B5WsZ2yjjOLrC", + "S4lGzlwadYx9VjDMCtvWmlStrqfrcGy3Q1+rKjR+OW0UO3fvL215myblpkV2TRXsJU4jk8BZLIlTr/Ip", + "E4NsZjCBM112xGCAXYuwklXmxEEg0tyf4UquGc03TSzfj5ggMuFMmkKMffQrWUoUU7hDyLqHyCGJsiDu", + "cMQ2hAv4zyL7E5xKEuofIJq266I29dCoAuxj/cGIyXkKFeQ2++iIM5nGRFhXD5pQ8ENvIpka4w7GC9SA", + "eqaShkSMmH7Ng532JVPUD/YHg8Egq2zXOdjR/x74uOmO+Hnu83Yqh/081zX8PLwObq8djt5dwctWlcM5", + "LxfCaW3R3aFaZKtgc6fT2TBz+9X4JndlBAU8jUJtJkz0wWC8OCS0ziZJVF5jCM6SD+ySaXYuTd0GuSmO", + "/kiJWKKPp6elCzZBpraESouJw4ZqWAee3GgZttcY1mtHc0tIu8eAsase6gVl6t5B64oefpfuZji0hac/", + "N668IcqUmaXRfLJiThUfbUgW4zT16ez6kUty//Dh5LjEHBjvD58Pnr/oPZ8M93u74WDYw8Od/d72Hh5M", + "d4JnOw1FzNqnKNw+68BrofmqUrnYw7GLgfQWHm6IQK0ckTaq7oqykF+1Ktmd9W5DqNZ1Xw+QbD0Eb1g1", + "VDqGlhqkxGmhnrEJK6zULGrwPO2/HwzXeJ7alWFukL/vRcoCgz4Fkjjz6RYLMBcXq16E9ebiFAbkwpfX", + "UavYeXuiDQ72Xhzs3ZVoLgR33Rir7PSIi9sUcOAgDCsxvi7PpOC/KFQwBn3DuFltSHCn28miluFvOGgr", + "EXHZ41ah+E0btusXI6vkd0NG2klJbYZbWINkFB5oLSBLZpqkCmWJjlq9OIp4GqKC78cAu8DFyElBhdbN", + "wD2FdQ0ZhC4TUqtVbYCEBEBiyrQghgsh3YhNXztAr+FdeIRjY13YQWAWVu5CcLg0d8F6f7muja6/esjn", + "Vs2Hb7TOj6CwuJ62JoN1Ea5uwmg+B+gNh28yo4Pxqq/RvA7afv31ql9yw+aVufxj6MyqcQfoVaa6Zcqf", + "VfY2JLF/jq3AytP+N0vJl3bFO5pb8pUr5BV2O4ainW7HEQryD+uZiB9yrq/tvyIr+oIkCI5gL+eZXqmi", + "kUW5hJlQKK1tQ4D14jbpFxYQnoRjY5g0hTyZ9CFrvGQfOfXl4ynaADyrvyPrSNX/2szCo0pn3faL3Rf7", + "z7Zf7LdCrcgHuF7tPILktvrg1uqgQZKOXdXahqkfnX0wJnhgjFvwzNu5F5KEE8G16NEzz8vg5p2/6L8o", + "gnWEPDUlwe2QLLLP10Lh+5U1ixtifP6g0YJOp+yPz8Hl9j8FjYfX+3J74nVo5hX2vd6fk+Jtb81VSiY9", + "U8TDj6cADCVkI+TIOyJhBuicKAT800M4ANMhy0mzLOeASSzFvYy1u7Oz8/zZ3nYrvrKjK2ycMfiCPIey", + "HUFhi8GbaOPd+TnaKjCcadMl6gJAKLNmpX+fIVsKbFBWSPvDwY6PSxoO7pxrbNuLuJHkH61pZidliQ6p", + "dZnZVtvlXmrv7Aye7e4932u3ja2XciyuV0sYF3huyGNxbIsrvwHa5PvDMwRpXVMclP0mw+2d3b39Z89v", + "NCp1o1EBBrPBTr3BwJ4/29/b3dketsPO8UUBWFSo0oYtyy7PpvMwhWc1PKSoi95u02nhU6cMg70jQYRp", + "fBi4CN7K6WMwUsfCvJYvQpuDwTrGawdXi29bOY4qJaeNasAFSlmGzN1ffwV4uxu9ZjFtzoP1Yrxu2UeY", + "aXJZkAdTiuMWtEsEWVCeyntoiCuT6jSNOBc3+rbJQnlHZBopc+1GJfp4+lcQIpq5kFQkKRtNlv1WQGHc", + "cnI32sAlnvBzdROxWq1Gm6VfNeFuwzbtrsqDLm3/RsSZUIuqlK0PvzvCUZAC+DzO1lPPCrAjIJMzSaKl", + "CVSNIs4ZCuaYzQiUETSlLtgMYTTnUdj3Bg/qJ+Op99qeX6GIG6zMS0ISi8tuBqE/0zoLXRC0UcgkRYaV", + "KuWZ9mIjVSzydpkb92J/ISAsfckPWQajpidWvADjaD4p+RgjPpNgBSoIwe1X0YMTLExkLWamzsAiNsZj", + "GXpnW5/2niFWpLfvCDVHJ59ai9bqGJAfaCiJA8GlRCSiM8C0/3haSTtbkUORJZ+tD6krD7YF65qLNM/Z", + "BWeabF2OxHcgeoLT73IkAg9DDsqKYDXnjYwxSwGpvcDI5DqhwrBHu4C0OZdqnMGJ3HCwUo0BhTsVJMcc", + "ypIlMweQe8d7LjrRdhty2cjPW31d4yp/U00DbJapXor6qdXNeNDHxnVAlZUYLjkoTBUB5CaQPzlsM5XQ", + "Ki2gzaANxlVJLBWghzfbBGf4bVTdT808tVWmftsdnLdF41kNvnOG1fyETbknZfsG15DO9WzjBRMiYgo4", + "9CgkjJLQGY/ZfaT1bUGeXyQJClNiKWcUUoEtwbHZ3pB2zZxTjLJZRdZXO2zjDzZjWA3SDf3aF9uE2Uh/", + "dth7kQKtTGCcRDjPE2sVZUjl2H9/VW9YkFkaYYGqiFMrhiyXcUTZZZvW5TKe8IgGSH9QvWSe8ijiV2P9", + "SP4Ec9lsNTv9wThPc6hcGpvB2SQXsyCVfvMp/KRnuVlJsQPXy5b5fgsS/dtELXljdV/RiFhQpg+MXhcY", + "vYxiu7s9aMq+bGi0lHdZB/S6qeS2LOvb8Q5r6zArmum5pTRRt5W70rIjcu1NH4R1r8o1rbti0IYLhHIo", + "wWW6FtB6W3lC2kWWV0P+3Gi2JAnKve8+33u23xIu+U6+TpPCft+ezUW8wqPZsFKnbdxmz/eev3ixs7v3", + "YvtGDioXHdqwPk0RosX1qdTGrTjN9iCSanCjQZn4UP+QGmJEywMq1bm99YC+rti6TcEF+d5suuSMiivp", + "7lnKHtB2PsYV2tJhSeUqVIHfINMpAaNybOjWywdTyRBsNYYAJzigaulxmOArU7Ive6UC99bGm1YerIek", + "tm0LDaQll0wneRLFhusc/c241iu88Lw16rpMJ01u/LfVXo0TP/cBFa+IWtzQ5IUh6+6CbD5XWJZizfTf", + "AURP5lX+qzGz5o3V0FNVwA+4BLTFBQqRFD7Iwsr5Zz8qLn9lOQtu35KSXKX4qiO0eQveyIb2nMgeEzpY", + "nwlTkQ/2ALzdV+NJsR7CyoITpeIJ+al7835bJPvUwUKzE+zm/RXSHm7yYRUYC/jRjsGSPG+7W2KJBm4q", + "hOl6zBEekV4W52BjeJFMjX9V73mLtejJwwgu+XRaBnzaawYIBGQ+CNl2vWClSJyoLiLXYKaTsIYu5+po", + "78lRB3GBRp1hPOpUnIDeJIgYX49tB+Vc5cEqxL6sElB1kNLNYBLx4NJA/StBieyjAYoJZhKlDDZ/xUc5", + "HKz2tXU7SWFtMnw8Ym6Ia2ILxjQhc7ygAMtqPVSzUhwLuaZKQrwNtHOAQg72bbnOkZ2hfs2kKBzkk4ZD", + "B7OlbVg3qN/jzAUE5e+CuTSF6krsMxG8a5PvtMR++/a0a+5/IHLDDKwUHuImakagBWTWRQVjNP/dH341", + "icgYxl3FrIzrdCymkoGfWhBJlLQgdjk7VJgABTxlqgpmGbcL4SxHvNePpJRBrIS9PQPIBtu7rY4YksBW", + "oK27l0qMfgvmroRdWkr74i53fCwMmwI8c37P+zvrXq8OQCIsSLFEmWmnGBZnfK5jqbjFtM929ZhcB4SE", + "VTge/yttQw3tl95Qw9+wzXnPqofZtyFcrD67/sPFnjNyvXqsb8i1ApC9MI308JqW3m51K8zXjfjWSQtN", + "G4KL9cUqH6D2gAl3u1X1ARsp9ygFCOzPD1J0oLYc50S5d88t3zTXmizBA5c8ai7A0L1SvuIzvNNF9kRE", + "w3izwnO7c78bwQJctcy3YDgm40SQKb1ewS3mBWNJlrN5851Tqkcq0UaMr9HuMxTMsZCVsTM6m6toWb7/", + "2/Uk09+pFIcgijB1g8qt+Wq6D+uX7XY5i637tMnzQuq7v6IsCcerYOCOstfcdWaCl+D2aPRRPtvZHQx2", + "tge3woG7r0K3hXaaQvgL39l7hlJUTLGFLGGqXg3pSlDIu8rIJJUgOD6AQN8EBwRFZAooKVkVurWHRa3r", + "1YO3GojNw8743y2UXTd3BVAGWs66shB6bhodF0BUTr0vPq8PewWUSiZmghqmiifGf6c32H8/3DnY2z8Y", + "Dh8COC4jUlN06bPPw6tn0Tae7kbPl8/+GM6fzbbjHa/h8gA1lcs5k9USy3YOCRHVMlfV8nCSRJSRnswi", + "stenRayQBSbIYe3+v5lj3MxgpbJwXp5kUWfAKidOibMeCWfCjn6ld786/JPj1cO+VYhzdSB+BqsOBfip", + "3WAAyHR4V9TMlLU8dz4UXmx98qwMu1939viyJGFre1e5geI+fi4JxtIOW3Vi1081j3dxxgVV83j18ZC9", + "lmHwQZzWZ6nCMq5BH53MGBS1K/6cXcsXjTz9cafbiT7vlveM/b09woXFnMsY0C51UQ1ocW0NNRNXUwFe", + "yU0LYSLntDWrx/zTsDd8AcFj0efdnwa9F330j0IQW9dQq0i+oXu79OugDQ2LlSIcwvjwxY0ivBw9V3HQ", + "r9RX5yA/iC0aneXxvKKYOytcRk9pgfPHtTWupOE3apx3Ve3saTYuakkhifDSVwe64MqUFfuwyGRoQmaU", + "yTaezZ1B5trci0edPjq0GI5greYFI0vNQ6nAAp/QOCYh1UqlMe6bIya3W3qrqsbDzYB93Vce9azv189e", + "rM/BXBfgve6Y7N8h4edO5m47E3dVejB4npxNCiVE4MUuolOEWaWOja3aajP1IPMCEFoOHCBJzrJWBshc", + "8XOekC6acYXyHL123jORsmbPWTZ+cg0eyRVJuYYhtu8l4zpD/6CrxNfJMUoED9MgT1CJYNB5SrFIK2CI", + "K7T69SFAD+nQgMyvKRdovUOjyYPRzgPZtN4V76Nm2OalHg7WL/WDeEG6nTQJ18sw81I7CXYjrM41KQ8e", + "n0yZ7BVNsDCZTy0k+rsiBetGrvEWB1olShN3BaF5qs5JngsJcML7YOuOSUT0MVVvBPEozKMyqcyl6HqR", + "Otx/Pm+6BIQ7m/pAfiUk0bYKACxAfzFmS+/AqkVK0cbAVZ6S5kqoZ7C+LbXKg3u2VhNrXKr29V4rXm2T", + "kF0sr5uBVt5vsVf75dpS3A/hh/uWStpbe7lQgSdz4HkZ3Kfr32ScALwhiyrn9a7vAtvHFu+tZdwEuVbN", + "Oim6mQ97/2XcymjcP9j66e//d+/T3/7ir/JfspslEb2QTCEU55Ise4AXj7SN3i8DjgHWrVambW18RXAM", + "TqPgkhgnVYyvi+PdG2RCY/kGx7UpQAxTTFn277UT+vtfmiOACmT8AHJyLcveGQr6IersKO6Oo42YiJmr", + "BOwC1zf7Iwax8JdkKVEBzN+qNI5R/yqzT7SKDk5LHKELowb2CVtcoAmFmihyxLRVi4OAJNqasKDm1BSz", + "4yB9BMFRsR1bVMAlmtkrR3PjTtDH0xpa3dsP739+++HN8fjt2cs3hyfjX1/+JwRBXPVMD2FP897u3r4t", + "YVek5NCzxHcAzr0TCp6P3QyWloe/IKkD6gJ6FGYqISXTXcgXXkYbJE7U0hXrcbkhmzfD9jrMGvSGg90z", + "ivngxX0UbfmwskrLgkc9rVE3gNB6HZiGFt5wZmjKhIl3mhzbM08Z7XPrTZzRGfb4sr0VPO+juIob0FrQ", + "tdr6N5ZG8AeXH1cReo00MKSqIMpW7FKpes2x57FWpMZ5rcQyyFPKbHoGLQQ8lXMxYqa2bBEkX0poqE/e", + "1Qk5+S5ziD89+Gh9nslKVb4ws8JImtfm1GmsFZ16BYHONGmu5kSQwkLABzny6Q1JZpMlWiQam1IlCRF5", + "IKHLtIDa4YJC9kXmbHAkyBJq6h7Y1ci2p/g66wG891jW7rhgHjnG/PD1z6POZh+9c6U+6dQ1AcOo2BN+", + "INEyF62iieOq+mIUuao+b/O+d+NZWbVC+jXtrQpz5n2UWNPHj//AVL3iAiyQ5rTeB8cjBesmJAJwTapo", + "o62gOmlMwnFW8Lhp/7saxyanNysnnZdBctYWBibWQm59qRqXeJqPoU5pTQ4SpIKq5TkURjURtgQLIg5T", + "s+FdfVX7c94xVBX6+hX8lFNPFP9rwoigATo8O4H9GGMGSjr6eFqoBGKKwtQwyEC9fHt0Yi1cB2MHFgtV", + "wHouGO7w7KTT7SyIMFZeZ9Df7g9gMyeE4YR2Djo7/WF/0DEVcWGKW1CEEP60CXqZpXQSWj3oZ/OK/krg", + "mCgiZOfgd0+iGwSywcug7+JZwWJJMBXWZEkiSL8zrEL1twBM647SA3Me24K2rR10Ui1tMgJJ3tpl/QTq", + "JOwamOL2YGBhOpU9eCGVwsRvb/3Thk7m/bbS54A8HpTWmkXhdEpL8q/dzu5geKPxrBoG7Fhftx8YTtWc", + "C/qZwDD3bkiEW3V6wkyGFDJYWzbQprjPgIWKO+z3T3q9ZBrHWCwduXJaJVw2KcNEIowYubIVNf/JJ31k", + "rx+g7Iqc8zTS0gSZ9C/naFBY9GefERbBnC7IiNlzOk4jRRMswI0QI30+G4OpvDVM12b1s8T8n3m4rFA3", + "a25LN9dzTuecwNWwfknGgOM7biqXm7ubKWNaTGJJbC2KrG5kPZpHi8uxDLgvH+c9YZipnkxIQKc0QPCy", + "3r3Wo+1tsBW0nRZ4sCxEAOKU89Bsb/pzOqFqhj8d+jh7hix5y+oEg2ugIErDXOdyaUZYTHAUebGPZhGf", + "4Ghs6HNJPCrqa3jDEqVYYMQpN4yHxBSLSJZqzpn5O52kTKXm74ngV5IIrQLZImCW1iQ0Zb8M614BVmYM", + "hbhMiVHd55YZ4taXS7L82h+xwzB25WOl+QRHkutT0xbts3H1Zksb3vWXNWmIKzlKpeKxZamsgmE+TJ6q", + "JFX2Tl0SZSuXwetUoiSVcxKOmOLoiyAzKpVYft36kvf4FWwXgkPNJ4VXzJS2vtDwa9Oo5Rjr2Y/hVY/1", + "R4AAo44+XUYd/fdMYG27pHIOThQJjpNZcUk3MjwarRduVikcYIYSnhgsH2CqOdYsV2oDqkDiKEIKtpL7", + "VmubsJIN87HpufGkMTfXJFNWthFl6PTnwmYa7D737ydJAkF8Do7/OH/7BsFRpdfAvJY7rMylNtOnKApT", + "0OSh9/6IvcTBHBm9CfBaRx0ajjqZdRFuwlhTaSO0ez1QcX/SQ/vJdNOl4U/9vm7KaM8H6PcvppUDvZeS", + "eKz4JWGjztcuKjyYUTVPJ9mzT36CNqU4npcEAdowsn/T1fAFqKX8GDTnBmYh4lbWRkuEUS6Bin6UCWVY", + "rCxA7CG9paA25fFMFonxZQS+21HnYOS8t6NOd9QhbAG/WRfvqPPVTwGrRDeDg5oazE7XzphofzDYXI89", + "YOnrUaFLL+rt97WmfW3fm+Jhla664mEm55CN9QqaatpG3XoEzednHLr6jD9UvDUqnvVcFJQ3+L54Dhj2", + "jYgxcCsamLZnI6eBrbRODFtAwQGwOBxSiDE4qNPgcuYtmh9Vc75uVuw27bIAhhg5/tt9BP6DfrNSfqbf", + "F4/VL44Ap9sBvz8xdoTFcozY9VvEr4n6Hjhu8Fii1IKKf0v+fSr885pYvS8nWkWabZGFu2/y4yFBsom0", + "rZiXta16DmPqnROm0Ev4tW//6yweyDa+iPjs4gAZEkZ8hiLK7D1g4bZIH4qWlvCRyTfJvrPpJw6McsOc", + "n//73/8Dg6Js9r///T9amzZ/wXbfMghhAF5/MSdYqAnB6uIA/UpI0sMRXRA3GYCXJgsilmhnAGpmIuBR", + "sSSR1U3kiI3YO6JSwQr3pQYXUtoGwfRgMB/KUiJtvo5+kU4taJVxMHtMeLeXDSkfdUd3PYnCMIPCBPSp", + "6HgAskRt+TJrf3X83jMz55L/rOorr3lM18sXRa6V4d6eGeANBQyQ2Lfv4IGdNNo4P3+52UdgYxiuAGAy", + "0JjzZqzy3P8hk9bLJCNRygIFqGxkk0liW+3/PbbvtHMA2xb/TB5gi1V5AxewcXlA3W23Aj9shRbuYD/d", + "nGvY5589dlmazQ7a28+32IWLY2plCN/fOjveq9PcPCmQ7FuYwGjDRcODG5ELdHZ04mrCbn4zpn+UU0PP", + "1Na6y44OxBlglz2aWXbE2TSigUI9NxYobBSTzFQrM8hTEQfv7KgRdvOqQgAXz7etEqJd40mXgdvlR97D", + "nx6VTm9yjOQwxTmv/ThJ1rHOMZUB198WuKUX4AQI6dSXbJ8WuWidQ8oE12dHzkp1yYrnk2O3IR/PNWW7", + "Tln1bHgEoXhcEYjfUBBWiq8XgL2fEjd/yFbRIVKs8Fx9X6w5eDwt6LG9WD42f0purLBCNi0FTVB34wH6", + "migTyt15wIW2PXgmfk6E29WuDgPMOpuW+dQUJjUTggvp1bbviXmlnelr2vszWb5AnptoLJbkP1SUFsZu", + "TqtVBu6JLfn9cPYt9HAj8/b+7nktg3mIDMEmE+exNnVrsVyyYPNPddX7KKeZIfaTPMzO0ihyNx4LIhR6", + "e3RidlbxDNj6AmFJ63V7t9tWHgcf3v3WIyzgEIeWxVD5lSj75J41fLNgZio/2KSNTWjSoqk7z5o0nDus", + "vwkXRCbCsU/5v22/iuhEYLH8t+1XOEooI/+2cxhhRaTafDBmGTyWaH5sjfsJM59WuGmZaCCaGNRLX6eh", + "Zm+1VFLd+38qPdVM+kaaakbXH8pqG2W1SK6V+qpdigfVWE0f3+hKJmM2H7XhkYtP/JNpqo/r5bMc6YCi", + "qSxfe9gidVyAnxceUYZSSZ5gACXNOK54bLR0V+cbcuXx4Vj35LgLhIS6AoDaZBNEHsl57cbx6Mqt7ffx", + "PdeH8YTOUp7KYu5JjFUwJ9ImK0WkLICfmtqdH8+Nivd3zKWDxzw6Hl2v/sH3D6TxVxfUCG9zA7VO53dv", + "tdX57fta5zcp1DZ3zUJLdR3s4GZDUKFLom7LxqVc83qwo29cPlsEfdCGSm4uILAgDkbs/2j743dFcPzp", + "J5ckkw4G2/vwO2GLTz+5PBl26liFMKgYBCixh2+O4dpvBtnnACSbp+RVx2EqTwDrOeicfzkDKb/5bG8h", + "OS78YSG1spAK5FptIdm1eFgTqQy/9eg2kuM3H8EtiMkPK+kxrCSZTqc0oISpvGZoLUjMlhx+grllzN4P", + "FYI7Sgdtaysp25RrFNC8LMCjB/ac5DiIj20cuQoETzNGnicW0tuaI/lh2GyPfG/8MHhc4fz4dshTZjGj", + "8NdJl2id0leKGjAm4xTKKqIcIQSiPpGw9RNdi32UV4CWaZJwoaTBqQQF2CDZz7UC7MO0LMNU+nApAYuR", + "EtkdMahUoB+bXP6tS7I0KJSUswxwslrP1Jd7VUYB/abb6P51LD/EaSsd65G3sQWt/nY61jcTHY+iaZ2U", + "agFsZBsDDMoJyXYyz5L76GfKZptPKgLVCKtsbgU8I4+qtQWV/iyu75bMqrE2HbQFaF9buvVf8MStT9Kn", + "tTss2gIBUUjxjHGpaFCsXFuEB/1xQrc+oVdT1svNU1ti3G/Qv+Lisu0R56kr9gROuuIMv0Nfgh4eoIF9", + "e5cCGNvmNNBM8+inYK1Y3LdMwaDVczGI0hCqutsD0amSU8Hjsf3R4NXqXWHRQMFFEdhWv7Ww0b0/gsPo", + "DVeIxklEtBZPQtQz3KRX06r+Dm6eykJpxZsJQ71tigkxBoxOutJEVkTC5ZpbsA24Z68vl1dqRny2HgQj", + "69whPnhQMEbMwOETh51/gTIhC8W7SEQCha7mNJgDIgYU9IKKrgBWgZPkIoPA2jxAr2GnFpHAoPMNSYQ2", + "hALOJI+IAbpYxPHFQR2x9ePpKXxkwDAMNuvFQVayPDsgpH6riHCR1Tx6Y3E7NjQnCR5FZkUvtNVYmN+m", + "xb7IIcpGzIeDwciVbZBO0UUBEuOiARPDCdTf+OybaVvdZmBJMxfFkQDCGd4kLOw0XcTQyI+GMRx468G0", + "ROYww3hgYI7aYH7jswzUssTKOEnasq8dJnDxIo5X8DDaKBRnlSrkqfq7VCERAj623N3E3GgDB+YfCl9q", + "RrWFhbLytsB+3utGgzLnJZUWqoUqOuZfizjudDt2PAV0uhto72sQTqoN1q/F9MoUYEx+6N03ASgpC/sC", + "Qknl5LDl/5tV7nfmhT+9f9YSKvwzeFnK91n5KCjLC0AJqODuqnA9KaQDWMiaLmYKI/n2iJtlTxaKh7a7", + "3qqVHf0OjNZ1t15ZDcmswOVjX3/VR/CUk2BkbTZTLqrp8evuxb57Rrq/JalNtQ2H/ODNm7vnWjFmkq6o", + "JQqlUCX4+aC+JuA6B3POZYHtJ2SOF5QLi8Buva4ZZ4LLwliPNnruQrPqhfXfXlj1/MD6mhAuPrJ99OFz", + "G3Pn/8I9yr94VbC2M4nfdSo1oEBKhNFEUDJFCU4l0dpSGhNkKoxYIG+Cg7mrFt4fsfdzgmx9zIIDISun", + "TCW6GMYXXTRJFYqwmIG1Yx6aSDpBAh7HhIWm5u2IzQleUG2qCRRhRViw7EkCNZAXJC9gok13e0NpSm1n", + "VVa7yBXnBQfDRaH07gVKBAEmMuYyK9W5HTGRsn83yJW62Qs30AtEpMKTiMp5VisiwCFhgRcW8vz7FmP3", + "78Q9J6penfab3FneSpZ+y0vMoi8zqw/+XdxvPrFALS5cZc0WYn6F0iubTcNy5ON5XpH3X3BLm7m6OX6j", + "m5mMxKt28fdxJVMqyf/jWkbZLRmmpjtSLlv/p71ryetIp6x03WJ9sre9cMkqIWRkvpHM2/ri/jy5hY/s", + "O5GE3UbDvglzO5/09yByLVVvJXO/kXPQ+pIKXrFvKILtoL6d+sRFQcp9F2LYbLhMGhdljhIYbCrOfgjj", + "qjC24QG3FcbO41q7AC+IZ8p6SYSb5HJetd4vgK1D4F80+rUyu4Ig/OaCL78ReDRhd5KJNyPwEryMOP6z", + "38sEXAiT0GnLET8dQLGCL7BwwbQBHrduJiG6Lpvk4+npZpOUEGqljBDqCUuIclnTIPZUa3y7IELQ0JWO", + "PDo9ttGrVCKRsj56G1Oo53hJSAKFYihPJYLM3L6en0ttrRfBK+WwdjuEKbFMOGVq7SjyVx9mMF9vVTrv", + "keWkhVT8018egxf+6QkpkB1aXbETWG1FKqwag/FccBplpt6l1rbwhKe6dS1ZXKHdGZxtUxoRuZSKxCYy", + "b5pGsIkAdNfWZLLfmYzSLqJKIr0fupCBlxARUykpZ3LEbPn3hAjdt/4civ/mQUZe573CmdQ8M6Lv+whg", + "04MxMVtYNVENoAWgDmjnoLOFk2QLykX7g6Ts8O4wpFcQkYbkMp7wiAYoouxSoo2IXhqjAy0kivQfmytD", + "2sbw3X1XnLr9ztKUPmFT7i3KYXg2Y+Y/RxJSWay5S8QnJ9Zek+JmcfIHFtov1uRauSYIjnqKxiRLfkep", + "ohH9bESdboRKRQOTV5OnXkIRZpt9OWKnRAn9DhYEBTyKSKCcc2UrETzYGqWDwU6QUEAp2SEwOBB4zY9j", + "6PHo7AO8ZwpFd0dM/wMafn94Zm5ip9j6CAoDZURdcXGJTrbergnyPQcy/QtHyZkJrsyB9C74j+u7m2c2", + "N+4h2bBFebLKAOLJnz6M02pwP7wFT9NbANAS2Ww2ZgIHoBTLeapCfsX8noEFj9JY/8P8cbIOoEThYP4R", + "Xv1utF0znLXduAk+iU1p5xQSUzTom1xQGII91fhSTTg3BVBiSpF73lPgUP0Zufv+nfJFOn6HV5OWoq4g", + "13eztx775LNjcLhbRXo8lW1uOM3NRPHV3qcrTJu9Tz9HPLiUKGWKRiVQA223AQ6o/jHHbbQXf6AmQHak", + "KyWOyHVCBSDYVOARENEzlggjRURMGY62YM6mEUCgdF4svOAUkpSDiEKaGA0JSngUAcrO1ZwwpGcDjirX", + "QOGeVtoKEMV3ileMiqMJCXhMHCrnps90+wem6hUXZYjN70Uuvi/QX89HT1XPcw2qaHOPd0IZPcXXENYc", + "pvaa2I1o4zXPfzSuoC6CtRl1dgZy1OmiUWc7HnX0ChxhcKFihfZQTFmqiOyjY+PfgjTU/QGSJOAslA4c", + "1HnwdgayKSnVsGVDhuM+fPeYao/lKiDlO9uJTzzo95D+HhJs0EZxw9k9GXZh04WIpwoCuN2+sm+FRIF7", + "ZPPRb2ALe+SHbd9Gkv/Dbt+SjIJV1uKysPRGsmfwkWu9bi6pYs5ljjqJApzggKplF+Eo4kHuPUhldjvQ", + "y4YyEQRfahuqP2LvMuBKmwiBjs4+dJ3TDIVUXpoWrF+sj94uiJDpJBscAmlgPHiwGCQcMcVRgKMgjTTf", + "kumUBJDDENGYKtngV8uG8pBlEPNOPAvvHmawNU/LmeTnCVi9nC1kheO2zFJvCRJEmMZFp1KVOKD6wpUu", + "uH0nulGuj+FpZK+3AsGlRLapHonojE4ie1kj++i9VjlwTEYsiTBjRKBUmrgjPfReIoiUqUmM0Q1AnVnD", + "UV2UA50kgivrJo44F9J4djWHfzxFUpFkBZu9My2fwpwfCCbYNG57+kYGQ2UMzceSfQXpBTGcYgiu+Ugf", + "098g2McM6FvDCT+Vjf9e0NmMCL0rsBGy5mrUbGtHTrPpS5kejRj559lb7TDys1YL0dyFSOeVQBVj9+IY", + "FOib3MB6Or+kjVgm9tHNsi9+1R+17Lsc5e8fhH10x1n+WUqPnReCq9si6+cc/tRA7gsjL23VUoLCejiC", + "1hkJD5kh0Bp34JvBDTxllAFcSjtoghP4/hhh8LjZcY8Ns/20eauEElAqrNOQKrUevvO74MCHwe38xtmh", + "t8Dt/K7ylQB38dvljX5XmUolP6ArHvKnR+Z8qAQlA88JMBZNCUpG6tlAgpWG0kf7Tjszybb4Z9Lg7d3z", + "DfR3R/YfVn8Lk6FALL/LzuRGO9wWEidq6S4X+bRyASjpZ0jG8AE/ZDEED4e3cIvr9ftjD8enjZfrP+pp", + "Pdr9fV50+OT46RfRKu650sGypU+dHhbBnC5Is9O9vIMtiRJBeglP4HIlNASz9HBnmcKiP/uMbPMWq8r+", + "C1EHcUxCFFJBAhUtEWWKg0QwffxVIsG1JQDPuVj6nOnFnftK8PjQzmbNeWj3lHWG5Xe+8bIXYoV7Cydt", + "VrjQ7nDT7u62tcBDlKHXP6MNcq2EQdxFU235IDrNSEquA0JCCTy5WRzwcNDg2aSfyXg2aTPKFdjJby02", + "NQpSqXjs1v7kGG1AsYUZYXottKo/BU02EXxBQ1OINCfqgkeGqsMGgt7U76qViqxShjMuzOC+iQ7T5kCa", + "faZJWSyY0IXOQWdCGYbBrUUpLu8pk1Cl+8MU0hryveM4p/PjCLOW34YzdjQnaiPHEVFxbqDxNn8cc0/5", + "mCsGprozrXTatSsV2S5WtWUI6UMA5mZxzI/rtv74/YRXUvkkIyut63yRGaRNbvPviwUHj3c+PLa7/OMT", + "Dsd/TZzxXXCVQwO6RR/D/MYDHKGQLEjEE6giad7tdDupiDoHnblSycHWVqTfm3OpDp4Png86Xz99/f8D", + "AAD//7vg+12fjQEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/restart-policy/README.md b/lib/restart-policy/README.md new file mode 100644 index 00000000..a91aeba6 --- /dev/null +++ b/lib/restart-policy/README.md @@ -0,0 +1,77 @@ +# Restart Policy + +Restart policy lets Hypeman keep an instance running after the guest program exits and the instance reaches `Stopped`. + +This supervises the whole instance, not an individual in-guest process. If the image runs systemd or multiple processes, a restart boots the instance again using the same persisted instance configuration. + +## Policies + +`never` is the default. Hypeman records exit information, but never restarts the instance. + +`on_failure` restarts when the last run failed. + +Failure means: + +- exit code is nonzero +- the guest was killed by signal or OOM +- the instance stopped unexpectedly and no clean exit code is available + +Exit code `0` does not restart under `on_failure`. + +`always` restarts after any guest exit, including exit code `0`. + +## Manual Stops + +Manual stop suppresses restart policy. + +When an instance is stopped through the API, Hypeman records an internal suppression marker before shutdown begins. The public instance `state` remains the single lifecycle state; the suppression marker is exposed only as `restart_status.blocked_reason=manual_stop`. + +Calling `start` clears manual suppression and retry status. + +Deleting an instance removes it entirely and no restart is attempted. + +Unexpected guest exit does not set manual suppression. If the instance has a restart policy, the controller may restart it. + +## Attempts And Backoff + +Each automatic restart waits for `backoff` before another restart attempt is allowed. + +`max_attempts` limits consecutive automatic restart attempts. `0` means unlimited attempts. + +If `max_attempts` is exceeded, restart policy is blocked for that failure window and `restart_status.blocked_reason` is set to `max_attempts_exceeded`. + +If an instance runs for `stable_after`, the consecutive attempt count resets. This prevents old transient failures from permanently consuming the retry budget. + +Manual `start` clears blocked restart status and starts a new failure window. + +Updating the restart policy clears retry status unless the instance was manually stopped. Changing the policy on a manually stopped instance does not start it; the user must call `start`. + +## Lifecycle Behavior + +Restart policy only starts instances from `Stopped`. + +It does not restore `Standby` instances, wake templates, or act on `Unknown` state. + +Automatic restart uses the normal instance start path. Resource validation, network allocation, config disk generation, volumes, GPU attachment, egress policy, and command/env metadata behave the same as a manual start. + +The restart keeps the same image, overlay disk, volumes, env, tags, entrypoint, and cmd stored on the instance. + +## Status + +Instance responses include the configured `restart_policy` and current `restart_status`. + +`restart_status.next_attempt_at` is set while waiting for backoff. + +`restart_status.attempts` counts consecutive automatic restart attempts in the current failure window. + +`restart_status.blocked_reason` explains why no more retries will happen despite a restart policy being configured. + +## Non-goals + +Restart policy is not a health check. + +It does not restart unhealthy-but-running workloads. + +It does not supervise individual processes inside the guest. + +It does not replace systemd for images that want in-guest service supervision. diff --git a/lib/restart-policy/controller.go b/lib/restart-policy/controller.go new file mode 100644 index 00000000..e6ac3fea --- /dev/null +++ b/lib/restart-policy/controller.go @@ -0,0 +1,213 @@ +package restartpolicy + +import ( + "context" + "log/slog" + "time" +) + +const ( + StateInitializing = "Initializing" + StateRunning = "Running" + StateStopped = "Stopped" + + DefaultReconcileInterval = 5 * time.Second +) + +type InstanceEventAction string + +const ( + InstanceEventCreate InstanceEventAction = "create" + InstanceEventUpdate InstanceEventAction = "update" + InstanceEventStart InstanceEventAction = "start" + InstanceEventStop InstanceEventAction = "stop" + InstanceEventDelete InstanceEventAction = "delete" +) + +type InstanceEvent struct { + Action InstanceEventAction + InstanceID string + Instance *Instance +} + +type Instance struct { + ID string + State string + StartedAt *time.Time + ExitCode *int + RestartPolicy *Policy + RestartStatus Status +} + +type Store interface { + ListInstances(ctx context.Context) ([]Instance, error) + RestartInstance(ctx context.Context, id string) error + SetRestartStatus(ctx context.Context, id string, status Status) error + SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) +} + +type ControllerOptions struct { + Log *slog.Logger + Now func() time.Time + ReconcileInterval time.Duration +} + +type Controller struct { + store Store + log *slog.Logger + now func() time.Time + reconcileInterval time.Duration +} + +func NewController(store Store, opts ControllerOptions) *Controller { + log := opts.Log + if log == nil { + log = slog.Default() + } + now := opts.Now + if now == nil { + now = time.Now + } + interval := opts.ReconcileInterval + if interval <= 0 { + interval = DefaultReconcileInterval + } + return &Controller{ + store: store, + log: log, + now: now, + reconcileInterval: interval, + } +} + +func (c *Controller) Run(ctx context.Context) error { + c.log.Info("restart policy controller started", "reconcile_interval", c.reconcileInterval) + if err := c.Reconcile(ctx); err != nil { + c.log.Warn("restart policy startup reconcile failed", "error", err) + } + + events, unsubscribe, err := c.store.SubscribeInstanceEvents() + if err != nil { + return err + } + defer unsubscribe() + + ticker := time.NewTicker(c.reconcileInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case event, ok := <-events: + if !ok { + return nil + } + if event.Action == InstanceEventDelete { + continue + } + if event.Instance == nil { + if err := c.Reconcile(ctx); err != nil { + c.log.Warn("restart policy event reconcile failed", "instance_id", event.InstanceID, "error", err) + } + continue + } + if err := c.reconcileInstance(ctx, *event.Instance); err != nil { + c.log.Warn("restart policy event handling failed", "instance_id", event.InstanceID, "error", err) + } + case <-ticker.C: + if err := c.Reconcile(ctx); err != nil { + c.log.Warn("restart policy reconcile failed", "error", err) + } + } + } +} + +func (c *Controller) Reconcile(ctx context.Context) error { + instances, err := c.store.ListInstances(ctx) + if err != nil { + return err + } + for _, inst := range instances { + if err := c.reconcileInstance(ctx, inst); err != nil { + c.log.Warn("restart policy reconcile failed for instance", "instance_id", inst.ID, "error", err) + } + } + return nil +} + +func (c *Controller) reconcileInstance(ctx context.Context, inst Instance) error { + policy, err := NormalizePolicy(inst.RestartPolicy) + if err != nil { + return err + } + if policy == nil { + return nil + } + + status := inst.RestartStatus + now := c.now().UTC() + + if shouldResetStableAttempts(policy, status, inst, now) { + c.log.Info("restart policy stable window reached", "instance_id", inst.ID, "attempts", status.Attempts) + return c.store.SetRestartStatus(ctx, inst.ID, Status{}) + } + + if inst.State != StateStopped { + return nil + } + if status.BlockedReason != "" { + return nil + } + if !ShouldRestart(policy, inst.ExitCode) { + return nil + } + if status.NextAttemptAt != nil && now.Before(status.NextAttemptAt.UTC()) { + return nil + } + if status.LastAttemptAt != nil { + nextAttemptAt := status.LastAttemptAt.UTC().Add(Backoff(policy)) + if now.Before(nextAttemptAt) { + status.NextAttemptAt = &nextAttemptAt + return c.store.SetRestartStatus(ctx, inst.ID, status) + } + } + if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { + status.NextAttemptAt = nil + status.BlockedReason = BlockedReasonMaxAttemptsExceeded + return c.store.SetRestartStatus(ctx, inst.ID, status) + } + + status.Attempts++ + status.LastAttemptAt = &now + status.NextAttemptAt = nil + if err := c.store.SetRestartStatus(ctx, inst.ID, status); err != nil { + return err + } + + c.log.Info("restart policy starting instance", "instance_id", inst.ID, "attempt", status.Attempts) + if err := c.store.RestartInstance(ctx, inst.ID); err != nil { + if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { + status.BlockedReason = BlockedReasonMaxAttemptsExceeded + status.NextAttemptAt = nil + } else { + nextAttemptAt := now.Add(Backoff(policy)) + status.NextAttemptAt = &nextAttemptAt + } + if statusErr := c.store.SetRestartStatus(ctx, inst.ID, status); statusErr != nil { + c.log.Warn("failed to persist restart status after restart failure", "instance_id", inst.ID, "error", statusErr) + } + return err + } + return nil +} + +func shouldResetStableAttempts(policy *Policy, status Status, inst Instance, now time.Time) bool { + if status.Attempts == 0 || inst.StartedAt == nil { + return false + } + if inst.State != StateRunning && inst.State != StateInitializing { + return false + } + return !now.Before(inst.StartedAt.UTC().Add(StableAfter(policy))) +} diff --git a/lib/restart-policy/controller_test.go b/lib/restart-policy/controller_test.go new file mode 100644 index 00000000..3fd8b1d0 --- /dev/null +++ b/lib/restart-policy/controller_test.go @@ -0,0 +1,116 @@ +package restartpolicy + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeStore struct { + instances []Instance + started []string + statuses map[string]Status + startErr error +} + +func (s *fakeStore) ListInstances(context.Context) ([]Instance, error) { + out := make([]Instance, len(s.instances)) + copy(out, s.instances) + return out, nil +} + +func (s *fakeStore) RestartInstance(_ context.Context, id string) error { + s.started = append(s.started, id) + return s.startErr +} + +func (s *fakeStore) SetRestartStatus(_ context.Context, id string, status Status) error { + if s.statuses == nil { + s.statuses = make(map[string]Status) + } + s.statuses[id] = status + return nil +} + +func (s *fakeStore) SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) { + ch := make(chan InstanceEvent) + close(ch) + return ch, func() {}, nil +} + +func TestReconcileRestartsFailedStoppedInstance(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + exitCode := 1 + store := &fakeStore{ + instances: []Instance{{ + ID: "inst-1", + State: StateStopped, + ExitCode: &exitCode, + RestartPolicy: &Policy{Policy: PolicyOnFailure}, + }}, + } + controller := NewController(store, ControllerOptions{Now: func() time.Time { return now }}) + + require.NoError(t, controller.Reconcile(context.Background())) + + assert.Equal(t, []string{"inst-1"}, store.started) + status := store.statuses["inst-1"] + assert.Equal(t, 1, status.Attempts) + require.NotNil(t, status.LastAttemptAt) + assert.Equal(t, now, *status.LastAttemptAt) +} + +func TestReconcileSkipsManualStop(t *testing.T) { + store := &fakeStore{ + instances: []Instance{{ + ID: "inst-1", + State: StateStopped, + RestartPolicy: &Policy{Policy: PolicyAlways}, + RestartStatus: Status{BlockedReason: BlockedReasonManualStop}, + }}, + } + controller := NewController(store, ControllerOptions{}) + + require.NoError(t, controller.Reconcile(context.Background())) + + assert.Empty(t, store.started) + assert.Nil(t, store.statuses) +} + +func TestReconcileBlocksAfterMaxAttempts(t *testing.T) { + store := &fakeStore{ + instances: []Instance{{ + ID: "inst-1", + State: StateStopped, + RestartPolicy: &Policy{Policy: PolicyAlways, MaxAttempts: 2}, + RestartStatus: Status{Attempts: 2}, + }}, + } + controller := NewController(store, ControllerOptions{}) + + require.NoError(t, controller.Reconcile(context.Background())) + + assert.Empty(t, store.started) + assert.Equal(t, BlockedReasonMaxAttemptsExceeded, store.statuses["inst-1"].BlockedReason) +} + +func TestReconcileSkipsCleanOnFailureExit(t *testing.T) { + exitCode := 0 + store := &fakeStore{ + instances: []Instance{{ + ID: "inst-1", + State: StateStopped, + ExitCode: &exitCode, + RestartPolicy: &Policy{Policy: PolicyOnFailure}, + }}, + } + controller := NewController(store, ControllerOptions{}) + + require.NoError(t, controller.Reconcile(context.Background())) + + assert.Empty(t, store.started) + assert.Nil(t, store.statuses) +} diff --git a/lib/restart-policy/policy.go b/lib/restart-policy/policy.go new file mode 100644 index 00000000..392490dc --- /dev/null +++ b/lib/restart-policy/policy.go @@ -0,0 +1,142 @@ +package restartpolicy + +import ( + "fmt" + "strings" + "time" +) + +const ( + DefaultBackoff = 5 * time.Second + DefaultStableAfter = 10 * time.Minute +) + +type PolicyMode string + +const ( + PolicyNever PolicyMode = "never" + PolicyAlways PolicyMode = "always" + PolicyOnFailure PolicyMode = "on_failure" +) + +type BlockedReason string + +const ( + BlockedReasonManualStop BlockedReason = "manual_stop" + BlockedReasonMaxAttemptsExceeded BlockedReason = "max_attempts_exceeded" +) + +type Policy struct { + Policy PolicyMode `json:"policy"` + Backoff string `json:"backoff,omitempty"` + MaxAttempts int `json:"max_attempts,omitempty"` + StableAfter string `json:"stable_after,omitempty"` +} + +type Status struct { + Attempts int `json:"attempts,omitempty"` + LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` + NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` + BlockedReason BlockedReason `json:"blocked_reason,omitempty"` +} + +func NormalizePolicy(policy *Policy) (*Policy, error) { + if policy == nil { + return nil, nil + } + + mode := policy.Policy + if mode == "" { + mode = PolicyNever + } + + switch mode { + case PolicyNever: + return nil, nil + case PolicyAlways, PolicyOnFailure: + default: + return nil, fmt.Errorf("restart_policy.policy must be one of never, always, on_failure") + } + + backoff, err := normalizeDuration(policy.Backoff, DefaultBackoff, "restart_policy.backoff") + if err != nil { + return nil, err + } + stableAfter, err := normalizeDuration(policy.StableAfter, DefaultStableAfter, "restart_policy.stable_after") + if err != nil { + return nil, err + } + if policy.MaxAttempts < 0 { + return nil, fmt.Errorf("restart_policy.max_attempts must be >= 0") + } + + return &Policy{ + Policy: mode, + Backoff: backoff.String(), + MaxAttempts: policy.MaxAttempts, + StableAfter: stableAfter.String(), + }, nil +} + +func Backoff(policy *Policy) time.Duration { + d, err := durationOrDefault(policy, func(p *Policy) string { return p.Backoff }, DefaultBackoff) + if err != nil { + return DefaultBackoff + } + return d +} + +func StableAfter(policy *Policy) time.Duration { + d, err := durationOrDefault(policy, func(p *Policy) string { return p.StableAfter }, DefaultStableAfter) + if err != nil { + return DefaultStableAfter + } + return d +} + +func Failure(exitCode *int) bool { + return exitCode == nil || *exitCode != 0 +} + +func ShouldRestart(policy *Policy, exitCode *int) bool { + if policy == nil { + return false + } + switch policy.Policy { + case PolicyAlways: + return true + case PolicyOnFailure: + return Failure(exitCode) + default: + return false + } +} + +func (s Status) IsZero() bool { + return s.Attempts == 0 && + s.LastAttemptAt == nil && + s.NextAttemptAt == nil && + s.BlockedReason == "" +} + +func normalizeDuration(raw string, fallback time.Duration, field string) (time.Duration, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback, nil + } + parsed, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("%s must be a valid duration: %w", field, err) + } + if parsed <= 0 { + return 0, fmt.Errorf("%s must be positive", field) + } + return parsed, nil +} + +func durationOrDefault(policy *Policy, selectValue func(*Policy) string, fallback time.Duration) (time.Duration, error) { + if policy == nil { + return fallback, nil + } + return normalizeDuration(selectValue(policy), fallback, "duration") +} diff --git a/lib/restart-policy/policy_test.go b/lib/restart-policy/policy_test.go new file mode 100644 index 00000000..1c0efe6c --- /dev/null +++ b/lib/restart-policy/policy_test.go @@ -0,0 +1,57 @@ +package restartpolicy + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePolicyDefaults(t *testing.T) { + policy, err := NormalizePolicy(&Policy{Policy: PolicyOnFailure}) + require.NoError(t, err) + + assert.Equal(t, PolicyOnFailure, policy.Policy) + assert.Equal(t, "5s", policy.Backoff) + assert.Equal(t, "10m0s", policy.StableAfter) + assert.Equal(t, 0, policy.MaxAttempts) +} + +func TestNormalizePolicyNeverBecomesNil(t *testing.T) { + policy, err := NormalizePolicy(&Policy{Policy: PolicyNever}) + require.NoError(t, err) + + assert.Nil(t, policy) +} + +func TestNormalizePolicyRejectsInvalidDuration(t *testing.T) { + _, err := NormalizePolicy(&Policy{Policy: PolicyAlways, Backoff: "0s"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "restart_policy.backoff") +} + +func TestShouldRestart(t *testing.T) { + exitZero := 0 + exitOne := 1 + + assert.False(t, ShouldRestart(nil, &exitOne)) + assert.True(t, ShouldRestart(&Policy{Policy: PolicyAlways}, &exitZero)) + assert.False(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, &exitZero)) + assert.True(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, &exitOne)) + assert.True(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, nil)) +} + +func TestStableAttemptReset(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + startedAt := now.Add(-11 * time.Minute) + + reset := shouldResetStableAttempts( + &Policy{Policy: PolicyAlways, StableAfter: "10m"}, + Status{Attempts: 2}, + Instance{State: StateRunning, StartedAt: &startedAt}, + now, + ) + + assert.True(t, reset) +} diff --git a/openapi.yaml b/openapi.yaml index 2e06e608..ceb7b469 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -375,6 +375,64 @@ components: description: Truncated error from the most recent failed check. example: "connection refused" + RestartPolicy: + type: object + description: Whole-instance restart supervision policy. + properties: + policy: + type: string + enum: [never, always, on_failure] + default: never + description: | + Restart behavior when the guest program exits: + - never: do not automatically restart + - always: restart after any guest exit + - on_failure: restart only for nonzero, signaled, OOM, or unknown exits + example: on_failure + backoff: + type: string + description: Delay before each restart attempt, expressed as a Go duration like "5s" or "1m". + default: "5s" + example: "5s" + max_attempts: + type: integer + minimum: 0 + default: 0 + description: Consecutive automatic restart attempts before blocking retries. 0 means unlimited. + example: 10 + stable_after: + type: string + description: Running this long resets the consecutive restart attempt count. + default: "10m" + example: "10m" + + RestartStatus: + type: object + description: Runtime status for restart policy decisions. + properties: + attempts: + type: integer + description: Consecutive automatic restart attempts in the current failure window. + example: 3 + last_attempt_at: + type: string + format: date-time + nullable: true + description: Last time Hypeman attempted an automatic restart. + example: "2025-01-15T12:30:00Z" + next_attempt_at: + type: string + format: date-time + nullable: true + description: Next scheduled automatic restart attempt after backoff. + example: "2025-01-15T12:30:05Z" + blocked_reason: + type: string + enum: [manual_stop, max_attempts_exceeded] + nullable: true + description: Reason automatic restarts are currently blocked. + example: max_attempts_exceeded + AutoStandbyStatus: type: object required: [supported, configured, enabled, eligible, status, reason, active_inbound_connections, tracking_mode] @@ -458,6 +516,8 @@ components: $ref: "#/components/schemas/AutoStandbyPolicy" health_check: $ref: "#/components/schemas/HealthCheck" + restart_policy: + $ref: "#/components/schemas/RestartPolicy" CreateInstanceRequest: type: object @@ -568,6 +628,8 @@ components: $ref: "#/components/schemas/AutoStandbyPolicy" health_check: $ref: "#/components/schemas/HealthCheck" + restart_policy: + $ref: "#/components/schemas/RestartPolicy" skip_kernel_headers: type: boolean description: | @@ -1026,6 +1088,10 @@ components: $ref: "#/components/schemas/HealthCheck" health_status: $ref: "#/components/schemas/InstanceHealthStatus" + restart_policy: + $ref: "#/components/schemas/RestartPolicy" + restart_status: + $ref: "#/components/schemas/RestartStatus" phase_durations_ms: type: object description: | diff --git a/stainless.yaml b/stainless.yaml index 4be9a032..947c2321 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -79,6 +79,8 @@ resources: health_check_tcp: "#/components/schemas/HealthCheckTCP" health_check_exec: "#/components/schemas/HealthCheckExec" instance_health_status: "#/components/schemas/InstanceHealthStatus" + restart_policy: "#/components/schemas/RestartPolicy" + restart_status: "#/components/schemas/RestartStatus" snapshot_policy: "#/components/schemas/SnapshotPolicy" standby_instance_request: "#/components/schemas/StandbyInstanceRequest" snapshot_schedule: "#/components/schemas/SnapshotSchedule" From f8a079bf0b88d6a25eff52ca4bc0895ab3b37e63 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 16 May 2026 03:00:01 +0000 Subject: [PATCH 02/13] Integrate health checks with restart policy --- cmd/api/api/restart_policy.go | 4 + lib/healthcheck/README.md | 8 +- lib/instances/health_check_controller.go | 11 + lib/instances/restart_policy.go | 67 +++ lib/oapi/oapi.go | 555 ++++++++++++----------- lib/restart-policy/README.md | 19 +- lib/restart-policy/controller.go | 41 +- lib/restart-policy/controller_test.go | 20 + lib/restart-policy/policy.go | 83 +++- openapi.yaml | 6 + 10 files changed, 508 insertions(+), 306 deletions(-) diff --git a/cmd/api/api/restart_policy.go b/cmd/api/api/restart_policy.go index 46c908f2..f50ce244 100644 --- a/cmd/api/api/restart_policy.go +++ b/cmd/api/api/restart_policy.go @@ -71,5 +71,9 @@ func toOAPIRestartStatus(status restartpolicy.Status) *oapi.RestartStatus { nextAttemptAt := status.NextAttemptAt.UTC() out.NextAttemptAt = &nextAttemptAt } + if status.LastReason != "" { + reason := oapi.RestartStatusLastReason(status.LastReason) + out.LastReason = &reason + } return out } diff --git a/lib/healthcheck/README.md b/lib/healthcheck/README.md index 46bd0412..1bdee903 100644 --- a/lib/healthcheck/README.md +++ b/lib/healthcheck/README.md @@ -75,6 +75,10 @@ Stopping, deleting, standing by, or restoring an instance stops active checks. S ## Restart Policy -Health checks only report health. They do not restart instances. +Health checks do not restart instances by themselves. -If Hypeman later adds restart-on-unhealthy behavior, it should consume `health_status=unhealthy` explicitly rather than making health checks mutate lifecycle state. +When an instance also has `restart_policy.policy=on_failure` or `restart_policy.policy=always`, an `unhealthy` health status becomes a restart-policy failure signal. The restart policy applies its normal backoff, max attempts, manual-stop suppression, and stable-window reset before Hypeman restarts the whole instance. + +With `restart_policy.policy=never` or no restart policy, health checks only report status. + +Health checks still do not mutate lifecycle state directly. The instance remains `Running` while unhealthy until restart policy chooses to stop and start it. diff --git a/lib/instances/health_check_controller.go b/lib/instances/health_check_controller.go index a675fd9c..e35a18de 100644 --- a/lib/instances/health_check_controller.go +++ b/lib/instances/health_check_controller.go @@ -22,6 +22,10 @@ type healthCheckControllerStore interface { SubscribeLifecycleEvents(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) } +type healthCheckUnhealthyHandler interface { + HandleHealthCheckUnhealthy(ctx context.Context, id string) error +} + type healthCheckControllerInstanceGetter interface { GetInstance(ctx context.Context, id string) (*Instance, error) } @@ -274,6 +278,13 @@ func (c *HealthCheckController) runCheck(ctx context.Context, id string, generat if err := c.store.SetHealthCheckRuntime(ctx, id, runtime); err != nil { c.log.Warn("failed to persist health check status", "instance_id", id, "error", err) } + if runtime.Status == healthcheck.StatusUnhealthy { + if handler, ok := c.store.(healthCheckUnhealthyHandler); ok { + if err := handler.HandleHealthCheckUnhealthy(ctx, id); err != nil { + c.log.Warn("failed to handle unhealthy instance", "instance_id", id, "error", err) + } + } + } c.mu.Lock() state = c.states[id] diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 9f1ea4bd..8594f149 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -3,6 +3,7 @@ package instances import ( "context" "fmt" + "time" "github.com/kernel/hypeman/lib/logger" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -76,6 +77,69 @@ func (m *manager) RestartInstance(ctx context.Context, id string) (*Instance, er return inst, err } +func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return err + } + if current.State != StateRunning { + return nil + } + + policy, err := restartpolicy.NormalizePolicy(current.RestartPolicy) + if err != nil { + return err + } + if !restartpolicy.ShouldRestartHealthCheck(policy) { + return nil + } + if current.RestartStatus.BlockedReason != "" { + return nil + } + + now := time.Now().UTC() + status := current.RestartStatus + status.LastReason = restartpolicy.RestartReasonHealthCheckFailed + nextStatus, shouldAttempt := restartpolicy.PrepareAttempt(policy, status, now) + if !shouldAttempt { + if !restartpolicy.EqualStatus(current.RestartStatus, nextStatus) { + return m.updateRestartStatusLocked(id, nextStatus) + } + return nil + } + + reason := nextStatus.LastReason + nextStatus.LastReason = "" + if err := m.updateRestartStatusLocked(id, nextStatus); err != nil { + return err + } + + stopped, err := m.stopInstance(ctx, id) + if err != nil { + _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedHealthAttempt(policy, nextStatus, reason, now)) + return err + } + m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) + + started, err := m.startInstance(ctx, id, StartInstanceRequest{}) + if err != nil { + _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedHealthAttempt(policy, nextStatus, reason, now)) + return err + } + m.notifyLifecycleEvent(ctx, LifecycleEventStart, started) + return nil +} + +func restartStatusAfterFailedHealthAttempt(policy *restartpolicy.Policy, status restartpolicy.Status, reason restartpolicy.RestartReason, now time.Time) restartpolicy.Status { + status = restartpolicy.AfterFailedAttempt(policy, status, now) + status.LastReason = reason + return status +} + func (m *manager) StartRestartPolicyController(ctx context.Context) error { controller := restartpolicy.NewController( restartPolicyStore{manager: m}, @@ -147,3 +211,6 @@ var _ interface { var _ interface { RestartInstance(context.Context, string) (*Instance, error) } = (*manager)(nil) +var _ interface { + HandleHealthCheckUnhealthy(context.Context, string) error +} = (*manager)(nil) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 7ef14c6e..1c053f62 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -198,6 +198,11 @@ const ( MaxAttemptsExceeded RestartStatusBlockedReason = "max_attempts_exceeded" ) +// Defines values for RestartStatusLastReason. +const ( + HealthCheckFailed RestartStatusLastReason = "health_check_failed" +) + // Defines values for RestoreSnapshotRequestTargetHypervisor. const ( RestoreSnapshotRequestTargetHypervisorCloudHypervisor RestoreSnapshotRequestTargetHypervisor = "cloud-hypervisor" @@ -1353,6 +1358,9 @@ type RestartStatus struct { // LastAttemptAt Last time Hypeman attempted an automatic restart. LastAttemptAt *time.Time `json:"last_attempt_at"` + // LastReason Most recent non-exit failure signal that entered restart policy. + LastReason *RestartStatusLastReason `json:"last_reason"` + // NextAttemptAt Next scheduled automatic restart attempt after backoff. NextAttemptAt *time.Time `json:"next_attempt_at"` } @@ -1360,6 +1368,9 @@ type RestartStatus struct { // RestartStatusBlockedReason Reason automatic restarts are currently blocked. type RestartStatusBlockedReason string +// RestartStatusLastReason Most recent non-exit failure signal that entered restart policy. +type RestartStatusLastReason string + // RestoreSnapshotRequest defines model for RestoreSnapshotRequest. type RestoreSnapshotRequest struct { // TargetHypervisor Optional hypervisor override. Allowed only when restoring from a Stopped snapshot. @@ -15856,13 +15867,13 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZI/+ioIntkYaYakSN1sa6Pjf9SS7dZ2y9axbM/ZbfpQYBVIYlQFVAMoSrTD", - "X/cB9hH3SU4gAdQVRZautqYduzEts6pwSSQSmYnMX37pBDxOOCNMyc7Bl44M5iTG8OehUjiYf+RRGpN3", + "H4sIAAAAAAAC/+y963IbOZI/+ioInt0YaYakSN1sa6Pjf9SS7dZ2y9axbM/ZbfpQYBVIYlQFVAMoSrTD", + "X/cB9hHnSU4gAdQVRZautqYduzEts6pwSSQSmYnMX37pBDxOOCNMyc7Bl44M5iTG8OehUjiYf+RRGpN3", "5I+USKV/TgRPiFCUwEsxT5kaJ1jN9b9CIgNBE0U56xx0zrCao6s5EQQtoBUk5zyNQjQhCL4jYafbIdc4", "TiLSOehsxUxthVjhTrejlon+SSpB2azztdsRBIecRUvTzRSnkeocTHEkSbfS7aluGmGJ9Cc9+CZrb8J5", - "RDDrfIUW/0ipIGHn4PfiND5lL/PJP0mgdOeHqeLnCrNwsjzjEQ2W9cn+Rll6Db0hnCoeY0UDJM03KIGP", + "RDDrfIUW/0ipIGHn4PfiND5lL/PJP0igdOeHqeLnCrNwsjzjEQ2W9cn+Rll6Db0hnCoeY0UDJM03KIGP", "0ARLEiLOEA4UXRBE2YSnLETvj85QwBkjgW5MjhifSCIWJERTwWOk5gTNuVTwjhI4uEQKTyLSH7FOt7Ie", - "hOkn4Xoq/WNO1JwIz2CpRLYVNOUCqTmViDL9NCD94oIpkZI6ZbsdGkZkrGhMeKrqhPqFX6GIsxlMy7WL", + "hOkn4Xoq/X1O1JwIz2CpRLYVNOUCqTmViDL9NCD94oIpkZI6ZbsdGkZkrGhMeKrqhPqFX6GIsxlMy7WL", "4lQqNMcLgj4TwdEfKY7odEnZrJlIEzLlgqBflgmJMUNJhAMiEVWIMsXdbAyNch7bi33MRWeMCzIOiVSU", "Yd3+OOHC7Ijy6N/CHzhChXdhaPA+UnOsHJczrtAlIUl5ovgKX5bJ+Pv2dvfFYDD41O1QRWKzrfA1jdO4", "c7C/t7ez1+3ElJl/D7PRU6bIjAg9fPsLFgIvC9ORPBUBGQc0FKtmEkSUMIWOTo7f3XICneGgD/+39bzT", @@ -15870,7 +15881,7 @@ var swaggerSpec = []string{ "pitNe+BbioCzKZ2lwm1B35YrkXOOJcLMCI1eRV7kjbXad4EWYiG/YmNBYkyZpnFtEO/cI6R3KLKbSA8p", "4EwJHkVaKChF4kRJt4u6WowzhJMkogGIntKm2o0HstPtsDSK9MPKCPPVJhGdUXihFWmoLCyS+xYpjghT", "RGQ7vA1pSmKxqeOc3N7VyOVieykoKQv802VVmsdawgsSmOlmJ0CJIhMS8Jgg3XR5BbYH2/u9wW5vsP9+", - "+OxgsHsw2PuvTrcz5SLGqnPQCbEiPb3gbZZptfw+yqmkX0T2xfyo8tCuX5HB7dglwlJluxo2OVXLMfaM", + "+OxgsHsw2PvvTrcz5SLGqnPQCbEiPb3gbZZptfw+yqmkX0T2xfyo8tCuX5HB7dglwlJluxo2OVXLMfaM", "6T2NiVQ4TvTG1mMoELNpW7sGq+vgKL+SwMM7EZiRazW2FPLOx8cf5DohgT5iuNue2Ymt2+siOkUYZTJA", "s6sRjCsn8uJOExEESz1grXfo0+n3TspkmuizkITjJMJKt6uVFGCDcUyl1J9mP4RUmo3Z7TgmHzOuxiJl", "zLzIiLri4rL4pm1lTJNOtzPHcryYJWmnu+ocKDM1dEEinEhoz664GBMhuOgYXXM5nnLhFkkfYjkJVzRV", @@ -15882,273 +15893,273 @@ var swaggerSpec = []string{ "ICzkopGk5rGfpMNBSFY02Yqktv0aSd98PDk+OURHXCRcgBG0duMUyVOcV5Ftyqvi4/+fUxqFda6f6J+J", "GGeHiI9gJ06NOjl2eoL9Dn08RRtahoRkks5mlM022/B7wDU59FHnO8RhqMi+o81E5bSUW5+3gSB4TXf6", "jVad1bdaalZyHMum1t0rWqLGNIqoJAFnoSz2QZna322eTGHDmBOq1tVL/TOKiZR4RtAGuFTA/DDCVCs2", - "U0wjEm62U2abJvNPPikcISX2Brbo4Ukw3N7xyo4Yz8g4pDPrE6seUfp3zWK6HYXgbf9E4DBvNw/oUpBp", - "vb9XILqhE0GmRBDN43fsLhF8QRi21stfoN/O/7WVOwu3rKdwC4h5lr/+tdv5IyUpGSdcUjPCmuSyTzQb", - "AakRfOEfMzxatdYFjpIKi9X7A964h52Y63VraWPdFlq1wbO1n7zX71RlJ4jGTJcoSIFGEflSKzUe7YAz", - "ZR9U3Jd8hiLKjMWhVTuzFqBXLRPyU8RBJN4THTLy1ze/HvcthJf5oaE1/aybKeARnxWpOSdYqAkpEbPh", - "CLMN5aNrJP9ZaftUziosyXi1BDmjjJEQ/MV2Y5s3tRrrNTNgF11SNV4QIb17Dob1K1XIvtHYVMSDyymN", - "yHiO5dw62MKQGmfhWWkmHm2t5IjHYI+7BkGLAPv1/JfD7b19ZDvw0NB6LvUL9ZkUvtbNm3eRwmKCo8jL", - "G83sdvMzus4hfg7InZVNZ0/GgY4xjaTr2NW0dnIq5+YvkN16VHD2aTGg2SvSf3/yTPoIhISxEhpvb/w6", - "YOYZnkVc03SJUkb/SEsKdh+dTMFBrA8KGpKwizA8AL+Dtv9mhBGh5VTuGSoowWiD9Gf9LhppvbCnteAe", - "3u4NBr3BqFNWY6PdnjHvE6wUEXqA/9/vuPf5sPdfg96LT/mf437v09//4mOAtpq50wrtPDfc3u8iN9ii", - "ul4d6DpV/tbSvzh8n8QxS32i5cRNV/ropK44mLmGPLgkok/5VkQnAovlFptRdn0QYUWkKs989bv3SguY", - "xwoisJkm0w3JUDF6gI03In5FRKAlcEQ048muFsJUyS7C2m4G4YX0KfnvKMBM7wWjXHCBCAvRFVVzhOG9", - "MrXiZQ8ntEfNUDvdToyvfyNspuadg/2dGp9rJt+wf/Q+/c39tPl/vKwu0oh4mPwdTxVlMwSPi9d6bgzZ", - "Fc2qFXHUTSNQ82LKTsxnw/od1N1W2E1k1UobY65xqbUQylxkawZSv9/VxlbsMR3eLogQNHTH8tHpMdqI", - "6CWx+wWJlKFROhjsBPAC/EnsLwGPY8xC89tmH72NqdLHYZqf8ubKtnK7RoI5B0UlivhNrtNAUwQDB0cr", - "z/FVpPFS+yhrt37q/8Kl6sWY4RkBc9S+iCaCXxI9UHMnQIlEl2SptZwlmulGewsq4YaHsAVaYON16I/Y", - "+zmXxLziHknw7dMFQTEPLs3V75yDJb/AUUpkF13NtcoBPkGCI/szMhdjIzbXg5QBT0iojRDzGkwNXRC2", - "uEAxTmCbY0Fgj6MYKyIojuhnc4UPtwwkpPqEGzECGwMlWO/5IOAihBs2jggO5gUq/FWiC6OwXEDzF5Rp", - "tr4wG7NyWf2l8/bD+5/ffnhzPH579vLN4cn415f/qX82H3UOfv/SMaEamabyM8GCCPSXLzDfr0a9DYno", - "HHQOUzXngn423pqv3Y6mgdT8hRPa5wlhmPYDHne6nb8V//np6yenkBk39kJvA8/AvnqVIXOWekTSsfMG", - "SmQ9TO5uQ5NMi6jXZx+29OmcYCnVXPB0Ni9vDKsa3GhLhFRejikfTxLfmKi8RCdbb5FWXFBE9QbNFJXh", - "YHD685YcdfQ/9tw/Nvvo2OxaGL6WQVxY/UnONftkUR9HZx8QjiIeWB/KtOmC13XlE/CEKbFMOPUZcRXh", - "lL9al1G9Xv70BqJoa0LZltTL0AtuRnfgm1ubEi/ZggrOYm3OLbCg+pyW5b3y5u3xy/HLNx87B/ogCNPA", - "eiXP3r573zno7AwGg46PQTUHrZGBr88+mFtP2DYER2o+DuYkuFz34S/w7hG8CjtOJVE6G0v62aOFHGak", - "QTGJuTDWt/0GbczLSorZ8gjWddTZef2z4cvha2BJt572eilrxTRcuRF8/bOP0ebLhIgFlT4X3S/ZM8c0", - "9Uih0rYwF2wZv8MG6BdMnyDiadgrdNntTKkgAURm6H/9QWJtAyw+l2+0PN/5PWetdN81Si2OEsrICq32", - "O9Eur7i4jDgOe8N7Vi7tXawnqsY8KK9vdinnWKIWrDbBLLyioZqPQ37F9JA9Itk+QdnLmVy+1jPB0f/+", - "9/98PM1NtOHrSWKF9HB7745CuiKWddNe90s2kTTxT+ND4p/Ex9P//e//cTP5tpMwOsyt9EG7/i9NC9V4", - "GxuGaDypDZfK2cGfxboobm1x+Bw53lt7g+yT8XxBRISXBcFrx9QZDkD6VUYlKARYIvudFqOXSH+8Rgzr", - "1px+8LrqH9ge+AWtILCzx0kWWbqK/u/M27mZ4pmTZ0o/a1Fjj5U2E8nmMdw+tX9u12fkn5C8pMkY9PUx", - "nmXe5lXBqOeXNLFGAHxhuCCKjBwJUzAbJpyr/oiZ2Bi99MAf5JoEIDKlwgodnp1IdEWjCHxTIJPqJ5M2", - "KQpBVfC6VPp/Rcq6aJIqbSdwRZC12KCTFMYCL08IShl2N/EVrd1OsB7YAGS5JIKRaGy0ctmSMuYjZD9q", - "JA5MdYqlDY4TKk3K9Dr+9fQcbRwvGY5pgH41rZ7yMI0IOjdxDZtl6nVHLBEQIKE70exIbb98iniqenza", - "U4IQN8QYGsu8e/aaePH67IMNNJCb/RF7RzRhCQttiLE7sGz4acjZX/WGJ2G52WL/FaI3BZNIhhM55203", - "17l9Pd9d7d0Y3c4iSNLykm53G8NPF1SoFEdaVJcUWW9ogQmf9xgsJjq/aDhZsZmH66rynXBbX49pGWLp", - "vYG6HpeNUbRau2wKToSa88ZZuF/aDXZN+yfMDWSlyyo3cu/Q17lppBY2ZH7uupndgkonGU0qjq77Ic+h", - "LDgFWoW9m+gvo1BKtHGBE9q3fNwPeHzRRRd/K/2g976zTLR6coUMNUCeMP1Tsf2qO2Sto+JGgebFxcHy", - "9utxKBtjrNBiiJTATJrouDlOSB/9AkIcKRInWpKxGaISZUFliPGrf0fc6ETu0xHTQ5MmQsWSI3NXSTpj", - "lM02tZWgDyYchsanNU1VKvR7CypzapZZx/mNavG0ZnTEyGPIzaAsiNKQoAvnW7ooq5V1z1PdorSuqJqB", - "ZEgChhHYimorTpXuXk84xiqYazrxVJmQNTv1cjhhxb+17irXjiW75LvF+p9n4qKagrPwWEh6cvZ6CRyS", - "Bc9okwPSKip+5+glWcKSO0corrlCiz5Qv6dSEMmjBbHHbtGLOoEkI24Up9yBalyh1vupt381vcbnF1y3", - "FJperclftjQ8yUVS9dxkc46xxoOLRndSSE/O9NfVdrUkQHywXA4QqGMXXWNqEXBgIKaZJUIhFSRQteYp", - "m40YRK9c2F/6trULvcm1jnIvKVuQAQFKe3FpUWFlndoHzeip8ZgqRcJuWTe4JCSR6yel1WvrMvf49QW5", - "EtQJMhfO3FI9I2zKRUBiayTcze58WWjMawXerIl6MImhb2HMLjME8mJIaCKXzHqAg7eUMFLNmwwrVpsJ", - "Xih3eYGj6AJt2Jc2kSD/hBwAu1aMs5zZ3x+dORbILtw/nnY1R2opcDFXKhnr/5FjvYsvqo3Zb90Oz3Pa", - "ng/Avtrd3bGran12ZsCVZsvuOW9ARvPSOPW78U5P84UepY1waaPKH+Wf5D7cS8rCtg38qt9tdO5lipGz", - "NB7av5cI0kuTmcAQ3Huf3r1b39gCNZsl+JoMYl+AZp6bmErF42Kk/0YluISWw1DKxFrwqBdihcET2tJd", - "a4ZbD3mOl6YpY4t5/R70MxnPJp6IJfoZkhBmdIYnS1W+uRh68wjven3uxuJblqbUAWNBknCs+OrgaTpF", - "7t02sZIm00Hx8WJK+erEEht5U8r8M8eRtWt1E70koNadADpOMDexrYYIoDR+PC3eGvZHrAfH7wE6zjrI", - "ms2axKBb4tBcvGxwURiESSFBk+UmwujjaR+9z0b7V4m0wbIgLpdijiWaEMJQCp5rOA175iwuDiCVcGiq", - "6ufWd2LSLjbhcpTbZ/0s2xm8NFnuNgRpTWhlPiZnExbK3kZjVvSCtfJarQo5f0dmVCpRCThHG+9eHe3s", - "7Lyouj+393qDYW+49344OBjo//+v9rHp959Z4mvrsCxbbNhbUfocfTg53rbO0nI/6vMufvH8+hqrF/v0", - "Sr74HE/E7J87+FFyT/yi7DiP10MbqSSi58Sk5ipflF4hGK4hCu/WwXUPFCuXh/6uetdQ4r1+8yGSanzh", - "2jZY+OZpL1WBuTbguzC5uiW/TMDuzHdJQYOzcZUB9UaQHlN5+bMg+BKSBevndoxnRI7NeeaPpEilCe8h", - "19a7IThXU2muXctez+Hus93nO/u7zwcDTy5JneF5QMeBPoFaDeDt0QmK8JIIBN+gDbgvC9Ek4pMyo+/t", - "7D9/Nngx3G47DnND1I4OmeHlvkIbliJ/dwgp7klpUNvbz/Z3dnYG+/vbu61GZf3FrQblfMslleTZzrPd", - "4fPt3VZU8Cn0L11uT1WB9+V0HhpcAf2vnkxIQKc0QJAdhPQHaCOGI4xkt1XlPTnBoct89Z8dCtNIrgy4", - "MJ3ZN42jLU4jRZOImGewIK180TDzY2jJi83BWJZpfLOWbEbU2gADN5fsFVTKbCuR7tSkUheUJ0qi8MDs", - "0LVyDlYzH9inJj6wc2jJDb9p06kXkQWJikxgji6T0ysIyvjELFppVpQtcETDMWVJ6mWJRlK+SgXooqZR", - "hCc8Veaa0aaG551AvDXYHlMtrtvZua+4uFwbuapP4iwDfq1X6BAc6VPrqoFTHCP7tUuOKCh92XWguTS1", - "zyV6Z74wHqL85yQt4+l0oSfrSWJIEKk4SFLrMLTNtNUu/XoLOEtd9IjpL5edjxQ605uaaIP7tbDFjADy", - "g1qrsWhOeQ/vn8PrrQPh9YdrHSkt6M7I1WMQHTIFeppte5Lh5GEoviqWLfM15C/BKSxoSPoIdhcE1bjM", - "xMpOO1c8SUiY+X/6I2YjybOfpLlB0R8aOqg5oQJxQWe03HHZwfaQQXE3YUXHTbdmx+KHdQ0VHkL4RvOm", - "x1NlUB4uXbIWKWZO2UXodDvnGSaGlURl0rzLcEVqFMmDPGtDfH324aahbYngU+pDOoJYCPvUWmYu6Ou3", - "3cF5b/j/mABOzW+golFm4idiHlYgLOz77U6e12cfzprGlIFKoOLoanPKIl5WwWo5ithLJXsraS0Yx/76", - "YMk6yXXvFz5ddipwTCbpdErEOPY4117p58i8YEKbKEOnP5f1Wa03t7Waz0qLA2bzFAcWE6Ad9T0Ouco0", - "ugVqfvIv1ztijuGmTEK9VMK+Y5MJ++hNBuOBXp99kCiPUvJ46srL2xipfzZfShrgyLRoEoMpKzrYgDlb", - "a8hn+YfWFenRk/3oL24joI3FLElhG56/6528/bgVh2TRLY0JIovmPCJ63JsFabFw+YR5WkFJSCyaPB2G", - "MWTbDVSgVbaDWxOpsF891FFc4WgsI+4L1nivHyJ4iDY+vjL5XnoEXZSUllL/XqBCib/3vTtGS6Smbs+h", - "w6rLtLTBvbZjGYfTuFcK0yt16tsqJsi+ruPUoZf4ZXmh+eV6uB/TSHO/Ry4PoOLUtookMukCCNIF3P0z", - "Mp8ar7V1jUiSYIEViZZGs8iOvohOSbAMIrPHSf0qkVyT4AaJCC/1619NHnEqyFjNBZFzHpUvoHe6ddg3", - "CUGQC2LhN8ycCo53xVGMxSUcjE6RRikzFCgnHeysg7ScK5XcYFK/vH9/ZqxrRcTCRJQVg3xl7Wr1mER4", - "iSZEXRHC3FSwRBi95hnOSTUrRzZAIAg1ToigvEzDzo6n33MTmIlmAgcEma8cCqNdEgmnZltS2l48UF5B", - "QKRsWN/hqvW1n07TqN0a+4Y1XAtaGtxkgd8fnblU/gyV0JF5u07lMyJ6Zss5eMLVS7stV4NKuK4YZ6Te", - "meATAigTNvGmmCLkAksARUN/XkrLKQgHWcyHsf3ALjCk6pp9/qmVslfd7r6L9Biz0IfuaCKeTe7jLI31", - "gughixTujmhook5MOqZRy4uB24LgkDIiZSVxLEhF1Ol2elM7q4OtrYgHOJpzqQ52d4bPt1bH760M3LRx", - "KuOQrrLvXDSLiXdw+VUGZREmXWaJLZwkLTxgho5rzgcQT/VAMUB01GdbQcNzkcODQS0H7hoHygHjgEus", - "dOWJi9s20SxZmg80mMEK7714UdyfA+8ddI7q7dh/q8b7emYmmEzziHE3VOhomPyzV6Piwod+woWyGS4T", - "4qLNsvPQxXLZS5VSZ88Hz4uzbAWeDMKmss3tvvNM1bxdznojBXLb/WsbgLiYss7htvRqrDRNlzU8pSVi", - "jaVWE5QnhN2Innu7O9s3o2fbiZy4pLyKXPLl3B+dHhudKOBMYcqIQDFR2CLFF4QM+JK0lNEGfohJDHkK", - "039fLVoa4heKSfSNN+BHNbi1B7n9boAJemfiN0MUY0anWiDbN4s9yzne3ts/MGBmIZnu7u33+/2bpha/", - "zHOJWy3FlkmfLGQZ9+X8buvwABnEbebypXN2+P4XLchSKcyhtSUnlB0U/p39M38Af5h/TijzZh63wr+j", - "0xruXTkeTBv85veDAlS503tawRH7ncEQFgpwB16YGIVnWrcxHHdXPJhbI8blsKWqgBRXzKZpgRpHP6++", - "RnZeJXjH9pkyRaMcUK9+gXwrSES5EjWqhhiVEJbhREWR+SvgbKF3hQ80qnQSuWd3Cr5YqXr9o65xrd9v", - "TvFaw7Z+L1sm/9qC5VlIG89J9M2l/m0ClMq9v539xx//rzx79s/hH799/Pifi9f/cfyG/ufH6OztnbLX", - "V6MZfVNIontDIYKonBIUUVtWOsUq8HijtKHTQGH7xNjWKpj30RF4zQ9GrId+o4oIHB2gUaeSXzXqoA0C", - "NgF8pRU73ZRNE93UH5+ZuzP98Ren8H2tthHafFBhFyTLIpfpJOQxpmxzxEbMtoXcRCRowPqvEAU4UeC4", - "oAxpS2+JJgKqsdi7jbzzLvqCk+Tr5ojB9QC5VkLPIMFCZfBrrgdgCjsqE3BpXyehw/Mx1wsjlp1LGZyP", - "ueDqZ2ouBDZU01X8RFltqVgb4fnAB3wEIfN6ISMqlVG2M87WbJTF8qPng8265bJGm854aAX7wU6ol2ly", - "TNliLxkGhq6N4B47Z9yaQAQtm8weQWArKQ7/PUeuoZwW2RIbD7lJoJDmglVFspA6sdnxgsTD6rackLlh", - "hM+iFinXL01uzfvfzpEiInbZjhuBJueUBnp+EDtJpUw1K1KMDo9OX272W9SZAtpm41+xju+zGVYzY+2N", - "Y9NFam7Y4Zh00ckx5DbZHZorcBCT/IoLFBkBk+/rA/RBkoqNCNlrEBJpVjJa5teW5gQYdTZdi0lVUhyg", - "d5neiLOhlGpblW9C830JzdqoFRMwXWu9W6saI5xdZEUbhEdjlWXY6RO3WRS0d1RYisOer5jVN97bxZvk", - "RqO5sPb3DXZ3/+rOzs3UHVe4IJlj6ePuefEqBF5aUS2IVvy7ovnev9Tv2oI9WXdQg0gffcXPfaVW9nrD", - "4fvh7s1t/pvilJUBPQpYMhlUWXuMsYfA6qrbv9dUjRsjQpF+bOM/nZX38RTNsWR/VfCwYusNd561Qp3X", - "vbaNpSxGUfKpGVImpRw6SBYDaHBSLmkUmdBaSWcMR+gF2jg/ef3ryW+/baIeevv2tLoUq77wrU8LyDIn", - "Kl6ffYDrNCzHLhypOQMH51ls5JpKJevgKa2i+u4CkWY+bQcr7iZp2sjxxVfjrP1SwkLzouFs3iNAmgvF", - "rJHxMaDPvmWOy/cHu7YSKO2uaGfWeHkgsLNG4e4DCivLefPz/cKWPchw1hbpK571LgHx1jhh3Q71JF8d", - "Si2CSYhOznKo8tzJ6JqvzMlWvBwOBv3hoI3LNcbBir5PD4/adz7YNqrFAZ4cBOEBmd7B5WsZ2yjjOLrC", - "S4lGzlwadYx9VjDMCtvWmlStrqfrcGy3Q1+rKjR+OW0UO3fvL215myblpkV2TRXsJU4jk8BZLIlTr/Ip", - "E4NsZjCBM112xGCAXYuwklXmxEEg0tyf4UquGc03TSzfj5ggMuFMmkKMffQrWUoUU7hDyLqHyCGJsiDu", - "cMQ2hAv4zyL7E5xKEuofIJq266I29dCoAuxj/cGIyXkKFeQ2++iIM5nGRFhXD5pQ8ENvIpka4w7GC9SA", - "eqaShkSMmH7Ng532JVPUD/YHg8Egq2zXOdjR/x74uOmO+Hnu83Yqh/081zX8PLwObq8djt5dwctWlcM5", - "LxfCaW3R3aFaZKtgc6fT2TBz+9X4JndlBAU8jUJtJkz0wWC8OCS0ziZJVF5jCM6SD+ySaXYuTd0GuSmO", - "/kiJWKKPp6elCzZBpraESouJw4ZqWAee3GgZttcY1mtHc0tIu8eAsase6gVl6t5B64oefpfuZji0hac/", - "N668IcqUmaXRfLJiThUfbUgW4zT16ez6kUty//Dh5LjEHBjvD58Pnr/oPZ8M93u74WDYw8Od/d72Hh5M", - "d4JnOw1FzNqnKNw+68BrofmqUrnYw7GLgfQWHm6IQK0ckTaq7oqykF+1Ktmd9W5DqNZ1Xw+QbD0Eb1g1", - "VDqGlhqkxGmhnrEJK6zULGrwPO2/HwzXeJ7alWFukL/vRcoCgz4Fkjjz6RYLMBcXq16E9ebiFAbkwpfX", - "UavYeXuiDQ72Xhzs3ZVoLgR33Rir7PSIi9sUcOAgDCsxvi7PpOC/KFQwBn3DuFltSHCn28miluFvOGgr", - "EXHZ41ah+E0btusXI6vkd0NG2klJbYZbWINkFB5oLSBLZpqkCmWJjlq9OIp4GqKC78cAu8DFyElBhdbN", - "wD2FdQ0ZhC4TUqtVbYCEBEBiyrQghgsh3YhNXztAr+FdeIRjY13YQWAWVu5CcLg0d8F6f7muja6/esjn", - "Vs2Hb7TOj6CwuJ62JoN1Ea5uwmg+B+gNh28yo4Pxqq/RvA7afv31ql9yw+aVufxj6MyqcQfoVaa6Zcqf", - "VfY2JLF/jq3AytP+N0vJl3bFO5pb8pUr5BV2O4ainW7HEQryD+uZiB9yrq/tvyIr+oIkCI5gL+eZXqmi", - "kUW5hJlQKK1tQ4D14jbpFxYQnoRjY5g0hTyZ9CFrvGQfOfXl4ynaADyrvyPrSNX/2szCo0pn3faL3Rf7", - "z7Zf7LdCrcgHuF7tPILktvrg1uqgQZKOXdXahqkfnX0wJnhgjFvwzNu5F5KEE8G16NEzz8vg5p2/6L8o", - "gnWEPDUlwe2QLLLP10Lh+5U1ixtifP6g0YJOp+yPz8Hl9j8FjYfX+3J74nVo5hX2vd6fk+Jtb81VSiY9", - "U8TDj6cADCVkI+TIOyJhBuicKAT800M4ANMhy0mzLOeASSzFvYy1u7Oz8/zZ3nYrvrKjK2ycMfiCPIey", - "HUFhi8GbaOPd+TnaKjCcadMl6gJAKLNmpX+fIVsKbFBWSPvDwY6PSxoO7pxrbNuLuJHkH61pZidliQ6p", - "dZnZVtvlXmrv7Aye7e4932u3ja2XciyuV0sYF3huyGNxbIsrvwHa5PvDMwRpXVMclP0mw+2d3b39Z89v", - "NCp1o1EBBrPBTr3BwJ4/29/b3dketsPO8UUBWFSo0oYtyy7PpvMwhWc1PKSoi95u02nhU6cMg70jQYRp", - "fBi4CN7K6WMwUsfCvJYvQpuDwTrGawdXi29bOY4qJaeNasAFSlmGzN1ffwV4uxu9ZjFtzoP1Yrxu2UeY", - "aXJZkAdTiuMWtEsEWVCeyntoiCuT6jSNOBc3+rbJQnlHZBopc+1GJfp4+lcQIpq5kFQkKRtNlv1WQGHc", - "cnI32sAlnvBzdROxWq1Gm6VfNeFuwzbtrsqDLm3/RsSZUIuqlK0PvzvCUZAC+DzO1lPPCrAjIJMzSaKl", - "CVSNIs4ZCuaYzQiUETSlLtgMYTTnUdj3Bg/qJ+Op99qeX6GIG6zMS0ISi8tuBqE/0zoLXRC0UcgkRYaV", - "KuWZ9mIjVSzydpkb92J/ISAsfckPWQajpidWvADjaD4p+RgjPpNgBSoIwe1X0YMTLExkLWamzsAiNsZj", - "GXpnW5/2niFWpLfvCDVHJ59ai9bqGJAfaCiJA8GlRCSiM8C0/3haSTtbkUORJZ+tD6krD7YF65qLNM/Z", - "BWeabF2OxHcgeoLT73IkAg9DDsqKYDXnjYwxSwGpvcDI5DqhwrBHu4C0OZdqnMGJ3HCwUo0BhTsVJMcc", - "ypIlMweQe8d7LjrRdhty2cjPW31d4yp/U00DbJapXor6qdXNeNDHxnVAlZUYLjkoTBUB5CaQPzlsM5XQ", - "Ki2gzaANxlVJLBWghzfbBGf4bVTdT808tVWmftsdnLdF41kNvnOG1fyETbknZfsG15DO9WzjBRMiYgo4", - "9CgkjJLQGY/ZfaT1bUGeXyQJClNiKWcUUoEtwbHZ3pB2zZxTjLJZRdZXO2zjDzZjWA3SDf3aF9uE2Uh/", - "dth7kQKtTGCcRDjPE2sVZUjl2H9/VW9YkFkaYYGqiFMrhiyXcUTZZZvW5TKe8IgGSH9QvWSe8ijiV2P9", - "SP4Ec9lsNTv9wThPc6hcGpvB2SQXsyCVfvMp/KRnuVlJsQPXy5b5fgsS/dtELXljdV/RiFhQpg+MXhcY", - "vYxiu7s9aMq+bGi0lHdZB/S6qeS2LOvb8Q5r6zArmum5pTRRt5W70rIjcu1NH4R1r8o1rbti0IYLhHIo", - "wWW6FtB6W3lC2kWWV0P+3Gi2JAnKve8+33u23xIu+U6+TpPCft+ezUW8wqPZsFKnbdxmz/eev3ixs7v3", - "YvtGDioXHdqwPk0RosX1qdTGrTjN9iCSanCjQZn4UP+QGmJEywMq1bm99YC+rti6TcEF+d5suuSMiivp", - "7lnKHtB2PsYV2tJhSeUqVIHfINMpAaNybOjWywdTyRBsNYYAJzigaulxmOArU7Ive6UC99bGm1YerIek", - "tm0LDaQll0wneRLFhusc/c241iu88Lw16rpMJ01u/LfVXo0TP/cBFa+IWtzQ5IUh6+6CbD5XWJZizfTf", - "AURP5lX+qzGz5o3V0FNVwA+4BLTFBQqRFD7Iwsr5Zz8qLn9lOQtu35KSXKX4qiO0eQveyIb2nMgeEzpY", - "nwlTkQ/2ALzdV+NJsR7CyoITpeIJ+al7835bJPvUwUKzE+zm/RXSHm7yYRUYC/jRjsGSPG+7W2KJBm4q", - "hOl6zBEekV4W52BjeJFMjX9V73mLtejJwwgu+XRaBnzaawYIBGQ+CNl2vWClSJyoLiLXYKaTsIYu5+po", - "78lRB3GBRp1hPOpUnIDeJIgYX49tB+Vc5cEqxL6sElB1kNLNYBLx4NJA/StBieyjAYoJZhKlDDZ/xUc5", - "HKz2tXU7SWFtMnw8Ym6Ia2ILxjQhc7ygAMtqPVSzUhwLuaZKQrwNtHOAQg72bbnOkZ2hfs2kKBzkk4ZD", - "B7OlbVg3qN/jzAUE5e+CuTSF6krsMxG8a5PvtMR++/a0a+5/IHLDDKwUHuImakagBWTWRQVjNP/dH341", - "icgYxl3FrIzrdCymkoGfWhBJlLQgdjk7VJgABTxlqgpmGbcL4SxHvNePpJRBrIS9PQPIBtu7rY4YksBW", - "oK27l0qMfgvmroRdWkr74i53fCwMmwI8c37P+zvrXq8OQCIsSLFEmWmnGBZnfK5jqbjFtM929ZhcB4SE", - "VTge/yttQw3tl95Qw9+wzXnPqofZtyFcrD67/sPFnjNyvXqsb8i1ApC9MI308JqW3m51K8zXjfjWSQtN", - "G4KL9cUqH6D2gAl3u1X1ARsp9ygFCOzPD1J0oLYc50S5d88t3zTXmizBA5c8ai7A0L1SvuIzvNNF9kRE", - "w3izwnO7c78bwQJctcy3YDgm40SQKb1ewS3mBWNJlrN5851Tqkcq0UaMr9HuMxTMsZCVsTM6m6toWb7/", - "2/Uk09+pFIcgijB1g8qt+Wq6D+uX7XY5i637tMnzQuq7v6IsCcerYOCOstfcdWaCl+D2aPRRPtvZHQx2", - "tge3woG7r0K3hXaaQvgL39l7hlJUTLGFLGGqXg3pSlDIu8rIJJUgOD6AQN8EBwRFZAooKVkVurWHRa3r", - "1YO3GojNw8743y2UXTd3BVAGWs66shB6bhodF0BUTr0vPq8PewWUSiZmghqmiifGf6c32H8/3DnY2z8Y", - "Dh8COC4jUlN06bPPw6tn0Tae7kbPl8/+GM6fzbbjHa/h8gA1lcs5k9USy3YOCRHVMlfV8nCSRJSRnswi", - "stenRayQBSbIYe3+v5lj3MxgpbJwXp5kUWfAKidOibMeCWfCjn6ld786/JPj1cO+VYhzdSB+BqsOBfip", - "3WAAyHR4V9TMlLU8dz4UXmx98qwMu1939viyJGFre1e5geI+fi4JxtIOW3Vi1081j3dxxgVV83j18ZC9", - "lmHwQZzWZ6nCMq5BH53MGBS1K/6cXcsXjTz9cafbiT7vlveM/b09woXFnMsY0C51UQ1ocW0NNRNXUwFe", - "yU0LYSLntDWrx/zTsDd8AcFj0efdnwa9F330j0IQW9dQq0i+oXu79OugDQ2LlSIcwvjwxY0ivBw9V3HQ", - "r9RX5yA/iC0aneXxvKKYOytcRk9pgfPHtTWupOE3apx3Ve3saTYuakkhifDSVwe64MqUFfuwyGRoQmaU", - "yTaezZ1B5trci0edPjq0GI5greYFI0vNQ6nAAp/QOCYh1UqlMe6bIya3W3qrqsbDzYB93Vce9azv189e", - "rM/BXBfgve6Y7N8h4edO5m47E3dVejB4npxNCiVE4MUuolOEWaWOja3aajP1IPMCEFoOHCBJzrJWBshc", - "8XOekC6acYXyHL123jORsmbPWTZ+cg0eyRVJuYYhtu8l4zpD/6CrxNfJMUoED9MgT1CJYNB5SrFIK2CI", - "K7T69SFAD+nQgMyvKRdovUOjyYPRzgPZtN4V76Nm2OalHg7WL/WDeEG6nTQJ18sw81I7CXYjrM41KQ8e", - "n0yZ7BVNsDCZTy0k+rsiBetGrvEWB1olShN3BaF5qs5JngsJcML7YOuOSUT0MVVvBPEozKMyqcyl6HqR", - "Otx/Pm+6BIQ7m/pAfiUk0bYKACxAfzFmS+/AqkVK0cbAVZ6S5kqoZ7C+LbXKg3u2VhNrXKr29V4rXm2T", - "kF0sr5uBVt5vsVf75dpS3A/hh/uWStpbe7lQgSdz4HkZ3Kfr32ScALwhiyrn9a7vAtvHFu+tZdwEuVbN", - "Oim6mQ97/2XcymjcP9j66e//d+/T3/7ir/JfspslEb2QTCEU55Ise4AXj7SN3i8DjgHWrVambW18RXAM", - "TqPgkhgnVYyvi+PdG2RCY/kGx7UpQAxTTFn277UT+vtfmiOACmT8AHJyLcveGQr6IersKO6Oo42YiJmr", - "BOwC1zf7Iwax8JdkKVEBzN+qNI5R/yqzT7SKDk5LHKELowb2CVtcoAmFmihyxLRVi4OAJNqasKDm1BSz", - "4yB9BMFRsR1bVMAlmtkrR3PjTtDH0xpa3dsP739+++HN8fjt2cs3hyfjX1/+JwRBXPVMD2FP897u3r4t", - "YVek5NCzxHcAzr0TCp6P3QyWloe/IKkD6gJ6FGYqISXTXcgXXkYbJE7U0hXrcbkhmzfD9jrMGvSGg90z", - "ivngxX0UbfmwskrLgkc9rVE3gNB6HZiGFt5wZmjKhIl3mhzbM08Z7XPrTZzRGfb4sr0VPO+juIob0FrQ", - "tdr6N5ZG8AeXH1cReo00MKSqIMpW7FKpes2x57FWpMZ5rcQyyFPKbHoGLQQ8lXMxYqa2bBEkX0poqE/e", - "1Qk5+S5ziD89+Gh9nslKVb4ws8JImtfm1GmsFZ16BYHONGmu5kSQwkLABzny6Q1JZpMlWiQam1IlCRF5", - "IKHLtIDa4YJC9kXmbHAkyBJq6h7Y1ci2p/g66wG891jW7rhgHjnG/PD1z6POZh+9c6U+6dQ1AcOo2BN+", - "INEyF62iieOq+mIUuao+b/O+d+NZWbVC+jXtrQpz5n2UWNPHj//AVL3iAiyQ5rTeB8cjBesmJAJwTapo", - "o62gOmlMwnFW8Lhp/7saxyanNysnnZdBctYWBibWQm59qRqXeJqPoU5pTQ4SpIKq5TkURjURtgQLIg5T", - "s+FdfVX7c94xVBX6+hX8lFNPFP9rwoigATo8O4H9GGMGSjr6eFqoBGKKwtQwyEC9fHt0Yi1cB2MHFgtV", - "wHouGO7w7KTT7SyIMFZeZ9Df7g9gMyeE4YR2Djo7/WF/0DEVcWGKW1CEEP60CXqZpXQSWj3oZ/OK/krg", - "mCgiZOfgd0+iGwSywcug7+JZwWJJMBXWZEkiSL8zrEL1twBM647SA3Me24K2rR10Ui1tMgJJ3tpl/QTq", - "JOwamOL2YGBhOpU9eCGVwsRvb/3Thk7m/bbS54A8HpTWmkXhdEpL8q/dzu5geKPxrBoG7Fhftx8YTtWc", - "C/qZwDD3bkiEW3V6wkyGFDJYWzbQprjPgIWKO+z3T3q9ZBrHWCwduXJaJVw2KcNEIowYubIVNf/JJ31k", - "rx+g7Iqc8zTS0gSZ9C/naFBY9GefERbBnC7IiNlzOk4jRRMswI0QI30+G4OpvDVM12b1s8T8n3m4rFA3", - "a25LN9dzTuecwNWwfknGgOM7biqXm7ubKWNaTGJJbC2KrG5kPZpHi8uxDLgvH+c9YZipnkxIQKc0QPCy", - "3r3Wo+1tsBW0nRZ4sCxEAOKU89Bsb/pzOqFqhj8d+jh7hix5y+oEg2ugIErDXOdyaUZYTHAUebGPZhGf", - "4Ghs6HNJPCrqa3jDEqVYYMQpN4yHxBSLSJZqzpn5O52kTKXm74ngV5IIrQLZImCW1iQ0Zb8M614BVmYM", - "hbhMiVHd55YZ4taXS7L82h+xwzB25WOl+QRHkutT0xbts3H1Zksb3vWXNWmIKzlKpeKxZamsgmE+TJ6q", - "JFX2Tl0SZSuXwetUoiSVcxKOmOLoiyAzKpVYft36kvf4FWwXgkPNJ4VXzJS2vtDwa9Oo5Rjr2Y/hVY/1", - "R4AAo44+XUYd/fdMYG27pHIOThQJjpNZcUk3MjwarRduVikcYIYSnhgsH2CqOdYsV2oDqkDiKEIKtpL7", - "VmubsJIN87HpufGkMTfXJFNWthFl6PTnwmYa7D737ydJAkF8Do7/OH/7BsFRpdfAvJY7rMylNtOnKApT", - "0OSh9/6IvcTBHBm9CfBaRx0ajjqZdRFuwlhTaSO0ez1QcX/SQ/vJdNOl4U/9vm7KaM8H6PcvppUDvZeS", - "eKz4JWGjztcuKjyYUTVPJ9mzT36CNqU4npcEAdowsn/T1fAFqKX8GDTnBmYh4lbWRkuEUS6Bin6UCWVY", - "rCxA7CG9paA25fFMFonxZQS+21HnYOS8t6NOd9QhbAG/WRfvqPPVTwGrRDeDg5oazE7XzphofzDYXI89", - "YOnrUaFLL+rt97WmfW3fm+Jhla664mEm55CN9QqaatpG3XoEzednHLr6jD9UvDUqnvVcFJQ3+L54Dhj2", - "jYgxcCsamLZnI6eBrbRODFtAwQGwOBxSiDE4qNPgcuYtmh9Vc75uVuw27bIAhhg5/tt9BP6DfrNSfqbf", - "F4/VL44Ap9sBvz8xdoTFcozY9VvEr4n6Hjhu8Fii1IKKf0v+fSr885pYvS8nWkWabZGFu2/y4yFBsom0", - "rZiXta16DmPqnROm0Ev4tW//6yweyDa+iPjs4gAZEkZ8hiLK7D1g4bZIH4qWlvCRyTfJvrPpJw6McsOc", - "n//73/8Dg6Js9r///T9amzZ/wXbfMghhAF5/MSdYqAnB6uIA/UpI0sMRXRA3GYCXJgsilmhnAGpmIuBR", - "sSSR1U3kiI3YO6JSwQr3pQYXUtoGwfRgMB/KUiJtvo5+kU4taJVxMHtMeLeXDSkfdUd3PYnCMIPCBPSp", - "6HgAskRt+TJrf3X83jMz55L/rOorr3lM18sXRa6V4d6eGeANBQyQ2Lfv4IGdNNo4P3+52UdgYxiuAGAy", - "0JjzZqzy3P8hk9bLJCNRygIFqGxkk0liW+3/PbbvtHMA2xb/TB5gi1V5AxewcXlA3W23Aj9shRbuYD/d", - "nGvY5589dlmazQ7a28+32IWLY2plCN/fOjveq9PcPCmQ7FuYwGjDRcODG5ELdHZ04mrCbn4zpn+UU0PP", - "1Na6y44OxBlglz2aWXbE2TSigUI9NxYobBSTzFQrM8hTEQfv7KgRdvOqQgAXz7etEqJd40mXgdvlR97D", - "nx6VTm9yjOQwxTmv/ThJ1rHOMZUB198WuKUX4AQI6dSXbJ8WuWidQ8oE12dHzkp1yYrnk2O3IR/PNWW7", - "Tln1bHgEoXhcEYjfUBBWiq8XgL2fEjd/yFbRIVKs8Fx9X6w5eDwt6LG9WD42f0purLBCNi0FTVB34wH6", - "migTyt15wIW2PXgmfk6E29WuDgPMOpuW+dQUJjUTggvp1bbviXmlnelr2vszWb5AnptoLJbkP1SUFsZu", - "TqtVBu6JLfn9cPYt9HAj8/b+7nktg3mIDMEmE+exNnVrsVyyYPNPddX7KKeZIfaTPMzO0ihyNx4LIhR6", - "e3RidlbxDNj6AmFJ63V7t9tWHgcf3v3WIyzgEIeWxVD5lSj75J41fLNgZio/2KSNTWjSoqk7z5o0nDus", - "vwkXRCbCsU/5v22/iuhEYLH8t+1XOEooI/+2cxhhRaTafDBmGTyWaH5sjfsJM59WuGmZaCCaGNRLX6eh", - "Zm+1VFLd+38qPdVM+kaaakbXH8pqG2W1SK6V+qpdigfVWE0f3+hKJmM2H7XhkYtP/JNpqo/r5bMc6YCi", - "qSxfe9gidVyAnxceUYZSSZ5gACXNOK54bLR0V+cbcuXx4Vj35LgLhIS6AoDaZBNEHsl57cbx6Mqt7ffx", - "PdeH8YTOUp7KYu5JjFUwJ9ImK0WkLICfmtqdH8+Nivd3zKWDxzw6Hl2v/sH3D6TxVxfUCG9zA7VO53dv", - "tdX57fta5zcp1DZ3zUJLdR3s4GZDUKFLom7LxqVc83qwo29cPlsEfdCGSm4uILAgDkbs/2j743dFcPzp", - "J5ckkw4G2/vwO2GLTz+5PBl26liFMKgYBCixh2+O4dpvBtnnACSbp+RVx2EqTwDrOeicfzkDKb/5bG8h", - "OS78YSG1spAK5FptIdm1eFgTqQy/9eg2kuM3H8EtiMkPK+kxrCSZTqc0oISpvGZoLUjMlhx+grllzN4P", - "FYI7Sgdtaysp25RrFNC8LMCjB/ac5DiIj20cuQoETzNGnicW0tuaI/lh2GyPfG/8MHhc4fz4dshTZjGj", - "8NdJl2id0leKGjAm4xTKKqIcIQSiPpGw9RNdi32UV4CWaZJwoaTBqQQF2CDZz7UC7MO0LMNU+nApAYuR", - "EtkdMahUoB+bXP6tS7I0KJSUswxwslrP1Jd7VUYB/abb6P51LD/EaSsd65G3sQWt/nY61jcTHY+iaZ2U", - "agFsZBsDDMoJyXYyz5L76GfKZptPKgLVCKtsbgU8I4+qtQWV/iyu75bMqrE2HbQFaF9buvVf8MStT9Kn", - "tTss2gIBUUjxjHGpaFCsXFuEB/1xQrc+oVdT1svNU1ti3G/Qv+Lisu0R56kr9gROuuIMv0Nfgh4eoIF9", - "e5cCGNvmNNBM8+inYK1Y3LdMwaDVczGI0hCqutsD0amSU8Hjsf3R4NXqXWHRQMFFEdhWv7Ww0b0/gsPo", - "DVeIxklEtBZPQtQz3KRX06r+Dm6eykJpxZsJQ71tigkxBoxOutJEVkTC5ZpbsA24Z68vl1dqRny2HgQj", - "69whPnhQMEbMwOETh51/gTIhC8W7SEQCha7mNJgDIgYU9IKKrgBWgZPkIoPA2jxAr2GnFpHAoPMNSYQ2", - "hALOJI+IAbpYxPHFQR2x9ePpKXxkwDAMNuvFQVayPDsgpH6riHCR1Tx6Y3E7NjQnCR5FZkUvtNVYmN+m", - "xb7IIcpGzIeDwciVbZBO0UUBEuOiARPDCdTf+OybaVvdZmBJMxfFkQDCGd4kLOw0XcTQyI+GMRx468G0", - "ROYww3hgYI7aYH7jswzUssTKOEnasq8dJnDxIo5X8DDaKBRnlSrkqfq7VCERAj623N3E3GgDB+YfCl9q", - "RrWFhbLytsB+3utGgzLnJZUWqoUqOuZfizjudDt2PAV0uhto72sQTqoN1q/F9MoUYEx+6N03ASgpC/sC", - "Qknl5LDl/5tV7nfmhT+9f9YSKvwzeFnK91n5KCjLC0AJqODuqnA9KaQDWMiaLmYKI/n2iJtlTxaKh7a7", - "3qqVHf0OjNZ1t15ZDcmswOVjX3/VR/CUk2BkbTZTLqrp8evuxb57Rrq/JalNtQ2H/ODNm7vnWjFmkq6o", - "JQqlUCX4+aC+JuA6B3POZYHtJ2SOF5QLi8Buva4ZZ4LLwliPNnruQrPqhfXfXlj1/MD6mhAuPrJ99OFz", - "G3Pn/8I9yr94VbC2M4nfdSo1oEBKhNFEUDJFCU4l0dpSGhNkKoxYIG+Cg7mrFt4fsfdzgmx9zIIDISun", - "TCW6GMYXXTRJFYqwmIG1Yx6aSDpBAh7HhIWm5u2IzQleUG2qCRRhRViw7EkCNZAXJC9gok13e0NpSm1n", - "VVa7yBXnBQfDRaH07gVKBAEmMuYyK9W5HTGRsn83yJW62Qs30AtEpMKTiMp5VisiwCFhgRcW8vz7FmP3", - "78Q9J6penfab3FneSpZ+y0vMoi8zqw/+XdxvPrFALS5cZc0WYn6F0iubTcNy5ON5XpH3X3BLm7m6OX6j", - "m5mMxKt28fdxJVMqyf/jWkbZLRmmpjtSLlv/p71ryetIp6x03WJ9sre9cMkqIWRkvpHM2/ri/jy5hY/s", - "O5GE3UbDvglzO5/09yByLVVvJXO/kXPQ+pIKXrFvKILtoL6d+sRFQcp9F2LYbLhMGhdljhIYbCrOfgjj", - "qjC24QG3FcbO41q7AC+IZ8p6SYSb5HJetd4vgK1D4F80+rUyu4Ig/OaCL78ReDRhd5KJNyPwEryMOP6z", - "38sEXAiT0GnLET8dQLGCL7BwwbQBHrduJiG6Lpvk4+npZpOUEGqljBDqCUuIclnTIPZUa3y7IELQ0JWO", - "PDo9ttGrVCKRsj56G1Oo53hJSAKFYihPJYLM3L6en0ttrRfBK+WwdjuEKbFMOGVq7SjyVx9mMF9vVTrv", - "keWkhVT8018egxf+6QkpkB1aXbETWG1FKqwag/FccBplpt6l1rbwhKe6dS1ZXKHdGZxtUxoRuZSKxCYy", - "b5pGsIkAdNfWZLLfmYzSLqJKIr0fupCBlxARUykpZ3LEbPn3hAjdt/4civ/mQUZe573CmdQ8M6Lv+whg", - "04MxMVtYNVENoAWgDmjnoLOFk2QLykX7g6Ts8O4wpFcQkYbkMp7wiAYoouxSoo2IXhqjAy0kivQfmytD", - "2sbw3X1XnLr9ztKUPmFT7i3KYXg2Y+Y/RxJSWay5S8QnJ9Zek+JmcfIHFtov1uRauSYIjnqKxiRLfkep", - "ohH9bESdboRKRQOTV5OnXkIRZpt9OWKnRAn9DhYEBTyKSKCcc2UrETzYGqWDwU6QUEAp2SEwOBB4zY9j", - "6PHo7AO8ZwpFd0dM/wMafn94Zm5ip9j6CAoDZURdcXGJTrbergnyPQcy/QtHyZkJrsyB9C74j+u7m2c2", - "N+4h2bBFebLKAOLJnz6M02pwP7wFT9NbANAS2Ww2ZgIHoBTLeapCfsX8noEFj9JY/8P8cbIOoEThYP4R", - "Xv1utF0znLXduAk+iU1p5xQSUzTom1xQGII91fhSTTg3BVBiSpF73lPgUP0Zufv+nfJFOn6HV5OWoq4g", - "13eztx775LNjcLhbRXo8lW1uOM3NRPHV3qcrTJu9Tz9HPLiUKGWKRiVQA223AQ6o/jHHbbQXf6AmQHak", - "KyWOyHVCBSDYVOARENEzlggjRURMGY62YM6mEUCgdF4svOAUkpSDiEKaGA0JSngUAcrO1ZwwpGcDjirX", - "QOGeVtoKEMV3ileMiqMJCXhMHCrnps90+wem6hUXZYjN70Uuvi/QX89HT1XPcw2qaHOPd0IZPcXXENYc", - "pvaa2I1o4zXPfzSuoC6CtRl1dgZy1OmiUWc7HnX0ChxhcKFihfZQTFmqiOyjY+PfgjTU/QGSJOAslA4c", - "1HnwdgayKSnVsGVDhuM+fPeYao/lKiDlO9uJTzzo95D+HhJs0EZxw9k9GXZh04WIpwoCuN2+sm+FRIF7", - "ZPPRb2ALe+SHbd9Gkv/Dbt+SjIJV1uKysPRGsmfwkWu9bi6pYs5ljjqJApzggKplF+Eo4kHuPUhldjvQ", - "y4YyEQRfahuqP2LvMuBKmwiBjs4+dJ3TDIVUXpoWrF+sj94uiJDpJBscAmlgPHiwGCQcMcVRgKMgjTTf", - "kumUBJDDENGYKtngV8uG8pBlEPNOPAvvHmawNU/LmeTnCVi9nC1kheO2zFJvCRJEmMZFp1KVOKD6wpUu", - "uH0nulGuj+FpZK+3AsGlRLapHonojE4ie1kj++i9VjlwTEYsiTBjRKBUmrgjPfReIoiUqUmM0Q1AnVnD", - "UV2UA50kgivrJo44F9J4djWHfzxFUpFkBZu9My2fwpwfCCbYNG57+kYGQ2UMzceSfQXpBTGcYgiu+Ugf", - "098g2McM6FvDCT+Vjf9e0NmMCL0rsBGy5mrUbGtHTrPpS5kejRj559lb7TDys1YL0dyFSOeVQBVj9+IY", - "FOib3MB6Or+kjVgm9tHNsi9+1R+17Lsc5e8fhH10x1n+WUqPnReCq9si6+cc/tRA7gsjL23VUoLCejiC", - "1hkJD5kh0Bp34JvBDTxllAFcSjtoghP4/hhh8LjZcY8Ns/20eauEElAqrNOQKrUevvO74MCHwe38xtmh", - "t8Dt/K7ylQB38dvljX5XmUolP6ArHvKnR+Z8qAQlA88JMBZNCUpG6tlAgpWG0kf7Tjszybb4Z9Lg7d3z", - "DfR3R/YfVn8Lk6FALL/LzuRGO9wWEidq6S4X+bRyASjpZ0jG8AE/ZDEED4e3cIvr9ftjD8enjZfrP+pp", - "Pdr9fV50+OT46RfRKu650sGypU+dHhbBnC5Is9O9vIMtiRJBeglP4HIlNASz9HBnmcKiP/uMbPMWq8r+", - "C1EHcUxCFFJBAhUtEWWKg0QwffxVIsG1JQDPuVj6nOnFnftK8PjQzmbNeWj3lHWG5Xe+8bIXYoV7Cydt", - "VrjQ7nDT7u62tcBDlKHXP6MNcq2EQdxFU235IDrNSEquA0JCCTy5WRzwcNDg2aSfyXg2aTPKFdjJby02", - "NQpSqXjs1v7kGG1AsYUZYXottKo/BU02EXxBQ1OINCfqgkeGqsMGgt7U76qViqxShjMuzOC+iQ7T5kCa", - "faZJWSyY0IXOQWdCGYbBrUUpLu8pk1Cl+8MU0hryveM4p/PjCLOW34YzdjQnaiPHEVFxbqDxNn8cc0/5", - "mCsGprozrXTatSsV2S5WtWUI6UMA5mZxzI/rtv74/YRXUvkkIyut63yRGaRNbvPviwUHj3c+PLa7/OMT", - "Dsd/TZzxXXCVQwO6RR/D/MYDHKGQLEjEE6giad7tdDupiDoHnblSycHWVqTfm3OpDp4Png86Xz99/f8D", - "AAD//7vg+12fjQEA", + "U0wjEm62U2abJvMPPikcISX2Brbo4Ukw3N7xyo4Yz8g4pDPrE6seUfp3zWK6HYXgbf9E4DBvNw/oUpBp", + "vb9XILqhE0GmRBDN43fsLhF8QRi21su/Qb+d/2srdxZuWU/hFhDzLH/9a7fzR0pSMk64pGaENclln2g2", + "AlIj+MI/Zni0aq0LHCUVFqv3B7xxDzsx1+vW0sa6LbRqg2drP3mv36nKThCNmS5RkAKNIvKlVmo82gFn", + "yj6ouC/5DEWUGYtDq3ZmLUCvWibkp4iDSLwnOmTkr29+Pe5bCC/zQ0Nr+lk3U8AjPitSc06wUBNSImbD", + "EWYbykfXSP6z0vapnFVYkvFqCXJGGSMh+IvtxjZvajXWa2bALrqkarwgQnr3HAzrV6qQfaOxqYgHl1Ma", + "kfEcy7l1sIUhNc7Cs9JMPNpayRGPwR53DYIWAfbr+S+H23v7yHbgoaH1XOoX6jMpfK2bN+8ihcUER5GX", + "N5rZ7eZndJ1D/ByQOyubzp6MAx1jGknXsatp7eRUzs1fILv1qODs02JAs1ek//7kmfQRCAljJTTe3vh1", + "wMwzPIu4pukSpYz+kZYU7D46mYKDWB8UNCRhF2F4AH4Hbf/NCCNCy6ncM1RQgtEG6c/6XTTSemFPa8E9", + "vN0bDHqDUaesxka7PWPeJ1gpIvQA/7/fce/zYe+/B70Xn/I/x/3ep7/9m48B2mrmTiu089xwe7+L3GCL", + "6np1oOtU+VtL/+LwfRLHLPWJlhM3Xemjk7riYOYa8uCSiD7lWxGdCCyWW2xG2fVBhBWRqjzz1e/eKy1g", + "HiuIwGaaTDckQ8XoATbeiPgVEYGWwBHRjCe7WghTJbsIa7sZhBfSp+R/oAAzvReMcsEFIixEV1TNEYb3", + "ytSKlz2c0B41Q+10OzG+/o2wmZp3DvZ3anyumXzD/tH79Ff30+b/8bK6SCPiYfJ3PFWUzRA8Ll7ruTFk", + "VzSrVsRRN41AzYspOzGfDet3UHdbYTeRVSttjLnGpdZCKHORrRlI/X5XG1uxx3R4uyBC0NAdy0enx2gj", + "opfE7hckUoZG6WCwE8AL8CexvwQ8jjELzW+bffQ2pkofh2l+ypsr28rtGgnmHBSVKOI3uU4DTREMHByt", + "PMdXkcZL7aOs3fqp/wuXqhdjhmcEzFH7IpoIfkn0QM2dACUSXZKl1nKWaKYb7S2ohBsewhZogY3XoT9i", + "7+dcEvOKeyTBt08XBMU8uDRXv3MOlvwCRymRXXQ11yoH+AQJjuzPyFyMjdhcD1IGPCGhNkLMazA1dEHY", + "4gLFOIFtjgWBPY5irIigOKKfzRU+3DKQkOoTbsQIbAyUYL3ng4CLEG7YOCI4mBeo8BeJLozCcgHNX1Cm", + "2frCbMzKZfWXztsP739+++HN8fjt2cs3hyfjX1/+l/7ZfNQ5+P1Lx4RqZJrKzwQLItC/fYH5fjXqbUhE", + "56BzmKo5F/Sz8dZ87XY0DaTmL5zQPk8Iw7Qf8LjT7fy1+M9PXz85hcy4sRd6G3gG9tWrDJmz1COSjp03", + "UCLrYXJ3G5pkWkS9PvuwpU/nBEup5oKns3l5Y1jV4EZbIqTyckz5eJL4xkTlJTrZeou04oIiqjdopqgM", + "B4PTn7fkqKP/sef+sdlHx2bXwvC1DOLC6k9yrtkni/o4OvuAcBTxwPpQpk0XvK4rn4AnTIllwqnPiKsI", + "p/zVuozq9fKnNxBFWxPKtqRehl5wM7oD39zalHjJFlRwFmtzboEF1ee0LO+VN2+PX45fvvnYOdAHQZgG", + "1it59vbd+85BZ2cwGHR8DKo5aI0MfH32wdx6wrYhOFLzcTAnweW6D3+Bd4/gVdhxKonS2VjSzx4t5DAj", + "DYpJzIWxvu03aGNeVlLMlkewrqPOzuufDV8OXwNLuvW010tZK6bhyo3g6599jDZfJkQsqPS56H7Jnjmm", + "qUcKlbaFuWDL+B02QL9g+gQRT8NeoctuZ0oFCSAyQ//rDxJrG2DxuXyj5fnO7zlrpfuuUWpxlFBGVmi1", + "34l2ecXFZcRx2Bves3Jp72I9UTXmQXl9s0s5xxK1YLUJZuEVDdV8HPIrpofsEcn2CcpezuTytZ4Jjv75", + "P//78TQ30YavJ4kV0sPtvTsK6YpY1k173S/ZRNLEP40PiX8SH0//+T//62bybSdhdJhb6YN2/V+aFqrx", + "NjYM0XhSGy6Vs4M/i3VR3Nri8DlyvLf2Btkn4/mCiAgvC4LXjqkzHID0q4xKUAiwRPY7LUYvkf54jRjW", + "rTn94HXVP7A98AtaQWBnj5MssnQV/d+Zt3MzxTMnz5R+1qLGHittJpLNY7h9av/crs/IPyF5SZMx6Otj", + "PMu8zauCUc8vaWKNAPjCcEEUGTkSpmA2TDhX/REzsTF66YE/yDUJQGRKhRU6PDuR6IpGEfimQCbVTyZt", + "UhSCquB1qfT/ipR10SRV2k7giiBrsUEnKYwFXp4QlDLsbuIrWrudYD2wAchySQQj0dho5bIlZcxHyH7U", + "SByY6hRLGxwnVJqU6XX86+k52jheMhzTAP1qWj3lYRoRdG7iGjbL1OuOWCIgQEJ3otmR2n75FPFU9fi0", + "pwQhbogxNJZ59+w18eL12QcbaCA3+yP2jmjCEhbaEGN3YNnw05Czv+gNT8Jys8X+K0RvCiaRDCdyzttu", + "rnP7er672rsxup1FkKTlJd3uNoafLqhQKY60qC4pst7QAhM+7zFYTHR+0XCyYjMP11XlO+G2vh7TMsTS", + "ewN1PS4bo2i1dtkUnAg1542zcL+0G+ya9k+YG8hKl1Vu5N6hr3PTSC1syPzcdTO7BZVOMppUHF33Q55D", + "WXAKtAp7N9FfRqGUaOMCJ7Rv+bgf8Piiiy7+WvpB731nmWj15AoZaoA8YfqnYvtVd8haR8WNAs2Li4Pl", + "7dfjUDbGWKHFECmBmTTRcXOckD76BYQ4UiROtCRjM0QlyoLKEONX/4G40YncpyOmhyZNhIolR+auknTG", + "KJttaitBH0w4DI1Pa5qqVOj3FlTm1CyzjvMb1eJpzeiIkceQm0FZEKUhQRfOt3RRVivrnqe6RWldUTUD", + "yZAEDCOwFdVWnCrdvZ5wjFUw13TiqTIha3bq5XDCin9r3VWuHUt2yXeL9T/PxEU1BWfhsZD05Oz1Ejgk", + "C57RJgekVVT8ztFLsoQld45QXHOFFn2gfk+lIJJHC2KP3aIXdQJJRtwoTrkD1bhCrfdTb/9qeo3PL7hu", + "KTS9WpO/bGl4kouk6rnJ5hxjjQcXje6kkJ6c6a+r7WpJgPhguRwgUMcuusbUIuDAQEwzS4RCKkigas1T", + "NhsxiF65sL/0bWsXepNrHeVeUrYgAwKU9uLSosLKOrUPmtFT4zFVioTdsm5wSUgi109Kq9fWZe7x6wty", + "JagTZC6cuaV6RtiUi4DE1ki4m935stCY1wq8WRP1YBJD38KYXWYI5MWQ0EQumfUAB28pYaSaNxlWrDYT", + "vFDu8gJH0QXasC9tIkH+ATkAdq0YZzmzvz86cyyQXbh/PO1qjtRS4GKuVDLW/yPHehdfVBuz37odnue0", + "PR+AfbW7u2NX1frszIArzZbdc96AjOalcep3452e5gs9Shvh0kaVP8o/yX24l5SFbRv4Vb/b6NzLFCNn", + "aTy0fy8RpJcmM4EhuPc+vXu3vrEFajZL8DUZxL4AzTw3MZWKx8VI/41KcAkth6GUibXgUS/ECoMntKW7", + "1gy3HvIcL01Txhbz+j3oZzKeTTwRS/QzJCHM6AxPlqp8czH05hHe9frcjcW3LE2pA8aCJOFY8dXB03SK", + "3LttYiVNpoPi48WU8tWJJTbyppT5Z44ja9fqJnpJQK07AXScYG5iWw0RQGn8eFq8NeyPWA+O3wN0nHWQ", + "NZs1iUG3xKG5eNngojAIk0KCJstNhNHH0z56n432LxJpg2VBXC7FHEs0IYShFDzXcBr2zFlcHEAq4dBU", + "1c+t78SkXWzC5Si3z/pZtjN4abLcbQjSmtDKfEzOJiyUvY3GrOgFa+W1WhVy/o7MqFSiEnCONt69OtrZ", + "2XlRdX9u7/UGw95w7/1wcDDQ///f7WPT7z+zxNfWYVm22LC3ovQ5+nByvG2dpeV+1Odd/OL59TVWL/bp", + "lXzxOZ6I2T928KPknvhF2XEer4c2UklEz4lJzVW+KL1CMFxDFN6tg+seKFYuD/1d9a6hxHv95kMk1fjC", + "tW2w8M3TXqoCc23Ad2FydUt+mYDdme+SggZn4yoD6o0gPaby8mdB8CUkC9bP7RjPiByb88wfSZFKE95D", + "rq13Q3CuptJcu5a9nsPdZ7vPd/Z3nw8GnlySOsPzgI4DfQK1GsDboxMU4SURCL5BG3BfFqJJxCdlRt/b", + "2X/+bPBiuN12HOaGqB0dMsPLfYU2LEX+5hBS3JPSoLa3n+3v7OwM9ve3d1uNyvqLWw3K+ZZLKsmznWe7", + "w+fbu62o4FPoX7rcnqoC78vpPDS4AvpfPZmQgE5pgCA7COkP0EYMRxjJbqvKe3KCQ5f56j87FKaRXBlw", + "YTqzbxpHW5xGiiYRMc9gQVr5omHmx9CSF5uDsSzT+GYt2YyotQEGbi7ZK6iU2VYi3alJpS4oT5RE4YHZ", + "oWvlHKxmPrBPTXxg59CSG37TplMvIgsSFZnAHF0mp1cQlPGJWbTSrChb4IiGY8qS1MsSjaR8lQrQRU2j", + "CE94qsw1o00NzzuBeGuwPaZaXLezc19xcbk2clWfxFkG/Fqv0CE40qfWVQOnOEb2a5ccUVD6sutAc2lq", + "n0v0znxhPET5z0laxtPpQk/Wk8SQIFJxkKTWYWibaatd+vUWcJa66BHTXy47Hyl0pjc10Qb3a2GLGQHk", + "B7VWY9Gc8h7eP4fXWwfC6w/XOlJa0J2Rq8cgOmQK9DTb9iTDycNQfFUsW+ZryF+CU1jQkPQR7C4IqnGZ", + "iZWddq54kpAw8//0R8xGkmc/SXODoj80dFBzQgXigs5oueOyg+0hg+JuwoqOm27NjsUP6xoqPITwjeZN", + "j6fKoDxcumQtUsycsovQ6XbOM0wMK4nKpHmX4YrUKJIHedaG+Prsw01D2xLBp9SHdASxEPaptcxc0Ndv", + "u4Pz3vD/MQGcmt9ARaPMxE/EPKxAWNj32508r88+nDWNKQOVQMXR1eaURbysgtVyFLGXSvZW0lowjv31", + "wZJ1kuveL3y67FTgmEzS6ZSIcexxrr3Sz5F5wYQ2UYZOfy7rs1pvbms1n5UWB8zmKQ4sJkA76nsccpVp", + "dAvU/ORfrnfEHMNNmYR6qYR9xyYT9tGbDMYDvT77IFEepeTx1JWXtzFS/2y+lDTAkWnRJAZTVnSwAXO2", + "1pDP8g+tK9KjJ/vRX9xGQBuLWZLCNjx/1zt5+3ErDsmiWxoTRBbNeUT0uDcL0mLh8gnztIKSkFg0eToM", + "Y8i2G6hAq2wHtyZSYb96qKO4wtFYRtwXrPFeP0TwEG18fGXyvfQIuigpLaX+vUCFEn/ve3eMlkhN3Z5D", + "h1WXaWmDe23HMg6nca8Uplfq1LdVTJB9XcepQy/xy/JC88v1cD+mkeZ+j1weQMWpbRVJZNIFEKQLuPtn", + "ZD41XmvrGpEkwQIrEi2NZpEdfRGdkmAZRGaPk/pVIrkmwQ0SEV7q17+aPOJUkLGaCyLnPCpfQO9067Bv", + "EoIgF8TCb5g5FRzviqMYi0s4GJ0ijVJmKFBOOthZB2k5Vyq5waR+ef/+zFjXioiFiSgrBvnK2tXqMYnw", + "Ek2IuiKEualgiTB6zTOck2pWjmyAQBBqnBBBeZmGnR1Pv+cmMBPNBA4IMl85FEa7JBJOzbaktL14oLyC", + "gEjZsL7DVetrP52mUbs19g1ruBa0NLjJAr8/OnOp/BkqoSPzdp3KZ0T0zJZz8ISrl3ZbrgaVcF0xzki9", + "M8EnBFAmbOJNMUXIBZYAiob+vJSWUxAOspgPY/uBXWBI1TX7/FMrZa+63X0X6TFmoQ/d0UQ8m9zHWRrr", + "BdFDFincHdHQRJ2YdEyjlhcDtwXBIWVEykriWJCKqNPt9KZ2VgdbWxEPcDTnUh3s7gyfb62O31sZuGnj", + "VMYhXWXfuWgWE+/g8qsMyiJMuswSWzhJWnjADB3XnA8gnuqBYoDoqM+2gobnIocHg1oO3DUOlAPGAZdY", + "6coTF7dtolmyNB9oMIMV3nvxorg/B9476BzV27H/Vo339cxMMJnmEeNuqNDRMPlnr0bFhQ/9hAtlM1wm", + "xEWbZeehi+Wylyqlzp4Pnhdn2Qo8GYRNZZvbfeeZqnm7nPVGCuS2+9c2AHExZZ3DbenVWGmaLmt4SkvE", + "GkutJihPCLsRPfd2d7ZvRs+2EzlxSXkVueTLuT86PTY6UcCZwpQRgWKisEWKLwgZ8CVpKaMN/BCTGPIU", + "pv+xWrQ0xC8Uk+gbb8CPanBrD3L73QAT9M7Eb4YoxoxOtUC2bxZ7lnO8vbd/YMDMQjLd3dvv9/s3TS1+", + "mecSt1qKLZM+Wcgy7sv53dbhATKI28zlS+fs8P0vWpClUphDa0tOKDso/Dv7Z/4A/jD/nFDmzTxuhX9H", + "pzXcu3I8mDb4ze8HBahyp/e0giP2O4MhLBTgDrwwMQrPtG5jOO6ueDC3RozLYUtVASmumE3TAjWOfl59", + "jey8SvCO7TNlikY5oF79AvlWkIhyJWpUDTEqISzDiYoi81fA2ULvCh9oVOkkcs/uFHyxUvX6e13jWr/f", + "nOK1hm39XrZM/rUFy7OQNp6T6JtL/dsEKJV7fzv7zz/+X3n27B/DP377+PG/Fq//8/gN/a+P0dnbO2Wv", + "r0Yz+qaQRPeGQgRROSUoorasdIpV4PFGaUOngcL2ibGtVTDvoyPwmh+MWA/9RhURODpAo04lv2rUQRsE", + "bAL4Sit2uimbJrqpPz4zd2f64y9O4ftabSO0+aDCLkiWRS7TSchjTNnmiI2YbQu5iUjQgPVfIQpwosBx", + "QRnSlt4STQRUY7F3G3nnXfQFJ8nXzRGD6wFyrYSeQYKFyuDXXA/AFHZUJuDSvk5Ch+djrhdGLDuXMjgf", + "c8HVz9RcCGyopqv4ibLaUrE2wvOBD/gIQub1QkZUKqNsZ5yt2SiL5UfPB5t1y2WNNp3x0Ar2g51QL9Pk", + "mLLFXjIMDF0bwT12zrg1gQhaNpk9gsBWUhz+e45cQzktsiU2HnKTQCHNBauKZCF1YrPjBYmH1W05IXPD", + "CJ9FLVKuX5rcmve/nSNFROyyHTcCTc4pDfT8IHaSSplqVqQYHR6dvtzst6gzBbTNxr9iHd9nM6xmxtob", + "x6aL1NywwzHpopNjyG2yOzRX4CAm+RUXKDICJt/XB+iDJBUbEbLXICTSrGS0zK8tzQkw6my6FpOqpDhA", + "7zK9EWdDKdW2Kt+E5vsSmrVRKyZgutZ6t1Y1Rji7yIo2CI/GKsuw0ydusyho76iwFIc9XzGrb7y3izfJ", + "jUZzYe3vG+zu/tWdnZupO65wQTLH0sfd8+JVCLy0oloQrfh3RfO9f6nftQV7su6gBpE++oqf+0qt7PWG", + "w/fD3Zvb/DfFKSsDehSwZDKosvYYYw+B1VW3f6+pGjdGhCL92MZ/Oivv4ymaY8n+ouBhxdYb7jxrhTqv", + "e20bS1mMouRTM6RMSjl0kCwG0OCkXNIoMqG1ks4YjtALtHF+8vrXk99+20Q99PbtaXUpVn3hW58WkGVO", + "VLw++wDXaViOXThScwYOzrPYyDWVStbBU1pF9d0FIs182g5W3E3StJHji6/GWfulhIXmRcPZvEeANBeK", + "WSPjY0Cffcscl+8Pdm0lUNpd0c6s8fJAYGeNwt0HFFaW8+bn+4Ute5DhrC3SVzzrXQLirXHCuh3qSb46", + "lFoEkxCdnOVQ5bmT0TVfmZOteDkcDPrDQRuXa4yDFX2fHh6173ywbVSLAzw5CMIDMr2Dy9cytlHGcXSF", + "lxKNnLk06hj7rGCYFbatNalaXU/X4dhuh75WVWj8ctoodu7eX9ryNk3KTYvsmirYS5xGJoGzWBKnXuVT", + "JgbZzGACZ7rsiMEAuxZhJavMiYNApLk/w5VcM5pvmli+HzFBZMKZNIUY++hXspQopnCHkHUPkUMSZUHc", + "4YhtCBfwn0X2JziVJNQ/QDRt10Vt6qFRBdjH+oMRk/MUKsht9tERZzKNibCuHjSh4IfeRDI1xh2MF6gB", + "9UwlDYkYMf2aBzvtS6aoH+wPBoNBVtmuc7Cj/z3wcdMd8fPc5+1UDvt5rmv4eXgd3F47HL27gpetKodz", + "Xi6E09qiu0O1yFbB5k6ns2Hm9qvxTe7KCAp4GoXaTJjog8F4cUhonU2SqLzGEJwlH9gl0+xcmroNclMc", + "/ZESsUQfT09LF2yCTG0JlRYThw3VsA48udEybK8xrNeO5paQdo8BY1c91AvK1L2D1hU9/C7dzXBoC09/", + "blx5Q5QpM0uj+WTFnCo+2pAsxmnq09n1I5fk/uHDyXGJOTDeHz4fPH/Rez4Z7vd2w8Gwh4c7+73tPTyY", + "7gTPdhqKmLVPUbh91oHXQvNVpXKxh2MXA+ktPNwQgVo5Im1U3RVlIb9qVbI7692GUK3rvh4g2XoI3rBq", + "qHQMLTVIidNCPWMTVlipWdTgedp/Pxiu8Ty1K8PcIH/fi5QFBn0KJHHm0y0WYC4uVr0I683FKQzIhS+v", + "o1ax8/ZEGxzsvTjYuyvRXAjuujFW2ekRF7cp4MBBGFZifF2eScF/UahgDPqGcbPakOBOt5NFLcPfcNBW", + "IuKyx61C8Zs2bNcvRlbJ74aMtJOS2gy3sAbJKDzQWkCWzDRJFcoSHbV6cRTxNEQF348BdoGLkZOCCq2b", + "gXsK6xoyCF0mpFar2gAJCYDElGlBDBdCuhGbvnaAXsO78AjHxrqwg8AsrNyF4HBp7oL1/nJdG11/9ZDP", + "rZoP32idH0FhcT1tTQbrIlzdhNF8DtAbDt9kRgfjVV+jeR20/frrVb/khs0rc/nH0JlV4w7Qq0x1y5Q/", + "q+xtSGL/HFuBlaf9b5aSL+2KdzS35CtXyCvsdgxFO92OIxTkH9YzET/kXF/bf0VW9AVJEBzBXs4zvVJF", + "I4tyCTOhUFrbhgDrxW3SLywgPAnHxjBpCnky6UPWeMk+curLx1O0AXhWf0PWkar/tZmFR5XOuu0Xuy/2", + "n22/2G+FWpEPcL3aeQTJbfXBrdVBgyQdu6q1DVM/OvtgTPDAGLfgmbdzLyQJJ4Jr0aNnnpfBzTt/0X9R", + "BOsIeWpKgtshWWSfr4XC9ytrFjfE+PxBowWdTtkfn4PL7X8IGg+v9+X2xOvQzCvse70/J8Xb3pqrlEx6", + "poiHH08BGErIRsiRd0TCDNA5UQj4p4dwAKZDlpNmWc4Bk1iKexlrd2dn5/mzve1WfGVHV9g4Y/AFeQ5l", + "O4LCFoM30ca783O0VWA406ZL1AWAUGbNSv8+Q7YU2KCskPaHgx0flzQc3DnX2LYXcSPJP1rTzE7KEh1S", + "6zKzrbbLvdTe2Rk82917vtduG1sv5Vhcr5YwLvDckMfi2BZXfgO0yfeHZwjSuqY4KPtNhts7u3v7z57f", + "aFTqRqMCDGaDnXqDgT1/tr+3u7M9bIed44sCsKhQpQ1bll2eTedhCs9qeEhRF73dptPCp04ZBntHggjT", + "+DBwEbyV08dgpI6FeS1fhDYHg3WM1w6uFt+2chxVSk4b1YALlLIMmbu//grwdjd6zWLanAfrxXjdso8w", + "0+SyIA+mFMctaJcIsqA8lffQEFcm1WkacS5u9G2ThfKOyDRS5tqNSvTx9C8gRDRzIalIUjaaLPutgMK4", + "5eRutIFLPOHn6iZitVqNNku/asLdhm3aXZUHXdr+jYgzoRZVKVsffneEoyAF8HmcraeeFWBHQCZnkkRL", + "E6gaRZwzFMwxmxEoI2hKXbAZwmjOo7DvDR7UT8ZT77U9v0IRN1iZl4QkFpfdDEJ/pnUWuiBoo5BJigwr", + "Vcoz7cVGqljk7TI37sX+QkBY+pIfsgxGTU+seAHG0XxS8jFGfCbBClQQgtuvogcnWJjIWsxMnYFFbIzH", + "MvTOtj7tPUOsSG/fEWqOTj61Fq3VMSA/0FASB4JLiUhEZ4Bp//G0kna2IociSz5bH1JXHmwL1jUXaZ6z", + "C8402bocie9A9ASn3+VIBB6GHJQVwWrOGxljlgJSe4GRyXVChWGPdgFpcy7VOIMTueFgpRoDCncqSI45", + "lCVLZg4g9473XHSi7TbkspGft/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuAKisxXHJQmCoCyE0gf3LYZiqh", + "VVpAm0EbjKuSWCpAD2+2Cc7w26i6n5p5aqtM/bY7OG+LxrMafOcMq/kJm3JPyvYNriGd69nGCyZExBRw", + "6FFIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJbgmOzvSHtmjmnGGWziqyvdtjGH2zGsBqkG/q1L7YJs5H+", + "7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6", + "kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4w", + "ehnFdnd70JR92dBoKe+yDuh1U8ltWda34x3W1mFWNNNzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQ", + "gst0LaD1tvKEtIssr4b8udFsSRKUe999vvdsvyVc8p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fv", + "xfaNHFQuOrRhfZoiRIvrU6mNW3Ga7UEk1eBGgzLxof4hNcSIlgdUqnN76wF9XbF1m4IL8r3ZdMkZFVfS", + "3bOUPaDtfIwrtKXDkspVqAK/QaZTAkbl2NCtlw+mkiHYagwBTnBA1dLjMMFXpmRf9koF7q2NN608WA9J", + "bdsWGkhLLplO8iSKDdc5+qtxrVd44Xlr1HWZTprc+G+rvRonfu4DKl4RtbihyQtD1t0F2XyusCzFmum/", + "A4iezKv8V2NmzRuroaeqgB9wCWiLCxQiKXyQhZXzz35UXP7KchbcviUluUrxVUdo8xa8kQ3tOZE9JnSw", + "PhOmIh/sAXi7r8aTYj2ElQUnSsUT8lP35v22SPapg4VmJ9jN+yukPdzkwyowFvCjHYMled52t8QSDdxU", + "CNP1mCM8Ir0szsHG8CKZGv+q3vMWa9GThxFc8um0DPi01wwQCMh8ELLtesFKkThRXUSuwUwnYQ1dztXR", + "3pOjDuICjTrDeNSpOAG9SRAxvh7bDsq5yoNViH1ZJaDqIKWbwSTiwaWB+leCEtlHAxQTzCRKGWz+io9y", + "OFjta+t2ksLaZPh4xNwQ18QWjGlC5nhBAZbVeqhmpTgWck2VhHgbaOcAhRzs23KdIztD/ZpJUTjIJw2H", + "DmZL27BuUL/HmQsIyt8Fc2kK1ZXYZyJ41ybfaYn99u1p19z/QOSGGVgpPMRN1IxAC8isiwrGaP67P/xq", + "EpExjLuKWRnX6VhMJQM/tSCSKGlB7HJ2qDABCnjKVBXMMm4XwlmOeK8fSSmDWAl7ewaQDbZ3Wx0xJIGt", + "QFt3L5UY/RbMXQm7tJT2xV3u+FgYNgV45vye93fWvV4dgERYkGKJMtNOMSzO+FzHUnGLaZ/t6jG5DggJ", + "q3A8/lfahhraL72hhr9hm/OeVQ+zb0O4WH12/YeLPYexNlG7GBLJOOtBtq1bUpsZa1BDbO51mdFK+HuF", + "DNSxD/zI90KbvClyvZrWb8i1ApDAMI00eZtY14oqexito/itky6aNjQX64ttPkDtBBOud6vqCTbS71EK", + "KNifH6RoQm05zoly755bvmmulVmCNy55BF2ApHulfEVpeKeL7ImOhvFmhed25343iAXoapkvwnBMxokg", + "U3q9glvMC8YSLmcj5zunVE9Voo0YX6PdZyiYYyErY2d0NlfRsnx/uesBA7hTKRFBFGHqBpVn89V0H9aD", + "BexyFlv3acPnhdR9f0VcEo5XwdgdZa+569gEL8Ft0+hjfbazOxjsbA9uhWN3X4V6C+00pSAUvrP3JKWo", + "nmILWcJXvZrTlaCQN5aRSSpBcHwAgcoJDgiKyBRQXrIqemsPi1rXqwdvNSibR57xv1sou27uCqMMFJ11", + "ZSEA3TQ6LgCqDB1QfF4f9goomEzMBDVMGE+Owk5vsP9+uHOwt38wHD4E8F1GpKbo2Gefh1fPom083Y2e", + "L5/9MZw/m23HO17D6wFqQpdzPqslou0cEiKqZbqq5e0kiSgjPZlFlK9P61ghC0yQxtr9fzPHvpnBSmXh", + "vDzJos6AVU6cEmc9Ek6GHf3K24nq8E+OVw/7ViHa1YH4Gaw6FOCndoMBINbhXVE/U9by3PlQeLH1ybMy", + "bWDd2ePL8oSt7V3lBor7+LkkGEs7bNWJXT/VPN7RGRdUzePVx0P2WoYhCHFmn6UKy7gMfXQyY1CUr/hz", + "FlZQNJP0x51uJ/q8W94z9vf2CB0WMy9jQLvURTWgxbU71HxcTQV4JTcthIn809a4HvNPw97wBQS/RZ93", + "fxr0XvTR3wtBeF1DrSL5hu7t0q+DNjQsVrpwCOnDFzeKUHP0XMVBv1JfnYb8ILZoepbH84po7qxwGUml", + "Bc4f19a4AiPQqHHeVbWzp9m4qCWFJMJLXx3rgitWVuzDIpOhCZlRJtt4ZncGmWt2Lx51+ujQYlCCtZoX", + "vCw1D6UOC3xC45iEVCuVxrhvjvjcbultqxoPNwMmdl951LO+Xz97sT6HdF2A+rpjsn+HhKU7mbvtTNxV", + "6c3gOXM2KZRAgRe7iE4RZpU6PLbqrM00hMwRQJg5cIAqOctaGSBzxc95QrpoxhXKcwxbetRS1uz5y8ZP", + "rsGjuiKp2DDE9r1kjGfoJXSV+Do5RongYRrkCTYRDDpPiRZpBcxxhVa/PoTpIR0akLk25QKtd2g0eTDa", + "eSCb1rvifdQM27zUw8H6pX4QL0i3kybhehlmXmonwW6ENbomZcPjkymTvaIJFibzqYVEf1ekYN3INd7i", + "QKtEaeKuUDRP1TnJc6EClwg+2L1jEhF9TNUbQTwK86hSKnMpul6kDvefz5suMeHOqT6QXwlJtK0CABHQ", + "X4zZ0juwapFVtDFwlbOkudLqGaxyS63y4J6t1cQal6p9vdqKV9sklBfLA2egm/dbrNZ+ubaU+EP44b6l", + "kvbWXi5U4NUc+F8GV+r6NxkzAM/Iosp5veu7gPexxXtrGTdBxlWzZopu5sPefxu3Mhr3D7Z++tv/3fv0", + "V697uWI3SyJ6IZlCKNElWfYA7x5pG71fBkwDrF6tTNva/orgGJxGwSUxTqoYXxfHuzfIhMbyDY5rU4AY", + "rJiy7N9rJ/S3f2uOYCqQ8QPIybUse2co64eoE6S4O442YiJmrpKxC7zf7I8YxPJfkqVEhWIEVqVxjPoX", + "mX2iVXRwWuIIXRg1sE/Y4gJNKNR0kSOmrVocBCTR1oQFZaemGB8H6SMIjort2KIILlHOXjmaiAGCPp7W", + "0Pbefnj/89sPb47Hb89evjk8Gf/68r8giOOqZ3oIe5r3dvf2bQm+IiWHniW+A/DvnVD8fOxmsMA8/AVJ", + "KVDX0KMwUwkppS6goPAy2iBxopau2JDLbdm8GTbZYdagN5ztnlHYBy/uo+jMh5VVZhY86mmNugFE1+vA", + "NLTwhmNDUybMvdPk2J55yoCfW2/ijM6wx5ftrUB6H8Vh3IDWgsbV1r+xtIM/OP64ijBspIEhVQURt2KX", + "StVrjp2PtSI1zms9liMyUmbTS2ghYKucSxIztWWLOPlSWkN98q5OKMp3mUMs6sFH6/NkVqryhZkVRtK8", + "NqdOY63o1CsIdKZJczUnghQWAj7IkVtvSDKb7NEiUdqUWkmIyAMhXaYI1D4XFLJHMmeDI0GWEFT3wK5G", + "5j3F11kP4L3HsnbHBfPIMfKHr38edTb76J0rVUqnrgkYRsWe8AOhlrloFU0cV9UXo8hV9Xmb970bz8qq", + "FdKvaW9VmDPvo8SaPn78O6bqFRdggTSnJT84nipYNyERgMtSRUttBTVKYxKOs4LNTfvf1Wg2OclZOey8", + "jJOztjAwsRZy60vtuMTZfAx1SmtykCAVVC3PobCriRAmWBBxmJoN7+rD2p/zjqEq0tev4KecerIQXhNG", + "BA3Q4dkJ7McYM1DS0cfTQiUTU9SmhqEG6uXboxNr4ToYPrBYqALWc8F8h2cnnW5nQYSx8jqD/nZ/AJs5", + "IQwntHPQ2ekP+4OOqegLU9yCIorwp00wzCylk9DqQT+bV/RXAsdEESE7B797EvUgkA1eBn0XzwoWS4Kp", + "sCZLEkH6oGEVqr8FYF13lB6Y89gW5G3toJNqaZMpSPLWLusnUCdh18AUtwcDCzOq7MELqSAm/nzrHzYY", + "Me+3lT4H5PGgzNYsCqdTWpJ/7XZ2B8MbjWfVMGDH+rr9wHCq5lzQzwSGuXdDItyq0xNmMryQwQqzgTbF", + "fQYsVNxhv3/S6yXTOMZi6ciV0yrhskkZJhJhxMiVrQj6Dz7pI3v9AGVj5JynkZYmyKSvOUeDwqI/+4yw", + "COZ0QUbMntNxGimaYAFuhBjp89kYTOWtYbo2q58BC/zMw2WFullzW7q5nnM65wSupiVIMgYc4nFTud/c", + "3UwZ02ISS2JraWR1L+vRPFpcjmXAfflE7wnDTPVkQgI6pQGCl/XutR5tb4OtoPm0wINlIQIQs5yHZnvT", + "n5MKVT/86dzH2TNkyVtWJxhcAwVRGuY6l0uTwmKCo8iL3TSL+ARHY0OfS+JRUV/DG5YoxQIpTrlhPCSm", + "2EWyVHPOzN/pJGUqNX9PBL+SRGgVyBYxs7QmoSlbZlj3CrA+YygkZkqk6j63zBC3vlyS5df+iB2GsSt/", + "K80nOJJcn5q26KDNCzBb2vCuvyxLQ1zJUSoVjy1LZRUY82HyVCWpsnfqkihbeQ1epxIlqZyTcMQUR18E", + "mVGpxPLr1pe8x69guxAcaj4pvGKmtPWFhl+bRi3HWM9+DK96rD8CBBh19Oky6ui/ZwJr2yWVc3CiSHCc", + "zIpLupHh6Wi9cLNK4QAzlPDEYBEBU82xZrlSGxCPjqMIKdhK7lutbcJKNszHphfHk8bcYpMMWtlGlKHT", + "nwubabD73L+fJAkE8Tk4/vP87RsER5VeA/Na7rAyl9pMn6IoTEGTh977I/YSB3Nk9CbAmx11aDjqZNZF", + "uAljTaWN0O71QMX9SQ/tJ9NNl4Y/9fu6KaM9H6Dfv5hWDvReSuKx4peEjTpfu6jwYEbVPJ1kzz75CdqU", + "onleEgRow8j+TVeDGKCi8mPQnBuYhYhbWRstEUa5BCr6USaUYbGygLKH9JaC2pTHM1kkxpcR+G5HnYOR", + "896OOt1Rh7AF/GZdvKPOVz8FrBLdDG5qakg7XTtjov3BYHM9doKlr0eFLr2ot9/Xmva1fW+Kh1W66oqH", + "mZxDZtYraKqBG3XrETSfn3Ho6kv+UPHWqHjWc1FQ3uD74jlg2DcixsCtaGDano2cBrbSOjFsAQUTwOJw", + "SCfG4KBOg8uZt2h+VM35ulmx27TLAhhi5Phv9xH4D/rNShGafl88Vr84ApxxB1z/xNgRFssxYtdvEb8m", + "6nvguMFjiVILiv4t+fep8M9rYvW+nGgVabZFFu6+yY/nBMkm0rZiXta26jmMqXdOmEIv4de+/a+zeCBb", + "+iLis4sDZEgY8RmKKLP3gIXbIn0oWlrCRybfJPvOpp84MM0Nc37+83/+FwZF2eyf//O/Wps2f8F23zKJ", + "kwC+fzEnWKgJweriAP1KSNLDEV0QNxmAxyYLIpZoZwBqZiLgUbGkktVN5IiN2DuiUsEK96UG11LaBsH0", + "YDAfylIibb6OfpFOLeiWcTB7THi3lw0pH3VHdz2JzjCDwgT0qeh4ALJEbfk1a391/N4zM+eS/6zqK695", + "TNfLF0WuleHenhngDQUMkNi37+CBnTTaOD9/udlHYGMYrgBgNdCY82as8tz/IZPWyyQjUcoCBahsZJNJ", + "Ylvt/z2277RzANsW/0weYIu1eQMXsHF5QO66W4EftkILd7Cfbs417PPPHrsszWYH7e3nW+zCxTG1MoTv", + "b50d79Vpbp4USPYtTGC04aLhwY3IBTo7OnE1bTe/GdM/yqmhZ2pr9WVHB+IMsNcezSw74mwa0UChnhsL", + "FGaKSWaqlRnkqYiDd3bUCLt5VSGMi+fbVgmRr/Gky8D58iPv4U+PSqc3OUZymOWc136cJOtY55jKgOtv", + "C9zSC3AChHTqS7ZPi1y0ziFlguuzI2elumTF88mx25CP55qyXaesejY8glA8rgjEbygIK8XjC8DkT4mb", + "P2Sr6BApVniuvi/WHDyeFvTYXiwfmz8lN1ZYIZuWgiaou/EAfU2UCeXuPOBC2x48Ez8nwu1qV0cCZp1N", + "y3xqCquaCcGF9Grb98S80s70Ne39mSxfIM9NNBZL8h8qSgtjN6fVKgP3xJYsfzj7Fnq4kXl7f/e8lsE8", + "RIZgk4nzWJu6u1guWbD5p7rqfZTTzBD7SR5mZ2kUuRuPBREKvT06MTureAZsfYGwpPW6vdttK4+DD+9+", + "6xEWcIhDy2Ko/EqUfXLPGr5ZMDOVH2zSxiY0adHUnWdNGs4d1t+ECyIT4din/N+3X0V0IrBY/vv2Kxwl", + "lJF/3zmMsCJSbT4YswweSzQ/tsb9hJlPK9y0TDQQTQzqva/TULO3Wiqp7v0/lZ5qJn0jTTWj6w9ltY2y", + "WiTXSn3VLsWDaqymj290JZMxm4/a8MjFJ/7JNNXH9fJZjnRA0VSWrz1skT0uwM8LjyhDqSRPMICSZhxX", + "PDZauqvzDbny+HCse3LcBUJCXQRAbbIJIo/kvHbjeHTl1vb7+J7rw3hCZylPZTH3JMYqmBNpk5UiUhbA", + "T03tzo/nRsX7O+bSwWMeHY+uV//g+wfS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2ahpboOdnCzIajQ", + "JVG3ZeNSrnk92NE3Lp8tgj5oQyU3FxBYEAcj9n+0/fG7Ijj+9JNLkkkHg+19+J2wxaefXJ4MO3WsQhhU", + "PAKU2MM3x3DtN4PscwCSzVPyquMwlSeA9Rx0zr+cgZTffLa3kBwX/rCQWllIBXKttpDsWjysiVSG33p0", + "G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF7ztBYkZksmP8HcMmbvhwrBHaWDtrWVlG3KNQpoXhbg0QN7TnIc", + "xMc2jlwFgqcZI88TC+ltzZH8MGy2R743fhg8rnB+fDvkKbOYUfjrpEu0TukrpQ0Yk3EKZSFRjhACUZ9I", + "2PqPrsU+yitYyzRJuFDS4FSCAmyQ7OdaAfZhWpZhKn24lIDFSInsjhhUKtCPTS7/1iVZGhRKylkGOFmt", + "x+rLvSqjgH7TbXT/OpYf4rSVjvXI29iCVn87HeubiY5H0bROSrUANrKNAQblhGQ7mWfJffQzZbPNJxWB", + "aoRVNrcCnpFH1dqCSn8W13dLZtVkmw7aArSvLT37L3ji1ifp09odFm2BgCikeMa4VDQoVt4twoP+OKFb", + "n9CrKevl5qktke436F9xcdn2iPPUFXsCJ11xht+hL0EPD9DAvr1LAYxtcxpopnn0U7BWLO5bpmDQ6rkY", + "RGkIVentgehUyang8dj+aPBq9a6waKDgoghsq99a2OjeH8Fh9IYrROMkIlqLJyHqGW7Sq2lVfwc3T2Wh", + "tOLNhKHeNsWEGANGJ11pIisi4XLNLdgG3LPXl8srNSM+Ww+CkXXuEB88KBgjZuDwicPOv0CZkIXiXSQi", + "gUJXcxrMAREDCnpBRVcAq8BJcpFBYG0eoNewU4tIYND5hiRCG0IBZ5JHxABdLOL44qCO2Prx9BQ+MmAY", + "Bpv14iAruZ4dEFK/VUS4yGoevbG4HRuakwSPIrOiF9pqLMxv02Jf5BBlI+bDwWDkyjZIp+iiAIlx0YCJ", + "4QTqb3z2zbStbjOwpJmL4kgA4QxvEhZ2mi5iaORHwxgOvPVgWiJzmGE8MDBHbTC/8VkGalliZZwkbdnX", + "DhO4eBHHK3gYbRSKs0oV8lT9TaqQCAEfW+5uYm60gQPzD4UvNaPawkJZeVtgP+91o0GZ85JKC9VCFR3z", + "r0Ucd7odO54COt0NtPc1CCfVBuvXYnplCjAmP/TumwCUlIV9AaGkcnLY8v/NKvc788Kf3j9rCRX+Gbws", + "5fusfBSU5QWgBFRwd1W4nhTSASxkTRczhZF8e8TNsicLxUPbXW/Vyo5+B0bruluvrIZkVuDysa+/6iN4", + "ykkwsjabKRfV9Ph192LfPSPd35LUptqGQ37w5s3dc60YM0lX1BKFUqgS/HxQXxNwnYM557LA9hMyxwvK", + "hUVgt17XjDPBZWGsRxs9d6FZ9cL6by+sen5gfU0IFx/ZPvrwuY2583/hHuVfvCpY25nE7zqVGlAgJcJo", + "IiiZogSnkmhtKY0JMhVGLJA3wcHcVQvvj9j7OUG2PmbBgZCVU6YSXQzjiy6apApFWMzA2jEPTSSdIAGP", + "Y8JCU/N2xOYEL6g21QSKsCIsWPYkgRrIC5IXMNGmu72hNKW2syqrXeSK84KD4aJQevcCJYIAExlzmZXq", + "3I6YSNl/GORK3eyFG+gFIlLhSUTlPKsVEeCQsMALC3n+fYux+3finhNVr077Te4sbyVLv+UlZtGXmdUH", + "/y7uN59YoBYXrrJmCzG/QumVzaZhOfLxPK/I+y+4pc1c3Ry/0c1MRuJVu/j7uJIpleT/cS2j7JYMU9Md", + "KZet/9PeteR1pFNWum6xPtnbXrhklRAyMt9I5m19cX+e3MJH9p1Iwm6jYd+EuZ1P+nsQuZaqt5K538g5", + "aH1JBa/YNxTBdlDfTn3ioiDlvgsxbDZcJo2LMkcJDDYVZz+EcVUY2/CA2wpj53GtXYAXxDNlvSTCTXI5", + "r1rvF8DWIfAvGv1amV1BEH5zwZffCDyasDvJxJsReAleRhz/2e9lAi6ESei05YifDqBYwRdYuGDaAI9b", + "N5MQXZdN8vH0dLNJSgi1UkYI9YQlRLmsaRB7qjW+XRAhaOhKRx6dHtvoVSqRSFkfvY0p1HO8JCSBQjGU", + "pxJBZm5fz8+lttaL4JVyWLsdwpRYJpwytXYU+asPM5ivtyqd98hy0kIq/ukvj8EL//SEFMgOra7YCay2", + "IhVWjcF4LjiNMlPvUmtbeMJT3bqWLK7Q7gzOtimNiFxKRWITmTdNI9hEALprazLZ70xGaRdRJZHeD13I", + "wEuIiKmUlDM5Yrb8e0KE7lt/DsV/8yAjr/Ne4UxqnhnR930EsOnBmJgtrJqoBtACUAe0c9DZwkmyBeWi", + "/UFSdnh3GNIriEhDchlPeEQDFFF2KdFGRC+N0YEWEkX6j82VIW1j+O6+K07dfmdpSp+wKfcW5TA8mzHz", + "nyMJqSzW3CXikxNrr0lxszj5AwvtF2tyrVwTBEc9RWOSJb+jVNGIfjaiTjdCpaKByavJUy+hCLPNvhyx", + "U6KEfgcLggIeRSRQzrmylQgebI3SwWAnSCiglOwQGBwIvObHMfR4dPYB3jOForsjpv8BDb8/PDM3sVNs", + "fQSFgTKirri4RCdbb9cE+Z4Dmf6Fo+TMBFfmQHoX/Mf13c0zmxv3kGzYojxZZQDx5E8fxmk1uB/egqfp", + "LQBoiWw2GzOBA1CK5TxVIb9ifs/AgkdprP9h/jhZB1CicDD/CK9+N9quGc7abtwEn8SmtHMKiSka9E0u", + "KAzBnmp8qSacmwIoMaXIPe8pcKj+jNx9/075Ih2/w6tJS1FXkOu72VuPffLZMTjcrSI9nso2N5zmZqL4", + "au/TFabN3qefIx5cSpQyRaMSqIG22wAHVP+Y4zbaiz9QEyA70pUSR+Q6oQIQbCrwCIjoGUuEkSIipgxH", + "WzBn0wggUDovFl5wCknKQUQhTYyGBCU8igBl52pOGNKzAUeVa6BwTyttBYjiO8UrRsXRhAQ8Jg6Vc9Nn", + "uv0dU/WKizLE5vciF98X6K/no6eq57kGVbS5xzuhjJ7iawhrDlN7TexGtPGa5z8aV1AXwdqMOjsDOep0", + "0aizHY86egWOMLhQsUJ7KKYsVUT20bHxb0Ea6v4ASRJwFkoHDuo8eDsD2ZSUatiyIcNxH757TLXHchWQ", + "8p3txCce9HtIfw8JNmijuOHsngy7sOlCxFMFAdxuX9m3QqLAPbL56DewhT3yw7ZvI8n/brdvSUbBKmtx", + "WVh6I9kz+Mi1XjeXVDHnMkedRAFOcEDVsotwFPEg9x6kMrsd6GVDmQiCL7UN1R+xdxlwpU2EQEdnH7rO", + "aYZCKi9NC9Yv1kdvF0TIdJINDoE0MB48WAwSjpjiKMBRkEaab8l0SgLIYYhoTJVs8KtlQ3nIMoh5J56F", + "dw8z2Jqn5Uzy8wSsXs4WssJxW2aptwQJIkzjolOpShxQfeFKF9y+E90o18fwNLLXW4HgUiLbVI9EdEYn", + "kb2skX30XqscOCYjlkSYMSJQKk3ckR56LxFEytQkxugGoM6s4aguyoFOEsGVdRNHnAtpPLuawz+eIqlI", + "soLN3pmWT2HODwQTbBq3PX0jg6EyhuZjyb6C9IIYTjEE13ykj+lvEOxjBvSt4YSfysZ/L+hsRoTeFdgI", + "WXM1ara1I6fZ9KVMj0aM/PPsrXYY+VmrhWjuQqTzSqCKsXtxDAr0TW5gPZ1f0kYsE/voZtkXv+qPWvZd", + "jvL3D8I+uuMs/yylx84LwdVtkfVzDn9qIPeFkZe2ailBYT0cQeuMhIfMEGiNO/DN4AaeMsoALqUdNMEJ", + "fH+MMHjc7LjHhtl+2rxVQgkoFdZpSJVaD9/5XXDgw+B2fuPs0Fvgdn5X+UqAu/jt8ka/q0ylkh/QFQ/5", + "0yNzPlSCkoHnBBiLpgQlI/VsIMFKQ+mjfaedmWRb/DNp8Pbu+Qb6uyP7D6u/hclQIJbfZWdyox1uC4kT", + "tXSXi3xauQCU9DMkY/iAH7IYgofDW7jF9fr9sYfj08bL9R/1tB7t/j4vOnxy/PSLaBX3XOlg2dKnTg+L", + "YE4XpNnpXt7BlkSJIL2EJ3C5EhqCWXq4s0xh0Z99RrZ5i1Vl/4WogzgmIQqpIIGKlogyxUEimD7+IpHg", + "2hKA51wsfc704s59JXh8aGez5jy0e8o6w/I733jZC7HCvYWTNitcaHe4aXd321rgIcrQ65/RBrlWwiDu", + "oqm2fBCdZiQl1wEhoQSe3CwOeDho8GzSz2Q8m7QZ5Qrs5LcWmxoFqVQ8dmt/cow2oNjCjDC9FlrVn4Im", + "mwi+oKEpRJoTdcEjQ9VhA0Fv6nfVSkVWKcMZF2Zw30SHaXMgzT7TpCwWTOhC56AzoQzD4NaiFJf3lEmo", + "0v1hCmkN+d5xnNP5cYRZy2/DGTuaE7WR44ioODfQeJs/jrmnfMwVA1PdmVY67dqVimwXq9oyhPQhAHOz", + "OObHdVt//H7CK6l8kpGV1nW+yAzSJrf598WCg8c7Hx7bXf7xCYfjvybO+C64yqEB3aKPYX7jAY5QSBYk", + "4glUkTTvdrqdVESdg85cqeRgayvS7825VAfPB88Hna+fvv7/AQAA//8QadjqX44BAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/restart-policy/README.md b/lib/restart-policy/README.md index a91aeba6..f8dd76bb 100644 --- a/lib/restart-policy/README.md +++ b/lib/restart-policy/README.md @@ -1,6 +1,6 @@ # Restart Policy -Restart policy lets Hypeman keep an instance running after the guest program exits and the instance reaches `Stopped`. +Restart policy lets Hypeman keep an instance running after the guest program exits, or after a configured health check marks a running workload unhealthy. This supervises the whole instance, not an individual in-guest process. If the image runs systemd or multiple processes, a restart boots the instance again using the same persisted instance configuration. @@ -15,6 +15,7 @@ Failure means: - exit code is nonzero - the guest was killed by signal or OOM - the instance stopped unexpectedly and no clean exit code is available +- a configured health check reached `unhealthy` Exit code `0` does not restart under `on_failure`. @@ -46,9 +47,19 @@ Manual `start` clears blocked restart status and starts a new failure window. Updating the restart policy clears retry status unless the instance was manually stopped. Changing the policy on a manually stopped instance does not start it; the user must call `start`. +## Health Checks + +Health checks are a separate feature. They report workload health without changing instance lifecycle state. + +When health status reaches `unhealthy`, restart policy treats that as a failure signal for `on_failure` and `always`. `never` still reports health without restarting. + +Health-triggered restart uses the same attempt counter, backoff, max-attempt limit, manual-stop suppression, and stable-window reset as exit-triggered restart. + ## Lifecycle Behavior -Restart policy only starts instances from `Stopped`. +Exit-triggered restart only starts instances from `Stopped`. + +Health-triggered restart acts on `Running` instances by stopping the instance and starting it again. The health check itself does not keep the lifecycle state in `Initializing` or otherwise hide the distinction between boot readiness and workload health. It does not restore `Standby` instances, wake templates, or act on `Unknown` state. @@ -66,12 +77,12 @@ Instance responses include the configured `restart_policy` and current `restart_ `restart_status.blocked_reason` explains why no more retries will happen despite a restart policy being configured. +`restart_status.last_reason=health_check_failed` means the current retry window was entered because health checks marked the workload unhealthy rather than because the guest process exited on its own. + ## Non-goals Restart policy is not a health check. -It does not restart unhealthy-but-running workloads. - It does not supervise individual processes inside the guest. It does not replace systemd for images that want in-guest service supervision. diff --git a/lib/restart-policy/controller.go b/lib/restart-policy/controller.go index e6ac3fea..2bd31f34 100644 --- a/lib/restart-policy/controller.go +++ b/lib/restart-policy/controller.go @@ -159,42 +159,29 @@ func (c *Controller) reconcileInstance(ctx context.Context, inst Instance) error if status.BlockedReason != "" { return nil } - if !ShouldRestart(policy, inst.ExitCode) { + if !ShouldRestartInstance(policy, inst.ExitCode, status) { return nil } - if status.NextAttemptAt != nil && now.Before(status.NextAttemptAt.UTC()) { - return nil - } - if status.LastAttemptAt != nil { - nextAttemptAt := status.LastAttemptAt.UTC().Add(Backoff(policy)) - if now.Before(nextAttemptAt) { - status.NextAttemptAt = &nextAttemptAt - return c.store.SetRestartStatus(ctx, inst.ID, status) + + nextStatus, shouldAttempt := PrepareAttempt(policy, status, now) + if !shouldAttempt { + if !EqualStatus(status, nextStatus) { + return c.store.SetRestartStatus(ctx, inst.ID, nextStatus) } - } - if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { - status.NextAttemptAt = nil - status.BlockedReason = BlockedReasonMaxAttemptsExceeded - return c.store.SetRestartStatus(ctx, inst.ID, status) + return nil } - status.Attempts++ - status.LastAttemptAt = &now - status.NextAttemptAt = nil - if err := c.store.SetRestartStatus(ctx, inst.ID, status); err != nil { + reason := nextStatus.LastReason + nextStatus.LastReason = "" + if err := c.store.SetRestartStatus(ctx, inst.ID, nextStatus); err != nil { return err } - c.log.Info("restart policy starting instance", "instance_id", inst.ID, "attempt", status.Attempts) + c.log.Info("restart policy starting instance", "instance_id", inst.ID, "attempt", nextStatus.Attempts) if err := c.store.RestartInstance(ctx, inst.ID); err != nil { - if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { - status.BlockedReason = BlockedReasonMaxAttemptsExceeded - status.NextAttemptAt = nil - } else { - nextAttemptAt := now.Add(Backoff(policy)) - status.NextAttemptAt = &nextAttemptAt - } - if statusErr := c.store.SetRestartStatus(ctx, inst.ID, status); statusErr != nil { + nextStatus = AfterFailedAttempt(policy, nextStatus, now) + nextStatus.LastReason = reason + if statusErr := c.store.SetRestartStatus(ctx, inst.ID, nextStatus); statusErr != nil { c.log.Warn("failed to persist restart status after restart failure", "instance_id", inst.ID, "error", statusErr) } return err diff --git a/lib/restart-policy/controller_test.go b/lib/restart-policy/controller_test.go index 3fd8b1d0..47c37a2c 100644 --- a/lib/restart-policy/controller_test.go +++ b/lib/restart-policy/controller_test.go @@ -114,3 +114,23 @@ func TestReconcileSkipsCleanOnFailureExit(t *testing.T) { assert.Empty(t, store.started) assert.Nil(t, store.statuses) } + +func TestReconcileRestartsStoppedHealthFailure(t *testing.T) { + exitCode := 0 + store := &fakeStore{ + instances: []Instance{{ + ID: "inst-1", + State: StateStopped, + ExitCode: &exitCode, + RestartPolicy: &Policy{Policy: PolicyOnFailure}, + RestartStatus: Status{LastReason: RestartReasonHealthCheckFailed}, + }}, + } + controller := NewController(store, ControllerOptions{}) + + require.NoError(t, controller.Reconcile(context.Background())) + + assert.Equal(t, []string{"inst-1"}, store.started) + assert.Equal(t, 1, store.statuses["inst-1"].Attempts) + assert.Empty(t, store.statuses["inst-1"].LastReason) +} diff --git a/lib/restart-policy/policy.go b/lib/restart-policy/policy.go index 392490dc..d012b239 100644 --- a/lib/restart-policy/policy.go +++ b/lib/restart-policy/policy.go @@ -26,6 +26,12 @@ const ( BlockedReasonMaxAttemptsExceeded BlockedReason = "max_attempts_exceeded" ) +type RestartReason string + +const ( + RestartReasonHealthCheckFailed RestartReason = "health_check_failed" +) + type Policy struct { Policy PolicyMode `json:"policy"` Backoff string `json:"backoff,omitempty"` @@ -38,6 +44,7 @@ type Status struct { LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` BlockedReason BlockedReason `json:"blocked_reason,omitempty"` + LastReason RestartReason `json:"last_reason,omitempty"` } func NormalizePolicy(policy *Policy) (*Policy, error) { @@ -112,11 +119,85 @@ func ShouldRestart(policy *Policy, exitCode *int) bool { } } +func ShouldRestartHealthCheck(policy *Policy) bool { + if policy == nil { + return false + } + switch policy.Policy { + case PolicyAlways, PolicyOnFailure: + return true + default: + return false + } +} + +func ShouldRestartInstance(policy *Policy, exitCode *int, status Status) bool { + if status.LastReason == RestartReasonHealthCheckFailed { + return ShouldRestartHealthCheck(policy) + } + return ShouldRestart(policy, exitCode) +} + +func PrepareAttempt(policy *Policy, status Status, now time.Time) (Status, bool) { + now = now.UTC() + if status.BlockedReason != "" { + return status, false + } + if status.NextAttemptAt != nil && now.Before(status.NextAttemptAt.UTC()) { + return status, false + } + if status.LastAttemptAt != nil { + nextAttemptAt := status.LastAttemptAt.UTC().Add(Backoff(policy)) + if now.Before(nextAttemptAt) { + status.NextAttemptAt = &nextAttemptAt + return status, false + } + } + if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { + status.NextAttemptAt = nil + status.BlockedReason = BlockedReasonMaxAttemptsExceeded + return status, false + } + + status.Attempts++ + status.LastAttemptAt = &now + status.NextAttemptAt = nil + return status, true +} + +func AfterFailedAttempt(policy *Policy, status Status, now time.Time) Status { + now = now.UTC() + if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { + status.BlockedReason = BlockedReasonMaxAttemptsExceeded + status.NextAttemptAt = nil + return status + } + nextAttemptAt := now.Add(Backoff(policy)) + status.NextAttemptAt = &nextAttemptAt + return status +} + +func EqualStatus(a, b Status) bool { + return a.Attempts == b.Attempts && + equalTime(a.LastAttemptAt, b.LastAttemptAt) && + equalTime(a.NextAttemptAt, b.NextAttemptAt) && + a.BlockedReason == b.BlockedReason && + a.LastReason == b.LastReason +} + func (s Status) IsZero() bool { return s.Attempts == 0 && s.LastAttemptAt == nil && s.NextAttemptAt == nil && - s.BlockedReason == "" + s.BlockedReason == "" && + s.LastReason == "" +} + +func equalTime(a, b *time.Time) bool { + if a == nil || b == nil { + return a == b + } + return a.UTC().Equal(b.UTC()) } func normalizeDuration(raw string, fallback time.Duration, field string) (time.Duration, error) { diff --git a/openapi.yaml b/openapi.yaml index ceb7b469..c6bce872 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -432,6 +432,12 @@ components: nullable: true description: Reason automatic restarts are currently blocked. example: max_attempts_exceeded + last_reason: + type: string + enum: [health_check_failed] + nullable: true + description: Most recent non-exit failure signal that entered restart policy. + example: health_check_failed AutoStandbyStatus: type: object From bf50d28ac23094f25a372cf358e6a475307622c0 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 16:17:08 +0000 Subject: [PATCH 03/13] Simplify restart policy controller --- lib/instances/restart_policy.go | 208 ++++++++++++++++---------- lib/instances/restart_policy_test.go | 18 +++ lib/restart-policy/controller.go | 200 ------------------------- lib/restart-policy/controller_test.go | 136 ----------------- lib/restart-policy/policy.go | 25 +--- lib/restart-policy/policy_test.go | 15 -- 6 files changed, 152 insertions(+), 450 deletions(-) delete mode 100644 lib/restart-policy/controller.go delete mode 100644 lib/restart-policy/controller_test.go diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 8594f149..21ad234d 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -3,15 +3,14 @@ package instances import ( "context" "fmt" + "log/slog" "time" "github.com/kernel/hypeman/lib/logger" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" ) -type restartPolicyStore struct { - manager *manager -} +const defaultRestartPolicyReconcileInterval = 5 * time.Second func cloneRestartPolicy(policy *restartpolicy.Policy) *restartpolicy.Policy { if policy == nil { @@ -65,6 +64,13 @@ func (m *manager) updateRestartStatusLocked(id string, status restartpolicy.Stat return m.saveMetadata(meta) } +func (m *manager) setRestartStatus(ctx context.Context, id string, status restartpolicy.Status) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + return m.updateRestartStatusLocked(id, status) +} + func (m *manager) RestartInstance(ctx context.Context, id string) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() @@ -94,7 +100,7 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err if err != nil { return err } - if !restartpolicy.ShouldRestartHealthCheck(policy) { + if !restartpolicy.ShouldRestart(policy, nil) { return nil } if current.RestartStatus.BlockedReason != "" { @@ -104,7 +110,7 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err now := time.Now().UTC() status := current.RestartStatus status.LastReason = restartpolicy.RestartReasonHealthCheckFailed - nextStatus, shouldAttempt := restartpolicy.PrepareAttempt(policy, status, now) + nextStatus, reason, shouldAttempt := prepareRestartAttempt(policy, status, now) if !shouldAttempt { if !restartpolicy.EqualStatus(current.RestartStatus, nextStatus) { return m.updateRestartStatusLocked(id, nextStatus) @@ -112,105 +118,157 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err return nil } - reason := nextStatus.LastReason - nextStatus.LastReason = "" if err := m.updateRestartStatusLocked(id, nextStatus); err != nil { return err } stopped, err := m.stopInstance(ctx, id) if err != nil { - _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedHealthAttempt(policy, nextStatus, reason, now)) + _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) return err } m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) started, err := m.startInstance(ctx, id, StartInstanceRequest{}) if err != nil { - _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedHealthAttempt(policy, nextStatus, reason, now)) + _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) return err } m.notifyLifecycleEvent(ctx, LifecycleEventStart, started) return nil } -func restartStatusAfterFailedHealthAttempt(policy *restartpolicy.Policy, status restartpolicy.Status, reason restartpolicy.RestartReason, now time.Time) restartpolicy.Status { - status = restartpolicy.AfterFailedAttempt(policy, status, now) - status.LastReason = reason - return status -} - func (m *manager) StartRestartPolicyController(ctx context.Context) error { - controller := restartpolicy.NewController( - restartPolicyStore{manager: m}, - restartpolicy.ControllerOptions{ - Log: logger.FromContext(ctx).With("controller", "restart_policy"), - }, - ) - return controller.Run(ctx) + log := logger.FromContext(ctx).With("controller", "restart_policy") + log.InfoContext(ctx, "restart policy controller started", "reconcile_interval", defaultRestartPolicyReconcileInterval) + if err := m.reconcileRestartPolicies(ctx, log); err != nil { + log.WarnContext(ctx, "restart policy startup reconcile failed", "error", err) + } + + events, unsubscribe := m.SubscribeLifecycleEvents(LifecycleEventConsumerRestartPolicy) + defer unsubscribe() + + ticker := time.NewTicker(defaultRestartPolicyReconcileInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case event, ok := <-events: + if !ok { + return nil + } + if event.Action == LifecycleEventDelete { + continue + } + if event.Instance == nil { + if err := m.reconcileRestartPolicies(ctx, log); err != nil { + log.WarnContext(ctx, "restart policy event reconcile failed", "instance_id", event.InstanceID, "error", err) + } + continue + } + if err := m.reconcileRestartPolicyInstance(ctx, event.Instance, log); err != nil { + log.WarnContext(ctx, "restart policy event handling failed", "instance_id", event.InstanceID, "error", err) + } + case <-ticker.C: + if err := m.reconcileRestartPolicies(ctx, log); err != nil { + log.WarnContext(ctx, "restart policy reconcile failed", "error", err) + } + } + } } -func (s restartPolicyStore) ListInstances(ctx context.Context) ([]restartpolicy.Instance, error) { - insts, err := s.manager.ListInstances(ctx, nil) +func (m *manager) reconcileRestartPolicies(ctx context.Context, log *slog.Logger) error { + insts, err := m.ListInstances(ctx, nil) if err != nil { - return nil, err + return err } - out := make([]restartpolicy.Instance, 0, len(insts)) - for _, inst := range insts { - out = append(out, *toRestartPolicyInstance(&inst)) + for i := range insts { + if err := m.reconcileRestartPolicyInstance(ctx, &insts[i], log); err != nil { + log.WarnContext(ctx, "restart policy reconcile failed for instance", "instance_id", insts[i].Id, "error", err) + } } - return out, nil + return nil } -func (s restartPolicyStore) RestartInstance(ctx context.Context, id string) error { - _, err := s.manager.RestartInstance(ctx, id) - return err +func (m *manager) reconcileRestartPolicyInstance(ctx context.Context, inst *Instance, log *slog.Logger) error { + if inst == nil { + return nil + } + policy, err := restartpolicy.NormalizePolicy(inst.RestartPolicy) + if err != nil { + return err + } + if policy == nil { + return nil + } + + status := inst.RestartStatus + now := time.Now().UTC() + if shouldResetRestartAttempts(policy, status, inst, now) { + log.InfoContext(ctx, "restart policy stable window reached", "instance_id", inst.Id, "attempts", status.Attempts) + return m.setRestartStatus(ctx, inst.Id, restartpolicy.Status{}) + } + if inst.State != StateStopped || status.BlockedReason != "" { + return nil + } + + exitCode := inst.ExitCode + if status.LastReason == restartpolicy.RestartReasonHealthCheckFailed { + exitCode = nil + } + if !restartpolicy.ShouldRestart(policy, exitCode) { + return nil + } + return m.startInstanceForRestartPolicy(ctx, inst.Id, policy, status, now, log) } -func (s restartPolicyStore) SetRestartStatus(ctx context.Context, id string, status restartpolicy.Status) error { - lock := s.manager.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - return s.manager.updateRestartStatusLocked(id, status) -} - -func (s restartPolicyStore) SubscribeInstanceEvents() (<-chan restartpolicy.InstanceEvent, func(), error) { - src, unsub := s.manager.SubscribeLifecycleEvents(LifecycleEventConsumerRestartPolicy) - dst := make(chan restartpolicy.InstanceEvent, 32) - go func() { - defer close(dst) - for event := range src { - dst <- restartpolicy.InstanceEvent{ - Action: restartpolicy.InstanceEventAction(event.Action), - InstanceID: event.InstanceID, - Instance: toRestartPolicyInstance(event.Instance), - } +func (m *manager) startInstanceForRestartPolicy(ctx context.Context, id string, policy *restartpolicy.Policy, status restartpolicy.Status, now time.Time, log *slog.Logger) error { + nextStatus, reason, shouldAttempt := prepareRestartAttempt(policy, status, now) + if !shouldAttempt { + if !restartpolicy.EqualStatus(status, nextStatus) { + return m.setRestartStatus(ctx, id, nextStatus) } - }() - return dst, unsub, nil + return nil + } + if err := m.setRestartStatus(ctx, id, nextStatus); err != nil { + return err + } + + log.InfoContext(ctx, "restart policy starting instance", "instance_id", id, "attempt", nextStatus.Attempts) + if _, err := m.RestartInstance(ctx, id); err != nil { + nextStatus = restartStatusAfterFailedAttempt(policy, nextStatus, reason, now) + if statusErr := m.setRestartStatus(ctx, id, nextStatus); statusErr != nil { + log.WarnContext(ctx, "failed to persist restart status after restart failure", "instance_id", id, "error", statusErr) + } + return err + } + return nil } -func toRestartPolicyInstance(inst *Instance) *restartpolicy.Instance { - if inst == nil { - return nil +func prepareRestartAttempt(policy *restartpolicy.Policy, status restartpolicy.Status, now time.Time) (restartpolicy.Status, restartpolicy.RestartReason, bool) { + nextStatus, shouldAttempt := restartpolicy.PrepareAttempt(policy, status, now) + if !shouldAttempt { + return nextStatus, "", false + } + reason := nextStatus.LastReason + nextStatus.LastReason = "" + return nextStatus, reason, true +} + +func restartStatusAfterFailedAttempt(policy *restartpolicy.Policy, status restartpolicy.Status, reason restartpolicy.RestartReason, now time.Time) restartpolicy.Status { + status = restartpolicy.AfterFailedAttempt(policy, status, now) + status.LastReason = reason + return status +} + +func shouldResetRestartAttempts(policy *restartpolicy.Policy, status restartpolicy.Status, inst *Instance, now time.Time) bool { + if status.Attempts == 0 || inst.StartedAt == nil { + return false } - return &restartpolicy.Instance{ - ID: inst.Id, - State: string(inst.State), - StartedAt: inst.StartedAt, - ExitCode: inst.ExitCode, - RestartPolicy: inst.RestartPolicy, - RestartStatus: inst.RestartStatus, - } -} - -var _ restartpolicy.Store = restartPolicyStore{} -var _ interface { - StartRestartPolicyController(context.Context) error -} = (*manager)(nil) -var _ interface { - RestartInstance(context.Context, string) (*Instance, error) -} = (*manager)(nil) -var _ interface { - HandleHealthCheckUnhealthy(context.Context, string) error -} = (*manager)(nil) + if inst.State != StateRunning && inst.State != StateInitializing { + return false + } + return !now.Before(inst.StartedAt.UTC().Add(restartpolicy.StableAfter(policy))) +} diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go index d500f578..2c939a87 100644 --- a/lib/instances/restart_policy_test.go +++ b/lib/instances/restart_policy_test.go @@ -3,6 +3,7 @@ package instances import ( "errors" "testing" + "time" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/stretchr/testify/assert" @@ -46,3 +47,20 @@ func TestRestartStatusAfterPolicyUpdateClearsRetryState(t *testing.T) { assert.True(t, status.IsZero()) } + +func TestShouldResetRestartAttempts(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + startedAt := now.Add(-11 * time.Minute) + + reset := shouldResetRestartAttempts( + &restartpolicy.Policy{Policy: restartpolicy.PolicyAlways, StableAfter: "10m"}, + restartpolicy.Status{Attempts: 2}, + &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{StartedAt: &startedAt}, + }, + now, + ) + + assert.True(t, reset) +} diff --git a/lib/restart-policy/controller.go b/lib/restart-policy/controller.go deleted file mode 100644 index 2bd31f34..00000000 --- a/lib/restart-policy/controller.go +++ /dev/null @@ -1,200 +0,0 @@ -package restartpolicy - -import ( - "context" - "log/slog" - "time" -) - -const ( - StateInitializing = "Initializing" - StateRunning = "Running" - StateStopped = "Stopped" - - DefaultReconcileInterval = 5 * time.Second -) - -type InstanceEventAction string - -const ( - InstanceEventCreate InstanceEventAction = "create" - InstanceEventUpdate InstanceEventAction = "update" - InstanceEventStart InstanceEventAction = "start" - InstanceEventStop InstanceEventAction = "stop" - InstanceEventDelete InstanceEventAction = "delete" -) - -type InstanceEvent struct { - Action InstanceEventAction - InstanceID string - Instance *Instance -} - -type Instance struct { - ID string - State string - StartedAt *time.Time - ExitCode *int - RestartPolicy *Policy - RestartStatus Status -} - -type Store interface { - ListInstances(ctx context.Context) ([]Instance, error) - RestartInstance(ctx context.Context, id string) error - SetRestartStatus(ctx context.Context, id string, status Status) error - SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) -} - -type ControllerOptions struct { - Log *slog.Logger - Now func() time.Time - ReconcileInterval time.Duration -} - -type Controller struct { - store Store - log *slog.Logger - now func() time.Time - reconcileInterval time.Duration -} - -func NewController(store Store, opts ControllerOptions) *Controller { - log := opts.Log - if log == nil { - log = slog.Default() - } - now := opts.Now - if now == nil { - now = time.Now - } - interval := opts.ReconcileInterval - if interval <= 0 { - interval = DefaultReconcileInterval - } - return &Controller{ - store: store, - log: log, - now: now, - reconcileInterval: interval, - } -} - -func (c *Controller) Run(ctx context.Context) error { - c.log.Info("restart policy controller started", "reconcile_interval", c.reconcileInterval) - if err := c.Reconcile(ctx); err != nil { - c.log.Warn("restart policy startup reconcile failed", "error", err) - } - - events, unsubscribe, err := c.store.SubscribeInstanceEvents() - if err != nil { - return err - } - defer unsubscribe() - - ticker := time.NewTicker(c.reconcileInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return nil - case event, ok := <-events: - if !ok { - return nil - } - if event.Action == InstanceEventDelete { - continue - } - if event.Instance == nil { - if err := c.Reconcile(ctx); err != nil { - c.log.Warn("restart policy event reconcile failed", "instance_id", event.InstanceID, "error", err) - } - continue - } - if err := c.reconcileInstance(ctx, *event.Instance); err != nil { - c.log.Warn("restart policy event handling failed", "instance_id", event.InstanceID, "error", err) - } - case <-ticker.C: - if err := c.Reconcile(ctx); err != nil { - c.log.Warn("restart policy reconcile failed", "error", err) - } - } - } -} - -func (c *Controller) Reconcile(ctx context.Context) error { - instances, err := c.store.ListInstances(ctx) - if err != nil { - return err - } - for _, inst := range instances { - if err := c.reconcileInstance(ctx, inst); err != nil { - c.log.Warn("restart policy reconcile failed for instance", "instance_id", inst.ID, "error", err) - } - } - return nil -} - -func (c *Controller) reconcileInstance(ctx context.Context, inst Instance) error { - policy, err := NormalizePolicy(inst.RestartPolicy) - if err != nil { - return err - } - if policy == nil { - return nil - } - - status := inst.RestartStatus - now := c.now().UTC() - - if shouldResetStableAttempts(policy, status, inst, now) { - c.log.Info("restart policy stable window reached", "instance_id", inst.ID, "attempts", status.Attempts) - return c.store.SetRestartStatus(ctx, inst.ID, Status{}) - } - - if inst.State != StateStopped { - return nil - } - if status.BlockedReason != "" { - return nil - } - if !ShouldRestartInstance(policy, inst.ExitCode, status) { - return nil - } - - nextStatus, shouldAttempt := PrepareAttempt(policy, status, now) - if !shouldAttempt { - if !EqualStatus(status, nextStatus) { - return c.store.SetRestartStatus(ctx, inst.ID, nextStatus) - } - return nil - } - - reason := nextStatus.LastReason - nextStatus.LastReason = "" - if err := c.store.SetRestartStatus(ctx, inst.ID, nextStatus); err != nil { - return err - } - - c.log.Info("restart policy starting instance", "instance_id", inst.ID, "attempt", nextStatus.Attempts) - if err := c.store.RestartInstance(ctx, inst.ID); err != nil { - nextStatus = AfterFailedAttempt(policy, nextStatus, now) - nextStatus.LastReason = reason - if statusErr := c.store.SetRestartStatus(ctx, inst.ID, nextStatus); statusErr != nil { - c.log.Warn("failed to persist restart status after restart failure", "instance_id", inst.ID, "error", statusErr) - } - return err - } - return nil -} - -func shouldResetStableAttempts(policy *Policy, status Status, inst Instance, now time.Time) bool { - if status.Attempts == 0 || inst.StartedAt == nil { - return false - } - if inst.State != StateRunning && inst.State != StateInitializing { - return false - } - return !now.Before(inst.StartedAt.UTC().Add(StableAfter(policy))) -} diff --git a/lib/restart-policy/controller_test.go b/lib/restart-policy/controller_test.go deleted file mode 100644 index 47c37a2c..00000000 --- a/lib/restart-policy/controller_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package restartpolicy - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakeStore struct { - instances []Instance - started []string - statuses map[string]Status - startErr error -} - -func (s *fakeStore) ListInstances(context.Context) ([]Instance, error) { - out := make([]Instance, len(s.instances)) - copy(out, s.instances) - return out, nil -} - -func (s *fakeStore) RestartInstance(_ context.Context, id string) error { - s.started = append(s.started, id) - return s.startErr -} - -func (s *fakeStore) SetRestartStatus(_ context.Context, id string, status Status) error { - if s.statuses == nil { - s.statuses = make(map[string]Status) - } - s.statuses[id] = status - return nil -} - -func (s *fakeStore) SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) { - ch := make(chan InstanceEvent) - close(ch) - return ch, func() {}, nil -} - -func TestReconcileRestartsFailedStoppedInstance(t *testing.T) { - now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) - exitCode := 1 - store := &fakeStore{ - instances: []Instance{{ - ID: "inst-1", - State: StateStopped, - ExitCode: &exitCode, - RestartPolicy: &Policy{Policy: PolicyOnFailure}, - }}, - } - controller := NewController(store, ControllerOptions{Now: func() time.Time { return now }}) - - require.NoError(t, controller.Reconcile(context.Background())) - - assert.Equal(t, []string{"inst-1"}, store.started) - status := store.statuses["inst-1"] - assert.Equal(t, 1, status.Attempts) - require.NotNil(t, status.LastAttemptAt) - assert.Equal(t, now, *status.LastAttemptAt) -} - -func TestReconcileSkipsManualStop(t *testing.T) { - store := &fakeStore{ - instances: []Instance{{ - ID: "inst-1", - State: StateStopped, - RestartPolicy: &Policy{Policy: PolicyAlways}, - RestartStatus: Status{BlockedReason: BlockedReasonManualStop}, - }}, - } - controller := NewController(store, ControllerOptions{}) - - require.NoError(t, controller.Reconcile(context.Background())) - - assert.Empty(t, store.started) - assert.Nil(t, store.statuses) -} - -func TestReconcileBlocksAfterMaxAttempts(t *testing.T) { - store := &fakeStore{ - instances: []Instance{{ - ID: "inst-1", - State: StateStopped, - RestartPolicy: &Policy{Policy: PolicyAlways, MaxAttempts: 2}, - RestartStatus: Status{Attempts: 2}, - }}, - } - controller := NewController(store, ControllerOptions{}) - - require.NoError(t, controller.Reconcile(context.Background())) - - assert.Empty(t, store.started) - assert.Equal(t, BlockedReasonMaxAttemptsExceeded, store.statuses["inst-1"].BlockedReason) -} - -func TestReconcileSkipsCleanOnFailureExit(t *testing.T) { - exitCode := 0 - store := &fakeStore{ - instances: []Instance{{ - ID: "inst-1", - State: StateStopped, - ExitCode: &exitCode, - RestartPolicy: &Policy{Policy: PolicyOnFailure}, - }}, - } - controller := NewController(store, ControllerOptions{}) - - require.NoError(t, controller.Reconcile(context.Background())) - - assert.Empty(t, store.started) - assert.Nil(t, store.statuses) -} - -func TestReconcileRestartsStoppedHealthFailure(t *testing.T) { - exitCode := 0 - store := &fakeStore{ - instances: []Instance{{ - ID: "inst-1", - State: StateStopped, - ExitCode: &exitCode, - RestartPolicy: &Policy{Policy: PolicyOnFailure}, - RestartStatus: Status{LastReason: RestartReasonHealthCheckFailed}, - }}, - } - controller := NewController(store, ControllerOptions{}) - - require.NoError(t, controller.Reconcile(context.Background())) - - assert.Equal(t, []string{"inst-1"}, store.started) - assert.Equal(t, 1, store.statuses["inst-1"].Attempts) - assert.Empty(t, store.statuses["inst-1"].LastReason) -} diff --git a/lib/restart-policy/policy.go b/lib/restart-policy/policy.go index d012b239..29d57abe 100644 --- a/lib/restart-policy/policy.go +++ b/lib/restart-policy/policy.go @@ -101,10 +101,6 @@ func StableAfter(policy *Policy) time.Duration { return d } -func Failure(exitCode *int) bool { - return exitCode == nil || *exitCode != 0 -} - func ShouldRestart(policy *Policy, exitCode *int) bool { if policy == nil { return false @@ -113,31 +109,12 @@ func ShouldRestart(policy *Policy, exitCode *int) bool { case PolicyAlways: return true case PolicyOnFailure: - return Failure(exitCode) - default: - return false - } -} - -func ShouldRestartHealthCheck(policy *Policy) bool { - if policy == nil { - return false - } - switch policy.Policy { - case PolicyAlways, PolicyOnFailure: - return true + return exitCode == nil || *exitCode != 0 default: return false } } -func ShouldRestartInstance(policy *Policy, exitCode *int, status Status) bool { - if status.LastReason == RestartReasonHealthCheckFailed { - return ShouldRestartHealthCheck(policy) - } - return ShouldRestart(policy, exitCode) -} - func PrepareAttempt(policy *Policy, status Status, now time.Time) (Status, bool) { now = now.UTC() if status.BlockedReason != "" { diff --git a/lib/restart-policy/policy_test.go b/lib/restart-policy/policy_test.go index 1c0efe6c..b02a8abc 100644 --- a/lib/restart-policy/policy_test.go +++ b/lib/restart-policy/policy_test.go @@ -2,7 +2,6 @@ package restartpolicy import ( "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -41,17 +40,3 @@ func TestShouldRestart(t *testing.T) { assert.True(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, &exitOne)) assert.True(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, nil)) } - -func TestStableAttemptReset(t *testing.T) { - now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) - startedAt := now.Add(-11 * time.Minute) - - reset := shouldResetStableAttempts( - &Policy{Policy: PolicyAlways, StableAfter: "10m"}, - Status{Attempts: 2}, - Instance{State: StateRunning, StartedAt: &startedAt}, - now, - ) - - assert.True(t, reset) -} From 6f58518a3a0b15f801d058726be6fea6ce4deea6 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 17:14:58 +0000 Subject: [PATCH 04/13] Add restart policy network integration coverage --- lib/instances/network_test.go | 87 +++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index 6ade0dde..47680f9c 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -15,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vishvananda/netlink" @@ -30,6 +31,7 @@ func TestCreateInstanceWithNetwork(t *testing.T) { manager, _ := setupTestManager(t) ctx := context.Background() startHealthCheckControllerForTest(t, ctx, manager) + startRestartPolicyControllerForTest(t, ctx, manager) // Pull nginx:alpine image (long-running workload) t.Log("Pulling nginx:alpine image...") @@ -241,6 +243,46 @@ func TestCreateInstanceWithNetwork(t *testing.T) { require.Contains(t, psOutput, "nginx: master process", "nginx master should still be running") t.Log("Nginx process confirmed running - restore was successful!") + // Flip health checks to a guaranteed failure and verify restart policy stop-starts the same instance. + t.Log("Triggering restart policy from failing health check...") + require.NotNil(t, inst.StartedAt) + startedAtBeforeRestart := *inst.StartedAt + inst, err = manager.UpdateInstance(ctx, inst.Id, UpdateInstanceRequest{ + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeHTTP, + Interval: "1s", + Timeout: "1s", + StartPeriod: "1s", + FailureThreshold: 1, + SuccessThreshold: 1, + HTTP: &healthcheck.HTTPCheck{ + Port: 80, + Path: "/definitely-missing", + ExpectedStatus: 200, + }, + }, + RestartPolicy: &restartpolicy.Policy{ + Policy: restartpolicy.PolicyOnFailure, + Strategy: restartpolicy.StrategyStopStart, + Backoff: "1s", + MaxAttempts: 1, + StableAfter: "30s", + }, + RestartPolicySet: true, + }) + require.NoError(t, err) + require.NotNil(t, inst.RestartPolicy) + assert.Equal(t, restartpolicy.PolicyOnFailure, inst.RestartPolicy.Policy) + assert.Equal(t, restartpolicy.StrategyStopStart, inst.RestartPolicy.Strategy) + + inst, err = waitForRestartPolicyBlocked(ctx, manager, inst.Id, restartpolicy.BlockedReasonMaxAttemptsExceeded, 60*time.Second) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) + require.Equal(t, 1, inst.RestartStatus.Attempts) + require.NotNil(t, inst.StartedAt) + assert.True(t, inst.StartedAt.After(startedAtBeforeRestart), "instance should have been started again") + t.Log("Restart policy performed stop-start and blocked after max attempts") + // Cleanup t.Log("Cleaning up instance...") err = manager.DeleteInstance(ctx, inst.Id) @@ -261,6 +303,21 @@ func TestCreateInstanceWithNetwork(t *testing.T) { t.Log("Network integration test complete!") } +func startRestartPolicyControllerForTest(t *testing.T, ctx context.Context, manager *manager) { + t.Helper() + + controllerCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + done <- manager.StartRestartPolicyController(controllerCtx) + }() + + t.Cleanup(func() { + cancel() + require.NoError(t, <-done) + }) +} + func startHealthCheckControllerForTest(t *testing.T, ctx context.Context, manager Manager) { t.Helper() @@ -309,6 +366,36 @@ func waitForInstanceHealthStatus(ctx context.Context, manager Manager, instanceI return nil, last, fmt.Errorf("instance %s health did not reach %s within %v (last state: %s, last health: %s)", instanceID, expected, timeout, lastState, last.Status) } +func waitForRestartPolicyBlocked(ctx context.Context, manager Manager, instanceID string, reason restartpolicy.BlockedReason, timeout time.Duration) (*Instance, error) { + timeout = integrationTestTimeout(timeout) + deadline := time.Now().Add(timeout) + lastState := StateUnknown + lastStatus := restartpolicy.Status{} + lastErr := error(nil) + + for time.Now().Before(deadline) { + inst, err := manager.GetInstance(ctx, instanceID) + if err != nil { + lastErr = err + time.Sleep(200 * time.Millisecond) + continue + } + + lastState = inst.State + lastStatus = inst.RestartStatus + if inst.State == StateRunning && inst.RestartStatus.BlockedReason == reason { + return inst, nil + } + + time.Sleep(200 * time.Millisecond) + } + + if lastErr != nil { + return nil, fmt.Errorf("instance %s restart policy did not block with %s within %v (last error: %w)", instanceID, reason, timeout, lastErr) + } + return nil, fmt.Errorf("instance %s restart policy did not block with %s within %v (last state: %s, attempts: %d, blocked_reason: %s)", instanceID, reason, timeout, lastState, lastStatus.Attempts, lastStatus.BlockedReason) +} + // TestDockerForwardChainRestored validates recovery from an external FORWARD-chain flush. // This test intentionally mutates host-global iptables state, so it must run non-parallel. func TestDockerForwardChainRestored(t *testing.T) { From d0b81b92d80016ddbb3b1bb049f33ccfbd5381d1 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 17:37:33 +0000 Subject: [PATCH 05/13] Expect restart policy lifecycle metrics --- lib/instances/metrics_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 72f4f578..811571e3 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -221,6 +221,8 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T assert.Equal(t, int64(1), point.Value) case string(LifecycleEventConsumerHealthCheck): assert.Equal(t, int64(0), point.Value) + case string(LifecycleEventConsumerRestartPolicy): + assert.Equal(t, int64(0), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } @@ -238,6 +240,8 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T assert.Equal(t, int64(m.lifecycleEvents.bufferSize), point.Value) case string(LifecycleEventConsumerHealthCheck): assert.Equal(t, int64(0), point.Value) + case string(LifecycleEventConsumerRestartPolicy): + assert.Equal(t, int64(0), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } From b956143f157d9e99edd4d98a94142e4063fd0b2f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 17:39:57 +0000 Subject: [PATCH 06/13] Update instance patch validation test --- lib/instances/update_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 3ec0e55c..f3f1e16f 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -30,7 +30,7 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{}) require.Error(t, err) assert.ErrorIs(t, err, ErrInvalidRequest) - assert.Contains(t, err.Error(), "env, auto_standby, and/or health_check") + assert.Contains(t, err.Error(), "env, auto_standby, health_check, and/or restart_policy") }) t.Run("rejects instances without credential backed envs", func(t *testing.T) { From 2a59bf38d1d480ab91b756aac32a6dd2dad641f0 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 17:51:13 +0000 Subject: [PATCH 07/13] Fix restart policy event reconciliation --- lib/instances/restart_policy.go | 43 +++++++++++++++-- lib/instances/restart_policy_test.go | 70 ++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 21ad234d..f1302d3d 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "log/slog" "time" @@ -40,7 +41,21 @@ func restartStatusAfterPolicyUpdate(status restartpolicy.Status) restartpolicy.S } func (m *manager) markRestartManualStopLocked(ctx context.Context, id string) error { - if err := m.updateRestartStatusLocked(id, restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop}); err != nil { + meta, err := m.loadMetadata(id) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to mark restart policy manual stop", "instance_id", id, "error", err) + return err + } + if meta.RestartPolicy == nil { + return nil + } + + status := restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop} + if restartpolicy.EqualStatus(meta.RestartStatus, status) { + return nil + } + meta.RestartStatus = status + if err := m.saveMetadata(meta); err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to mark restart policy manual stop", "instance_id", id, "error", err) return err } @@ -48,7 +63,16 @@ func (m *manager) markRestartManualStopLocked(ctx context.Context, id string) er } func (m *manager) clearRestartStatusLocked(ctx context.Context, id string) error { - if err := m.updateRestartStatusLocked(id, restartpolicy.Status{}); err != nil { + meta, err := m.loadMetadata(id) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to clear restart policy status", "instance_id", id, "error", err) + return err + } + if meta.RestartStatus.IsZero() { + return nil + } + meta.RestartStatus = restartpolicy.Status{} + if err := m.saveMetadata(meta); err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to clear restart policy status", "instance_id", id, "error", err) return err } @@ -162,13 +186,13 @@ func (m *manager) StartRestartPolicyController(ctx context.Context) error { if event.Action == LifecycleEventDelete { continue } - if event.Instance == nil { + if event.InstanceID == "" { if err := m.reconcileRestartPolicies(ctx, log); err != nil { log.WarnContext(ctx, "restart policy event reconcile failed", "instance_id", event.InstanceID, "error", err) } continue } - if err := m.reconcileRestartPolicyInstance(ctx, event.Instance, log); err != nil { + if err := m.reconcileRestartPolicyInstanceID(ctx, event.InstanceID, log); err != nil { log.WarnContext(ctx, "restart policy event handling failed", "instance_id", event.InstanceID, "error", err) } case <-ticker.C: @@ -192,6 +216,17 @@ func (m *manager) reconcileRestartPolicies(ctx context.Context, log *slog.Logger return nil } +func (m *manager) reconcileRestartPolicyInstanceID(ctx context.Context, id string, log *slog.Logger) error { + inst, err := m.currentInstanceWithoutHydration(ctx, id) + if errors.Is(err, ErrNotFound) { + return nil + } + if err != nil { + return err + } + return m.reconcileRestartPolicyInstance(ctx, inst, log) +} + func (m *manager) reconcileRestartPolicyInstance(ctx context.Context, inst *Instance, log *slog.Logger) error { if inst == nil { return nil diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go index 2c939a87..3ee43eb6 100644 --- a/lib/instances/restart_policy_test.go +++ b/lib/instances/restart_policy_test.go @@ -1,10 +1,14 @@ package instances import ( + "context" "errors" + "log/slog" + "os" "testing" "time" + "github.com/kernel/hypeman/lib/hypervisor" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -64,3 +68,69 @@ func TestShouldResetRestartAttempts(t *testing.T) { assert.True(t, reset) } + +func TestMarkRestartManualStopLockedSkipsInstancesWithoutPolicy(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-no-policy" + require.NoError(t, manager.ensureDirectories(id)) + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: manager.paths.InstanceDir(id), + SocketPath: manager.paths.InstanceSocket(id, "cloud-hypervisor.sock"), + }, + })) + + require.NoError(t, manager.markRestartManualStopLocked(context.Background(), id)) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.True(t, loaded.RestartStatus.IsZero()) +} + +func TestReconcileRestartPolicyInstanceIDUsesCurrentState(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-current-state" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + socketPath := manager.paths.InstanceSocket(id, "cloud-hypervisor.sock") + require.NoError(t, os.WriteFile(socketPath, nil, 0o600)) + manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + + status := restartpolicy.Status{ + Attempts: 1, + LastReason: restartpolicy.RestartReasonHealthCheckFailed, + } + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + StartedAt: &now, + ProgramStartedAt: &now, + GuestAgentReadyAt: &now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: socketPath, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartPolicy: &restartpolicy.Policy{ + Policy: restartpolicy.PolicyOnFailure, + Backoff: "1s", + MaxAttempts: 1, + StableAfter: "10m", + }, + RestartStatus: status, + }, + })) + + err := manager.reconcileRestartPolicyInstanceID(context.Background(), id, slog.Default()) + require.NoError(t, err) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, status.Attempts, loaded.RestartStatus.Attempts) + assert.Equal(t, status.LastReason, loaded.RestartStatus.LastReason) + assert.Empty(t, loaded.RestartStatus.BlockedReason) +} From 65d6ff31118cb5d9bd8b8bef640d98c1cb8e6bf0 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 18:08:41 +0000 Subject: [PATCH 08/13] Preserve manual restart stop status --- lib/instances/restart_policy.go | 31 ++++++++++++++++---- lib/instances/restart_policy_test.go | 42 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index f1302d3d..a89198b2 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -88,11 +88,24 @@ func (m *manager) updateRestartStatusLocked(id string, status restartpolicy.Stat return m.saveMetadata(meta) } -func (m *manager) setRestartStatus(ctx context.Context, id string, status restartpolicy.Status) error { +func (m *manager) setRestartStatus(ctx context.Context, id string, status restartpolicy.Status) (bool, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.updateRestartStatusLocked(id, status) + + meta, err := m.loadMetadata(id) + if err != nil { + return false, err + } + if meta.RestartStatus.BlockedReason == restartpolicy.BlockedReasonManualStop && + status.BlockedReason != restartpolicy.BlockedReasonManualStop { + return false, nil + } + if restartpolicy.EqualStatus(meta.RestartStatus, status) { + return false, nil + } + meta.RestartStatus = status + return true, m.saveMetadata(meta) } func (m *manager) RestartInstance(ctx context.Context, id string) (*Instance, error) { @@ -243,7 +256,8 @@ func (m *manager) reconcileRestartPolicyInstance(ctx context.Context, inst *Inst now := time.Now().UTC() if shouldResetRestartAttempts(policy, status, inst, now) { log.InfoContext(ctx, "restart policy stable window reached", "instance_id", inst.Id, "attempts", status.Attempts) - return m.setRestartStatus(ctx, inst.Id, restartpolicy.Status{}) + _, err := m.setRestartStatus(ctx, inst.Id, restartpolicy.Status{}) + return err } if inst.State != StateStopped || status.BlockedReason != "" { return nil @@ -263,18 +277,23 @@ func (m *manager) startInstanceForRestartPolicy(ctx context.Context, id string, nextStatus, reason, shouldAttempt := prepareRestartAttempt(policy, status, now) if !shouldAttempt { if !restartpolicy.EqualStatus(status, nextStatus) { - return m.setRestartStatus(ctx, id, nextStatus) + _, err := m.setRestartStatus(ctx, id, nextStatus) + return err } return nil } - if err := m.setRestartStatus(ctx, id, nextStatus); err != nil { + wrote, err := m.setRestartStatus(ctx, id, nextStatus) + if err != nil { return err } + if !wrote { + return nil + } log.InfoContext(ctx, "restart policy starting instance", "instance_id", id, "attempt", nextStatus.Attempts) if _, err := m.RestartInstance(ctx, id); err != nil { nextStatus = restartStatusAfterFailedAttempt(policy, nextStatus, reason, now) - if statusErr := m.setRestartStatus(ctx, id, nextStatus); statusErr != nil { + if _, statusErr := m.setRestartStatus(ctx, id, nextStatus); statusErr != nil { log.WarnContext(ctx, "failed to persist restart status after restart failure", "instance_id", id, "error", statusErr) } return err diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go index 3ee43eb6..d96c12a3 100644 --- a/lib/instances/restart_policy_test.go +++ b/lib/instances/restart_policy_test.go @@ -90,6 +90,48 @@ func TestMarkRestartManualStopLockedSkipsInstancesWithoutPolicy(t *testing.T) { assert.True(t, loaded.RestartStatus.IsZero()) } +func TestStartInstanceForRestartPolicyPreservesConcurrentManualStop(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-manual-race" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + policy := &restartpolicy.Policy{ + Policy: restartpolicy.PolicyAlways, + Backoff: "1s", + StableAfter: "10m", + } + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: manager.paths.InstanceSocket(id, "cloud-hypervisor.sock"), + RestartPolicy: policy, + RestartStatus: restartpolicy.Status{ + BlockedReason: restartpolicy.BlockedReasonManualStop, + }, + }, + })) + + err := manager.startInstanceForRestartPolicy( + context.Background(), + id, + policy, + restartpolicy.Status{}, + now, + slog.Default(), + ) + require.NoError(t, err) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, restartpolicy.BlockedReasonManualStop, loaded.RestartStatus.BlockedReason) + assert.Zero(t, loaded.RestartStatus.Attempts) + assert.Nil(t, loaded.StartedAt) +} + func TestReconcileRestartPolicyInstanceIDUsesCurrentState(t *testing.T) { manager, _ := setupTestManager(t) id := "restart-current-state" From ddeb7fc75290dbadc254d73a537157405d3173bb Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 18:31:09 +0000 Subject: [PATCH 09/13] Address restart policy review comments --- cmd/api/api/instances_test.go | 2 +- cmd/api/api/restart_policy.go | 5 +++-- lib/instances/restart_policy.go | 20 ++++++++++++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 4fadadee..4e7863aa 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -741,7 +741,7 @@ func TestCreateInstance_MapsRestartPolicy(t *testing.T) { require.NotNil(t, mockMgr.lastReq.RestartPolicy) assert.Equal(t, restartpolicy.PolicyOnFailure, mockMgr.lastReq.RestartPolicy.Policy) assert.Equal(t, "7s", mockMgr.lastReq.RestartPolicy.Backoff) - assert.Equal(t, "2m", mockMgr.lastReq.RestartPolicy.StableAfter) + assert.Equal(t, "2m0s", mockMgr.lastReq.RestartPolicy.StableAfter) assert.Equal(t, 4, mockMgr.lastReq.RestartPolicy.MaxAttempts) instance := oapi.Instance(created) diff --git a/cmd/api/api/restart_policy.go b/cmd/api/api/restart_policy.go index f50ce244..07e0477e 100644 --- a/cmd/api/api/restart_policy.go +++ b/cmd/api/api/restart_policy.go @@ -24,10 +24,11 @@ func toDomainRestartPolicy(policy *oapi.RestartPolicy) (*restartpolicy.Policy, e if policy.StableAfter != nil { out.StableAfter = *policy.StableAfter } - if _, err := restartpolicy.NormalizePolicy(out); err != nil { + normalized, err := restartpolicy.NormalizePolicy(out) + if err != nil { return nil, err } - return out, nil + return normalized, nil } func toOAPIRestartPolicy(policy *restartpolicy.Policy) *oapi.RestartPolicy { diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index a89198b2..f8d5761c 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "path/filepath" "time" "github.com/kernel/hypeman/lib/logger" @@ -217,13 +218,24 @@ func (m *manager) StartRestartPolicyController(ctx context.Context) error { } func (m *manager) reconcileRestartPolicies(ctx context.Context, log *slog.Logger) error { - insts, err := m.ListInstances(ctx, nil) + files, err := m.listMetadataFiles() if err != nil { return err } - for i := range insts { - if err := m.reconcileRestartPolicyInstance(ctx, &insts[i], log); err != nil { - log.WarnContext(ctx, "restart policy reconcile failed for instance", "instance_id", insts[i].Id, "error", err) + + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + log.WarnContext(ctx, "skipping restart policy reconcile for invalid instance metadata", "instance_id", id, "error", err) + continue + } + if meta.RestartPolicy == nil { + continue + } + inst := m.toInstanceWithoutHydration(ctx, meta) + if err := m.reconcileRestartPolicyInstance(ctx, &inst, log); err != nil { + log.WarnContext(ctx, "restart policy reconcile failed for instance", "instance_id", inst.Id, "error", err) } } return nil From 5936a3a42be8cf0e2e7dabe65068a0167fa8c2c3 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 18:34:31 +0000 Subject: [PATCH 10/13] Preserve restart reason during attempts --- lib/instances/restart_policy.go | 1 - lib/instances/restart_policy_test.go | 13 +++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index f8d5761c..19834b31 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -319,7 +319,6 @@ func prepareRestartAttempt(policy *restartpolicy.Policy, status restartpolicy.St return nextStatus, "", false } reason := nextStatus.LastReason - nextStatus.LastReason = "" return nextStatus, reason, true } diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go index d96c12a3..18a24f72 100644 --- a/lib/instances/restart_policy_test.go +++ b/lib/instances/restart_policy_test.go @@ -69,6 +69,19 @@ func TestShouldResetRestartAttempts(t *testing.T) { assert.True(t, reset) } +func TestPrepareRestartAttemptPreservesLastReason(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + nextStatus, reason, shouldAttempt := prepareRestartAttempt( + &restartpolicy.Policy{Policy: restartpolicy.PolicyAlways}, + restartpolicy.Status{LastReason: restartpolicy.RestartReasonHealthCheckFailed}, + now, + ) + + require.True(t, shouldAttempt) + assert.Equal(t, restartpolicy.RestartReasonHealthCheckFailed, reason) + assert.Equal(t, restartpolicy.RestartReasonHealthCheckFailed, nextStatus.LastReason) +} + func TestMarkRestartManualStopLockedSkipsInstancesWithoutPolicy(t *testing.T) { manager, _ := setupTestManager(t) id := "restart-no-policy" From 943bddc3950c32e3675eefe74ee4c6a06e1cc6c7 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 19:19:36 +0000 Subject: [PATCH 11/13] Clear restart status before start no-op --- lib/instances/manager.go | 6 ++--- lib/instances/restart_policy_test.go | 38 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index e8e26655..b0842e6a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -499,6 +499,9 @@ func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanc lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() + if err := m.clearRestartStatusLocked(ctx, id); err != nil { + return nil, err + } if !startRequestHasOverrides(req) { current, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { @@ -508,9 +511,6 @@ func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanc return current, nil } } - if err := m.clearRestartStatusLocked(ctx, id); err != nil { - return nil, err - } inst, err := m.startInstance(ctx, id, req) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventStart, inst) diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go index 18a24f72..e925f838 100644 --- a/lib/instances/restart_policy_test.go +++ b/lib/instances/restart_policy_test.go @@ -103,6 +103,44 @@ func TestMarkRestartManualStopLockedSkipsInstancesWithoutPolicy(t *testing.T) { assert.True(t, loaded.RestartStatus.IsZero()) } +func TestStartInstanceClearsRestartStatusWhenAlreadyRunning(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-running-clear" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + socketPath := manager.paths.InstanceSocket(id, "cloud-hypervisor.sock") + require.NoError(t, os.WriteFile(socketPath, nil, 0o600)) + manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + StartedAt: &now, + ProgramStartedAt: &now, + GuestAgentReadyAt: &now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: socketPath, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartStatus: restartpolicy.Status{ + Attempts: 2, + BlockedReason: restartpolicy.BlockedReasonMaxAttemptsExceeded, + LastReason: restartpolicy.RestartReasonHealthCheckFailed, + }, + }, + })) + + inst, err := manager.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) + assert.True(t, inst.RestartStatus.IsZero()) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.True(t, loaded.RestartStatus.IsZero()) +} + func TestStartInstanceForRestartPolicyPreservesConcurrentManualStop(t *testing.T) { manager, _ := setupTestManager(t) id := "restart-manual-race" From 606061296060f0388773ff5c522242042bd1b0fe Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sun, 17 May 2026 19:34:53 +0000 Subject: [PATCH 12/13] Recheck state before policy restart --- lib/instances/restart_policy.go | 19 ++++++++++-- lib/instances/restart_policy_test.go | 44 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 19834b31..32f01376 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -90,6 +90,16 @@ func (m *manager) updateRestartStatusLocked(id string, status restartpolicy.Stat } func (m *manager) setRestartStatus(ctx context.Context, id string, status restartpolicy.Status) (bool, error) { + return m.setRestartStatusIf(ctx, id, status, nil) +} + +func (m *manager) setRestartStatusIfStopped(ctx context.Context, id string, status restartpolicy.Status) (bool, error) { + return m.setRestartStatusIf(ctx, id, status, func(meta *metadata) bool { + return m.toInstanceWithoutHydration(ctx, meta).State == StateStopped + }) +} + +func (m *manager) setRestartStatusIf(ctx context.Context, id string, status restartpolicy.Status, shouldWrite func(*metadata) bool) (bool, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() @@ -105,6 +115,9 @@ func (m *manager) setRestartStatus(ctx context.Context, id string, status restar if restartpolicy.EqualStatus(meta.RestartStatus, status) { return false, nil } + if shouldWrite != nil && !shouldWrite(meta) { + return false, nil + } meta.RestartStatus = status return true, m.saveMetadata(meta) } @@ -289,12 +302,12 @@ func (m *manager) startInstanceForRestartPolicy(ctx context.Context, id string, nextStatus, reason, shouldAttempt := prepareRestartAttempt(policy, status, now) if !shouldAttempt { if !restartpolicy.EqualStatus(status, nextStatus) { - _, err := m.setRestartStatus(ctx, id, nextStatus) + _, err := m.setRestartStatusIfStopped(ctx, id, nextStatus) return err } return nil } - wrote, err := m.setRestartStatus(ctx, id, nextStatus) + wrote, err := m.setRestartStatusIfStopped(ctx, id, nextStatus) if err != nil { return err } @@ -305,7 +318,7 @@ func (m *manager) startInstanceForRestartPolicy(ctx context.Context, id string, log.InfoContext(ctx, "restart policy starting instance", "instance_id", id, "attempt", nextStatus.Attempts) if _, err := m.RestartInstance(ctx, id); err != nil { nextStatus = restartStatusAfterFailedAttempt(policy, nextStatus, reason, now) - if _, statusErr := m.setRestartStatus(ctx, id, nextStatus); statusErr != nil { + if _, statusErr := m.setRestartStatusIfStopped(ctx, id, nextStatus); statusErr != nil { log.WarnContext(ctx, "failed to persist restart status after restart failure", "instance_id", id, "error", statusErr) } return err diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go index e925f838..c07795a5 100644 --- a/lib/instances/restart_policy_test.go +++ b/lib/instances/restart_policy_test.go @@ -183,6 +183,50 @@ func TestStartInstanceForRestartPolicyPreservesConcurrentManualStop(t *testing.T assert.Nil(t, loaded.StartedAt) } +func TestStartInstanceForRestartPolicySkipsConcurrentStart(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-concurrent-start" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + socketPath := manager.paths.InstanceSocket(id, "cloud-hypervisor.sock") + require.NoError(t, os.WriteFile(socketPath, nil, 0o600)) + manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + policy := &restartpolicy.Policy{ + Policy: restartpolicy.PolicyAlways, + Backoff: "1s", + StableAfter: "10m", + } + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + StartedAt: &now, + ProgramStartedAt: &now, + GuestAgentReadyAt: &now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: socketPath, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartPolicy: policy, + }, + })) + + err := manager.startInstanceForRestartPolicy( + context.Background(), + id, + policy, + restartpolicy.Status{}, + now, + slog.Default(), + ) + require.NoError(t, err) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.True(t, loaded.RestartStatus.IsZero()) +} + func TestReconcileRestartPolicyInstanceIDUsesCurrentState(t *testing.T) { manager, _ := setupTestManager(t) id := "restart-current-state" From bb9b4f00afdbdbc9e93d762f45938bb12b2b5b88 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 18 May 2026 15:07:45 +0000 Subject: [PATCH 13/13] Drop stale restart strategy test fields --- lib/instances/network_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index 47680f9c..2611cb2b 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -263,7 +263,6 @@ func TestCreateInstanceWithNetwork(t *testing.T) { }, RestartPolicy: &restartpolicy.Policy{ Policy: restartpolicy.PolicyOnFailure, - Strategy: restartpolicy.StrategyStopStart, Backoff: "1s", MaxAttempts: 1, StableAfter: "30s", @@ -273,7 +272,6 @@ func TestCreateInstanceWithNetwork(t *testing.T) { require.NoError(t, err) require.NotNil(t, inst.RestartPolicy) assert.Equal(t, restartpolicy.PolicyOnFailure, inst.RestartPolicy.Policy) - assert.Equal(t, restartpolicy.StrategyStopStart, inst.RestartPolicy.Strategy) inst, err = waitForRestartPolicyBlocked(ctx, manager, inst.Id, restartpolicy.BlockedReasonMaxAttemptsExceeded, 60*time.Second) require.NoError(t, err)