diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index be399f86..2c85284b 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -293,6 +293,13 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Message: err.Error(), }, nil } + restartPolicy, err := toDomainRestartPolicy(request.Body.RestartPolicy) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_restart_policy", + Message: err.Error(), + }, nil + } domainReq := instances.CreateInstanceRequest{ Name: request.Body.Name, @@ -319,6 +326,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, AutoStandby: autoStandby, HealthCheck: healthCheck, + RestartPolicy: restartPolicy, } if request.Body.SnapshotPolicy != nil { snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) @@ -970,11 +978,20 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst Message: err.Error(), }, nil } + restartPolicy, err := toDomainRestartPolicy(request.Body.RestartPolicy) + if err != nil { + return oapi.UpdateInstance400JSONResponse{ + Code: "invalid_restart_policy", + Message: err.Error(), + }, nil + } result, err := s.InstanceManager.UpdateInstance(ctx, inst.Id, instances.UpdateInstanceRequest{ - Env: env, - AutoStandby: autoStandby, - HealthCheck: healthCheck, + Env: env, + AutoStandby: autoStandby, + HealthCheck: healthCheck, + RestartPolicy: restartPolicy, + RestartPolicySet: request.Body.RestartPolicy != nil, }) if err != nil { switch { @@ -1108,6 +1125,8 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { oapiInst.AutoStandby = toOAPIAutoStandbyPolicy(inst.AutoStandby) oapiInst.HealthCheck = toOAPIHealthCheck(inst.HealthCheck) oapiInst.HealthStatus = toOAPIHealthStatus(healthcheck.Snapshot(inst.HealthCheck, string(inst.State), inst.HealthCheckRuntime)) + oapiInst.RestartPolicy = toOAPIRestartPolicy(inst.RestartPolicy) + oapiInst.RestartStatus = toOAPIRestartStatus(inst.RestartStatus) // Convert volume attachments if len(inst.Volumes) > 0 { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b28fbc58..4e7863aa 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -16,6 +16,7 @@ import ( mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/kernel/hypeman/lib/system" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -281,6 +282,7 @@ func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, re Env: req.Env, AutoStandby: req.AutoStandby, HealthCheck: req.HealthCheck, + RestartPolicy: req.RestartPolicy, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -304,6 +306,7 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances Vcpus: req.Vcpus, AutoStandby: req.AutoStandby, HealthCheck: req.HealthCheck, + RestartPolicy: req.RestartPolicy, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -705,6 +708,48 @@ func TestCreateInstance_MapsHealthCheckPolicy(t *testing.T) { assert.Equal(t, oapi.InstanceHealthStatusStatusStarting, instance.HealthStatus.Status) } +func TestCreateInstance_MapsRestartPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + policy := oapi.OnFailure + backoff := "7s" + stableAfter := "2m" + maxAttempts := 4 + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-restart-policy", + Image: "docker.io/library/alpine:latest", + RestartPolicy: &oapi.RestartPolicy{ + Policy: &policy, + Backoff: &backoff, + StableAfter: &stableAfter, + MaxAttempts: &maxAttempts, + }, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.RestartPolicy) + assert.Equal(t, restartpolicy.PolicyOnFailure, mockMgr.lastReq.RestartPolicy.Policy) + assert.Equal(t, "7s", mockMgr.lastReq.RestartPolicy.Backoff) + assert.Equal(t, "2m0s", mockMgr.lastReq.RestartPolicy.StableAfter) + assert.Equal(t, 4, mockMgr.lastReq.RestartPolicy.MaxAttempts) + + instance := oapi.Instance(created) + require.NotNil(t, instance.RestartPolicy) + require.NotNil(t, instance.RestartPolicy.Policy) + assert.Equal(t, oapi.OnFailure, *instance.RestartPolicy.Policy) +} + func TestUpdateInstance_MapsEnvPatch(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -883,6 +928,108 @@ func TestUpdateInstance_MapsHealthCheckPatch(t *testing.T) { assert.Equal(t, oapi.InstanceHealthStatusStatusUnknown, instance.HealthStatus.Status) } +func TestUpdateInstance_MapsRestartPolicyPatch(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{ + Manager: origMgr, + result: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-restart-policy", + Name: "inst-update-restart-policy", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartPolicy: &restartpolicy.Policy{ + Policy: restartpolicy.PolicyAlways, + Backoff: "5s", + StableAfter: "10m0s", + }, + RestartStatus: restartpolicy.Status{ + BlockedReason: restartpolicy.BlockedReasonManualStop, + }, + }, + State: instances.StateStopped, + }, + } + svc.InstanceManager = mockMgr + + policy := oapi.Always + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-restart-policy", + Name: "inst-update-restart-policy", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + RestartPolicy: &oapi.RestartPolicy{Policy: &policy}, + }, + }) + require.NoError(t, err) + updated, ok := resp.(oapi.UpdateInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + assert.True(t, mockMgr.lastReq.RestartPolicySet) + require.NotNil(t, mockMgr.lastReq.RestartPolicy) + assert.Equal(t, restartpolicy.PolicyAlways, mockMgr.lastReq.RestartPolicy.Policy) + + instance := oapi.Instance(updated) + require.NotNil(t, instance.RestartPolicy) + require.NotNil(t, instance.RestartStatus) + require.NotNil(t, instance.RestartStatus.BlockedReason) + assert.Equal(t, oapi.ManualStop, *instance.RestartStatus.BlockedReason) +} + +func TestUpdateInstance_RejectsInvalidRestartPolicy(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureUpdateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + now := time.Now() + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-restart-policy", + Name: "inst-update-restart-policy", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + policy := oapi.OnFailure + backoff := "0s" + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + RestartPolicy: &oapi.RestartPolicy{ + Policy: &policy, + Backoff: &backoff, + }, + }, + }) + require.NoError(t, err) + + badReq, ok := resp.(oapi.UpdateInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_restart_policy", badReq.Code) + assert.Nil(t, mockMgr.lastReq) +} + func TestUpdateInstance_RejectsZeroAutoStandbyIgnoreDestinationPort(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/cmd/api/api/restart_policy.go b/cmd/api/api/restart_policy.go new file mode 100644 index 00000000..07e0477e --- /dev/null +++ b/cmd/api/api/restart_policy.go @@ -0,0 +1,80 @@ +package api + +import ( + "github.com/kernel/hypeman/lib/oapi" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + "github.com/samber/lo" +) + +func toDomainRestartPolicy(policy *oapi.RestartPolicy) (*restartpolicy.Policy, error) { + if policy == nil { + return nil, nil + } + + out := &restartpolicy.Policy{} + if policy.Policy != nil { + out.Policy = restartpolicy.PolicyMode(*policy.Policy) + } + if policy.Backoff != nil { + out.Backoff = *policy.Backoff + } + if policy.MaxAttempts != nil { + out.MaxAttempts = *policy.MaxAttempts + } + if policy.StableAfter != nil { + out.StableAfter = *policy.StableAfter + } + normalized, err := restartpolicy.NormalizePolicy(out) + if err != nil { + return nil, err + } + return normalized, nil +} + +func toOAPIRestartPolicy(policy *restartpolicy.Policy) *oapi.RestartPolicy { + if policy == nil { + return nil + } + + mode := oapi.RestartPolicyPolicy(policy.Policy) + out := &oapi.RestartPolicy{ + Policy: &mode, + } + if policy.Backoff != "" { + out.Backoff = lo.ToPtr(policy.Backoff) + } + if policy.MaxAttempts > 0 { + out.MaxAttempts = lo.ToPtr(policy.MaxAttempts) + } + if policy.StableAfter != "" { + out.StableAfter = lo.ToPtr(policy.StableAfter) + } + return out +} + +func toOAPIRestartStatus(status restartpolicy.Status) *oapi.RestartStatus { + if status.IsZero() { + return nil + } + + out := &oapi.RestartStatus{ + Attempts: lo.ToPtr(status.Attempts), + } + if status.BlockedReason != "" { + reason := oapi.RestartStatusBlockedReason(status.BlockedReason) + out.BlockedReason = &reason + } + if status.LastAttemptAt != nil { + lastAttemptAt := status.LastAttemptAt.UTC() + out.LastAttemptAt = &lastAttemptAt + } + if status.NextAttemptAt != nil { + nextAttemptAt := status.NextAttemptAt.UTC() + out.NextAttemptAt = &nextAttemptAt + } + if status.LastReason != "" { + reason := oapi.RestartStatusLastReason(status.LastReason) + out.LastReason = &reason + } + return out +} diff --git a/cmd/api/main.go b/cmd/api/main.go index f7e24c4e..da2a2326 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -571,6 +571,14 @@ func run() error { return app.HealthCheckController.Run(gctx) }) } + if restartController, ok := app.InstanceManager.(interface { + StartRestartPolicyController(context.Context) error + }); ok { + grp.Go(func() error { + logger.Info("starting restart policy controller") + return restartController.StartRestartPolicyController(gctx) + }) + } // Run the server grp.Go(func() error { diff --git a/lib/healthcheck/README.md b/lib/healthcheck/README.md index 46bd0412..1bdee903 100644 --- a/lib/healthcheck/README.md +++ b/lib/healthcheck/README.md @@ -75,6 +75,10 @@ Stopping, deleting, standing by, or restoring an instance stops active checks. S ## Restart Policy -Health checks only report health. They do not restart instances. +Health checks do not restart instances by themselves. -If Hypeman later adds restart-on-unhealthy behavior, it should consume `health_status=unhealthy` explicitly rather than making health checks mutate lifecycle state. +When an instance also has `restart_policy.policy=on_failure` or `restart_policy.policy=always`, an `unhealthy` health status becomes a restart-policy failure signal. The restart policy applies its normal backoff, max attempts, manual-stop suppression, and stable-window reset before Hypeman restarts the whole instance. + +With `restart_policy.policy=never` or no restart policy, health checks only report status. + +Health checks still do not mutate lifecycle state directly. The instance remains `Running` while unhealthy until restart policy chooses to stop and start it. diff --git a/lib/instances/create.go b/lib/instances/create.go index dc2f57b0..df991674 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -375,6 +375,7 @@ func (m *manager) createInstance( SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), + RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } // 12. Ensure directories @@ -638,6 +639,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { if err := validateHealthCheckCompatibility(req.HealthCheck, req.NetworkEnabled, req.SkipGuestAgent); err != nil { return err } + normalizedRestartPolicy, err := normalizeRestartPolicy(req.RestartPolicy) + if err != nil { + return err + } + req.RestartPolicy = normalizedRestartPolicy // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 7c0b25db..c29d7189 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -16,6 +16,7 @@ import ( "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "gvisor.dev/gvisor/pkg/cleanup" @@ -281,6 +282,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.VsockSocket = m.paths.InstanceSocket(forkID, hypervisor.VsockSocketNameForType(forkMeta.HypervisorType)) forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + forkMeta.RestartStatus = restartpolicy.Status{} // Forks are new instances; phase accounting must not inherit the source's // cumulative durations. The first transition into the fork's runtime // phase (Standby for snapshot forks, Stopped for stopped forks) will be @@ -481,6 +483,9 @@ func cloneStoredMetadata(src StoredMetadata) StoredMetadata { if src.HealthCheck != nil { dst.HealthCheck = cloneHealthCheckPolicy(src.HealthCheck) } + if src.RestartPolicy != nil { + dst.RestartPolicy = cloneRestartPolicy(src.RestartPolicy) + } if src.SnapshotPolicy != nil { dst.SnapshotPolicy = cloneSnapshotPolicy(src.SnapshotPolicy) } diff --git a/lib/instances/health_check_controller.go b/lib/instances/health_check_controller.go index a675fd9c..e35a18de 100644 --- a/lib/instances/health_check_controller.go +++ b/lib/instances/health_check_controller.go @@ -22,6 +22,10 @@ type healthCheckControllerStore interface { SubscribeLifecycleEvents(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) } +type healthCheckUnhealthyHandler interface { + HandleHealthCheckUnhealthy(ctx context.Context, id string) error +} + type healthCheckControllerInstanceGetter interface { GetInstance(ctx context.Context, id string) (*Instance, error) } @@ -274,6 +278,13 @@ func (c *HealthCheckController) runCheck(ctx context.Context, id string, generat if err := c.store.SetHealthCheckRuntime(ctx, id, runtime); err != nil { c.log.Warn("failed to persist health check status", "instance_id", id, "error", err) } + if runtime.Status == healthcheck.StatusUnhealthy { + if handler, ok := c.store.(healthCheckUnhealthyHandler); ok { + if err := handler.HandleHealthCheckUnhealthy(ctx, id); err != nil { + c.log.Warn("failed to handle unhealthy instance", "instance_id", id, "error", err) + } + } + } c.mu.Lock() state = c.states[id] diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go index 4bba4851..9eb15b75 100644 --- a/lib/instances/lifecycle_events.go +++ b/lib/instances/lifecycle_events.go @@ -12,15 +12,17 @@ const defaultLifecycleEventBufferSize = 256 type LifecycleEventConsumer string const ( - LifecycleEventConsumerWaitForState LifecycleEventConsumer = "wait_for_state" - LifecycleEventConsumerAutoStandby LifecycleEventConsumer = "auto_standby" - LifecycleEventConsumerHealthCheck LifecycleEventConsumer = "health_check" + LifecycleEventConsumerWaitForState LifecycleEventConsumer = "wait_for_state" + LifecycleEventConsumerAutoStandby LifecycleEventConsumer = "auto_standby" + LifecycleEventConsumerHealthCheck LifecycleEventConsumer = "health_check" + LifecycleEventConsumerRestartPolicy LifecycleEventConsumer = "restart_policy" ) var allLifecycleEventConsumers = []LifecycleEventConsumer{ LifecycleEventConsumerWaitForState, LifecycleEventConsumerAutoStandby, LifecycleEventConsumerHealthCheck, + LifecycleEventConsumerRestartPolicy, } // LifecycleEventAction identifies which instance lifecycle action occurred. diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 36e7a627..b0842e6a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -475,7 +475,17 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error return nil, err } if current.State == StateStopped { - return current, nil + if err := m.markRestartManualStopLocked(ctx, id); err != nil { + return nil, err + } + updated, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return nil, err + } + return updated, nil + } + if err := m.markRestartManualStopLocked(ctx, id); err != nil { + return nil, err } inst, err := m.stopInstance(ctx, id) if err == nil { @@ -489,6 +499,9 @@ func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanc lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() + if err := m.clearRestartStatusLocked(ctx, id); err != nil { + return nil, err + } if !startRequestHasOverrides(req) { current, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 72f4f578..811571e3 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -221,6 +221,8 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T assert.Equal(t, int64(1), point.Value) case string(LifecycleEventConsumerHealthCheck): assert.Equal(t, int64(0), point.Value) + case string(LifecycleEventConsumerRestartPolicy): + assert.Equal(t, int64(0), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } @@ -238,6 +240,8 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T assert.Equal(t, int64(m.lifecycleEvents.bufferSize), point.Value) case string(LifecycleEventConsumerHealthCheck): assert.Equal(t, int64(0), point.Value) + case string(LifecycleEventConsumerRestartPolicy): + assert.Equal(t, int64(0), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index 6ade0dde..2611cb2b 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -15,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vishvananda/netlink" @@ -30,6 +31,7 @@ func TestCreateInstanceWithNetwork(t *testing.T) { manager, _ := setupTestManager(t) ctx := context.Background() startHealthCheckControllerForTest(t, ctx, manager) + startRestartPolicyControllerForTest(t, ctx, manager) // Pull nginx:alpine image (long-running workload) t.Log("Pulling nginx:alpine image...") @@ -241,6 +243,44 @@ func TestCreateInstanceWithNetwork(t *testing.T) { require.Contains(t, psOutput, "nginx: master process", "nginx master should still be running") t.Log("Nginx process confirmed running - restore was successful!") + // Flip health checks to a guaranteed failure and verify restart policy stop-starts the same instance. + t.Log("Triggering restart policy from failing health check...") + require.NotNil(t, inst.StartedAt) + startedAtBeforeRestart := *inst.StartedAt + inst, err = manager.UpdateInstance(ctx, inst.Id, UpdateInstanceRequest{ + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeHTTP, + Interval: "1s", + Timeout: "1s", + StartPeriod: "1s", + FailureThreshold: 1, + SuccessThreshold: 1, + HTTP: &healthcheck.HTTPCheck{ + Port: 80, + Path: "/definitely-missing", + ExpectedStatus: 200, + }, + }, + RestartPolicy: &restartpolicy.Policy{ + Policy: restartpolicy.PolicyOnFailure, + Backoff: "1s", + MaxAttempts: 1, + StableAfter: "30s", + }, + RestartPolicySet: true, + }) + require.NoError(t, err) + require.NotNil(t, inst.RestartPolicy) + assert.Equal(t, restartpolicy.PolicyOnFailure, inst.RestartPolicy.Policy) + + inst, err = waitForRestartPolicyBlocked(ctx, manager, inst.Id, restartpolicy.BlockedReasonMaxAttemptsExceeded, 60*time.Second) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) + require.Equal(t, 1, inst.RestartStatus.Attempts) + require.NotNil(t, inst.StartedAt) + assert.True(t, inst.StartedAt.After(startedAtBeforeRestart), "instance should have been started again") + t.Log("Restart policy performed stop-start and blocked after max attempts") + // Cleanup t.Log("Cleaning up instance...") err = manager.DeleteInstance(ctx, inst.Id) @@ -261,6 +301,21 @@ func TestCreateInstanceWithNetwork(t *testing.T) { t.Log("Network integration test complete!") } +func startRestartPolicyControllerForTest(t *testing.T, ctx context.Context, manager *manager) { + t.Helper() + + controllerCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + done <- manager.StartRestartPolicyController(controllerCtx) + }() + + t.Cleanup(func() { + cancel() + require.NoError(t, <-done) + }) +} + func startHealthCheckControllerForTest(t *testing.T, ctx context.Context, manager Manager) { t.Helper() @@ -309,6 +364,36 @@ func waitForInstanceHealthStatus(ctx context.Context, manager Manager, instanceI return nil, last, fmt.Errorf("instance %s health did not reach %s within %v (last state: %s, last health: %s)", instanceID, expected, timeout, lastState, last.Status) } +func waitForRestartPolicyBlocked(ctx context.Context, manager Manager, instanceID string, reason restartpolicy.BlockedReason, timeout time.Duration) (*Instance, error) { + timeout = integrationTestTimeout(timeout) + deadline := time.Now().Add(timeout) + lastState := StateUnknown + lastStatus := restartpolicy.Status{} + lastErr := error(nil) + + for time.Now().Before(deadline) { + inst, err := manager.GetInstance(ctx, instanceID) + if err != nil { + lastErr = err + time.Sleep(200 * time.Millisecond) + continue + } + + lastState = inst.State + lastStatus = inst.RestartStatus + if inst.State == StateRunning && inst.RestartStatus.BlockedReason == reason { + return inst, nil + } + + time.Sleep(200 * time.Millisecond) + } + + if lastErr != nil { + return nil, fmt.Errorf("instance %s restart policy did not block with %s within %v (last error: %w)", instanceID, reason, timeout, lastErr) + } + return nil, fmt.Errorf("instance %s restart policy did not block with %s within %v (last state: %s, attempts: %d, blocked_reason: %s)", instanceID, reason, timeout, lastState, lastStatus.Attempts, lastStatus.BlockedReason) +} + // TestDockerForwardChainRestored validates recovery from an external FORWARD-chain flush. // This test intentionally mutates host-global iptables state, so it must run non-parallel. func TestDockerForwardChainRestored(t *testing.T) { diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go new file mode 100644 index 00000000..32f01376 --- /dev/null +++ b/lib/instances/restart_policy.go @@ -0,0 +1,352 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/logger" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" +) + +const defaultRestartPolicyReconcileInterval = 5 * time.Second + +func cloneRestartPolicy(policy *restartpolicy.Policy) *restartpolicy.Policy { + if policy == nil { + return nil + } + return &restartpolicy.Policy{ + Policy: policy.Policy, + Backoff: policy.Backoff, + MaxAttempts: policy.MaxAttempts, + StableAfter: policy.StableAfter, + } +} + +func normalizeRestartPolicy(policy *restartpolicy.Policy) (*restartpolicy.Policy, error) { + normalized, err := restartpolicy.NormalizePolicy(policy) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return normalized, nil +} + +func restartStatusAfterPolicyUpdate(status restartpolicy.Status) restartpolicy.Status { + if status.BlockedReason == restartpolicy.BlockedReasonManualStop { + return restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop} + } + return restartpolicy.Status{} +} + +func (m *manager) markRestartManualStopLocked(ctx context.Context, id string) error { + meta, err := m.loadMetadata(id) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to mark restart policy manual stop", "instance_id", id, "error", err) + return err + } + if meta.RestartPolicy == nil { + return nil + } + + status := restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop} + if restartpolicy.EqualStatus(meta.RestartStatus, status) { + return nil + } + meta.RestartStatus = status + if err := m.saveMetadata(meta); err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to mark restart policy manual stop", "instance_id", id, "error", err) + return err + } + return nil +} + +func (m *manager) clearRestartStatusLocked(ctx context.Context, id string) error { + meta, err := m.loadMetadata(id) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to clear restart policy status", "instance_id", id, "error", err) + return err + } + if meta.RestartStatus.IsZero() { + return nil + } + meta.RestartStatus = restartpolicy.Status{} + if err := m.saveMetadata(meta); err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to clear restart policy status", "instance_id", id, "error", err) + return err + } + return nil +} + +func (m *manager) updateRestartStatusLocked(id string, status restartpolicy.Status) error { + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.RestartStatus = status + return m.saveMetadata(meta) +} + +func (m *manager) setRestartStatus(ctx context.Context, id string, status restartpolicy.Status) (bool, error) { + return m.setRestartStatusIf(ctx, id, status, nil) +} + +func (m *manager) setRestartStatusIfStopped(ctx context.Context, id string, status restartpolicy.Status) (bool, error) { + return m.setRestartStatusIf(ctx, id, status, func(meta *metadata) bool { + return m.toInstanceWithoutHydration(ctx, meta).State == StateStopped + }) +} + +func (m *manager) setRestartStatusIf(ctx context.Context, id string, status restartpolicy.Status, shouldWrite func(*metadata) bool) (bool, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return false, err + } + if meta.RestartStatus.BlockedReason == restartpolicy.BlockedReasonManualStop && + status.BlockedReason != restartpolicy.BlockedReasonManualStop { + return false, nil + } + if restartpolicy.EqualStatus(meta.RestartStatus, status) { + return false, nil + } + if shouldWrite != nil && !shouldWrite(meta) { + return false, nil + } + meta.RestartStatus = status + return true, m.saveMetadata(meta) +} + +func (m *manager) RestartInstance(ctx context.Context, id string) (*Instance, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + inst, err := m.startInstance(ctx, id, StartInstanceRequest{}) + if err == nil { + m.notifyLifecycleEvent(ctx, LifecycleEventStart, inst) + } + return inst, err +} + +func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return err + } + if current.State != StateRunning { + return nil + } + + policy, err := restartpolicy.NormalizePolicy(current.RestartPolicy) + if err != nil { + return err + } + if !restartpolicy.ShouldRestart(policy, nil) { + return nil + } + if current.RestartStatus.BlockedReason != "" { + return nil + } + + now := time.Now().UTC() + status := current.RestartStatus + status.LastReason = restartpolicy.RestartReasonHealthCheckFailed + nextStatus, reason, shouldAttempt := prepareRestartAttempt(policy, status, now) + if !shouldAttempt { + if !restartpolicy.EqualStatus(current.RestartStatus, nextStatus) { + return m.updateRestartStatusLocked(id, nextStatus) + } + return nil + } + + if err := m.updateRestartStatusLocked(id, nextStatus); err != nil { + return err + } + + stopped, err := m.stopInstance(ctx, id) + if err != nil { + _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) + return err + } + m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) + + started, err := m.startInstance(ctx, id, StartInstanceRequest{}) + if err != nil { + _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) + return err + } + m.notifyLifecycleEvent(ctx, LifecycleEventStart, started) + return nil +} + +func (m *manager) StartRestartPolicyController(ctx context.Context) error { + log := logger.FromContext(ctx).With("controller", "restart_policy") + log.InfoContext(ctx, "restart policy controller started", "reconcile_interval", defaultRestartPolicyReconcileInterval) + if err := m.reconcileRestartPolicies(ctx, log); err != nil { + log.WarnContext(ctx, "restart policy startup reconcile failed", "error", err) + } + + events, unsubscribe := m.SubscribeLifecycleEvents(LifecycleEventConsumerRestartPolicy) + defer unsubscribe() + + ticker := time.NewTicker(defaultRestartPolicyReconcileInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case event, ok := <-events: + if !ok { + return nil + } + if event.Action == LifecycleEventDelete { + continue + } + if event.InstanceID == "" { + if err := m.reconcileRestartPolicies(ctx, log); err != nil { + log.WarnContext(ctx, "restart policy event reconcile failed", "instance_id", event.InstanceID, "error", err) + } + continue + } + if err := m.reconcileRestartPolicyInstanceID(ctx, event.InstanceID, log); err != nil { + log.WarnContext(ctx, "restart policy event handling failed", "instance_id", event.InstanceID, "error", err) + } + case <-ticker.C: + if err := m.reconcileRestartPolicies(ctx, log); err != nil { + log.WarnContext(ctx, "restart policy reconcile failed", "error", err) + } + } + } +} + +func (m *manager) reconcileRestartPolicies(ctx context.Context, log *slog.Logger) error { + files, err := m.listMetadataFiles() + if err != nil { + return err + } + + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + log.WarnContext(ctx, "skipping restart policy reconcile for invalid instance metadata", "instance_id", id, "error", err) + continue + } + if meta.RestartPolicy == nil { + continue + } + inst := m.toInstanceWithoutHydration(ctx, meta) + if err := m.reconcileRestartPolicyInstance(ctx, &inst, log); err != nil { + log.WarnContext(ctx, "restart policy reconcile failed for instance", "instance_id", inst.Id, "error", err) + } + } + return nil +} + +func (m *manager) reconcileRestartPolicyInstanceID(ctx context.Context, id string, log *slog.Logger) error { + inst, err := m.currentInstanceWithoutHydration(ctx, id) + if errors.Is(err, ErrNotFound) { + return nil + } + if err != nil { + return err + } + return m.reconcileRestartPolicyInstance(ctx, inst, log) +} + +func (m *manager) reconcileRestartPolicyInstance(ctx context.Context, inst *Instance, log *slog.Logger) error { + if inst == nil { + return nil + } + policy, err := restartpolicy.NormalizePolicy(inst.RestartPolicy) + if err != nil { + return err + } + if policy == nil { + return nil + } + + status := inst.RestartStatus + now := time.Now().UTC() + if shouldResetRestartAttempts(policy, status, inst, now) { + log.InfoContext(ctx, "restart policy stable window reached", "instance_id", inst.Id, "attempts", status.Attempts) + _, err := m.setRestartStatus(ctx, inst.Id, restartpolicy.Status{}) + return err + } + if inst.State != StateStopped || status.BlockedReason != "" { + return nil + } + + exitCode := inst.ExitCode + if status.LastReason == restartpolicy.RestartReasonHealthCheckFailed { + exitCode = nil + } + if !restartpolicy.ShouldRestart(policy, exitCode) { + return nil + } + return m.startInstanceForRestartPolicy(ctx, inst.Id, policy, status, now, log) +} + +func (m *manager) startInstanceForRestartPolicy(ctx context.Context, id string, policy *restartpolicy.Policy, status restartpolicy.Status, now time.Time, log *slog.Logger) error { + nextStatus, reason, shouldAttempt := prepareRestartAttempt(policy, status, now) + if !shouldAttempt { + if !restartpolicy.EqualStatus(status, nextStatus) { + _, err := m.setRestartStatusIfStopped(ctx, id, nextStatus) + return err + } + return nil + } + wrote, err := m.setRestartStatusIfStopped(ctx, id, nextStatus) + if err != nil { + return err + } + if !wrote { + return nil + } + + log.InfoContext(ctx, "restart policy starting instance", "instance_id", id, "attempt", nextStatus.Attempts) + if _, err := m.RestartInstance(ctx, id); err != nil { + nextStatus = restartStatusAfterFailedAttempt(policy, nextStatus, reason, now) + if _, statusErr := m.setRestartStatusIfStopped(ctx, id, nextStatus); statusErr != nil { + log.WarnContext(ctx, "failed to persist restart status after restart failure", "instance_id", id, "error", statusErr) + } + return err + } + return nil +} + +func prepareRestartAttempt(policy *restartpolicy.Policy, status restartpolicy.Status, now time.Time) (restartpolicy.Status, restartpolicy.RestartReason, bool) { + nextStatus, shouldAttempt := restartpolicy.PrepareAttempt(policy, status, now) + if !shouldAttempt { + return nextStatus, "", false + } + reason := nextStatus.LastReason + return nextStatus, reason, true +} + +func restartStatusAfterFailedAttempt(policy *restartpolicy.Policy, status restartpolicy.Status, reason restartpolicy.RestartReason, now time.Time) restartpolicy.Status { + status = restartpolicy.AfterFailedAttempt(policy, status, now) + status.LastReason = reason + return status +} + +func shouldResetRestartAttempts(policy *restartpolicy.Policy, status restartpolicy.Status, inst *Instance, now time.Time) bool { + if status.Attempts == 0 || inst.StartedAt == nil { + return false + } + if inst.State != StateRunning && inst.State != StateInitializing { + return false + } + return !now.Before(inst.StartedAt.UTC().Add(restartpolicy.StableAfter(policy))) +} diff --git a/lib/instances/restart_policy_test.go b/lib/instances/restart_policy_test.go new file mode 100644 index 00000000..c07795a5 --- /dev/null +++ b/lib/instances/restart_policy_test.go @@ -0,0 +1,273 @@ +package instances + +import ( + "context" + "errors" + "log/slog" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateUpdateInstanceRequestAllowsRestartPolicyOnly(t *testing.T) { + err := validateUpdateInstanceRequest(&metadata{}, UpdateInstanceRequest{ + RestartPolicy: &restartpolicy.Policy{Policy: restartpolicy.PolicyAlways}, + RestartPolicySet: true, + }) + + require.NoError(t, err) +} + +func TestNormalizeRestartPolicyWrapsInvalidRequest(t *testing.T) { + _, err := normalizeRestartPolicy(&restartpolicy.Policy{ + Policy: restartpolicy.PolicyOnFailure, + Backoff: "0s", + }) + + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestRestartStatusAfterPolicyUpdatePreservesManualStop(t *testing.T) { + status := restartStatusAfterPolicyUpdate(restartpolicy.Status{ + Attempts: 3, + BlockedReason: restartpolicy.BlockedReasonManualStop, + }) + + assert.Equal(t, restartpolicy.BlockedReasonManualStop, status.BlockedReason) + assert.Zero(t, status.Attempts) +} + +func TestRestartStatusAfterPolicyUpdateClearsRetryState(t *testing.T) { + status := restartStatusAfterPolicyUpdate(restartpolicy.Status{ + Attempts: 3, + BlockedReason: restartpolicy.BlockedReasonMaxAttemptsExceeded, + }) + + assert.True(t, status.IsZero()) +} + +func TestShouldResetRestartAttempts(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + startedAt := now.Add(-11 * time.Minute) + + reset := shouldResetRestartAttempts( + &restartpolicy.Policy{Policy: restartpolicy.PolicyAlways, StableAfter: "10m"}, + restartpolicy.Status{Attempts: 2}, + &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{StartedAt: &startedAt}, + }, + now, + ) + + assert.True(t, reset) +} + +func TestPrepareRestartAttemptPreservesLastReason(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + nextStatus, reason, shouldAttempt := prepareRestartAttempt( + &restartpolicy.Policy{Policy: restartpolicy.PolicyAlways}, + restartpolicy.Status{LastReason: restartpolicy.RestartReasonHealthCheckFailed}, + now, + ) + + require.True(t, shouldAttempt) + assert.Equal(t, restartpolicy.RestartReasonHealthCheckFailed, reason) + assert.Equal(t, restartpolicy.RestartReasonHealthCheckFailed, nextStatus.LastReason) +} + +func TestMarkRestartManualStopLockedSkipsInstancesWithoutPolicy(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-no-policy" + require.NoError(t, manager.ensureDirectories(id)) + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: manager.paths.InstanceDir(id), + SocketPath: manager.paths.InstanceSocket(id, "cloud-hypervisor.sock"), + }, + })) + + require.NoError(t, manager.markRestartManualStopLocked(context.Background(), id)) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.True(t, loaded.RestartStatus.IsZero()) +} + +func TestStartInstanceClearsRestartStatusWhenAlreadyRunning(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-running-clear" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + socketPath := manager.paths.InstanceSocket(id, "cloud-hypervisor.sock") + require.NoError(t, os.WriteFile(socketPath, nil, 0o600)) + manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + StartedAt: &now, + ProgramStartedAt: &now, + GuestAgentReadyAt: &now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: socketPath, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartStatus: restartpolicy.Status{ + Attempts: 2, + BlockedReason: restartpolicy.BlockedReasonMaxAttemptsExceeded, + LastReason: restartpolicy.RestartReasonHealthCheckFailed, + }, + }, + })) + + inst, err := manager.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) + assert.True(t, inst.RestartStatus.IsZero()) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.True(t, loaded.RestartStatus.IsZero()) +} + +func TestStartInstanceForRestartPolicyPreservesConcurrentManualStop(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-manual-race" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + policy := &restartpolicy.Policy{ + Policy: restartpolicy.PolicyAlways, + Backoff: "1s", + StableAfter: "10m", + } + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: manager.paths.InstanceSocket(id, "cloud-hypervisor.sock"), + RestartPolicy: policy, + RestartStatus: restartpolicy.Status{ + BlockedReason: restartpolicy.BlockedReasonManualStop, + }, + }, + })) + + err := manager.startInstanceForRestartPolicy( + context.Background(), + id, + policy, + restartpolicy.Status{}, + now, + slog.Default(), + ) + require.NoError(t, err) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, restartpolicy.BlockedReasonManualStop, loaded.RestartStatus.BlockedReason) + assert.Zero(t, loaded.RestartStatus.Attempts) + assert.Nil(t, loaded.StartedAt) +} + +func TestStartInstanceForRestartPolicySkipsConcurrentStart(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-concurrent-start" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + socketPath := manager.paths.InstanceSocket(id, "cloud-hypervisor.sock") + require.NoError(t, os.WriteFile(socketPath, nil, 0o600)) + manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + policy := &restartpolicy.Policy{ + Policy: restartpolicy.PolicyAlways, + Backoff: "1s", + StableAfter: "10m", + } + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + StartedAt: &now, + ProgramStartedAt: &now, + GuestAgentReadyAt: &now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: socketPath, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartPolicy: policy, + }, + })) + + err := manager.startInstanceForRestartPolicy( + context.Background(), + id, + policy, + restartpolicy.Status{}, + now, + slog.Default(), + ) + require.NoError(t, err) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.True(t, loaded.RestartStatus.IsZero()) +} + +func TestReconcileRestartPolicyInstanceIDUsesCurrentState(t *testing.T) { + manager, _ := setupTestManager(t) + id := "restart-current-state" + require.NoError(t, manager.ensureDirectories(id)) + + now := time.Now().UTC() + socketPath := manager.paths.InstanceSocket(id, "cloud-hypervisor.sock") + require.NoError(t, os.WriteFile(socketPath, nil, 0o600)) + manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + + status := restartpolicy.Status{ + Attempts: 1, + LastReason: restartpolicy.RestartReasonHealthCheckFailed, + } + require.NoError(t, manager.saveMetadata(&metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: now, + StartedAt: &now, + ProgramStartedAt: &now, + GuestAgentReadyAt: &now, + DataDir: manager.paths.InstanceDir(id), + SocketPath: socketPath, + HypervisorType: hypervisor.TypeCloudHypervisor, + RestartPolicy: &restartpolicy.Policy{ + Policy: restartpolicy.PolicyOnFailure, + Backoff: "1s", + MaxAttempts: 1, + StableAfter: "10m", + }, + RestartStatus: status, + }, + })) + + err := manager.reconcileRestartPolicyInstanceID(context.Background(), id, slog.Default()) + require.NoError(t, err) + + loaded, err := manager.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, status.Attempts, loaded.RestartStatus.Attempts) + assert.Equal(t, status.LastReason, loaded.RestartStatus.LastReason) + assert.Empty(t, loaded.RestartStatus.BlockedReason) +} diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 707d071c..05d8084e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/tags" "github.com/nrednav/cuid2" @@ -448,6 +449,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS forkMeta.VsockSocket = m.paths.InstanceSocket(forkID, hypervisor.VsockSocketNameForType(targetHypervisor)) forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + forkMeta.RestartStatus = restartpolicy.Status{} if rec.Snapshot.Kind == SnapshotKindStandby { forkMeta.VsockCID = rec.StoredMetadata.VsockCID } else { diff --git a/lib/instances/types.go b/lib/instances/types.go index 892191cb..8c031fc9 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -7,6 +7,7 @@ import ( "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances/phasetracking" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/tags" ) @@ -152,6 +153,10 @@ type StoredMetadata struct { // Workload health check policy. Health is reported separately from lifecycle state. HealthCheck *healthcheck.Policy + // Whole-instance restart supervision policy and runtime status. + RestartPolicy *restartpolicy.Policy + RestartStatus restartpolicy.Status + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -243,6 +248,7 @@ type CreateInstanceRequest struct { SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance AutoStandby *autostandby.Policy // Optional automatic standby policy HealthCheck *healthcheck.Policy // Optional workload health check policy + RestartPolicy *restartpolicy.Policy // Optional whole-instance restart policy } // StartInstanceRequest is the domain request for starting a stopped instance @@ -253,9 +259,11 @@ type StartInstanceRequest struct { // UpdateInstanceRequest is the domain request for updating mutable instance properties. type UpdateInstanceRequest struct { - Env map[string]string // Updated environment variables (merged with existing) - AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil - HealthCheck *healthcheck.Policy // Replaces the persisted health check policy when non-nil + Env map[string]string // Updated environment variables (merged with existing) + AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil + HealthCheck *healthcheck.Policy // Replaces the persisted health check policy when non-nil + RestartPolicy *restartpolicy.Policy // Replaces the persisted restart policy when non-nil + RestartPolicySet bool // True when restart policy was present in the update request } // ForkInstanceRequest is the domain request for forking an instance. diff --git a/lib/instances/update.go b/lib/instances/update.go index d8f97833..cdea9e4c 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -41,6 +41,13 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta return nil, err } req.HealthCheck = normalizedHealthCheck + if req.RestartPolicySet { + normalizedRestartPolicy, err := normalizeRestartPolicy(req.RestartPolicy) + if err != nil { + return nil, err + } + req.RestartPolicy = normalizedRestartPolicy + } if err := validateUpdateInstanceRequest(meta, req); err != nil { return nil, err @@ -56,6 +63,10 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta nextMeta.HealthCheck = cloneHealthCheckPolicy(req.HealthCheck) nextMeta.HealthCheckRuntime = nil } + if req.RestartPolicySet { + nextMeta.RestartPolicy = cloneRestartPolicy(req.RestartPolicy) + nextMeta.RestartStatus = restartStatusAfterPolicyUpdate(nextMeta.RestartStatus) + } if len(req.Env) == 0 { if err := m.saveMetadata(nextMeta); err != nil { return nil, fmt.Errorf("save metadata: %w", err) @@ -103,8 +114,8 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta } func validateUpdateInstanceRequest(meta *metadata, req UpdateInstanceRequest) error { - if len(req.Env) == 0 && req.AutoStandby == nil && req.HealthCheck == nil { - return fmt.Errorf("%w: request must include env, auto_standby, and/or health_check", ErrInvalidRequest) + if len(req.Env) == 0 && req.AutoStandby == nil && req.HealthCheck == nil && !req.RestartPolicySet { + return fmt.Errorf("%w: request must include env, auto_standby, health_check, and/or restart_policy", ErrInvalidRequest) } if req.HealthCheck != nil { if meta == nil { diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 3ec0e55c..f3f1e16f 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -30,7 +30,7 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{}) require.Error(t, err) assert.ErrorIs(t, err, ErrInvalidRequest) - assert.Contains(t, err.Error(), "env, auto_standby, and/or health_check") + assert.Contains(t, err.Error(), "env, auto_standby, health_check, and/or restart_policy") }) t.Run("rejects instances without credential backed envs", func(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 607c5a4a..1c053f62 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -185,6 +185,24 @@ const ( MemoryReclaimResponseHostPressureStatePressure MemoryReclaimResponseHostPressureState = "pressure" ) +// Defines values for RestartPolicyPolicy. +const ( + Always RestartPolicyPolicy = "always" + Never RestartPolicyPolicy = "never" + OnFailure RestartPolicyPolicy = "on_failure" +) + +// Defines values for RestartStatusBlockedReason. +const ( + ManualStop RestartStatusBlockedReason = "manual_stop" + MaxAttemptsExceeded RestartStatusBlockedReason = "max_attempts_exceeded" +) + +// Defines values for RestartStatusLastReason. +const ( + HealthCheckFailed RestartStatusLastReason = "health_check_failed" +) + // Defines values for RestoreSnapshotRequestTargetHypervisor. const ( RestoreSnapshotRequestTargetHypervisorCloudHypervisor RestoreSnapshotRequestTargetHypervisor = "cloud-hypervisor" @@ -506,6 +524,9 @@ type CreateInstanceRequest struct { // OverlaySize Writable overlay disk size (human-readable format like "10GB", "50G") OverlaySize *string `json:"overlay_size,omitempty"` + // RestartPolicy Whole-instance restart supervision policy. + RestartPolicy *RestartPolicy `json:"restart_policy,omitempty"` + // Size Base memory size (human-readable format like "1GB", "512MB", "2G") Size *string `json:"size,omitempty"` @@ -1025,6 +1046,12 @@ type Instance struct { // billable. PhaseDurationsMs *map[string]int64 `json:"phase_durations_ms,omitempty"` + // RestartPolicy Whole-instance restart supervision policy. + RestartPolicy *RestartPolicy `json:"restart_policy,omitempty"` + + // RestartStatus Runtime status for restart policy decisions. + RestartStatus *RestartStatus `json:"restart_status,omitempty"` + // Size Base memory size (human-readable) Size *string `json:"size,omitempty"` SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` @@ -1296,6 +1323,54 @@ type Resources struct { Network ResourceStatus `json:"network"` } +// RestartPolicy Whole-instance restart supervision policy. +type RestartPolicy struct { + // Backoff Delay before each restart attempt, expressed as a Go duration like "5s" or "1m". + Backoff *string `json:"backoff,omitempty"` + + // MaxAttempts Consecutive automatic restart attempts before blocking retries. 0 means unlimited. + MaxAttempts *int `json:"max_attempts,omitempty"` + + // Policy Restart behavior when the guest program exits: + // - never: do not automatically restart + // - always: restart after any guest exit + // - on_failure: restart only for nonzero, signaled, OOM, or unknown exits + Policy *RestartPolicyPolicy `json:"policy,omitempty"` + + // StableAfter Running this long resets the consecutive restart attempt count. + StableAfter *string `json:"stable_after,omitempty"` +} + +// RestartPolicyPolicy Restart behavior when the guest program exits: +// - never: do not automatically restart +// - always: restart after any guest exit +// - on_failure: restart only for nonzero, signaled, OOM, or unknown exits +type RestartPolicyPolicy string + +// RestartStatus Runtime status for restart policy decisions. +type RestartStatus struct { + // Attempts Consecutive automatic restart attempts in the current failure window. + Attempts *int `json:"attempts,omitempty"` + + // BlockedReason Reason automatic restarts are currently blocked. + BlockedReason *RestartStatusBlockedReason `json:"blocked_reason"` + + // LastAttemptAt Last time Hypeman attempted an automatic restart. + LastAttemptAt *time.Time `json:"last_attempt_at"` + + // LastReason Most recent non-exit failure signal that entered restart policy. + LastReason *RestartStatusLastReason `json:"last_reason"` + + // NextAttemptAt Next scheduled automatic restart attempt after backoff. + NextAttemptAt *time.Time `json:"next_attempt_at"` +} + +// RestartStatusBlockedReason Reason automatic restarts are currently blocked. +type RestartStatusBlockedReason string + +// RestartStatusLastReason Most recent non-exit failure signal that entered restart policy. +type RestartStatusLastReason string + // RestoreSnapshotRequest defines model for RestoreSnapshotRequest. type RestoreSnapshotRequest struct { // TargetHypervisor Optional hypervisor override. Allowed only when restoring from a Stopped snapshot. @@ -1472,6 +1547,9 @@ type UpdateInstanceRequest struct { // HealthCheck Workload health check policy. Health is reported separately from instance lifecycle state. HealthCheck *HealthCheck `json:"health_check,omitempty"` + + // RestartPolicy Whole-instance restart supervision policy. + RestartPolicy *RestartPolicy `json:"restart_policy,omitempty"` } // Volume defines model for Volume. @@ -15789,291 +15867,299 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZI/+ioIntkYaYakSN1sa6Pjf9SS7dZ2y9axZM/ZbfpQYBVIYlQFVAMoSrTD", - "X/cB9hH3SU4gAdQVRZZkSbamHbsxLbOqcEkkEpmJzF9+7gQ8TjgjTMnOweeODOYkxvDnoVI4mH/gURqT", - "d+SPlEilf04ET4hQlMBLMU+ZGidYzfW/QiIDQRNFOescdM6wmqPrOREELaAVJOc8jUI0IQi+I2Gn2yE3", - "OE4i0jnobMVMbYVY4U63o5aJ/kkqQdms86XbEQSHnEVL080Up5HqHExxJEm30u2pbhphifQnPfgma2/C", - "eUQw63yBFv9IqSBh5+D34jQ+Zi/zyT9JoHTnh6ni5wqzcLI84xENlvXJ/kZZegO9IZwqHmNFAyTNNyiB", - "j9AESxIizhAOFF0QRNmEpyxEF0dnKOCMkUA3JkeMTyQRCxKiqeAxUnOC5lwqeEcJHFwhhScR6Y9Yp1tZ", - "D8L0k3A9lf4xJ2pOhGewVCLbCppygdScSkSZfhqQfnHBlEhJnbLdDg0jMlY0JjxVdUL9wq9RxNkMpuXa", - "RXEqFZrjBUGfiODojxRHdLqkbNZMpAmZckHQL8uExJihJMIBkYgqRJnibjaGRjmP7cU+5qIzxgUZh0Qq", - "yrBuf5xwYXZEefRv4Q8cocK7MDR4H6k5Vo7LGVfoipCkPFF8ja/KZPx9e7v7YjAYfOx2qCKx2Vb4hsZp", - "3DnY39vb2et2YsrMv4fZ6ClTZEaEHr79BQuBl4XpSJ6KgIwDGopVMwkiSphCRyfH7+44gc5w0If/23re", - "6XaGL7b7w/3n8O/hfqc4rRrhyyP/snrrnSusUlmXQWY3jS2jjAtMUp/1mzSeEIH4FAWpEISpaIlgS5Gw", - "BdOVpj3wLUXA2ZTOUuG2oG/Llcg5xxJhZoRGryIv8sZa7btAC7GQX7OxIDGmTNO4Noh37hHSOxTZTaSH", - "FHCmBI8iLRSUInGipNtFXS3GGcJJEtEARE9pU+3GA9npdlgaRfphZYT5apOIzii80Io0VBYWyX2LFEeE", - "KSKyHd6GNCWx2NRxTm7vauRysb0UlJQF/umyKs1jLeEFCcx0sxOgRJEJCXhMkG66vALbg+393mC3N9i/", - "GD47GOweDPb+q9PtTLmIseocdEKsSE8veJtlWi2/j3Iq6ReRfTE/qjy061dkcDt2ibBU2a6GTU7Vcow9", - "Y7qgMZEKx4ne2HoMBWI2bWvXYHUdHOVXEnj4VQRm5EaNLYW88/HxB7lJSKCPGO62Z3Zi6/a6iE4RRpkM", - "0OxqBOPKibz4qokIgqUesNY79On0eydlMk30WUjCcRJhpdvVSgqwwTimUupPsx9CKs3G7HYck48ZV2OR", - "MmZeZERdc3FVfNO2MqZJp9uZYzlezJK00111DpSZGrogEU4ktGdXXIyJEFx0jK65HE+5cIukD7GchCua", - "qlFIZmeWh0KdbqdEgEw+urm4cWer6h0c9AK8JIyabvRqmEx94MW26sPNhrZaUhqxbLRSt8zIfizLEiCk", - "eMa4VDSQreQmnMZ6eWMeekTncdYcoiFhik4pEVZRJUikDI411wjSjSDKUCor+yDTpcdkoY2f8WJ3rIKk", - "TpSKpVBcvMJhnx8xhWMuW/5sp6xh0vLcvZbIAlPYk8dkQc3RUlaG7NKMQ0EXRHjEd3aiGlFo3kMbeq9r", - "EcI4I5slSrEFDSluIw5CGNOYerjn7OgEmcfo5BhtzMlNuZPtZ5PnneYmGY49vPBLGmPW0xtCD8u1D+8W", - "2/5t16vz8zhOxzPB06Te8snb09P3CB4iBipjscXn2z7VLwnoGIehIFL65+8eFsc2GAwGB3j7YDDoD3yj", - "XBAWctFIUvPYT9LhICQrmmxFUtt+jaRvPpwcnxyiIy4SLsAIWrtxiuQpzqvINuVV8fH/zymNwjrXT/TP", - "RIyzQ8RHsBOnRp0cOz3Bfoc+nKINLUNCMklnM8pmm234PeCaHPqo8x3iMFRk39FmonJayp3P20AQvKY7", - "/UarzupbLTUrOY5lU+vuFS1RYxpFVJKAs1AW+6BM7e82T6awYcwJVevqpf4ZxURKPCNoA1wqYH4YYaoV", - "mymmEQk32ymzTZP5J58UjpASewNb9PAkGG7veGVHjGdkHNKZ9YlVjyj9u2Yx3Y5C8LZ/InCYt5sHdCnI", - "tN7fKxDd0IkgUyKI5vGv7C4RfEEYttbLX6Dfzv+1lTsLt6yncAuIeZa//qXb+SMlKRknXFIzwprksk80", - "GwGpEXzhHzM8WrXWBY6SCovV+wPeuIedmOt1a2lj3RZatcGztZ9c6HeqshNEY6ZLFKRAo4h8qZUaj3bA", - "mbIPKu5LPkMRZcbi0KqdWQvQq5YJ+SniIBLviQ4Z+eubX4/7DsLL/NDQmn7WzRTwiM+K1JwTLNSElIjZ", - "cITZhvLRNZL/rLR9KmcVlmS8WoKcUcZICP5iu7HNm1qN9ZoZsIuuqBoviJDePQfD+pUqZN9obCriwdWU", - "RmQ8x3JuHWxhSI2z8Kw0E4+2VnLEY7DHXYOgRYD9ev7L4fbePrIdeGhoPZf6hfpMCl/r5s27SGExwVHk", - "5Y1mdrv9GV3nED8H5M7KprMn40DHmEbSdexqWjs5lXPzF8huPSo4+7QY0OwV6b8/eiZ9BELCWAmNtzd+", - "HTDzDM8irmm6RCmjf6QlBbuPTqbgINYHBQ1J2EUYHoDfQdt/M8KI0HIq9wwVlGC0QfqzfheNtF7Y01pw", - "D2/3BoPeYNQpq7HRbs+Y9wlWigg9wP/vd9z7dNj7r0Hvxcf8z3G/9/Hvf/ExQFvN3GmFdp4bbu93kRts", - "UV2vDnSdKn9n6V8cvk/imKU+0XLitit9dFJXHMxcQx5cEdGnfCuiE4HFcovNKLs5iLAiUpVnvvrde6UF", - "zGMFEdhMk+mWZKgYPcDGGxG/JiLQEjgimvFkVwthqmQXYW03g/BC+pT8dxRgpveCUS64QISF6JqqOcLw", - "Xpla8bKHE9qjZqidbifGN78RNlPzzsH+To3PNZNv2D96H//mftr8P15WF2lEPEz+jqeKshmCx8VrPTeG", - "7Ipm1Yo46qYRqHkxZSfms2H9DurrVthNZNVKG2Oucam1EMpcZGsGUr/f1cZW7DEd3i6IEDR0x/LR6THa", - "iOgVsfsFiZShUToY7ATwAvxJ7C8Bj2PMQvPbZh+9janSx2Gan/LmyrZyu0aCOQdFJYr4ba7TQFMEAwdH", - "K8/xVaTxUvsoa7d+6v/CperFmOEZAXPUvogmgl8RPVBzJ0CJRFdkqbWcJZrpRnsLKuGGh7AFWmDjdeiP", - "2MWcS2JecY8k+PbpgqCYB1fm6nfOwZJf4Cglsouu51rlAJ8gwZH9GZmLsRGb60HKgCck1EaIeQ2mhi4J", - "W1yiGCewzbEgsMdRjBURFEf0k7nCh1sGElJ9wo0YgY2BEqz3fBBwEcING0cEB/MCFf4q0aVRWC6h+UvK", - "NFtfmo1Zuaz+3Hn7/uLnt+/fHI/fnr18c3gy/vXlf+qfzUedg98/d0yoRqap/EywIAL95TPM94tRb0Mi", - "Ogedw1TNuaCfjLfmS7ejaSA1f+GE9nlCGKb9gMedbudvxX9+/PLRKWTGjb3Q28AzsC9eZcicpR6RdOy8", - "gRJZD5O729Ak0yLq9dn7LX06J1hKNRc8nc3LG8OqBrfaEiGVV2PKx5PENyYqr9DJ1lukFRcUUb1BM0Vl", - "OBic/rwlRx39jz33j80+Oja7FoavZRAXVn+Sc80+WdTH0dl7hKOIB9aHMm264HVd+QQ8YUosE059RlxF", - "OOWv1mVUr5c/vYUo2ppQtiX1MvSC29Ed+ObOpsRLtqCCs1ibcwssqD6nZXmvvHl7/HL88s2HzoE+CMI0", - "sF7Js7fvLjoHnZ3BYNDxMajmoDUy8PXZe3PrCduG4EjNx8GcBFfrPvwF3j2CV2HHqSRKZ2NJP3m0kMOM", - "NCgmMRfG+rbfoI15WUkxWx7Buo46O69/Nnw5fA0s6dbTXi9lrZiGKzeCr3/2Mdp8mRCxoNLnovsle+aY", - "ph4pVNoW5oIt43fYAP2C6RNEPA17hS67nSkVJIDIDP2vP0isbYDFp/KNluc7v+esle67RqnFUUIZWaHV", - "fifa5TUXVxHHYW94z8qlvYv1RNWYB+X1zS7lHEvUgtUmmIXXNFTzccivmR6yRyTbJyh7OZPLN3omOPrf", - "//6fD6e5iTZ8PUmskB5u732lkK6IZd201/2STSRN/NN4n/gn8eH0f//7f9xMvu0kjA5zJ33Qrv9L00I1", - "3saGIRpPasOlcnbwZ7EuiltbHD5HjvfW3iD7ZDxfEBHhZUHw2jF1hgOQfpVRCQoBlsh+p8XoFdIfrxHD", - "ujWnH7yu+ge2B35B6xmUZ0w/a1lhz4U2I8kGMtw+tX9u14fUMKIrmoxB4R7jWeYuXhVNen5FE6vFwxdm", - "GaPICIIwBb1/wrnqj5gJbtFrBwtMbkgAMk8qrNDh2YlE1zSKwLkEQqV+tGiboBAVBa9Lpf9XpKyLJqnS", - "ij5XBFmTCzpJYSzw8oSglGF3lV5Ru+0E65EJQJYrIhiJxkatli0pYz5C9qNG4sBUp1ja6Dah0qRMr+Nf", - "T8/RxvGS4ZgG6FfT6ikP04igcxOYsFmmXnfEEgERDroTvZ+p7ZdPEU9Vj097ShDihhhDY5l7zt7zLl6f", - "vbeRAnKzP2LviCYsYaGNEXYnjo0fDTn7q96xJCw3W+y/QvSmaBDJcCLnXI2TLO56lXQ6t6/nVnx7P0S3", - "swiStLyk293G+NEFFSrFkZa1JU3UGxtg4t89FocJry9aPlbu5fG2qnyp29ZZY1qGYHhvpK3H52I0pdY+", - "l4IXoOZ9cSbq53aDXdP+CXMDWelzyq3Ur+jr3DRSi/sxP3fdzO5ApZOMJhVP1f2Q51AWrPpWcesmfMto", - "hBJtXOKE9i0fa9P/sosu/1b6Qe99Z1po/eIaGWqAPGH6p2L7VX/GWk/DrSLFi4uD5d3X41A2BkmhxRAp", - "gZk04W1znJA++gWEOFIkTrQkYzNEJcqiwhDj1/+OuFFq3KcjpocmTYiJJUfmb5J0xiibbWo1Xx9MOAyN", - "U2qaqlTo9xZU5tQss45z/NQCYs3oiJHHkFxBWRClIUGXzjl0WdYL666juklofUk1C8eQBCwbMPbUVpwq", - "3b2ecIxVMNd04qkyMWd26uV4wIqDat1drB1Ldkt3h/U/z8RFNYdm4TFx9OTs/RB4FAuuzSYPolVU/N7N", - "K7KEJXeeTFzzZRadmH5XoyCSRwtij92iG3QCWULcKE65B9T4Mq37Um//an6Mz7G3bik0vVqTv2wqeLKD", - "pOq5yeYcY7V/F07upJCenOmvqw1jSYD4YHocIFDHLrvGViLggUBMM0uEQipIoGrNUzYbMQg/ubS/9G1r", - "l3qTax3lXnKuIIUBlPbi0qLCyjq1D5rRU+MxVYqE3bJucEVIItdPSqvX1uftccwLci2oE2QuHrmlekbY", - "lIuAxNZI+DrD8WWhMa8Zd7sm6tEghr6FMbvUDkhsIaEJPTLrAR7aUsZHNfExrFhtJvqg3OUljqJLtGFf", - "2kSC/BOC+O1aMc5yZr84OnMskN2Yfzjtao7UUuByrlQy1v8jx3oXX1Ybs9+6HZ4npT0fgH21u7tjV9U6", - "3cyAK82W/WveiIrmpXHqd+OlnOYLPUobotJGlT/KP8mdsFeUhW0b+FW/2+idyxQjZ2k8tIMuEaSXJjOB", - "ITr3Pt1zd75yBWo2S/A1KcC+CMs8uTCVisfFUP2NSnQILceRlIm14FEvxAqDK7Olv9UMtx6zHC9NU8YW", - "a/LEjGcTT8gR/QRZBDM6w5OlKl89DL2JgF97/+3G4luWpth/Y0GScKz46uhnOkXu3TbBjiZVQfHxYkr5", - "6swQGzpTSt0zx5G1a3UTvSSg1p0AOk4wN8GphgigNH44LV779UesB8fvATrOOsiazZrEoFvi0NycbHBR", - "GITJAUGT5SbC6MNpH11ko/2rRNpgWRCXDDHHEk0IYSgF1zOchj1zFhcHkEo4NFX1c+s7MXkTm3C7ye2z", - "fpauDF6aLPkaoqwmtDIfk3QJC2WvkzEresFaea1WxYy/IzMqlahEjKONd6+OdnZ2XlT9l9t7vcGwN9y7", - "GA4OBvr//6t9cPn9p4b42josyxYbt1aUPkfvT463rbO03I/6tItfPL+5werFPr2WLz7FEzH75w5+lOQR", - "vyg7zgPu0EYqieg5Mam5yhdmV4hmawiju3N03AMFu+Wxu6veNZS40G8+RFaML97aRvvePm+lKjDXRmwX", - "Jle35JcJ2J35LilocDYwMqDeENBjKq9+FgRfQbZf/dyO8YzIsTnP/KEQqTTxOeTGejcE52oqzb1p2es5", - "3H22+3xnf/f5YOBJBqkzPA/oONAnUKsBvD06QRFeEoHgG7QBF14hmkR8Umb0vZ39588GL4bbbcdhrnja", - "0SEzvNxXaMNS5O8O4sQ9KQ1qe/vZ/s7OzmB/f3u31aisv7jVoJxvuaSSPNt5tjt8vr3bigo+hf6lS86p", - "KvC+pMxDAwyg/9WTCQnolAYI0nuQ/gBtxHCEkey2qrwnJzh0qav+s0NhGsmVEROmM/umcbTFaaRoEhHz", - "DBaklS8aZn4MLXnBNRjLUoVv15JNaVobIeDmkr2CSqlpJdKdmlzogvJESRQemB26Vs7BauYD+9jEB3YO", - "LbnhN2069SKyIFGRCczRZZJyBUEZn5hFK82KsgWOaDimLEm9LNFIylepAF3UNIrwhKfKXDPa3O68EwiY", - "BttjqsV1Ozv3FRdXa0NP9UmcpbCv9QodgiN9al01cIpjZL922Q0FpS+7DjSXpva5RO/MF8ZDlP+cpGVA", - "nC70ZD1JDAkiFQdJah2Gtpm22qVfbwFnqQv/MP3lsvORYl96UxMucL8WtpgRgG5QazUWzSkX8P45vN46", - "kl1/uNaR0oLujFw/BtEh1L+n2bYnGU4ehuKrgtEyX0P+EpzCgoakj2B3QVSMSy2s7LRzxZOEhJn/pz9i", - "NhQ8+0maGxT9oaGDmhMqEBd0Rssdlx1sDxnVdhtWdNx0Z3YsfljXUOEhhG80b3o8VQam4cplW5Fi6pNd", - "hE63c56BWlhJVCbNuwwYpEaRPEqzNsTXZ+9vG5uWCD6lPqgiiIWwT61l5qK2ftsdnPeG/4+JwNT8Bioa", - "ZSZ+IuZhBYPCvt/u5Hl99v6saUwZKgQqjq42pyziZRUulqOIvVSyt5LWgnHsrw+WrJNc937h02WnAsdk", - "kk6nRIxjj3PtlX6OzAsmtIkydPpzWZ/VenNbq/mstDhgNk9xYJP621Hf45CrTKNboOZH/3K9I+YYbkoF", - "1Esl7Ds2G7CP3mQ4HOj12XuJ8iglj6euvLyNofZn86WkAY5Miyazl7Kigw2Ys7WGfJZ/aF2RHj3ZD9/i", - "NgLaWMySFLbh+bveydsPW3FIFt3SmCCyaM4jose9WZAWC5cQmOcFlITEosnTYRhDtt1ABVplO7g1kQr7", - "1UMdxRWOxjLivmCNC/0QwUO08eGVSdjSI+iipLSU+vcCFUr8ve/dMVoiNXV7Dh1WXaalDe61HctAmsa9", - "UpheqVPfVjFR8nUdp46dxK/KC82v1uP1mEaa+z1ygfwVp7ZVJJGJ90cQ7+/un5H51HitrWtEkgQLrEi0", - "NJpFdvRFdEqCZRCZPU7qV4nkhgS3yCR4qV//YhKBU0HGai6InPOofAG9063jtkkIglwQi59h5lRwvCuO", - "Yiyu4GB0ijRKmaFAOWtgZx0m5Vyp5BaT+uXi4sxY14qIhYkoK0bpytrV6jGJ8BJNiLomhLmpYIkwes0z", - "oJJqWo1swDAQapwQQXmZhp0dT7/nJjATzQQOCDJfORhFuyQSTs22pLS9eLC4goBI2bC+w1Xraz+dplG7", - "NfYNa7gWdTS4zQJfHJ25XPwMVtCRebtO5TMiembLOXzB1Uu7LVejQriuGGek3pngEwIwETZzppjj4wJL", - "AAZDf17KqykIB1lMaLH9wC4wpOqaff6xlbJX3e6+i/QYs9AHz2gink3y4iyN9YLoIYsU7o5oaKJOTD6l", - "UcuLgduC4JAyImUl8ytIRdTpdnpTO6uDra2IBziac6kOdneGz7dWx++tDNy0cSrjkK6y71w0i4l3cAlS", - "BiYRJl1miS2cJC08YIaOa84HEE/1QDGAZNRnW0HDc5HDg0Etie0GB8oh24BLrHTliYvbNtEsWZoPNJjh", - "Au+9eFHcnwPvHXQOy+3Yf6vG+3pmJphM84hxN1ToaJj8k1ej4sIHX8KFsikqE+KizbLz0MVy2UuVUmfP", - "B8+Ls2yFfgzCprLN7b7zTNW8XU5bIwVy2/1rG4C4mLLO4bb0arAzTZc1PKUlYo2lVhOUJ4Tdip57uzvb", - "t6Nn24mcuKy6ilzyJc0fnR4bnSjgTGHKiEAxUdhCvReEDPiStJTRBn6ISQx5CtN/Xy1aGuIXilnwjTfg", - "RzW8tAe5/W7A+Xln4jdDFGNGp1og2zeLPcs53t7bPzBoZCGZ7u7t9/v92+YGv8yTgVstxZbJfyykCffl", - "/OvW4QFSgNvM5XPn7PDiFy3IUinMobUlJ5QdFP6d/TN/AH+Yf04o86YOtwKwo9MacF05Hkwb/Ob3gwLW", - "uNN7WuEJ+53BEBYKeAVenBeFZ1q3MRz3tYAud4Z8y3FHVQHqrZhN0wL2jX5afY3svErwju0zZYpGOSJe", - "/QL5TpiGciXsUw3yKSEsA3qKIvNXwNlC7wof6lPpJHLPvir4YqXq9Y+6xrV+vznFaw3b+r1smfxri3Zn", - "MWk8J9E3l/p3CVAq9/529h9//L/y7Nk/h3/89uHDfy5e/8fxG/qfH6Kzt1+Vfr4ajuibYgrdG4wQROWU", - "sITastIpVoHHG6UNnQYK2yfGtlbBvI+OwGt+MGI99BtVRODoAI06lfyqUQdtELAJ4Cut2OmmbJropv74", - "zNyd6Y8/O4XvS7WN0OaDCrsgWRq4TCchjzFlmyM2YrYt5CYiQQPWf4UowIkCxwVlSFt6SzQRUE7F3m3k", - "nXfRZ5wkXzZHDK4HyI0SegYJFirDT3M9AFPYUZmAS/s6CR0gj7leGLHsXMrweMwFVz9TcyGwoZqu4ifK", - "akvF2gjPBz7kIgiZ1wsZUamMsp1xtmajLJYfPR9s1i2XNdp0xkMr2A92Qr3OkmPKFnvJMDB0bQT32Dnj", - "1gQiaNlk9ggCW0lx+O85cg3ltMiW2HjITQKFNBesKpKF1InNjhflHVa35YTMDSN8FrVIuX5pcmsufjtH", - "iojYZTtuBJqcUxro+UHsJJUy1axIMTo8On252W9RKApom41/xTpeZDOsZsbaG8emi9TcsMMx6aKTY8ht", - "sjs0V+AgJvkVFygyAibf1wfovSQVGxGy1yAk0qxktMyvLc0JMOpsuhaTqqQ4QO8yvRFnQykVpyrfhOb7", - "Epq1USsmYLrWerdW9kU4u8iKNgiPxirLsNMnbrMoaO+osBSHPV8xq2+9t4s3yY1Gc2Ht7xut7v7VnZ3b", - "qTuu8kAyx9LH3fPiVQi8tKLcD634d0XzvX+p37UVd7LuoIiQPvqKn/tqpez1hsOL4e7tbf7bAo2VAT0K", - "YDAZ1lh7kLCHANuq2783VI0bI0KRfmzjP52V9+EUzbFkf1XwsGLrDXeetYKN1722jaUsRlHyqRlSJqUc", - "OkgWA2hwUq5oFJnQWklnDEfoBdo4P3n968lvv22iHnr79rS6FKu+8K1PC8wxJypen72H6zQsxy4cqTkD", - "B+dZbOSGSiXr4Cmtovq+BuPMfNoOF9xN0rSRA4SvBkr7pQRm5kXD2bxHhDMXilkj42Ngl33LHJfvDzdt", - "JdLZ18KVWePlgdDKGoW7D+mrLOfNz/eLO/Ygw1lbZa941rsExDsDfXU71JN8dSi1CCYhOjnLscZzJ6Nr", - "vjInW7JyOBj0h4M2LtcYByv6Pj08at/5YNuoFgd4chCEB2T6FS5fy9hGGcfRNV5KNHLm0qhj7LOCYVbY", - "ttakanU9XcdTuxt8WlWh8ctpo9i5e39p69M0KTctsmuqYC9xGpkEzmJNm3qZTpkYZDMD6pvpsiMGA+xa", - "hJWstCYOApHm/gxXM81ovmli+X7EBJEJZ9JUUuyjX8lSopjCHULWPUQOSZQFcYcjtiFcwH8W2Z/gVJJQ", - "/wDRtF0XtamHRhWAF+sPRkzOUygBt9lHR5zJNCbCunrQhIIfehPJ1Bh3MF6gBhQklTQkYsT0ax7stM+Z", - "on6wPxgMBllpus7Bjv73wMdNfi5aB3jXDsnua+HDVlWUOS/XkmltU31FwcVW4d5Oq7KB3var8W1uqwgK", - "eBqFWlGfaNFs/CgktO4eSVRepgek+Xt2xTRDlaZuw8wUR3+kRCzRh9PT0hWXIFNbhaTFxIGlG9aBJ7da", - "hu01pu3a0dwRVO4xgOSqx2pBnbl32Liij90lnBkObeFrz80bb5AwZWZpNJ+smFPFSxqSxThNfVqzfuTS", - "zN+/PzkuMQfG+8Png+cves8nw/3ebjgY9vBwZ7+3vYcH053g2U5DHbD2SQJ3j/v32ki+wk4u+m/sohC9", - "tXsbYkArh5SNa7umLOTXrapeZ73bIKZ13ddDFFsPwRvYDMWCoaUGKXFaKAlsAvsqZX8afD/7F4PhGt9P", - "u0rGDfL3QqQsMPhPIIkzr2qxhnFxsep1TG8vTmFALoB4HbWKnbcn2uBg78XB3tcSzQXBrhtjlZ0ecXGb", - "rvwdiGAlytZlehQ8CIUiwKBvGEenDcrtdDtZ3DD8DQdtJSYte9wqGL5pw3b9YmSV/G7ICTspKa5wD2qw", - "hMIDrQVk6USTVKEs1VCrF0cRT0NU8L4YaBW4mjgpKLG6GbgpsM4Zg5Flglq1sgugjAAJTJkWxHAloxux", - "CWQH6DW8C49wbPR7OwjMwsptBA6X5jZW7y/XtdG2Vw/53Cra8I3WuhHU5tbT1mSwTrrVTRjN5wC94fBN", - "pvYzXvX2mddB366/XvUMbtjMLpcBDJ1ZNe4AvcpUt0z5s8rehiT2z7EVWHni/WYp/dGueEdzS75yhcy+", - "bsdQtNPtOEJBBmA9F/B9zvW1/VdkRV+YAsER7OU81ypVNLI4kzATCtWpbRCuXtwm/cJiqpNwbAyTpqAj", - "k8BjjZfsI6e+fDhFG4Ao9XdkXZn6X5tZgFLprNt+sfti/9n2i/1WuBH5ANernUeQXlYf3FodNEjSsSv8", - "2jD1o7P3xggOjHkJvnE790KabiK4Fj165nkl2bzzF/0XRbiMkKemqrYdksXW+VKoHb+y7G9DlM0fNFrQ", - "6ZT98Sm42v6noPHwZl9uT7wuxbxIvdf/clK8b605K8mkZ+pg+BENgKGEbAT9eEckzACdE4WAf3oIB2A6", - "ZFlhluUcNIiluJexdnd2dp4/29tuxVd2dIWNMwZvjOdQtiMobDF4E228Oz9HWwWGM226VFmA6GTWrPTv", - "M2SraQ3KCml/ONjxcUnDwZ1zjW17ETeS/IM1zeykLNEhuS0z22q73EvtnZ3Bs92953vttrH1E47FzWoJ", - "40K/DXkskmxx5TdAm7w4PEOQWDXFQdlvMtze2d3bf/b8VqNStxoVoCAb9NJbDOz5s/293Z3tYTv0Gt89", - "vMVlKm3YsuzybDoPU3hWw0OKuujtNp0WPnXKMNg7EkSYxoeBi6GtnD4GpXQszGv5IrQ5GKxrunZwtfi2", - "leOoUrXZqAZcoJRl2Nj99Zdwd7tTaxbT5jxYL8brln2EmSaXhVkwxTDuQLtEkAXlqbyHhrgyyUbTiHNx", - "q2+bLJR3RKaRMhdfVKIPp38FIaKZC0lFkrLRZNlvBRjFHSd3qw1c4gk/VzcRq9VqtFn6VRPuNmzT7qpM", - "5NL2b8R8CbWoStn6ALgjHAUpwL/jbD31rAC9AXIpkyRamlDRKOKcoWCO2YxAJT5TbILNEEZzHoV9b/ie", - "fjKeei/O+TWKuEGrvCIkscjoZhD6M62z0AVBG4VcTmRYqVLhaC82UsViX5e5cc8b5SkIlr70gyyHUNMT", - "K14AUjSflHyMEZ9JsAIVBMH2q/i9CRYmthUzg/S/iI3xWAa/2danvWeIFentO0LN0cmn1qK1OgZk6BlK", - "4kBwKRGJ6AxQ5T+cVhK/VmQxZOlf64PayoNtwbrmKstzdsGZJlsXBPEdiJ7w8K85EoGHIQtkRbiY80bG", - "mKWAlV5gZHKTUGHYo11I2JxLNc4APW45WKnGgIOdCpKj/mTpipkDyL3jPRedaLsLuWzs5Z2+rnGVv6mm", - "ATbLVC9F/dTqZjzoY+M6pMlKFJUclqWKwXEb0J0cOJlKaJUW8F7QBuOqJJYK4L+bbcIj/Daq7qdmnto6", - "T7/tDs7b4uGshr85w2p+wqbckzR9i2tI53q2EXsJETEFJHgUEkZJ6IzH7D7S+rYg0y6SBIUpsZQzCqnA", - "luDYbG9IfGbOKUbZrCLrqx228QebMayGyYZ+7YttAl2kPz/rQqRAKxOaJhHOM7VaxflROfbfX9UbFmSW", - "RligKubTiiHLZRxRdtWmdbmMJzyiAdIfVC+ZpzyK+PVYP5I/wVw2W81OfzDOEw0ql8ZmcDbNxCxIpd98", - "Cj/pWW5WktzA9bJlvt+CVPs2cUPeaNlXNCIWFuk9ozcFRi/jyO5uD5ryHxsaLWU+1iG1biu5Lcv6drxD", - "uzrM6k56bilN3GvlrrTsiFx70weB1auyPeuuGLThQpEcTm+ZrgW83FaekHax3dWgOzeaLUmCcu+7z/ee", - "7bcELP4qX6dJIr9vz+YiXuHRbFip0zZus+d7z1+82Nnde7F9KweVi89sWJ+mGM3i+lTKy1acZnsQyzS4", - "1aBMhKZ/SA1RmuUBlUrF3nlAX1Zs3abggnxvNl1yRsWVdPcsZQ9oOx/jCm3psKRyFQqpb5DplIBROTZ0", - "6+WDqeTotRpDgBMcULX0OEzwtSmal71SAVxr400rD9ZDUtu2BefRkkumkzyNYcN1jv5mXOsVXnjeGvdc", - "ppMmN/7baq/GiZ/7gIpXRC1uaPLSjHV3QTafayxLsWb67wDiF/NC+dWoVfPGavCnKuQGXAJaeP9CJIUP", - "NLBy/tmPistfWc6C27ekJFcpvuoIbd6Ct7KhPSeyx4QO1ueiVOSDPQDv9tV4UqxIsLLkQ6l8QX7q3r7f", - "diX+69+ZE+z2/RUSD27zYRWaCvjRjsGSPG+7W2KJBm5SXKyvyfUAEMsmpuBOIMs2HOFRcJbtzw+CrVxb", - "jnOi3Lvn2qJPoxUltUooiCWzxUVxuFfKflQjibvIuvjQMN6sRD7tzv26msXxaBnUqrXAcSLIlN6s4Bbz", - "gjmuy0lL0lIgLJddk2gjxjdo9xkK5ljIytgZnc1VtCw7WXc9OYNfhTguiNKqc/sCdflqug/rNxp2OYut", - "+7bseSHDz184j4TjVWg3R9lrzmec4CXolo2G4LOd3cFgZ3twJ7ib+6rnV2inKU6y8J115pSuHostZFHp", - "9aIP14KasvCOTFIJguMDiKZKcEBQRKaQDJ4V21lr09e6Xj14e0lq080y/ncLZdfN+VnKeJJZVxYpyE2j", - "425pyxmGxef1Ya/IGM/ETFBLHfcEUu70BvsXw52Dvf2D4fAh8HEyIjWF8Dz7NLx+Fm3j6W70fPnsj+H8", - "2Ww73vEmqT1A6chyYkq1kqSdQ0JEtZpHtQqOJBFlpCezsLf1sacrZIG5SVq7/2/nfTAzWKksnJcnWdQZ", - "sMqJUy1q/xjptHb0K10o1eGfHK8e9p3iyKoD8TNYdSjAT+0GA3htw68FB0tZy3PnfeHF1ifPytjGdWeP", - "LxUFtrZ3lRso7uPnkmAs7bBVJ3b9VPOYcDMuqJrHq4+H7LUMagguwz9JFZbTN/voZMagdk/x5+zuoxhg", - "rj/udDvRp93ynrG/t0/ktdA6GQPapS6qAS3uBqA01GoqwCu5aSFMeAIWBAjx07A3fAE39NGn3Z8GvRd9", - "9I9CpEDXUKtIvqF7u/TroA0Ni4DYDkh1+OJW1+iOnqs46Ffqg3POD2ILumN5PC+c4s4KFzZdWuD8cW2N", - "K7mOD1aq2Z5m46KWFJIIL33lLg2kO2Cpy4p9WGQyNCEzymQXkRsndKrw4CiiVwSNOjsDOeogLtCosxeP", - "On10aKGqwFrN62KVmoeKSAU+obGt0m1rIjWHpWzH7bK6qsbD7fAL3Vce9azv189erE90WRdFt+6Y7H9F", - "VPVXmbvtTNxVOVi/YakymxSQ0uHFLhRKZhW4fluczqZDQHgrJKIfuLzrnGWtDJC54uc8IV004wrliRDt", - "sqFEyrz8UB4/uYEUnhWZT4Yhtu8lrS1LsaarxNfJMUoED9MgjwKOYNB53pZIK5hPK7T69fesD+nQgPD6", - "KRdovUOjyYOxHuCB3DSv9xtyowpdaoZtXurhYP1SP4gXpNtJk3C9DDMvtZNgt4IkWxNX6vHJlMle0QQL", - "k/nYQqK/K1KwbuRC2W8UaJUoTWxdGeCpOifJesmYGN+Mveg8xyQi+piqN4J4FOahL1TmUnS9SB3uP/e7", - "DfHNOIBU8dpAfiUk0bYKZLFCfzFmS+/AqrXY0MbAFdiQCJrvGUhTS63y4J6t1cQal6p9WbuKV9tkvRWr", - "CGbYXPdb085+ubbi6EP44b6lkvbWXi5UUFgcRlCGaub6N2G9gOLEosp5vbsn2+lkF9YybkKWqYb2Ft3M", - "h73/Mm5lNO4fbP309/+79/Fvf/EXMy7ZzZKIXkimcN95RZY9gMVF2kbvl3FVANJPK9O2BLAiOAanUXBF", - "jJMqxjfF8e4NMqGxfIPj2hTgojimLPv32gn9/S/N16wFMr4HObmWZb8a8fIhygko7o6jjZiImSt46KID", - "N/sjBgGHV2QpUQGz2Ko0jlH/KrNPtIoOTkscoUujBvYJW1yiCQXodzli2qrFQUASbU1Y7FZqavZwkD6C", - "4KjYjsVOdtH8Wi6ZrHGbXVUD5Xn7/uLnt+/fHI/fnr18c3gy/vXlf2pthFz3TA9hT/Pe7t6+rdRTpOTQ", - "s8R3xgf08YtBHPEwCIS+Qv0ij8ZLJSSuuPjswstog8SJWrqiAi6CdvN2CCiHWYPeS/N7RlsdvLgPcPn3", - "K9HkFzzqaZW4ASzP64E0tPAGfUFTJpiu0+SZnnnKfZ5bd+CMzrDHGe2tNHYfIPBuQGuhaWrr3wjh7A/B", - "O64iCZrtbEhVQb6rGJZS9Zoj9GKtCY3zmk5lKIyU2SBWC6MyS6t1ZLZiprZssQZf4kyoj87VYcv5LnO4", - "CD34aH007kpdvDCzwkia1+bUqZwVpXgFgc40aa7nRJDCQsAHOULbLUlmQ0pbpGMZSPWEiF7GEi4eFWqc", - "Cgoxqpm3wJEgCzuuu1BXI/Cd4pusB3C/Y1m7pIJ55Fi4w9c/Q93id64kGZ26JmAYFYPAD7dW5qJVNHFc", - "VV+MIlfV523e9248K6tWSL+mvVVhzryPEmv6+PEfmKpXXIAJ0Zz89OCobWCehERA9ncVk60VoBmNSTjO", - "CjM27X9Xi9FkPmVlL/NyDc5cwsDEWsith9R36Tn5GOqU1uQgQSqoWp5DATeDyEqwIOIwNRve1YGzP+cd", - "Q/WDL1/A0Tj1xDq+JowIGqDDsxPYjzFmoGWjD6cFxHIDXl9DagH98O3RiTVRHdgPmBxUAevpt2PMdPud", - "bmdBhDHTOoP+dn8AmzkhDCe0c9DZ6Q/7g46p3AdT3IJiSfCnTWPITJ2T0OpBP5tX9FcCx0QRITsHv3vS", - "ARQRpviSBIUVzwomR4KpsDZHEkGSgmEVqr8F+D53lB6Y89gW3mvtYZNqaUM2SfLWLutHzQlm18AUtwcD", - "C2am7MELAacmym3rnza1M++3lT4H5PFg2dVMAqdTWpJ/6XZ2B8NbjWfVMGDH+rp9z3Cq5lzQTwSGuXdL", - "Ityp0xNm4siRQSSxkTLFfQYsVNxhv3/U6yXTOMZi6ciV0yrhskkZJtrsZ+TaVv76J5/0kb0/AHh4Oedp", - "pKUJMkHyzlOgsOjPPiEsgjldkBGz53ScRoomWIAfIEb6fDYWT3lrmK7N6mfpiz/zcFmhbtbclm6u57zG", - "OYGrMNCSjAHtcNxU1i/3F1OmDXv9icXMzupb1cNxtLgcy4D7opYvCMNM9WRCAjqlAYKX9e61Lmlvg60A", - "gLTAg2UhAnA5nItle9Of+QLo3v6ksePsGbLkLasTDO5xgigNc53LBWNjMcFR5EWImEV8gqOxoc8V8aio", - "r+ENS5QiELpTbhgPiQG1TpZqzpn5O52kTKXm74ng15IIrQLZYiWW1iQ05UkM614DolgMBUNMKTTd55YZ", - "4tbnK7L80h+xwzB2Ze6k+QRHkutT0xYXAoPARe8a3vXDrzcEhhylUvHYslRWaSkfJk9Vkip7KS6JshVW", - "4HUqUZLKOQlHTHH0WZAZlUosv2x9znv8ArYLwaHmk8IrZkpbn2n4pWnUcoz17Mfwqsf6I0CAUUefLqOO", - "/nsmsLZdUjkHL4gEz8esuKQbWda+1gs3qxQOMEMJTwziATDVHGuWK7UB1apwFCEFW8l9q7VNWMmG+dgk", - "pnjSmMFkUk4q24gydPpzYTMNdp/795MkgSA+B8d/nL99g+Co0mtgXss9TuZWmulTFIUpaPLQe3/EXuJg", - "jozeBKh2ow4NR53Mugg3YayptCHWvR6ouD9BeWvTTZeGP/X7uimjPR+g3z+bVg70XkriseJXhI06X7qo", - "8GBG1TydZM8++gnalAhyXhIEaMPI/k1XaxAAKfJj0JwbmIWIW1kbLRFGuQQq+lEmlGGxslCih/SWgtqU", - "xzNZJMbnEThfR52DkXO/jjrdUYewBfxmfbSjzhc/BawS3QyhZmpFOl07Y6L9wWBzfYampa9HhS69qLff", - "l5r2tX1viodVuuqKh5mcw3/UK2iqfhp16xE0n59x6OpI/VDx1qh41nNRUN7g++I5YNg3IsbArWhg2p6N", - "nAa20joxbAGwzGBxuHxqY3BQp8HlzFs0P6rmfN2s2G3aZQEMMXL8t/sI/Af9ZiWHTL8vHqtfHAGaqYPH", - "fWLsCIvlGLHrt4hfE/U9cNzgsUSphV79lvz7VPjnNbF6X060ijTbIgt33+RHjYBsEWlbMS9rW/UcxtQ7", - "J0yhl/Br3/7XWTyAgXwZ8dnlATIkjPgMRZTZi7zCbZE+FC0t4SOTMJJ9Z/NHHGTXhjk///e//wcGRdns", - "f//7f7Q2bf6C7b5lcFQA4vdyTrBQE4LV5QH6lZCkhyO6IG4yAMJJFkQs0c4A1MxEwCNPbXI5YiP2jqhU", - "sMKFp0HPkrZBMD0YzIeylEibcKNfpFML7WEczB4T3u1lQ8pH3dFdD3g9zKAwAX0qOh6AXG1bZsXaXx2/", - "98zMueQ/q/rKax7T9fJFkRtluLdnBnhLAQMk9u07eGAnjTbOz19u9hHYGIYrAL4FNOa8Gas893/IpPUy", - "yUiUskABKhvZZLLQVvt/j+077RzAtsU/kwfYInrdwgVsXB5QH9StwA9boYU72E835xr2+WePXZpls4P2", - "7vMtduECkVoZwve3zo736jQ3Twok+xYmMNpw4eyuRPTZ0YmrXbf5zZj+UU4NPVNbESg7OhA3hakfzSw7", - "4mwa0UChnhsLlH+ISWaqlRnkqYiDd3bUCLt5VYESi+fbVgn3p/GkyyCA8iPv4U+PSqe3OUZyMMec136c", - "JOtY55jKgOtvC9zSC3BiCzob9SXbp0UuWueQMtHx2ZGzUl2y4vnk2G3Ix3NN2a5TVj0bHkEoHlcE4jcU", - "hJUisQX406fEze+zVXSQEis8V98Xaw4eTwt6bC+Wj82fkhsrrJBNS0ETld14gL4mysRidx5woW0Pnomf", - "E+F2tUOrhlln0zKfmvJtZkJwIb3a9j0xr7QzfU17fybLF8hzG43FkvyHitLC2M1ptcrAPbGFUR/OvoUe", - "bmXe3t89r2UwD5Eh2GTiPNamuh+WSxZs/qmueh/lNDPEfpKH2VkaRe7GY0GEQm+PTszOKp4BW58hLGm9", - "bu9228rj4P2733qEBRzi0LIYKr8SZZ/cs4ZvFsxM5QebtLEJTV4zdedZk4bzFetvC+CbCMc+5f+2/Sqi", - "E4HF8t+2X+EooYz8285hhBWRavPBmGXwWKL5sTXuJ8x8WuGmZaKBaGJQVXadhpq91VJJde//qfRUM+lb", - "aaoZXX8oq22U1SK5VuqrdikeVGM1fXyjK5mM2XzUhkcuPvFPpqk+rpfPcqRDeqayfO1hS/lwAX5eeEQZ", - "SiV5ggGUNOO44rHR0l2db8iVx4dj3ZPjLhCyq0kHsEs2QeSRnNduHI+u3Np+H99zfRhP6CzlqSzmnsRY", - "BXMibbJSRMoC+Kmp3fnx3Kh4f8dcOnjMo+PR9eoffP9AGn91QY3wNjdQ63R+91Zbnd++r3V+k0Jtc9cs", - "NlTX4QZuNgQVuiTqtmxcyjWvBzv6xuWzRdB7bajk5gICC+JgxP6Ptj9+VwTHH39ySTLpYLC9D78Ttvj4", - "k8uTYaeOVQhTglqY18M3x3DtN4Psc0CCzVPyquMwpSOA9Rz2zb+cgZTffLa3kBwX/rCQWllIBXKttpDs", - "WjysiVTGz3p0G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF5ZrRYkZgszPsHcMmbvhwrBHaWDtrWVlG3KNQpo", - "juv/6IE9JzmQ4WMbR66EwNOMkeeJxeS25kh+GDbbI98bPwweVzg/vh3ylFnMKPx10iVap/QV7ASQyDhV", - "EJSYI4RA1CcSRmvPWuyjvE6mTJOECyUN0CQowAaKfq4VYB8oZRln0gcsCfDBlMjuiEGpAf3Y5PJvXZGl", - "gZGknGWIkdlMLXSkL/eqDOP5TbfR/etYfozSVjrWI29jizr97XSsbyY6HkXTOimB+W9kGwMMygnJdjLP", - "kvvoJ8pmm08qAtUIq2xuBTwjj6q1BaX6LDDvlsxKAjcdtAVsXlvI8l/wxK1P0qe1OyzaAgFRSPGMcalo", - "4BJ3q3DnP07o1if0asp6uXlqC7H6DfpXXFy1PeI8hcGewElXnOF36EvQwwM0sG/vUgBj25wGmmke/RSs", - "VXv7likYtHouBlEa6oPQHYhOlZwKHo/tjwavVu8KiwYKLorAtvqthY3u/REcRm+4QjROIqK1eBKinuEm", - "vZpW9Xd48VQWaiPeThjqbVNMiDFgdNLVFrIiEi7X3IJtwD17fbm8UjPis/UgGFnnDvHBg4IxYgbPnjjw", - "+0uUCVmovkUiEih0PafBHBAxoCIXlGQFsAqcJJcZBNbmAXoNO7WIBAadb0gitCEUcCZ5RAzQxSKOLw/q", - "iK0fTk/hIwOGYbBZLw+QQ2nNDgip3yoiXGRFi95Y3I4NzUmCR5FZ0UttNRbmt2mxL3KIshHz4WAwcm0b", - "pFN0WYDEuGzAxHAC9Tc++2baVrcZWNLMRXEkgHCGNwkLO00XMTTyo2EMB96CLi2ROcwwHhiYozaY3/gs", - "A7UssTJOkrbsa4cJXLyI4xU8jDYK1VWlCnmq/i5VSISAjy13NzE32sCB+YfCV5pRbWWgrD4tsJ/3utGg", - "zHlJpYVqoQyO+dcijjvdjh1PAZ3uFtr7GoSTaoP1azG9MgUYkx96920ASsrCvoBQUjk5bP3+ZpX7nXnh", - "T++ftYQK/wxelvJ9Vj4KyvIKTgJKsLsyWk8K6QAWsqaLmcpGvj3iZtmTheqf7a63anVDvwOjdd2tV1YE", - "MqtQ+djXX/URPOUkGFmbzZSLanr8unux756R7m9JalNtwyE/ePP27rlWjJmkK4qBQi1TCX4+KJAJuM7B", - "nHNZYPsJmeMF5cIisFuva8aZ4LIw1qONnrvUrHpp/beXVj0/sL4mhIuPbB99+NzG3Pm/cI/yL14VrO1M", - "4nedSg0okBJhNBGUTFGCU0m0tpTGBJkKIxbIm+Bg7sp990fsYk6QLXBZcCBk9ZCpRJfD+LKLJqlCERYz", - "sHbMQxNJJ0jA45iw0BStHbE5wQuqTTWBIqwIC5Y9SaCI8YLkBUy06W5vKE2t7KxMahe56rrgYLgs1M69", - "RIkgwETGXGalQrUjJlL27wa5Ujd76QZ6iYhUeBJROc9qRQQ4JCzwwkKef99i7P6duOdE1cvLfpM7yzvJ", - "0m95iVn0ZWYFvr+L+80nFqjFhSuN2ULMr1B6ZbNpWI58PM9L6v4LbmkzVzfHb3Qzk5F41S7+Pq5kSjX1", - "f1zLKLslw9R0R8p15/+0dy15IeiUla5brE/2rhcuWSWEjMy3knlbn92fJ3fwkX0nkrDbaNg3YW7nk/4e", - "RK6l6p1k7jdyDlpfUsEr9g1FsB3Ut1OfuChIue9CDJsNl0njosxRAoNNxdkPYVwVxjY84K7C2Hlcaxfg", - "BfFMWS+JcJNczsvO+wWwdQj8i0a/VmZXEITfXPDlNwKPJuxOMvFmBF6ClxHHf/Z7mYALYRI6bTnipwMo", - "VvAFFi6YNsDj1s0kRNdlk3w4Pd1skhJCrZQRQj1hCVEuaxrEnmqNbxdECBq60pFHp8c2epVKJFLWR29j", - "CvUcrwhJoFAM5alEkJnb1/Nzqa31InilHNZuhzAllgmnTK0dRf7qwwzmy51K5z2ynLSQin/6y2Pwwj89", - "IQWyQ6srdgKrrUiFVWMwngtOo8zUu9TaFp7wVLeuJYsrtDuDs21KIyKXUpHYROZN0wg2EYDu2ppM9juT", - "UdpFVEmk90MXMvASImIqJeVMjpgt/54QofvWn0Px3zzIyOu8VziTmmdG9H0fAWx6MCZmC6smqgG0ANQB", - "7Rx0tnCSbEG5aH+QlB3eVwzpFUSkIbmMJzyiAYoou5JoI6JXxuhAC4ki/cfmypC2MXx33xWn7r6zNKVP", - "2JR7i3IYns2Y+c+RhFQWa+4S8cmJtdekuFmc/IGF9os1uVauCYKjnqIxyZLfUapoRD8ZUacboVLRwOTV", - "5KmXUITZZl+O2ClRQr+DBUEBjyISKOdc2UoED7ZG6WCwEyQUUEp2CAwOBF7z4xh6PDp7D++ZQtHdEdP/", - "gIYvDs/MTewUWx9BYaCMqGsurtDJ1ts1Qb7nQKZ/4Sg5M8GVOZDeBf9xfXf7zObGPSQbtihPVhlAPPnT", - "h3FaDe6Ht+BpegsAWiKbzcZM4ACUYjlPVcivmd8zsOBRGut/mD9O1gGUKBzMP8Cr3422a4azths3wSex", - "Ke2cQmKKBn2TCwpDsKcaX6oJ56YASkwpcs97ChyqPyN3379TvkjH7/Bq0lLUFeT6bvbWY598dgwOd6tI", - "j6eyzQ2nuZkovtr7dI1ps/fp54gHVxKlTNGoBGqg7TbAAdU/5riN9uIP1ATIjnSlxBG5SagABJsKPAIi", - "esYSYaSIiCnD0RbM2TQCCJTOi4UXnEKSchBRSBOjIUEJjyJA2bmeE4b0bMBR5Roo3NNKWwGi+E7xilFx", - "NCEBj4lD5dz0mW7/wFS94qIMsfm9yMWLAv31fPRU9TzXoIo29/hVKKOn+AbCmsPUXhO7EW285vmPxhXU", - "RbA2o87OQI46XTTqbMejjl6BIwwuVKzQHoopSxWRfXRs/FuQhro/QJIEnIXSgYM6D97OQDYlpRq2bMhw", - "3IfvHlPtsVwFpHxnO/GJB/0e0t9Dgg3aKG44uyfDLmy6EPFUQQC321f2rZAocI9sPvoNbGGP/LDt20jy", - "f9jtW5JRsMpaXBaW3kj2DD5yrdfNJVXMucxRJ1GAExxQtewiHEU8yL0HqcxuB3rZUCaC4CttQ/VH7F0G", - "XGkTIdDR2fuuc5qhkMor04L1i/XR2wURMp1kg0MgDYwHDxaDhCOmOApwFKSR5lsynZIAchgiGlMlG/xq", - "2VAesgxi3oln4d3DDLbmaTmT/DwBq5ezhaxw3JZZ6i1BggjTuOhUqhIHVF+40gW370Q3yvUxPI3s9VYg", - "uJTINtUjEZ3RSWQva2QfXWiVA8dkxJIIM0YESqWJO9JD7yWCSJmaxBjdANSZNRzVRTnQSSK4sm7iiHMh", - "jWdXc/iHUyQVSVaw2TvT8inM+YFggk3jtqdvZDBUxtB8LNlXkF4QwymG4JqP9DH9DYJ9zIC+NZzwU9n4", - "F4LOZkToXYGNkDVXo2ZbO3KaTV/K9GjEyD/P3mqHkZ+1WojmLkQ6rwSqGLsXx6BA3+YG1tP5FW3EMrGP", - "bpd98av+qGXf5Sh//yDso6+c5Z+l9Nh5Ibi6LbJ+zuFPDeS+MPLSVi0lKKyHI2idkfCQGQKtcQe+GdzA", - "U0YZwKW0gyY4ge+PEQaPmx332DDbT5u3SigBpcI6DalS6+E7vwsOfBjczm+cHXoH3M7vKl8JcBe/Xd7o", - "d5WpVPIDuuIhf3pkzodKUDLwnABj0ZSgZKSeDSRYaSh9sO+0M5Nsi38mDd7ePd9Cf3dk/2H1tzAZCsTy", - "u+xMbrTDbSFxopbucpFPKxeAkn6CZAwf8EMWQ/BweAt3uF6/P/ZwfNp4uf6jntaj3d/nRYdPjp9+Ea3i", - "nisdLFv61OlhEczpgjQ73cs72JIoEaSX8AQuV0JDMEsPd5YpLPqzT8g2b7Gq7L8QdRDHJEQhFSRQ0RJR", - "pjhIBNPHXyUSXFsC8JyLpc+ZXty5rwSPD+1s1pyHdk9ZZ1h+5xsveyFWuLdw0maFC+0rbtrd3bYWeIgy", - "9PpntEFulDCIu2iqLR9EpxlJyU1ASCiBJzeLAx4OGjyb9BMZzyZtRrkCO/mtxaZGQSoVj93anxyjDSi2", - "MCNMr4VW9aegySaCL2hoCpHmRF3wyFB12EDQ2/pdtVKRVcpwxoUZ3DfRYdocSLNPNCmLBRO60DnoTCjD", - "MLi1KMXlPWUSqnR/mEJaQ753HOd0fhxh1vLbcMaO5kRt5DgiKs4NNN7mj2PuKR9zxcBUd6aVTrt2pSLb", - "xaq2DCF9CMDcLI75cd3WH76f8Eoqn2RkpXWdLzKDtMlt/n2x4ODxzofHdpd/eMLh+K+JM74LrnJoQLfo", - "Y5jfeIAjFJIFiXgCVSTNu51uJxVR56AzVyo52NqK9HtzLtXB88HzQefLxy//fwAAAP//QhsHTwiGAQA=", + "H4sIAAAAAAAC/+y963IbOZI/+ioInt0YaYakSN1sa6Pjf9SS7dZ2y9axbM/ZbfpQYBVIYlQFVAMoSrTD", + "X/cB9hHnSU4gAdQVRZautqYduzEts6pwSSQSmYnMX37pBDxOOCNMyc7Bl44M5iTG8OehUjiYf+RRGpN3", + "5I+USKV/TgRPiFCUwEsxT5kaJ1jN9b9CIgNBE0U56xx0zrCao6s5EQQtoBUk5zyNQjQhCL4jYafbIdc4", + "TiLSOehsxUxthVjhTrejlon+SSpB2azztdsRBIecRUvTzRSnkeocTHEkSbfS7aluGmGJ9Cc9+CZrb8J5", + "RDDrfIUW/0ipIGHn4PfiND5lL/PJP0igdOeHqeLnCrNwsjzjEQ2W9cn+Rll6Db0hnCoeY0UDJM03KIGP", + "0ARLEiLOEA4UXRBE2YSnLETvj85QwBkjgW5MjhifSCIWJERTwWOk5gTNuVTwjhI4uEQKTyLSH7FOt7Ie", + "hOkn4Xoq/X1O1JwIz2CpRLYVNOUCqTmViDL9NCD94oIpkZI6ZbsdGkZkrGhMeKrqhPqFX6GIsxlMy7WL", + "4lQqNMcLgj4TwdEfKY7odEnZrJlIEzLlgqBflgmJMUNJhAMiEVWIMsXdbAyNch7bi33MRWeMCzIOiVSU", + "Yd3+OOHC7Ijy6N/CHzhChXdhaPA+UnOsHJczrtAlIUl5ovgKX5bJ+Pv2dvfFYDD41O1QRWKzrfA1jdO4", + "c7C/t7ez1+3ElJl/D7PRU6bIjAg9fPsLFgIvC9ORPBUBGQc0FKtmEkSUMIWOTo7f3XICneGgD/+39bzT", + "7QxfbPeH+8/h38P9TnFaNcKXR/519dY7V1ilsi6DzG4aW0YZF5ikPus3aTwhAvEpClIhCFPREsGWImEL", + "pitNe+BbioCzKZ2lwm1B35YrkXOOJcLMCI1eRV7kjbXad4EWYiG/YmNBYkyZpnFtEO/cI6R3KLKbSA8p", + "4EwJHkVaKChF4kRJt4u6WowzhJMkogGIntKm2o0HstPtsDSK9MPKCPPVJhGdUXihFWmoLCyS+xYpjghT", + "RGQ7vA1pSmKxqeOc3N7VyOVieykoKQv802VVmsdawgsSmOlmJ0CJIhMS8Jgg3XR5BbYH2/u9wW5vsP9+", + "+OxgsHsw2PvvTrcz5SLGqnPQCbEiPb3gbZZptfw+yqmkX0T2xfyo8tCuX5HB7dglwlJluxo2OVXLMfaM", + "6T2NiVQ4TvTG1mMoELNpW7sGq+vgKL+SwMM7EZiRazW2FPLOx8cf5DohgT5iuNue2Ymt2+siOkUYZTJA", + "s6sRjCsn8uJOExEESz1grXfo0+n3TspkmuizkITjJMJKt6uVFGCDcUyl1J9mP4RUmo3Z7TgmHzOuxiJl", + "zLzIiLri4rL4pm1lTJNOtzPHcryYJWmnu+ocKDM1dEEinEhoz664GBMhuOgYXXM5nnLhFkkfYjkJVzRV", + "o5DMziwPhTrdTokAmXx0c3HjzlbVOzjoBXhJGDXd6NUwmfrAi23Vh5sNbbWkNGLZaKVumZH9WJYlQEjx", + "jHGpaCBbyU04jfXyxjz0iM7jrDlEQ8IUnVIirKJKkEgZHGuuEaQbQZShVFb2QaZLj8lCGz/jxe5YBUmd", + "KBVLobh4hcM+P2IKx1y2/NlOWcOk5bl7LZEFprAnj8mCmqOlrAzZpRmHgi6I8Ijv7EQ1otC8hzb0Xtci", + "hHFGNkuUYgsaUtxGHIQwpjH1cM/Z0Qkyj9HJMdqYk+tyJ9vPJs87zU0yHHt44Zc0xqynN4Qelmsf3i22", + "/duuV+fncZyOZ4KnSb3lk7enpx8QPEQMVMZii8+3fapfEtAxDkNBpPTP3z0sjm0wGAwO8PbBYNAf+Ea5", + "ICzkopGk5rGfpMNBSFY02Yqktv0aSd98PDk+OURHXCRcgBG0duMUyVOcV5Ftyqvi4/+fUxqFda6f6J+J", + "GGeHiI9gJ06NOjl2eoL9Dn08RRtahoRkks5mlM022/B7wDU59FHnO8RhqMi+o81E5bSUW5+3gSB4TXf6", + "jVad1bdaalZyHMum1t0rWqLGNIqoJAFnoSz2QZna322eTGHDmBOq1tVL/TOKiZR4RtAGuFTA/DDCVCs2", + "U0wjEm62U2abJvMPPikcISX2Brbo4Ukw3N7xyo4Yz8g4pDPrE6seUfp3zWK6HYXgbf9E4DBvNw/oUpBp", + "vb9XILqhE0GmRBDN43fsLhF8QRi21su/Qb+d/2srdxZuWU/hFhDzLH/9a7fzR0pSMk64pGaENclln2g2", + "AlIj+MI/Zni0aq0LHCUVFqv3B7xxDzsx1+vW0sa6LbRqg2drP3mv36nKThCNmS5RkAKNIvKlVmo82gFn", + "yj6ouC/5DEWUGYtDq3ZmLUCvWibkp4iDSLwnOmTkr29+Pe5bCC/zQ0Nr+lk3U8AjPitSc06wUBNSImbD", + "EWYbykfXSP6z0vapnFVYkvFqCXJGGSMh+IvtxjZvajXWa2bALrqkarwgQnr3HAzrV6qQfaOxqYgHl1Ma", + "kfEcy7l1sIUhNc7Cs9JMPNpayRGPwR53DYIWAfbr+S+H23v7yHbgoaH1XOoX6jMpfK2bN+8ihcUER5GX", + "N5rZ7eZndJ1D/ByQOyubzp6MAx1jGknXsatp7eRUzs1fILv1qODs02JAs1ek//7kmfQRCAljJTTe3vh1", + "wMwzPIu4pukSpYz+kZYU7D46mYKDWB8UNCRhF2F4AH4Hbf/NCCNCy6ncM1RQgtEG6c/6XTTSemFPa8E9", + "vN0bDHqDUaesxka7PWPeJ1gpIvQA/7/fce/zYe+/B70Xn/I/x/3ep7/9m48B2mrmTiu089xwe7+L3GCL", + "6np1oOtU+VtL/+LwfRLHLPWJlhM3Xemjk7riYOYa8uCSiD7lWxGdCCyWW2xG2fVBhBWRqjzz1e/eKy1g", + "HiuIwGaaTDckQ8XoATbeiPgVEYGWwBHRjCe7WghTJbsIa7sZhBfSp+R/oAAzvReMcsEFIixEV1TNEYb3", + "ytSKlz2c0B41Q+10OzG+/o2wmZp3DvZ3anyumXzD/tH79Ff30+b/8bK6SCPiYfJ3PFWUzRA8Ll7ruTFk", + "VzSrVsRRN41AzYspOzGfDet3UHdbYTeRVSttjLnGpdZCKHORrRlI/X5XG1uxx3R4uyBC0NAdy0enx2gj", + "opfE7hckUoZG6WCwE8AL8CexvwQ8jjELzW+bffQ2pkofh2l+ypsr28rtGgnmHBSVKOI3uU4DTREMHByt", + "PMdXkcZL7aOs3fqp/wuXqhdjhmcEzFH7IpoIfkn0QM2dACUSXZKl1nKWaKYb7S2ohBsewhZogY3XoT9i", + "7+dcEvOKeyTBt08XBMU8uDRXv3MOlvwCRymRXXQ11yoH+AQJjuzPyFyMjdhcD1IGPCGhNkLMazA1dEHY", + "4gLFOIFtjgWBPY5irIigOKKfzRU+3DKQkOoTbsQIbAyUYL3ng4CLEG7YOCI4mBeo8BeJLozCcgHNX1Cm", + "2frCbMzKZfWXztsP739+++HN8fjt2cs3hyfjX1/+l/7ZfNQ5+P1Lx4RqZJrKzwQLItC/fYH5fjXqbUhE", + "56BzmKo5F/Sz8dZ87XY0DaTmL5zQPk8Iw7Qf8LjT7fy1+M9PXz85hcy4sRd6G3gG9tWrDJmz1COSjp03", + "UCLrYXJ3G5pkWkS9PvuwpU/nBEup5oKns3l5Y1jV4EZbIqTyckz5eJL4xkTlJTrZeou04oIiqjdopqgM", + "B4PTn7fkqKP/sef+sdlHx2bXwvC1DOLC6k9yrtkni/o4OvuAcBTxwPpQpk0XvK4rn4AnTIllwqnPiKsI", + "p/zVuozq9fKnNxBFWxPKtqRehl5wM7oD39zalHjJFlRwFmtzboEF1ee0LO+VN2+PX45fvvnYOdAHQZgG", + "1it59vbd+85BZ2cwGHR8DKo5aI0MfH32wdx6wrYhOFLzcTAnweW6D3+Bd4/gVdhxKonS2VjSzx4t5DAj", + "DYpJzIWxvu03aGNeVlLMlkewrqPOzuufDV8OXwNLuvW010tZK6bhyo3g6599jDZfJkQsqPS56H7Jnjmm", + "qUcKlbaFuWDL+B02QL9g+gQRT8NeoctuZ0oFCSAyQ//rDxJrG2DxuXyj5fnO7zlrpfuuUWpxlFBGVmi1", + "34l2ecXFZcRx2Bves3Jp72I9UTXmQXl9s0s5xxK1YLUJZuEVDdV8HPIrpofsEcn2CcpezuTytZ4Jjv75", + "P//78TQ30YavJ4kV0sPtvTsK6YpY1k173S/ZRNLEP40PiX8SH0//+T//62bybSdhdJhb6YN2/V+aFqrx", + "NjYM0XhSGy6Vs4M/i3VR3Nri8DlyvLf2Btkn4/mCiAgvC4LXjqkzHID0q4xKUAiwRPY7LUYvkf54jRjW", + "rTn94HXVP7A98AtaQWBnj5MssnQV/d+Zt3MzxTMnz5R+1qLGHittJpLNY7h9av/crs/IPyF5SZMx6Otj", + "PMu8zauCUc8vaWKNAPjCcEEUGTkSpmA2TDhX/REzsTF66YE/yDUJQGRKhRU6PDuR6IpGEfimQCbVTyZt", + "UhSCquB1qfT/ipR10SRV2k7giiBrsUEnKYwFXp4QlDLsbuIrWrudYD2wAchySQQj0dho5bIlZcxHyH7U", + "SByY6hRLGxwnVJqU6XX86+k52jheMhzTAP1qWj3lYRoRdG7iGjbL1OuOWCIgQEJ3otmR2n75FPFU9fi0", + "pwQhbogxNJZ59+w18eL12QcbaCA3+yP2jmjCEhbaEGN3YNnw05Czv+gNT8Jys8X+K0RvCiaRDCdyzttu", + "rnP7er672rsxup1FkKTlJd3uNoafLqhQKY60qC4pst7QAhM+7zFYTHR+0XCyYjMP11XlO+G2vh7TMsTS", + "ewN1PS4bo2i1dtkUnAg1542zcL+0G+ya9k+YG8hKl1Vu5N6hr3PTSC1syPzcdTO7BZVOMppUHF33Q55D", + "WXAKtAp7N9FfRqGUaOMCJ7Rv+bgf8Piiiy7+WvpB731nmWj15AoZaoA8YfqnYvtVd8haR8WNAs2Li4Pl", + "7dfjUDbGWKHFECmBmTTRcXOckD76BYQ4UiROtCRjM0QlyoLKEONX/4G40YncpyOmhyZNhIolR+auknTG", + "KJttaitBH0w4DI1Pa5qqVOj3FlTm1CyzjvMb1eJpzeiIkceQm0FZEKUhQRfOt3RRVivrnqe6RWldUTUD", + "yZAEDCOwFdVWnCrdvZ5wjFUw13TiqTIha3bq5XDCin9r3VWuHUt2yXeL9T/PxEU1BWfhsZD05Oz1Ejgk", + "C57RJgekVVT8ztFLsoQld45QXHOFFn2gfk+lIJJHC2KP3aIXdQJJRtwoTrkD1bhCrfdTb/9qeo3PL7hu", + "KTS9WpO/bGl4kouk6rnJ5hxjjQcXje6kkJ6c6a+r7WpJgPhguRwgUMcuusbUIuDAQEwzS4RCKkigas1T", + "NhsxiF65sL/0bWsXepNrHeVeUrYgAwKU9uLSosLKOrUPmtFT4zFVioTdsm5wSUgi109Kq9fWZe7x6wty", + "JagTZC6cuaV6RtiUi4DE1ki4m935stCY1wq8WRP1YBJD38KYXWYI5MWQ0EQumfUAB28pYaSaNxlWrDYT", + "vFDu8gJH0QXasC9tIkH+ATkAdq0YZzmzvz86cyyQXbh/PO1qjtRS4GKuVDLW/yPHehdfVBuz37odnue0", + "PR+AfbW7u2NX1frszIArzZbdc96AjOalcep3452e5gs9Shvh0kaVP8o/yX24l5SFbRv4Vb/b6NzLFCNn", + "aTy0fy8RpJcmM4EhuPc+vXu3vrEFajZL8DUZxL4AzTw3MZWKx8VI/41KcAkth6GUibXgUS/ECoMntKW7", + "1gy3HvIcL01Txhbz+j3oZzKeTTwRS/QzJCHM6AxPlqp8czH05hHe9frcjcW3LE2pA8aCJOFY8dXB03SK", + "3LttYiVNpoPi48WU8tWJJTbyppT5Z44ja9fqJnpJQK07AXScYG5iWw0RQGn8eFq8NeyPWA+O3wN0nHWQ", + "NZs1iUG3xKG5eNngojAIk0KCJstNhNHH0z56n432LxJpg2VBXC7FHEs0IYShFDzXcBr2zFlcHEAq4dBU", + "1c+t78SkXWzC5Si3z/pZtjN4abLcbQjSmtDKfEzOJiyUvY3GrOgFa+W1WhVy/o7MqFSiEnCONt69OtrZ", + "2XlRdX9u7/UGw95w7/1wcDDQ///f7WPT7z+zxNfWYVm22LC3ovQ5+nByvG2dpeV+1Odd/OL59TVWL/bp", + "lXzxOZ6I2T928KPknvhF2XEer4c2UklEz4lJzVW+KL1CMFxDFN6tg+seKFYuD/1d9a6hxHv95kMk1fjC", + "tW2w8M3TXqoCc23Ad2FydUt+mYDdme+SggZn4yoD6o0gPaby8mdB8CUkC9bP7RjPiByb88wfSZFKE95D", + "rq13Q3CuptJcu5a9nsPdZ7vPd/Z3nw8GnlySOsPzgI4DfQK1GsDboxMU4SURCL5BG3BfFqJJxCdlRt/b", + "2X/+bPBiuN12HOaGqB0dMsPLfYU2LEX+5hBS3JPSoLa3n+3v7OwM9ve3d1uNyvqLWw3K+ZZLKsmznWe7", + "w+fbu62o4FPoX7rcnqoC78vpPDS4AvpfPZmQgE5pgCA7COkP0EYMRxjJbqvKe3KCQ5f56j87FKaRXBlw", + "YTqzbxpHW5xGiiYRMc9gQVr5omHmx9CSF5uDsSzT+GYt2YyotQEGbi7ZK6iU2VYi3alJpS4oT5RE4YHZ", + "oWvlHKxmPrBPTXxg59CSG37TplMvIgsSFZnAHF0mp1cQlPGJWbTSrChb4IiGY8qS1MsSjaR8lQrQRU2j", + "CE94qsw1o00NzzuBeGuwPaZaXLezc19xcbk2clWfxFkG/Fqv0CE40qfWVQOnOEb2a5ccUVD6sutAc2lq", + "n0v0znxhPET5z0laxtPpQk/Wk8SQIFJxkKTWYWibaatd+vUWcJa66BHTXy47Hyl0pjc10Qb3a2GLGQHk", + "B7VWY9Gc8h7eP4fXWwfC6w/XOlJa0J2Rq8cgOmQK9DTb9iTDycNQfFUsW+ZryF+CU1jQkPQR7C4IqnGZ", + "iZWddq54kpAw8//0R8xGkmc/SXODoj80dFBzQgXigs5oueOyg+0hg+JuwoqOm27NjsUP6xoqPITwjeZN", + "j6fKoDxcumQtUsycsovQ6XbOM0wMK4nKpHmX4YrUKJIHedaG+Prsw01D2xLBp9SHdASxEPaptcxc0Ndv", + "u4Pz3vD/MQGcmt9ARaPMxE/EPKxAWNj32508r88+nDWNKQOVQMXR1eaURbysgtVyFLGXSvZW0lowjv31", + "wZJ1kuveL3y67FTgmEzS6ZSIcexxrr3Sz5F5wYQ2UYZOfy7rs1pvbms1n5UWB8zmKQ4sJkA76nsccpVp", + "dAvU/ORfrnfEHMNNmYR6qYR9xyYT9tGbDMYDvT77IFEepeTx1JWXtzFS/2y+lDTAkWnRJAZTVnSwAXO2", + "1pDP8g+tK9KjJ/vRX9xGQBuLWZLCNjx/1zt5+3ErDsmiWxoTRBbNeUT0uDcL0mLh8gnztIKSkFg0eToM", + "Y8i2G6hAq2wHtyZSYb96qKO4wtFYRtwXrPFeP0TwEG18fGXyvfQIuigpLaX+vUCFEn/ve3eMlkhN3Z5D", + "h1WXaWmDe23HMg6nca8Uplfq1LdVTJB9XcepQy/xy/JC88v1cD+mkeZ+j1weQMWpbRVJZNIFEKQLuPtn", + "ZD41XmvrGpEkwQIrEi2NZpEdfRGdkmAZRGaPk/pVIrkmwQ0SEV7q17+aPOJUkLGaCyLnPCpfQO9067Bv", + "EoIgF8TCb5g5FRzviqMYi0s4GJ0ijVJmKFBOOthZB2k5Vyq5waR+ef/+zFjXioiFiSgrBvnK2tXqMYnw", + "Ek2IuiKEualgiTB6zTOck2pWjmyAQBBqnBBBeZmGnR1Pv+cmMBPNBA4IMl85FEa7JBJOzbaktL14oLyC", + "gEjZsL7DVetrP52mUbs19g1ruBa0NLjJAr8/OnOp/BkqoSPzdp3KZ0T0zJZz8ISrl3ZbrgaVcF0xzki9", + "M8EnBFAmbOJNMUXIBZYAiob+vJSWUxAOspgPY/uBXWBI1TX7/FMrZa+63X0X6TFmoQ/d0UQ8m9zHWRrr", + "BdFDFincHdHQRJ2YdEyjlhcDtwXBIWVEykriWJCKqNPt9KZ2VgdbWxEPcDTnUh3s7gyfb62O31sZuGnj", + "VMYhXWXfuWgWE+/g8qsMyiJMuswSWzhJWnjADB3XnA8gnuqBYoDoqM+2gobnIocHg1oO3DUOlAPGAZdY", + "6coTF7dtolmyNB9oMIMV3nvxorg/B9476BzV27H/Vo339cxMMJnmEeNuqNDRMPlnr0bFhQ/9hAtlM1wm", + "xEWbZeehi+Wylyqlzp4Pnhdn2Qo8GYRNZZvbfeeZqnm7nPVGCuS2+9c2AHExZZ3DbenVWGmaLmt4SkvE", + "GkutJihPCLsRPfd2d7ZvRs+2EzlxSXkVueTLuT86PTY6UcCZwpQRgWKisEWKLwgZ8CVpKaMN/BCTGPIU", + "pv+xWrQ0xC8Uk+gbb8CPanBrD3L73QAT9M7Eb4YoxoxOtUC2bxZ7lnO8vbd/YMDMQjLd3dvv9/s3TS1+", + "mecSt1qKLZM+Wcgy7sv53dbhATKI28zlS+fs8P0vWpClUphDa0tOKDso/Dv7Z/4A/jD/nFDmzTxuhX9H", + "pzXcu3I8mDb4ze8HBahyp/e0giP2O4MhLBTgDrwwMQrPtG5jOO6ueDC3RozLYUtVASmumE3TAjWOfl59", + "jey8SvCO7TNlikY5oF79AvlWkIhyJWpUDTEqISzDiYoi81fA2ULvCh9oVOkkcs/uFHyxUvX6e13jWr/f", + "nOK1hm39XrZM/rUFy7OQNp6T6JtL/dsEKJV7fzv7zz/+X3n27B/DP377+PG/Fq//8/gN/a+P0dnbO2Wv", + "r0Yz+qaQRPeGQgRROSUoorasdIpV4PFGaUOngcL2ibGtVTDvoyPwmh+MWA/9RhURODpAo04lv2rUQRsE", + "bAL4Sit2uimbJrqpPz4zd2f64y9O4ftabSO0+aDCLkiWRS7TSchjTNnmiI2YbQu5iUjQgPVfIQpwosBx", + "QRnSlt4STQRUY7F3G3nnXfQFJ8nXzRGD6wFyrYSeQYKFyuDXXA/AFHZUJuDSvk5Ch+djrhdGLDuXMjgf", + "c8HVz9RcCGyopqv4ibLaUrE2wvOBD/gIQub1QkZUKqNsZ5yt2SiL5UfPB5t1y2WNNp3x0Ar2g51QL9Pk", + "mLLFXjIMDF0bwT12zrg1gQhaNpk9gsBWUhz+e45cQzktsiU2HnKTQCHNBauKZCF1YrPjBYmH1W05IXPD", + "CJ9FLVKuX5rcmve/nSNFROyyHTcCTc4pDfT8IHaSSplqVqQYHR6dvtzst6gzBbTNxr9iHd9nM6xmxtob", + "x6aL1NywwzHpopNjyG2yOzRX4CAm+RUXKDICJt/XB+iDJBUbEbLXICTSrGS0zK8tzQkw6my6FpOqpDhA", + "7zK9EWdDKdW2Kt+E5vsSmrVRKyZgutZ6t1Y1Rji7yIo2CI/GKsuw0ydusyho76iwFIc9XzGrb7y3izfJ", + "jUZzYe3vG+zu/tWdnZupO65wQTLH0sfd8+JVCLy0oloQrfh3RfO9f6nftQV7su6gBpE++oqf+0qt7PWG", + "w/fD3Zvb/DfFKSsDehSwZDKosvYYYw+B1VW3f6+pGjdGhCL92MZ/Oivv4ymaY8n+ouBhxdYb7jxrhTqv", + "e20bS1mMouRTM6RMSjl0kCwG0OCkXNIoMqG1ks4YjtALtHF+8vrXk99+20Q99PbtaXUpVn3hW58WkGVO", + "VLw++wDXaViOXThScwYOzrPYyDWVStbBU1pF9d0FIs182g5W3E3StJHji6/GWfulhIXmRcPZvEeANBeK", + "WSPjY0Cffcscl+8Pdm0lUNpd0c6s8fJAYGeNwt0HFFaW8+bn+4Ute5DhrC3SVzzrXQLirXHCuh3qSb46", + "lFoEkxCdnOVQ5bmT0TVfmZOteDkcDPrDQRuXa4yDFX2fHh6173ywbVSLAzw5CMIDMr2Dy9cytlHGcXSF", + "lxKNnLk06hj7rGCYFbatNalaXU/X4dhuh75WVWj8ctoodu7eX9ryNk3KTYvsmirYS5xGJoGzWBKnXuVT", + "JgbZzGACZ7rsiMEAuxZhJavMiYNApLk/w5VcM5pvmli+HzFBZMKZNIUY++hXspQopnCHkHUPkUMSZUHc", + "4YhtCBfwn0X2JziVJNQ/QDRt10Vt6qFRBdjH+oMRk/MUKsht9tERZzKNibCuHjSh4IfeRDI1xh2MF6gB", + "9UwlDYkYMf2aBzvtS6aoH+wPBoNBVtmuc7Cj/z3wcdMd8fPc5+1UDvt5rmv4eXgd3F47HL27gpetKodz", + "Xi6E09qiu0O1yFbB5k6ns2Hm9qvxTe7KCAp4GoXaTJjog8F4cUhonU2SqLzGEJwlH9gl0+xcmroNclMc", + "/ZESsUQfT09LF2yCTG0JlRYThw3VsA48udEybK8xrNeO5paQdo8BY1c91AvK1L2D1hU9/C7dzXBoC09/", + "blx5Q5QpM0uj+WTFnCo+2pAsxmnq09n1I5fk/uHDyXGJOTDeHz4fPH/Rez4Z7vd2w8Gwh4c7+73tPTyY", + "7gTPdhqKmLVPUbh91oHXQvNVpXKxh2MXA+ktPNwQgVo5Im1U3RVlIb9qVbI7692GUK3rvh4g2XoI3rBq", + "qHQMLTVIidNCPWMTVlipWdTgedp/Pxiu8Ty1K8PcIH/fi5QFBn0KJHHm0y0WYC4uVr0I683FKQzIhS+v", + "o1ax8/ZEGxzsvTjYuyvRXAjuujFW2ekRF7cp4MBBGFZifF2eScF/UahgDPqGcbPakOBOt5NFLcPfcNBW", + "IuKyx61C8Zs2bNcvRlbJ74aMtJOS2gy3sAbJKDzQWkCWzDRJFcoSHbV6cRTxNEQF348BdoGLkZOCCq2b", + "gXsK6xoyCF0mpFar2gAJCYDElGlBDBdCuhGbvnaAXsO78AjHxrqwg8AsrNyF4HBp7oL1/nJdG11/9ZDP", + "rZoP32idH0FhcT1tTQbrIlzdhNF8DtAbDt9kRgfjVV+jeR20/frrVb/khs0rc/nH0JlV4w7Qq0x1y5Q/", + "q+xtSGL/HFuBlaf9b5aSL+2KdzS35CtXyCvsdgxFO92OIxTkH9YzET/kXF/bf0VW9AVJEBzBXs4zvVJF", + "I4tyCTOhUFrbhgDrxW3SLywgPAnHxjBpCnky6UPWeMk+curLx1O0AXhWf0PWkar/tZmFR5XOuu0Xuy/2", + "n22/2G+FWpEPcL3aeQTJbfXBrdVBgyQdu6q1DVM/OvtgTPDAGLfgmbdzLyQJJ4Jr0aNnnpfBzTt/0X9R", + "BOsIeWpKgtshWWSfr4XC9ytrFjfE+PxBowWdTtkfn4PL7X8IGg+v9+X2xOvQzCvse70/J8Xb3pqrlEx6", + "poiHH08BGErIRsiRd0TCDNA5UQj4p4dwAKZDlpNmWc4Bk1iKexlrd2dn5/mzve1WfGVHV9g4Y/AFeQ5l", + "O4LCFoM30ca783O0VWA406ZL1AWAUGbNSv8+Q7YU2KCskPaHgx0flzQc3DnX2LYXcSPJP1rTzE7KEh1S", + "6zKzrbbLvdTe2Rk82917vtduG1sv5Vhcr5YwLvDckMfi2BZXfgO0yfeHZwjSuqY4KPtNhts7u3v7z57f", + "aFTqRqMCDGaDnXqDgT1/tr+3u7M9bIed44sCsKhQpQ1bll2eTedhCs9qeEhRF73dptPCp04ZBntHggjT", + "+DBwEbyV08dgpI6FeS1fhDYHg3WM1w6uFt+2chxVSk4b1YALlLIMmbu//grwdjd6zWLanAfrxXjdso8w", + "0+SyIA+mFMctaJcIsqA8lffQEFcm1WkacS5u9G2ThfKOyDRS5tqNSvTx9C8gRDRzIalIUjaaLPutgMK4", + "5eRutIFLPOHn6iZitVqNNku/asLdhm3aXZUHXdr+jYgzoRZVKVsffneEoyAF8HmcraeeFWBHQCZnkkRL", + "E6gaRZwzFMwxmxEoI2hKXbAZwmjOo7DvDR7UT8ZT77U9v0IRN1iZl4QkFpfdDEJ/pnUWuiBoo5BJigwr", + "Vcoz7cVGqljk7TI37sX+QkBY+pIfsgxGTU+seAHG0XxS8jFGfCbBClQQgtuvogcnWJjIWsxMnYFFbIzH", + "MvTOtj7tPUOsSG/fEWqOTj61Fq3VMSA/0FASB4JLiUhEZ4Bp//G0kna2IociSz5bH1JXHmwL1jUXaZ6z", + "C8402bocie9A9ASn3+VIBB6GHJQVwWrOGxljlgJSe4GRyXVChWGPdgFpcy7VOIMTueFgpRoDCncqSI45", + "lCVLZg4g9473XHSi7TbkspGft/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuAKisxXHJQmCoCyE0gf3LYZiqh", + "VVpAm0EbjKuSWCpAD2+2Cc7w26i6n5p5aqtM/bY7OG+LxrMafOcMq/kJm3JPyvYNriGd69nGCyZExBRw", + "6FFIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJbgmOzvSHtmjmnGGWziqyvdtjGH2zGsBqkG/q1L7YJs5H+", + "7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6", + "kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4w", + "ehnFdnd70JR92dBoKe+yDuh1U8ltWda34x3W1mFWNNNzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQ", + "gst0LaD1tvKEtIssr4b8udFsSRKUe999vvdsvyVc8p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fv", + "xfaNHFQuOrRhfZoiRIvrU6mNW3Ga7UEk1eBGgzLxof4hNcSIlgdUqnN76wF9XbF1m4IL8r3ZdMkZFVfS", + "3bOUPaDtfIwrtKXDkspVqAK/QaZTAkbl2NCtlw+mkiHYagwBTnBA1dLjMMFXpmRf9koF7q2NN608WA9J", + "bdsWGkhLLplO8iSKDdc5+qtxrVd44Xlr1HWZTprc+G+rvRonfu4DKl4RtbihyQtD1t0F2XyusCzFmum/", + "A4iezKv8V2NmzRuroaeqgB9wCWiLCxQiKXyQhZXzz35UXP7KchbcviUluUrxVUdo8xa8kQ3tOZE9JnSw", + "PhOmIh/sAXi7r8aTYj2ElQUnSsUT8lP35v22SPapg4VmJ9jN+yukPdzkwyowFvCjHYMled52t8QSDdxU", + "CNP1mCM8Ir0szsHG8CKZGv+q3vMWa9GThxFc8um0DPi01wwQCMh8ELLtesFKkThRXUSuwUwnYQ1dztXR", + "3pOjDuICjTrDeNSpOAG9SRAxvh7bDsq5yoNViH1ZJaDqIKWbwSTiwaWB+leCEtlHAxQTzCRKGWz+io9y", + "OFjta+t2ksLaZPh4xNwQ18QWjGlC5nhBAZbVeqhmpTgWck2VhHgbaOcAhRzs23KdIztD/ZpJUTjIJw2H", + "DmZL27BuUL/HmQsIyt8Fc2kK1ZXYZyJ41ybfaYn99u1p19z/QOSGGVgpPMRN1IxAC8isiwrGaP67P/xq", + "EpExjLuKWRnX6VhMJQM/tSCSKGlB7HJ2qDABCnjKVBXMMm4XwlmOeK8fSSmDWAl7ewaQDbZ3Wx0xJIGt", + "QFt3L5UY/RbMXQm7tJT2xV3u+FgYNgV45vye93fWvV4dgERYkGKJMtNOMSzO+FzHUnGLaZ/t6jG5DggJ", + "q3A8/lfahhraL72hhr9hm/OeVQ+zb0O4WH12/YeLPYexNlG7GBLJOOtBtq1bUpsZa1BDbO51mdFK+HuF", + "DNSxD/zI90KbvClyvZrWb8i1ApDAMI00eZtY14oqexito/itky6aNjQX64ttPkDtBBOud6vqCTbS71EK", + "KNifH6RoQm05zoly755bvmmulVmCNy55BF2ApHulfEVpeKeL7ImOhvFmhed25343iAXoapkvwnBMxokg", + "U3q9glvMC8YSLmcj5zunVE9Voo0YX6PdZyiYYyErY2d0NlfRsnx/uesBA7hTKRFBFGHqBpVn89V0H9aD", + "BexyFlv3acPnhdR9f0VcEo5XwdgdZa+569gEL8Ft0+hjfbazOxjsbA9uhWN3X4V6C+00pSAUvrP3JKWo", + "nmILWcJXvZrTlaCQN5aRSSpBcHwAgcoJDgiKyBRQXrIqemsPi1rXqwdvNSibR57xv1sou27uCqMMFJ11", + "ZSEA3TQ6LgCqDB1QfF4f9goomEzMBDVMGE+Owk5vsP9+uHOwt38wHD4E8F1GpKbo2Gefh1fPom083Y2e", + "L5/9MZw/m23HO17D6wFqQpdzPqslou0cEiKqZbqq5e0kiSgjPZlFlK9P61ghC0yQxtr9fzPHvpnBSmXh", + "vDzJos6AVU6cEmc9Ek6GHf3K24nq8E+OVw/7ViHa1YH4Gaw6FOCndoMBINbhXVE/U9by3PlQeLH1ybMy", + "bWDd2ePL8oSt7V3lBor7+LkkGEs7bNWJXT/VPN7RGRdUzePVx0P2WoYhCHFmn6UKy7gMfXQyY1CUr/hz", + "FlZQNJP0x51uJ/q8W94z9vf2CB0WMy9jQLvURTWgxbU71HxcTQV4JTcthIn809a4HvNPw97wBQS/RZ93", + "fxr0XvTR3wtBeF1DrSL5hu7t0q+DNjQsVrpwCOnDFzeKUHP0XMVBv1JfnYb8ILZoepbH84po7qxwGUml", + "Bc4f19a4AiPQqHHeVbWzp9m4qCWFJMJLXx3rgitWVuzDIpOhCZlRJtt4ZncGmWt2Lx51+ujQYlCCtZoX", + "vCw1D6UOC3xC45iEVCuVxrhvjvjcbultqxoPNwMmdl951LO+Xz97sT6HdF2A+rpjsn+HhKU7mbvtTNxV", + "6c3gOXM2KZRAgRe7iE4RZpU6PLbqrM00hMwRQJg5cIAqOctaGSBzxc95QrpoxhXKcwxbetRS1uz5y8ZP", + "rsGjuiKp2DDE9r1kjGfoJXSV+Do5RongYRrkCTYRDDpPiRZpBcxxhVa/PoTpIR0akLk25QKtd2g0eTDa", + "eSCb1rvifdQM27zUw8H6pX4QL0i3kybhehlmXmonwW6ENbomZcPjkymTvaIJFibzqYVEf1ekYN3INd7i", + "QKtEaeKuUDRP1TnJc6EClwg+2L1jEhF9TNUbQTwK86hSKnMpul6kDvefz5suMeHOqT6QXwlJtK0CABHQ", + "X4zZ0juwapFVtDFwlbOkudLqGaxyS63y4J6t1cQal6p9vdqKV9sklBfLA2egm/dbrNZ+ubaU+EP44b6l", + "kvbWXi5U4NUc+F8GV+r6NxkzAM/Iosp5veu7gPexxXtrGTdBxlWzZopu5sPefxu3Mhr3D7Z++tv/3fv0", + "V697uWI3SyJ6IZlCKNElWfYA7x5pG71fBkwDrF6tTNva/orgGJxGwSUxTqoYXxfHuzfIhMbyDY5rU4AY", + "rJiy7N9rJ/S3f2uOYCqQ8QPIybUse2co64eoE6S4O442YiJmrpKxC7zf7I8YxPJfkqVEhWIEVqVxjPoX", + "mX2iVXRwWuIIXRg1sE/Y4gJNKNR0kSOmrVocBCTR1oQFZaemGB8H6SMIjort2KIILlHOXjmaiAGCPp7W", + "0Pbefnj/89sPb47Hb89evjk8Gf/68r8giOOqZ3oIe5r3dvf2bQm+IiWHniW+A/DvnVD8fOxmsMA8/AVJ", + "KVDX0KMwUwkppS6goPAy2iBxopau2JDLbdm8GTbZYdagN5ztnlHYBy/uo+jMh5VVZhY86mmNugFE1+vA", + "NLTwhmNDUybMvdPk2J55yoCfW2/ijM6wx5ftrUB6H8Vh3IDWgsbV1r+xtIM/OP64ijBspIEhVQURt2KX", + "StVrjp2PtSI1zms9liMyUmbTS2ghYKucSxIztWWLOPlSWkN98q5OKMp3mUMs6sFH6/NkVqryhZkVRtK8", + "NqdOY63o1CsIdKZJczUnghQWAj7IkVtvSDKb7NEiUdqUWkmIyAMhXaYI1D4XFLJHMmeDI0GWEFT3wK5G", + "5j3F11kP4L3HsnbHBfPIMfKHr38edTb76J0rVUqnrgkYRsWe8AOhlrloFU0cV9UXo8hV9Xmb970bz8qq", + "FdKvaW9VmDPvo8SaPn78O6bqFRdggTSnJT84nipYNyERgMtSRUttBTVKYxKOs4LNTfvf1Wg2OclZOey8", + "jJOztjAwsRZy60vtuMTZfAx1SmtykCAVVC3PobCriRAmWBBxmJoN7+rD2p/zjqEq0tev4KecerIQXhNG", + "BA3Q4dkJ7McYM1DS0cfTQiUTU9SmhqEG6uXboxNr4ToYPrBYqALWc8F8h2cnnW5nQYSx8jqD/nZ/AJs5", + "IQwntHPQ2ekP+4OOqegLU9yCIorwp00wzCylk9DqQT+bV/RXAsdEESE7B797EvUgkA1eBn0XzwoWS4Kp", + "sCZLEkH6oGEVqr8FYF13lB6Y89gW5G3toJNqaZMpSPLWLusnUCdh18AUtwcDCzOq7MELqSAm/nzrHzYY", + "Me+3lT4H5PGgzNYsCqdTWpJ/7XZ2B8MbjWfVMGDH+rr9wHCq5lzQzwSGuXdDItyq0xNmMryQwQqzgTbF", + "fQYsVNxhv3/S6yXTOMZi6ciV0yrhskkZJhJhxMiVrQj6Dz7pI3v9AGVj5JynkZYmyKSvOUeDwqI/+4yw", + "COZ0QUbMntNxGimaYAFuhBjp89kYTOWtYbo2q58BC/zMw2WFullzW7q5nnM65wSupiVIMgYc4nFTud/c", + "3UwZ02ISS2JraWR1L+vRPFpcjmXAfflE7wnDTPVkQgI6pQGCl/XutR5tb4OtoPm0wINlIQIQs5yHZnvT", + "n5MKVT/86dzH2TNkyVtWJxhcAwVRGuY6l0uTwmKCo8iL3TSL+ARHY0OfS+JRUV/DG5YoxQIpTrlhPCSm", + "2EWyVHPOzN/pJGUqNX9PBL+SRGgVyBYxs7QmoSlbZlj3CrA+YygkZkqk6j63zBC3vlyS5df+iB2GsSt/", + "K80nOJJcn5q26KDNCzBb2vCuvyxLQ1zJUSoVjy1LZRUY82HyVCWpsnfqkihbeQ1epxIlqZyTcMQUR18E", + "mVGpxPLr1pe8x69guxAcaj4pvGKmtPWFhl+bRi3HWM9+DK96rD8CBBh19Oky6ui/ZwJr2yWVc3CiSHCc", + "zIpLupHh6Wi9cLNK4QAzlPDEYBEBU82xZrlSGxCPjqMIKdhK7lutbcJKNszHphfHk8bcYpMMWtlGlKHT", + "nwubabD73L+fJAkE8Tk4/vP87RsER5VeA/Na7rAyl9pMn6IoTEGTh977I/YSB3Nk9CbAmx11aDjqZNZF", + "uAljTaWN0O71QMX9SQ/tJ9NNl4Y/9fu6KaM9H6Dfv5hWDvReSuKx4peEjTpfu6jwYEbVPJ1kzz75CdqU", + "onleEgRow8j+TVeDGKCi8mPQnBuYhYhbWRstEUa5BCr6USaUYbGygLKH9JaC2pTHM1kkxpcR+G5HnYOR", + "896OOt1Rh7AF/GZdvKPOVz8FrBLdDG5qakg7XTtjov3BYHM9doKlr0eFLr2ot9/Xmva1fW+Kh1W66oqH", + "mZxDZtYraKqBG3XrETSfn3Ho6kv+UPHWqHjWc1FQ3uD74jlg2DcixsCtaGDano2cBrbSOjFsAQUTwOJw", + "SCfG4KBOg8uZt2h+VM35ulmx27TLAhhi5Phv9xH4D/rNShGafl88Vr84ApxxB1z/xNgRFssxYtdvEb8m", + "6nvguMFjiVILiv4t+fep8M9rYvW+nGgVabZFFu6+yY/nBMkm0rZiXta26jmMqXdOmEIv4de+/a+zeCBb", + "+iLis4sDZEgY8RmKKLP3gIXbIn0oWlrCRybfJPvOpp84MM0Nc37+83/+FwZF2eyf//O/Wps2f8F23zKJ", + "kwC+fzEnWKgJweriAP1KSNLDEV0QNxmAxyYLIpZoZwBqZiLgUbGkktVN5IiN2DuiUsEK96UG11LaBsH0", + "YDAfylIibb6OfpFOLeiWcTB7THi3lw0pH3VHdz2JzjCDwgT0qeh4ALJEbfk1a391/N4zM+eS/6zqK695", + "TNfLF0WuleHenhngDQUMkNi37+CBnTTaOD9/udlHYGMYrgBgNdCY82as8tz/IZPWyyQjUcoCBahsZJNJ", + "Ylvt/z2277RzANsW/0weYIu1eQMXsHF5QO66W4EftkILd7Cfbs417PPPHrsszWYH7e3nW+zCxTG1MoTv", + "b50d79Vpbp4USPYtTGC04aLhwY3IBTo7OnE1bTe/GdM/yqmhZ2pr9WVHB+IMsNcezSw74mwa0UChnhsL", + "FGaKSWaqlRnkqYiDd3bUCLt5VSGMi+fbVgmRr/Gky8D58iPv4U+PSqc3OUZymOWc136cJOtY55jKgOtv", + "C9zSC3AChHTqS7ZPi1y0ziFlguuzI2elumTF88mx25CP55qyXaesejY8glA8rgjEbygIK8XjC8DkT4mb", + "P2Sr6BApVniuvi/WHDyeFvTYXiwfmz8lN1ZYIZuWgiaou/EAfU2UCeXuPOBC2x48Ez8nwu1qV0cCZp1N", + "y3xqCquaCcGF9Grb98S80s70Ne39mSxfIM9NNBZL8h8qSgtjN6fVKgP3xJYsfzj7Fnq4kXl7f/e8lsE8", + "RIZgk4nzWJu6u1guWbD5p7rqfZTTzBD7SR5mZ2kUuRuPBREKvT06MTureAZsfYGwpPW6vdttK4+DD+9+", + "6xEWcIhDy2Ko/EqUfXLPGr5ZMDOVH2zSxiY0adHUnWdNGs4d1t+ECyIT4din/N+3X0V0IrBY/vv2Kxwl", + "lJF/3zmMsCJSbT4YswweSzQ/tsb9hJlPK9y0TDQQTQzqva/TULO3Wiqp7v0/lZ5qJn0jTTWj6w9ltY2y", + "WiTXSn3VLsWDaqymj290JZMxm4/a8MjFJ/7JNNXH9fJZjnRA0VSWrz1skT0uwM8LjyhDqSRPMICSZhxX", + "PDZauqvzDbny+HCse3LcBUJCXQRAbbIJIo/kvHbjeHTl1vb7+J7rw3hCZylPZTH3JMYqmBNpk5UiUhbA", + "T03tzo/nRsX7O+bSwWMeHY+uV//g+wfS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2ahpboOdnCzIajQ", + "JVG3ZeNSrnk92NE3Lp8tgj5oQyU3FxBYEAcj9n+0/fG7Ijj+9JNLkkkHg+19+J2wxaefXJ4MO3WsQhhU", + "PAKU2MM3x3DtN4PscwCSzVPyquMwlSeA9Rx0zr+cgZTffLa3kBwX/rCQWllIBXKttpDsWjysiVSG33p0", + "G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF7ztBYkZksmP8HcMmbvhwrBHaWDtrWVlG3KNQpoXhbg0QN7TnIc", + "xMc2jlwFgqcZI88TC+ltzZH8MGy2R743fhg8rnB+fDvkKbOYUfjrpEu0TukrpQ0Yk3EKZSFRjhACUZ9I", + "2PqPrsU+yitYyzRJuFDS4FSCAmyQ7OdaAfZhWpZhKn24lIDFSInsjhhUKtCPTS7/1iVZGhRKylkGOFmt", + "x+rLvSqjgH7TbXT/OpYf4rSVjvXI29iCVn87HeubiY5H0bROSrUANrKNAQblhGQ7mWfJffQzZbPNJxWB", + "aoRVNrcCnpFH1dqCSn8W13dLZtVkmw7aArSvLT37L3ji1ifp09odFm2BgCikeMa4VDQoVt4twoP+OKFb", + "n9CrKevl5qktke436F9xcdn2iPPUFXsCJ11xht+hL0EPD9DAvr1LAYxtcxpopnn0U7BWLO5bpmDQ6rkY", + "RGkIVentgehUyang8dj+aPBq9a6waKDgoghsq99a2OjeH8Fh9IYrROMkIlqLJyHqGW7Sq2lVfwc3T2Wh", + "tOLNhKHeNsWEGANGJ11pIisi4XLNLdgG3LPXl8srNSM+Ww+CkXXuEB88KBgjZuDwicPOv0CZkIXiXSQi", + "gUJXcxrMAREDCnpBRVcAq8BJcpFBYG0eoNewU4tIYND5hiRCG0IBZ5JHxABdLOL44qCO2Prx9BQ+MmAY", + "Bpv14iAruZ4dEFK/VUS4yGoevbG4HRuakwSPIrOiF9pqLMxv02Jf5BBlI+bDwWDkyjZIp+iiAIlx0YCJ", + "4QTqb3z2zbStbjOwpJmL4kgA4QxvEhZ2mi5iaORHwxgOvPVgWiJzmGE8MDBHbTC/8VkGalliZZwkbdnX", + "DhO4eBHHK3gYbRSKs0oV8lT9TaqQCAEfW+5uYm60gQPzD4UvNaPawkJZeVtgP+91o0GZ85JKC9VCFR3z", + "r0Ucd7odO54COt0NtPc1CCfVBuvXYnplCjAmP/TumwCUlIV9AaGkcnLY8v/NKvc788Kf3j9rCRX+Gbws", + "5fusfBSU5QWgBFRwd1W4nhTSASxkTRczhZF8e8TNsicLxUPbXW/Vyo5+B0bruluvrIZkVuDysa+/6iN4", + "ykkwsjabKRfV9Ph192LfPSPd35LUptqGQ37w5s3dc60YM0lX1BKFUqgS/HxQXxNwnYM557LA9hMyxwvK", + "hUVgt17XjDPBZWGsRxs9d6FZ9cL6by+sen5gfU0IFx/ZPvrwuY2583/hHuVfvCpY25nE7zqVGlAgJcJo", + "IiiZogSnkmhtKY0JMhVGLJA3wcHcVQvvj9j7OUG2PmbBgZCVU6YSXQzjiy6apApFWMzA2jEPTSSdIAGP", + "Y8JCU/N2xOYEL6g21QSKsCIsWPYkgRrIC5IXMNGmu72hNKW2syqrXeSK84KD4aJQevcCJYIAExlzmZXq", + "3I6YSNl/GORK3eyFG+gFIlLhSUTlPKsVEeCQsMALC3n+fYux+3finhNVr077Te4sbyVLv+UlZtGXmdUH", + "/y7uN59YoBYXrrJmCzG/QumVzaZhOfLxPK/I+y+4pc1c3Ry/0c1MRuJVu/j7uJIpleT/cS2j7JYMU9Md", + "KZet/9PeteR1pFNWum6xPtnbXrhklRAyMt9I5m19cX+e3MJH9p1Iwm6jYd+EuZ1P+nsQuZaqt5K538g5", + "aH1JBa/YNxTBdlDfTn3ioiDlvgsxbDZcJo2LMkcJDDYVZz+EcVUY2/CA2wpj53GtXYAXxDNlvSTCTXI5", + "r1rvF8DWIfAvGv1amV1BEH5zwZffCDyasDvJxJsReAleRhz/2e9lAi6ESei05YifDqBYwRdYuGDaAI9b", + "N5MQXZdN8vH0dLNJSgi1UkYI9YQlRLmsaRB7qjW+XRAhaOhKRx6dHtvoVSqRSFkfvY0p1HO8JCSBQjGU", + "pxJBZm5fz8+lttaL4JVyWLsdwpRYJpwytXYU+asPM5ivtyqd98hy0kIq/ukvj8EL//SEFMgOra7YCay2", + "IhVWjcF4LjiNMlPvUmtbeMJT3bqWLK7Q7gzOtimNiFxKRWITmTdNI9hEALprazLZ70xGaRdRJZHeD13I", + "wEuIiKmUlDM5Yrb8e0KE7lt/DsV/8yAjr/Ne4UxqnhnR930EsOnBmJgtrJqoBtACUAe0c9DZwkmyBeWi", + "/UFSdnh3GNIriEhDchlPeEQDFFF2KdFGRC+N0YEWEkX6j82VIW1j+O6+K07dfmdpSp+wKfcW5TA8mzHz", + "nyMJqSzW3CXikxNrr0lxszj5AwvtF2tyrVwTBEc9RWOSJb+jVNGIfjaiTjdCpaKByavJUy+hCLPNvhyx", + "U6KEfgcLggIeRSRQzrmylQgebI3SwWAnSCiglOwQGBwIvObHMfR4dPYB3jOForsjpv8BDb8/PDM3sVNs", + "fQSFgTKirri4RCdbb9cE+Z4Dmf6Fo+TMBFfmQHoX/Mf13c0zmxv3kGzYojxZZQDx5E8fxmk1uB/egqfp", + "LQBoiWw2GzOBA1CK5TxVIb9ifs/AgkdprP9h/jhZB1CicDD/CK9+N9quGc7abtwEn8SmtHMKiSka9E0u", + "KAzBnmp8qSacmwIoMaXIPe8pcKj+jNx9/075Ih2/w6tJS1FXkOu72VuPffLZMTjcrSI9nso2N5zmZqL4", + "au/TFabN3qefIx5cSpQyRaMSqIG22wAHVP+Y4zbaiz9QEyA70pUSR+Q6oQIQbCrwCIjoGUuEkSIipgxH", + "WzBn0wggUDovFl5wCknKQUQhTYyGBCU8igBl52pOGNKzAUeVa6BwTyttBYjiO8UrRsXRhAQ8Jg6Vc9Nn", + "uv0dU/WKizLE5vciF98X6K/no6eq57kGVbS5xzuhjJ7iawhrDlN7TexGtPGa5z8aV1AXwdqMOjsDOep0", + "0aizHY86egWOMLhQsUJ7KKYsVUT20bHxb0Ea6v4ASRJwFkoHDuo8eDsD2ZSUatiyIcNxH757TLXHchWQ", + "8p3txCce9HtIfw8JNmijuOHsngy7sOlCxFMFAdxuX9m3QqLAPbL56DewhT3yw7ZvI8n/brdvSUbBKmtx", + "WVh6I9kz+Mi1XjeXVDHnMkedRAFOcEDVsotwFPEg9x6kMrsd6GVDmQiCL7UN1R+xdxlwpU2EQEdnH7rO", + "aYZCKi9NC9Yv1kdvF0TIdJINDoE0MB48WAwSjpjiKMBRkEaab8l0SgLIYYhoTJVs8KtlQ3nIMoh5J56F", + "dw8z2Jqn5Uzy8wSsXs4WssJxW2aptwQJIkzjolOpShxQfeFKF9y+E90o18fwNLLXW4HgUiLbVI9EdEYn", + "kb2skX30XqscOCYjlkSYMSJQKk3ckR56LxFEytQkxugGoM6s4aguyoFOEsGVdRNHnAtpPLuawz+eIqlI", + "soLN3pmWT2HODwQTbBq3PX0jg6EyhuZjyb6C9IIYTjEE13ykj+lvEOxjBvSt4YSfysZ/L+hsRoTeFdgI", + "WXM1ara1I6fZ9KVMj0aM/PPsrXYY+VmrhWjuQqTzSqCKsXtxDAr0TW5gPZ1f0kYsE/voZtkXv+qPWvZd", + "jvL3D8I+uuMs/yylx84LwdVtkfVzDn9qIPeFkZe2ailBYT0cQeuMhIfMEGiNO/DN4AaeMsoALqUdNMEJ", + "fH+MMHjc7LjHhtl+2rxVQgkoFdZpSJVaD9/5XXDgw+B2fuPs0Fvgdn5X+UqAu/jt8ka/q0ylkh/QFQ/5", + "0yNzPlSCkoHnBBiLpgQlI/VsIMFKQ+mjfaedmWRb/DNp8Pbu+Qb6uyP7D6u/hclQIJbfZWdyox1uC4kT", + "tXSXi3xauQCU9DMkY/iAH7IYgofDW7jF9fr9sYfj08bL9R/1tB7t/j4vOnxy/PSLaBX3XOlg2dKnTg+L", + "YE4XpNnpXt7BlkSJIL2EJ3C5EhqCWXq4s0xh0Z99RrZ5i1Vl/4WogzgmIQqpIIGKlogyxUEimD7+IpHg", + "2hKA51wsfc704s59JXh8aGez5jy0e8o6w/I733jZC7HCvYWTNitcaHe4aXd321rgIcrQ65/RBrlWwiDu", + "oqm2fBCdZiQl1wEhoQSe3CwOeDho8GzSz2Q8m7QZ5Qrs5LcWmxoFqVQ8dmt/cow2oNjCjDC9FlrVn4Im", + "mwi+oKEpRJoTdcEjQ9VhA0Fv6nfVSkVWKcMZF2Zw30SHaXMgzT7TpCwWTOhC56AzoQzD4NaiFJf3lEmo", + "0v1hCmkN+d5xnNP5cYRZy2/DGTuaE7WR44ioODfQeJs/jrmnfMwVA1PdmVY67dqVimwXq9oyhPQhAHOz", + "OObHdVt//H7CK6l8kpGV1nW+yAzSJrf598WCg8c7Hx7bXf7xCYfjvybO+C64yqEB3aKPYX7jAY5QSBYk", + "4glUkTTvdrqdVESdg85cqeRgayvS7825VAfPB88Hna+fvv7/AQAA//8QadjqX44BAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/restart-policy/README.md b/lib/restart-policy/README.md new file mode 100644 index 00000000..f8dd76bb --- /dev/null +++ b/lib/restart-policy/README.md @@ -0,0 +1,88 @@ +# Restart Policy + +Restart policy lets Hypeman keep an instance running after the guest program exits, or after a configured health check marks a running workload unhealthy. + +This supervises the whole instance, not an individual in-guest process. If the image runs systemd or multiple processes, a restart boots the instance again using the same persisted instance configuration. + +## Policies + +`never` is the default. Hypeman records exit information, but never restarts the instance. + +`on_failure` restarts when the last run failed. + +Failure means: + +- exit code is nonzero +- the guest was killed by signal or OOM +- the instance stopped unexpectedly and no clean exit code is available +- a configured health check reached `unhealthy` + +Exit code `0` does not restart under `on_failure`. + +`always` restarts after any guest exit, including exit code `0`. + +## Manual Stops + +Manual stop suppresses restart policy. + +When an instance is stopped through the API, Hypeman records an internal suppression marker before shutdown begins. The public instance `state` remains the single lifecycle state; the suppression marker is exposed only as `restart_status.blocked_reason=manual_stop`. + +Calling `start` clears manual suppression and retry status. + +Deleting an instance removes it entirely and no restart is attempted. + +Unexpected guest exit does not set manual suppression. If the instance has a restart policy, the controller may restart it. + +## Attempts And Backoff + +Each automatic restart waits for `backoff` before another restart attempt is allowed. + +`max_attempts` limits consecutive automatic restart attempts. `0` means unlimited attempts. + +If `max_attempts` is exceeded, restart policy is blocked for that failure window and `restart_status.blocked_reason` is set to `max_attempts_exceeded`. + +If an instance runs for `stable_after`, the consecutive attempt count resets. This prevents old transient failures from permanently consuming the retry budget. + +Manual `start` clears blocked restart status and starts a new failure window. + +Updating the restart policy clears retry status unless the instance was manually stopped. Changing the policy on a manually stopped instance does not start it; the user must call `start`. + +## Health Checks + +Health checks are a separate feature. They report workload health without changing instance lifecycle state. + +When health status reaches `unhealthy`, restart policy treats that as a failure signal for `on_failure` and `always`. `never` still reports health without restarting. + +Health-triggered restart uses the same attempt counter, backoff, max-attempt limit, manual-stop suppression, and stable-window reset as exit-triggered restart. + +## Lifecycle Behavior + +Exit-triggered restart only starts instances from `Stopped`. + +Health-triggered restart acts on `Running` instances by stopping the instance and starting it again. The health check itself does not keep the lifecycle state in `Initializing` or otherwise hide the distinction between boot readiness and workload health. + +It does not restore `Standby` instances, wake templates, or act on `Unknown` state. + +Automatic restart uses the normal instance start path. Resource validation, network allocation, config disk generation, volumes, GPU attachment, egress policy, and command/env metadata behave the same as a manual start. + +The restart keeps the same image, overlay disk, volumes, env, tags, entrypoint, and cmd stored on the instance. + +## Status + +Instance responses include the configured `restart_policy` and current `restart_status`. + +`restart_status.next_attempt_at` is set while waiting for backoff. + +`restart_status.attempts` counts consecutive automatic restart attempts in the current failure window. + +`restart_status.blocked_reason` explains why no more retries will happen despite a restart policy being configured. + +`restart_status.last_reason=health_check_failed` means the current retry window was entered because health checks marked the workload unhealthy rather than because the guest process exited on its own. + +## Non-goals + +Restart policy is not a health check. + +It does not supervise individual processes inside the guest. + +It does not replace systemd for images that want in-guest service supervision. diff --git a/lib/restart-policy/policy.go b/lib/restart-policy/policy.go new file mode 100644 index 00000000..29d57abe --- /dev/null +++ b/lib/restart-policy/policy.go @@ -0,0 +1,200 @@ +package restartpolicy + +import ( + "fmt" + "strings" + "time" +) + +const ( + DefaultBackoff = 5 * time.Second + DefaultStableAfter = 10 * time.Minute +) + +type PolicyMode string + +const ( + PolicyNever PolicyMode = "never" + PolicyAlways PolicyMode = "always" + PolicyOnFailure PolicyMode = "on_failure" +) + +type BlockedReason string + +const ( + BlockedReasonManualStop BlockedReason = "manual_stop" + BlockedReasonMaxAttemptsExceeded BlockedReason = "max_attempts_exceeded" +) + +type RestartReason string + +const ( + RestartReasonHealthCheckFailed RestartReason = "health_check_failed" +) + +type Policy struct { + Policy PolicyMode `json:"policy"` + Backoff string `json:"backoff,omitempty"` + MaxAttempts int `json:"max_attempts,omitempty"` + StableAfter string `json:"stable_after,omitempty"` +} + +type Status struct { + Attempts int `json:"attempts,omitempty"` + LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` + NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` + BlockedReason BlockedReason `json:"blocked_reason,omitempty"` + LastReason RestartReason `json:"last_reason,omitempty"` +} + +func NormalizePolicy(policy *Policy) (*Policy, error) { + if policy == nil { + return nil, nil + } + + mode := policy.Policy + if mode == "" { + mode = PolicyNever + } + + switch mode { + case PolicyNever: + return nil, nil + case PolicyAlways, PolicyOnFailure: + default: + return nil, fmt.Errorf("restart_policy.policy must be one of never, always, on_failure") + } + + backoff, err := normalizeDuration(policy.Backoff, DefaultBackoff, "restart_policy.backoff") + if err != nil { + return nil, err + } + stableAfter, err := normalizeDuration(policy.StableAfter, DefaultStableAfter, "restart_policy.stable_after") + if err != nil { + return nil, err + } + if policy.MaxAttempts < 0 { + return nil, fmt.Errorf("restart_policy.max_attempts must be >= 0") + } + + return &Policy{ + Policy: mode, + Backoff: backoff.String(), + MaxAttempts: policy.MaxAttempts, + StableAfter: stableAfter.String(), + }, nil +} + +func Backoff(policy *Policy) time.Duration { + d, err := durationOrDefault(policy, func(p *Policy) string { return p.Backoff }, DefaultBackoff) + if err != nil { + return DefaultBackoff + } + return d +} + +func StableAfter(policy *Policy) time.Duration { + d, err := durationOrDefault(policy, func(p *Policy) string { return p.StableAfter }, DefaultStableAfter) + if err != nil { + return DefaultStableAfter + } + return d +} + +func ShouldRestart(policy *Policy, exitCode *int) bool { + if policy == nil { + return false + } + switch policy.Policy { + case PolicyAlways: + return true + case PolicyOnFailure: + return exitCode == nil || *exitCode != 0 + default: + return false + } +} + +func PrepareAttempt(policy *Policy, status Status, now time.Time) (Status, bool) { + now = now.UTC() + if status.BlockedReason != "" { + return status, false + } + if status.NextAttemptAt != nil && now.Before(status.NextAttemptAt.UTC()) { + return status, false + } + if status.LastAttemptAt != nil { + nextAttemptAt := status.LastAttemptAt.UTC().Add(Backoff(policy)) + if now.Before(nextAttemptAt) { + status.NextAttemptAt = &nextAttemptAt + return status, false + } + } + if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { + status.NextAttemptAt = nil + status.BlockedReason = BlockedReasonMaxAttemptsExceeded + return status, false + } + + status.Attempts++ + status.LastAttemptAt = &now + status.NextAttemptAt = nil + return status, true +} + +func AfterFailedAttempt(policy *Policy, status Status, now time.Time) Status { + now = now.UTC() + if policy.MaxAttempts > 0 && status.Attempts >= policy.MaxAttempts { + status.BlockedReason = BlockedReasonMaxAttemptsExceeded + status.NextAttemptAt = nil + return status + } + nextAttemptAt := now.Add(Backoff(policy)) + status.NextAttemptAt = &nextAttemptAt + return status +} + +func EqualStatus(a, b Status) bool { + return a.Attempts == b.Attempts && + equalTime(a.LastAttemptAt, b.LastAttemptAt) && + equalTime(a.NextAttemptAt, b.NextAttemptAt) && + a.BlockedReason == b.BlockedReason && + a.LastReason == b.LastReason +} + +func (s Status) IsZero() bool { + return s.Attempts == 0 && + s.LastAttemptAt == nil && + s.NextAttemptAt == nil && + s.BlockedReason == "" && + s.LastReason == "" +} + +func equalTime(a, b *time.Time) bool { + if a == nil || b == nil { + return a == b + } + return a.UTC().Equal(b.UTC()) +} + +func normalizeDuration(raw string, fallback time.Duration, field string) (time.Duration, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback, nil + } + parsed, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("%s must be a valid duration: %w", field, err) + } + if parsed <= 0 { + return 0, fmt.Errorf("%s must be positive", field) + } + return parsed, nil +} + +func durationOrDefault(policy *Policy, selectValue func(*Policy) string, fallback time.Duration) (time.Duration, error) { + if policy == nil { + return fallback, nil + } + return normalizeDuration(selectValue(policy), fallback, "duration") +} diff --git a/lib/restart-policy/policy_test.go b/lib/restart-policy/policy_test.go new file mode 100644 index 00000000..b02a8abc --- /dev/null +++ b/lib/restart-policy/policy_test.go @@ -0,0 +1,42 @@ +package restartpolicy + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePolicyDefaults(t *testing.T) { + policy, err := NormalizePolicy(&Policy{Policy: PolicyOnFailure}) + require.NoError(t, err) + + assert.Equal(t, PolicyOnFailure, policy.Policy) + assert.Equal(t, "5s", policy.Backoff) + assert.Equal(t, "10m0s", policy.StableAfter) + assert.Equal(t, 0, policy.MaxAttempts) +} + +func TestNormalizePolicyNeverBecomesNil(t *testing.T) { + policy, err := NormalizePolicy(&Policy{Policy: PolicyNever}) + require.NoError(t, err) + + assert.Nil(t, policy) +} + +func TestNormalizePolicyRejectsInvalidDuration(t *testing.T) { + _, err := NormalizePolicy(&Policy{Policy: PolicyAlways, Backoff: "0s"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "restart_policy.backoff") +} + +func TestShouldRestart(t *testing.T) { + exitZero := 0 + exitOne := 1 + + assert.False(t, ShouldRestart(nil, &exitOne)) + assert.True(t, ShouldRestart(&Policy{Policy: PolicyAlways}, &exitZero)) + assert.False(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, &exitZero)) + assert.True(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, &exitOne)) + assert.True(t, ShouldRestart(&Policy{Policy: PolicyOnFailure}, nil)) +} diff --git a/openapi.yaml b/openapi.yaml index 2e06e608..c6bce872 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -375,6 +375,70 @@ components: description: Truncated error from the most recent failed check. example: "connection refused" + RestartPolicy: + type: object + description: Whole-instance restart supervision policy. + properties: + policy: + type: string + enum: [never, always, on_failure] + default: never + description: | + Restart behavior when the guest program exits: + - never: do not automatically restart + - always: restart after any guest exit + - on_failure: restart only for nonzero, signaled, OOM, or unknown exits + example: on_failure + backoff: + type: string + description: Delay before each restart attempt, expressed as a Go duration like "5s" or "1m". + default: "5s" + example: "5s" + max_attempts: + type: integer + minimum: 0 + default: 0 + description: Consecutive automatic restart attempts before blocking retries. 0 means unlimited. + example: 10 + stable_after: + type: string + description: Running this long resets the consecutive restart attempt count. + default: "10m" + example: "10m" + + RestartStatus: + type: object + description: Runtime status for restart policy decisions. + properties: + attempts: + type: integer + description: Consecutive automatic restart attempts in the current failure window. + example: 3 + last_attempt_at: + type: string + format: date-time + nullable: true + description: Last time Hypeman attempted an automatic restart. + example: "2025-01-15T12:30:00Z" + next_attempt_at: + type: string + format: date-time + nullable: true + description: Next scheduled automatic restart attempt after backoff. + example: "2025-01-15T12:30:05Z" + blocked_reason: + type: string + enum: [manual_stop, max_attempts_exceeded] + nullable: true + description: Reason automatic restarts are currently blocked. + example: max_attempts_exceeded + last_reason: + type: string + enum: [health_check_failed] + nullable: true + description: Most recent non-exit failure signal that entered restart policy. + example: health_check_failed + AutoStandbyStatus: type: object required: [supported, configured, enabled, eligible, status, reason, active_inbound_connections, tracking_mode] @@ -458,6 +522,8 @@ components: $ref: "#/components/schemas/AutoStandbyPolicy" health_check: $ref: "#/components/schemas/HealthCheck" + restart_policy: + $ref: "#/components/schemas/RestartPolicy" CreateInstanceRequest: type: object @@ -568,6 +634,8 @@ components: $ref: "#/components/schemas/AutoStandbyPolicy" health_check: $ref: "#/components/schemas/HealthCheck" + restart_policy: + $ref: "#/components/schemas/RestartPolicy" skip_kernel_headers: type: boolean description: | @@ -1026,6 +1094,10 @@ components: $ref: "#/components/schemas/HealthCheck" health_status: $ref: "#/components/schemas/InstanceHealthStatus" + restart_policy: + $ref: "#/components/schemas/RestartPolicy" + restart_status: + $ref: "#/components/schemas/RestartStatus" phase_durations_ms: type: object description: | diff --git a/stainless.yaml b/stainless.yaml index 4be9a032..947c2321 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -79,6 +79,8 @@ resources: health_check_tcp: "#/components/schemas/HealthCheckTCP" health_check_exec: "#/components/schemas/HealthCheckExec" instance_health_status: "#/components/schemas/InstanceHealthStatus" + restart_policy: "#/components/schemas/RestartPolicy" + restart_status: "#/components/schemas/RestartStatus" snapshot_policy: "#/components/schemas/SnapshotPolicy" standby_instance_request: "#/components/schemas/StandbyInstanceRequest" snapshot_schedule: "#/components/schemas/SnapshotSchedule"