From 03e3eb19d100c069f922d6e43d252d1bc34dc71a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 1 Sep 2026 09:32:40 +0200 Subject: [PATCH] feat: reconciler plans the start phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan learns the start vocabulary — inert until a caller opts in (ReconcileOptions.Scope, zero value keeps today's create-only plans byte-identical): - OpWaitCondition, one node per (awaited service, condition), deduplicated across dependents like networkNodes deduplicates networks; required:false marks the shared node best-effort, one required dependent upgrades it. service_started needs no node — a plain DAG edge to the dependency's chain end expresses it. Health is deliberately re-observed at execution time: the plan encodes what to wait for, never a stale observation. - OpRunPreStart, emitted at plan time only when no replica was running at observation — the imperative gating — targeting the lowest-numbered replica. - OpRunPostStart per container, after its start. - replica chains: inject+start+post_start of replica n+1 depends on the end of replica n's chain, today's sequential start order made visible in golden plans; startChainEnds points at the chain end so a service_started dependent waits for the whole service, matching InDependencyOrder semantics. - scope Start plans starting observed exited/created containers without converging them (the future compose start); scope CreateStart appends the start phase to the create plan, start nodes resolving their target from the create node that materializes the replica (CreateNodeID, the mechanism OpRenameContainer already uses). Lifecycle parity with the imperative engine is load-bearing and golden-locked: - dependency conditions are evaluated even when nothing has to start (waitDependencies runs for every visited service before looking at what to start), so an up with everything running still fails on an unhealthy required dependency; - an exceptional-state replica takes NO start-phase node: its bare create-phase restart leaves it running when the start phase looks, so the imperative engine neither re-starts nor injects — and it gates pre_start like any running replica; - startChainEnds carries the end-of-visit node set (waits included when nothing started), so a service_started dependent begins only once the dependency's whole visit completed, matching InDependencyOrder; - under scope Start, a scale>0 service with no container at all fails the plan with startService's exact error. The "service::" resource-ID format is built and parsed in one place (serviceReplicaID/serviceReplicaPrefix/startGroupID), and the replica sort deliberately carries the plan's determinism over the unordered containerNodes iteration. Golden tests only; no executor support yet and no caller passes the scope. Epic #14081, Lot 1 — reconciler (first item). Signed-off-by: Nicolas De Loof --- pkg/compose/plan.go | 30 +- pkg/compose/reconcile.go | 417 ++++++++++++++++++++-- pkg/compose/reconcile_start_test.go | 524 ++++++++++++++++++++++++++++ 3 files changed, 948 insertions(+), 23 deletions(-) create mode 100644 pkg/compose/reconcile_start_test.go diff --git a/pkg/compose/plan.go b/pkg/compose/plan.go index 6a118f12e0..d1b8624af1 100644 --- a/pkg/compose/plan.go +++ b/pkg/compose/plan.go @@ -53,6 +53,23 @@ const ( // Provider operations OpRunProvider OperationType = 30 + + // Start-phase operations + OpWaitCondition OperationType = 40 + OpRunPreStart OperationType = 41 + OpRunPostStart OperationType = 42 +) + +// PlanPhase situates a node in the plan lifecycle. The Create phase converges +// resources and containers to their desired shape; the Start phase brings +// containers to running — dependency waits, pre_start hooks, starts, +// post_start hooks. The zero value is Create, so plans built before the start +// phase existed render unchanged. +type PlanPhase int + +const ( + PhaseCreate PlanPhase = iota + PhaseStart ) // String returns the human-readable name of an OperationType. @@ -82,6 +99,12 @@ func (o OperationType) String() string { return "RenameContainer" case OpRunProvider: return "RunProvider" + case OpWaitCondition: + return "WaitCondition" + case OpRunPreStart: + return "RunPreStart" + case OpRunPostStart: + return "RunPostStart" default: return fmt.Sprintf("Unknown(%d)", int(o)) } @@ -102,7 +125,8 @@ type Operation struct { Network *types.NetworkConfig // for network operations Volume *types.VolumeConfig // for volume operations Timeout *time.Duration // for stop operations - CreateNodeID int // for OpRenameContainer: ID of the CreateContainer node whose result to rename + CreateNodeID int // for OpRenameContainer/start-phase ops: ID of the CreateContainer node whose result to target + Condition string // for OpWaitCondition: depends_on condition to wait for (service_healthy, ...) // BestEffort marks an operation whose failure must not abort the plan. It is // used for the optional removal of the old network on a rename: if the // network is still in use (by non-Compose containers) the removal is skipped @@ -118,6 +142,7 @@ type PlanNode struct { Operation Operation DependsOn []*PlanNode // prerequisite operations Group string // event grouping key (e.g. "recreate:web:1"); empty for ungrouped nodes + Phase PlanPhase // lifecycle phase this node belongs to; zero is Create } // Plan is a directed acyclic graph of operations produced by the reconciler. @@ -171,6 +196,9 @@ func (p *Plan) String() string { if node.Group != "" { fmt.Fprintf(&sb, " [%s]", node.Group) } + if node.Phase == PhaseStart { + sb.WriteString(" {start}") + } sb.WriteByte('\n') } return sb.String() diff --git a/pkg/compose/reconcile.go b/pkg/compose/reconcile.go index 1d6b7f98e5..91b1c7bfc0 100644 --- a/pkg/compose/reconcile.go +++ b/pkg/compose/reconcile.go @@ -17,6 +17,7 @@ package compose import ( + "cmp" "context" "fmt" "slices" @@ -45,8 +46,22 @@ func toReconcileOptions(options api.CreateOptions) ReconcileOptions { } } +// ReconcileScope selects which lifecycle phases the plan covers. The zero +// value plans the Create phase only — the historical behavior, and what +// `compose create` keeps using. ScopeCreateStart adds the Start phase to the +// same plan; ScopeStart plans starting the observed containers without +// converging them first. +type ReconcileScope int + +const ( + ScopeCreate ReconcileScope = iota + ScopeCreateStart + ScopeStart +) + // ReconcileOptions controls how the reconciler compares desired and observed state. type ReconcileOptions struct { + Scope ReconcileScope // lifecycle phases to plan; zero = Create only Services []string // targeted services (empty = all) Recreate string // "diverged", "force", "never" for targeted services RecreateDependencies string // same for non-targeted services @@ -77,6 +92,26 @@ type reconciler struct { // serviceNodes tracks the last plan node per service, so dependent // services can order their operations after dependencies. serviceNodes map[string]*PlanNode + // containerNodes tracks, per service then replica number, the + // create-phase node that materializes that container (creation, + // recreation, or the exceptional-state restart), so start-phase nodes + // can depend on it and resolve their target from its result. + containerNodes map[string]map[int]*PlanNode + // startChainEnds tracks the nodes ending each service's start-phase + // visit: the last node of the replica chain, or — when nothing had to + // start — the dependency prerequisites the visit still evaluated. What a + // service_started dependent (or a wait node) hooks onto, matching + // InDependencyOrder semantics: a dependent's visit begins only once the + // dependency's whole visit (waits included) completed. + startChainEnds map[string][]*PlanNode + // waitNodes deduplicates OpWaitCondition nodes per (service, condition): + // several dependents awaiting the same condition share one node, like + // networkNodes deduplicates network creations. + waitNodes map[string]*PlanNode + // removedByPlan records replicas the create phase condemns (scale-down + // stop+remove): the start phase must never plan a start for them — the + // imperative engine only ever starts what survives the convergence. + removedByPlan map[string]bool // stoppedByPlan records containers already stopped by an earlier stage // of the plan (typically planRecreateNetwork) so that downstream stages // can chain on the existing OpStopContainer instead of emitting a second @@ -123,6 +158,10 @@ func reconcile(_ context.Context, project *types.Project, observed *ObservedStat networkNodes: map[string]*PlanNode{}, volumeNodes: map[string]*PlanNode{}, serviceNodes: map[string]*PlanNode{}, + containerNodes: map[string]map[int]*PlanNode{}, + startChainEnds: map[string][]*PlanNode{}, + waitNodes: map[string]*PlanNode{}, + removedByPlan: map[string]bool{}, stoppedByPlan: map[string]*PlanNode{}, connectNodes: map[string][]*PlanNode{}, recreatedServices: map[string]bool{}, @@ -131,20 +170,28 @@ func reconcile(_ context.Context, project *types.Project, observed *ObservedStat r.resolveObserved() - if err := r.reconcileNetworks(); err != nil { - return nil, err - } + if options.Scope != ScopeStart { + if err := r.reconcileNetworks(); err != nil { + return nil, err + } - if err := r.reconcileVolumes(); err != nil { - return nil, err - } + if err := r.reconcileVolumes(); err != nil { + return nil, err + } - if err := r.reconcileContainers(); err != nil { - return nil, err + if err := r.reconcileContainers(); err != nil { + return nil, err + } + + if r.options.RemoveOrphans { + r.reconcileOrphans() + } } - if r.options.RemoveOrphans { - r.reconcileOrphans() + if options.Scope != ScopeCreate { + if err := r.planStartPhase(); err != nil { + return nil, err + } } return r.plan, nil @@ -261,7 +308,7 @@ func (r *reconciler) planRecreateNetworks(keys []string) { var disconnectNodes []*PlanNode for i := range containers { oc := &containers[i] - resID := fmt.Sprintf("service:%s:%d", oc.Summary.Labels[api.ServiceLabel], oc.Number) + resID := serviceReplicaID(oc.Summary.Labels[api.ServiceLabel], oc.Number) stopNode, alreadyStopped := r.stoppedByPlan[oc.ID] if !alreadyStopped { stopNode = r.plan.addNode(Operation{ @@ -324,7 +371,7 @@ func (r *reconciler) planRecreateNetworks(keys []string) { // transitively through remove → create). for i := range containers { oc := &containers[i] - resID := fmt.Sprintf("service:%s:%d", oc.Summary.Labels[api.ServiceLabel], oc.Number) + resID := serviceReplicaID(oc.Summary.Labels[api.ServiceLabel], oc.Number) deps := []*PlanNode{createNode} if rename { deps = append(deps, disconnectNodes[i]) @@ -449,7 +496,7 @@ func (r *reconciler) planRecreateVolumes(keys []string) { var removeNodes []*PlanNode for i := range containers { oc := &containers[i] - resID := fmt.Sprintf("service:%s:%d", oc.Summary.Labels[api.ServiceLabel], oc.Number) + resID := serviceReplicaID(oc.Summary.Labels[api.ServiceLabel], oc.Number) stopNode, alreadyStopped := r.stoppedByPlan[oc.ID] if !alreadyStopped { stopNode = r.plan.addNode(Operation{ @@ -593,12 +640,12 @@ func (r *reconciler) reconcileContainers() error { } // Visit in dependency order (leaves first = services with no deps) - return r.visitInDependencyOrder(graph) + return r.visitInDependencyOrder(graph, r.reconcileService) } // visitInDependencyOrder processes services from leaves to roots so that // dependencies are reconciled before the services that depend on them. -func (r *reconciler) visitInDependencyOrder(g *Graph) error { +func (r *reconciler) visitInDependencyOrder(g *Graph, visit func(types.ServiceConfig) error) error { visited := map[string]bool{} // Sort vertex keys for deterministic plan output in tests keys := sortedKeys(g.Vertices) @@ -631,7 +678,7 @@ func (r *reconciler) visitInDependencyOrder(g *Graph) error { if err != nil { return err } - if err := r.reconcileService(service); err != nil { + if err := visit(service); err != nil { return err } } @@ -695,24 +742,27 @@ func (r *reconciler) reconcileService(service types.ServiceConfig) error { // Scale down: stop + remove excess containers. Track the remove // node so dependent services wait for the scale-down to finish // even when no other operation runs on this service. + resID := serviceReplicaID(service.Name, oc.Number) stopNode := r.plan.addNode(Operation{ Type: OpStopContainer, - ResourceID: fmt.Sprintf("service:%s:%d", service.Name, oc.Number), + ResourceID: resID, Cause: "scale down", Container: &containers[i].Summary, Timeout: r.options.Timeout, }, "") lastNode = r.plan.addNode(Operation{ Type: OpRemoveContainer, - ResourceID: fmt.Sprintf("service:%s:%d", service.Name, oc.Number), + ResourceID: resID, Cause: "scale down", Container: &containers[i].Summary, }, "", stopNode) + r.removedByPlan[resID] = true continue } if r.mustRecreate(service, expectedHash, parentRecreated, oc, strategy) { lastNode = r.planRecreateContainer(service, &containers[i], infraDeps) + r.setContainerNode(service.Name, oc.Number, lastNode) r.recreatedServices[service.Name] = true continue } @@ -729,10 +779,11 @@ func (r *reconciler) reconcileService(service types.ServiceConfig) error { // Any other state (paused, dead, ...): attempt to (re)start lastNode = r.plan.addNode(Operation{ Type: OpStartContainer, - ResourceID: fmt.Sprintf("service:%s:%d", service.Name, oc.Number), + ResourceID: serviceReplicaID(service.Name, oc.Number), Cause: "not running", Container: &containers[i].Summary, }, "", infraDeps...) + r.setContainerNode(service.Name, oc.Number, lastNode) } } @@ -744,12 +795,13 @@ func (r *reconciler) reconcileService(service types.ServiceConfig) error { serviceCopy := service // copy for pointer stability lastNode = r.plan.addNode(Operation{ Type: OpCreateContainer, - ResourceID: fmt.Sprintf("service:%s:%d", service.Name, number), + ResourceID: serviceReplicaID(service.Name, number), Cause: "no existing container", Service: &serviceCopy, Number: number, Name: name, }, "", infraDeps...) + r.setContainerNode(service.Name, number, lastNode) } if lastNode != nil { @@ -877,10 +929,331 @@ func (r *reconciler) hasVolumeMismatch(expected types.ServiceConfig, oc Observed return false } +// planStartPhase appends the Start phase to the plan: for every service, a +// replica chain of start-phase operations reproducing the imperative +// engine's semantics — dependency conditions first, one optional +// OpRunPreStart when no replica was running at observation, then per +// replica inject+start (OpStartContainer, enriched by the executor) and +// post_start hooks, each replica chained after the previous one to keep +// today's sequential start order, now visible in the plan. +// +// Conditions other than service_started materialize as OpWaitCondition +// nodes, deduplicated per (service, condition) across dependents; health is +// deliberately re-observed at execution time — the plan only encodes what to +// wait for, never a stale observation. service_started needs no node: a +// plain DAG edge to the dependency's chain end expresses it. +func (r *reconciler) planStartPhase() error { + graph, err := NewGraph(r.project, ServiceStopped) + if err != nil { + return err + } + return r.visitInDependencyOrder(graph, r.planServiceStart) +} + +// startReplica is a container the start phase must bring to running: either +// materialized by a create-phase node (create) or already observed (container). +type startReplica struct { + resID string + number int // replica number, the start order within the service + // container is the observed container to start; nil when the create + // phase materializes it. + container *container.Summary + // after is the create-phase node this replica's start must follow, when + // the create phase planned one. + after *PlanNode + // createNodeID is the ID of the node whose execution result carries the + // materialized container (the OpCreateContainer node); 0 when the + // container is observed. + createNodeID int +} + +// plannedReplica builds the startReplica for a container the create phase +// already planned a node for. The node registered in containerNodes is not +// always the one whose execution result carries the container ID: a recreate +// chain registers its final rename node, whose CreateNodeID names the actual +// create node. +func plannedReplica(service string, number int, node *PlanNode) startReplica { + rep := startReplica{resID: serviceReplicaID(service, number), number: number, after: node} + switch node.Operation.Type { + case OpCreateContainer: + rep.createNodeID = node.ID + case OpRenameContainer: + rep.createNodeID = node.Operation.CreateNodeID + default: + // no other node type is registered in containerNodes today + // (exceptional-state restarts are filtered out before this point); + // leave the target unresolved so execution fails with a clean "no + // materialized container" error instead of panicking here + } + return rep +} + +// serviceReplicaID is the canonical resource ID of one service replica: +// every producer of the "service::" form goes through it so +// the format lives in exactly one place (nothing parses it back — plan +// bookkeeping is keyed by service and number, not by resource ID). +func serviceReplicaID(service string, number int) string { + return fmt.Sprintf("service:%s:%d", service, number) +} + +// startGroupID is the event-group key of one replica's start chain. +func startGroupID(resID string) string { + return "start:" + strings.TrimPrefix(resID, "service:") +} + +// setContainerNode records the create-phase node materializing a replica. +func (r *reconciler) setContainerNode(service string, number int, node *PlanNode) { + if r.containerNodes[service] == nil { + r.containerNodes[service] = map[int]*PlanNode{} + } + r.containerNodes[service][number] = node +} + +// startPhaseReplicas collects the replicas to start, ascending number: +// containers materialized by the create phase plus observed up-to-date +// containers not running — exactly the isNotRunning set the imperative start +// phase acts on. anyRunning reports whether a replica will be running when +// the start phase looks: running at observation and left untouched by the +// plan, or brought to running by the create phase itself (the bare restart +// of exceptional states). Both gate pre_start and take no start node — the +// imperative engine neither re-starts nor injects into a running container. +func (r *reconciler) startPhaseReplicas(service types.ServiceConfig) (replicas []startReplica, anyRunning bool) { + planned := r.containerNodes[service.Name] + seen := map[int]bool{} + for i := range r.observed.Containers[service.Name] { + oc := &r.observed.Containers[service.Name][i] + resID := serviceReplicaID(service.Name, oc.Number) + if r.removedByPlan[resID] { + // condemned by the create phase (scale-down): gone by the time + // the imperative start phase would look, so it neither starts + // nor counts as running for the pre_start gating + seen[oc.Number] = true + continue + } + if node, ok := planned[oc.Number]; ok { + seen[oc.Number] = true + if node.Operation.Type == OpStartContainer { + // exceptional-state restart (paused, dead, ...): the + // create-phase bare start already brings it to running + anyRunning = true + continue + } + replicas = append(replicas, plannedReplica(service.Name, oc.Number, node)) + continue + } + if oc.State == container.StateRunning { + if stopNode, stopped := r.stoppedByPlan[oc.ID]; stopped { + // the create phase stops this container WITHOUT recreating + // it (network recreate): it will be down when the start + // phase looks, and the imperative engine restarts it from + // its second snapshot — so does the plan, ordered after the + // container's last reconnect (else its stop). Not running + // for the pre_start gating, like in the imperative engine. + after := stopNode + if reconnects := r.connectNodes[oc.ID]; len(reconnects) > 0 { + after = reconnects[len(reconnects)-1] + } + seen[oc.Number] = true + replicas = append(replicas, startReplica{resID: resID, number: oc.Number, container: &oc.Summary, after: after}) + continue + } + anyRunning = true + continue + } + seen[oc.Number] = true + replicas = append(replicas, startReplica{resID: resID, number: oc.Number, container: &oc.Summary}) + } + for number, node := range planned { + if !seen[number] { + replicas = append(replicas, plannedReplica(service.Name, number, node)) + } + } + // the map iteration above is unordered: this sort CARRIES the plan's + // determinism, on top of expressing the numeric start order + slices.SortFunc(replicas, func(a, b startReplica) int { return cmp.Compare(a.number, b.number) }) + return replicas, anyRunning +} + +// startPhaseDependencies turns the service's depends_on into plan +// prerequisites: a service_started condition is a plain edge to the +// dependency's chain end, any other condition a deduplicated OpWaitCondition +// node re-evaluated at execution time. +func (r *reconciler) startPhaseDependencies(service types.ServiceConfig) []*PlanNode { + var depNodes []*PlanNode + for _, dep := range sortedKeys(service.DependsOn) { + cfg := service.DependsOn[dep] + targets := r.startChainEnds[dep] + if len(targets) == 0 { + if node, ok := r.serviceNodes[dep]; ok { + targets = []*PlanNode{node} + } + } + depService, err := r.project.GetService(dep) + waitless := cfg.Condition == types.ServiceConditionStarted || + // mirror shouldWaitForDependency: nothing to wait for on + // disabled, scale-0 or provider dependencies + err != nil || depService.GetScale() == 0 || depService.Provider != nil + if waitless { + depNodes = append(depNodes, targets...) + continue + } + depNodes = append(depNodes, r.waitConditionNode(dep, cfg, targets)) + } + return depNodes +} + +// waitConditionNode returns the shared wait node for (dep, condition), +// creating it on first use. required:false marks it best-effort — a missing +// dependency is skipped, not fatal; one required dependent upgrades the +// shared node for everyone. +func (r *reconciler) waitConditionNode(dep string, cfg types.ServiceDependency, targets []*PlanNode) *PlanNode { + key := dep + ":" + cfg.Condition + wait, ok := r.waitNodes[key] + if !ok { + wait = r.plan.addNode(Operation{ + Type: OpWaitCondition, + ResourceID: fmt.Sprintf("wait:%s:%s", dep, cfg.Condition), + Cause: "depends_on condition", + Name: dep, + Condition: cfg.Condition, + BestEffort: !cfg.Required, + }, "", targets...) + wait.Phase = PhaseStart + r.waitNodes[key] = wait + return wait + } + if cfg.Required && wait.Operation.BestEffort { + wait.Operation.BestEffort = false + } + // merge this caller's prerequisites into the shared node. Today every + // caller passes the same targets (startChainEnds[dep] is fixed before + // any dependent is visited), so this is defensive — but relying on that + // silently would break the day the targets diverge per caller. + for _, target := range targets { + if !slices.Contains(wait.DependsOn, target) { + wait.DependsOn = append(wait.DependsOn, target) + } + } + return wait +} + +func (r *reconciler) planServiceStart(service types.ServiceConfig) error { + if service.Provider != nil { + // a provider has no container to start: its create-phase RunProvider + // node is what dependents hook onto + if node, ok := r.serviceNodes[service.Name]; ok { + r.startChainEnds[service.Name] = []*PlanNode{node} + } + return nil + } + if service.GetScale() == 0 { + return nil + } + + replicas, anyRunning := r.startPhaseReplicas(service) + if len(replicas) == 0 && !anyRunning && r.options.Scope == ScopeStart { + // imperative parity (startService): a scale>0 service with no + // container at all cannot be started — only reachable under scope + // Start, since CreateStart would have planned the missing creates + return fmt.Errorf("service %q has no container to start", service.Name) + } + // dependency conditions are evaluated even when nothing has to start: + // the imperative engine calls waitDependencies for every visited service + // before looking at what to start, so an up with everything running + // still fails on an unhealthy required dependency + depNodes := r.startPhaseDependencies(service) + if len(replicas) == 0 { + // the visit still happened: dependents order after its prerequisites + // (waits, service_started edges), else after the create phase + ends := depNodes + if len(ends) == 0 { + if node, ok := r.serviceNodes[service.Name]; ok { + ends = []*PlanNode{node} + } + } + if len(ends) > 0 { + r.startChainEnds[service.Name] = ends + } + // no entry otherwise (ScopeStart, all running, no depends_on): the + // visit had no effect, so dependents have nothing to order after — + // a service_started condition on a running dependency is already + // satisfied, exactly like the imperative engine's empty visit + return nil + } + + prev := depNodes + + // pre_start runs once per service, only when no replica was running at + // observation — the imperative gating (initial up, force-recreate, or + // spec change), decided at plan time + preStarted := len(service.PreStart) > 0 && !anyRunning + if preStarted { + serviceCopy := service + first := replicas[0] + op := Operation{ + Type: OpRunPreStart, + ResourceID: first.resID, + Cause: "pre_start hooks", + Service: &serviceCopy, + Container: first.container, + CreateNodeID: first.createNodeID, + } + // clone: appending into prev's backing array would silently mutate + // depNodes' hidden capacity (they share it) + deps := slices.Clone(prev) + if first.after != nil { + deps = append(deps, first.after) + } + preStart := r.plan.addNode(op, startGroupID(first.resID), deps...) + preStart.Phase = PhaseStart + prev = []*PlanNode{preStart} + } + + // the replica chain: inject+start then post_start of replica n+1 waits + // for the end of replica n's chain — today's sequential start order + var chainEnd *PlanNode + for i, rep := range replicas { + serviceCopy := service + group := startGroupID(rep.resID) + op := Operation{ + Type: OpStartContainer, + ResourceID: rep.resID, + Cause: "start", + Service: &serviceCopy, + Container: rep.container, + CreateNodeID: rep.createNodeID, + } + deps := slices.Clone(prev) + if rep.after != nil && (i > 0 || !preStarted) { + // the first replica's create node is already carried through the + // pre_start node when one was planned: no redundant direct edge + deps = append(deps, rep.after) + } + start := r.plan.addNode(op, group, deps...) + start.Phase = PhaseStart + chainEnd = start + if len(service.PostStart) > 0 { + post := r.plan.addNode(Operation{ + Type: OpRunPostStart, + ResourceID: rep.resID, + Cause: "post_start hooks", + Service: &serviceCopy, + Container: rep.container, + CreateNodeID: op.CreateNodeID, + }, group, start) + post.Phase = PhaseStart + chainEnd = post + } + prev = []*PlanNode{chainEnd} + } + r.startChainEnds[service.Name] = []*PlanNode{chainEnd} + return nil +} + // planRecreateContainer decomposes container recreation into 4 atomic operations: // CreateContainer(tmpName) → StopContainer → RemoveContainer → RenameContainer func (r *reconciler) planRecreateContainer(service types.ServiceConfig, oc *ObservedContainer, infraDeps []*PlanNode) *PlanNode { - resID := fmt.Sprintf("service:%s:%d", service.Name, oc.Number) + resID := serviceReplicaID(service.Name, oc.Number) group := fmt.Sprintf("recreate:%s:%d", service.Name, oc.Number) tmpName := fmt.Sprintf("%s_%s", oc.ID[:min(12, len(oc.ID))], getContainerName(r.project.Name, service, oc.Number)) serviceCopy := service // copy for pointer stability @@ -979,7 +1352,7 @@ func (r *reconciler) planStopDependents(service types.ServiceConfig) []*PlanNode } node := r.plan.addNode(Operation{ Type: OpStopContainer, - ResourceID: fmt.Sprintf("service:%s:%d", depName, oc.Number), + ResourceID: serviceReplicaID(depName, oc.Number), Cause: fmt.Sprintf("dependency %s being recreated", service.Name), Container: &r.observed.Containers[depName][i].Summary, Timeout: r.options.Timeout, diff --git a/pkg/compose/reconcile_start_test.go b/pkg/compose/reconcile_start_test.go new file mode 100644 index 0000000000..b5bd133521 --- /dev/null +++ b/pkg/compose/reconcile_start_test.go @@ -0,0 +1,524 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "slices" + "strconv" + "strings" + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "github.com/moby/moby/api/types/container" + "gotest.tools/v3/assert" + + "github.com/docker/compose/v5/pkg/api" +) + +func startScopeOptions(scope ReconcileScope) ReconcileOptions { + options := defaultReconcileOptions() + options.Scope = scope + return options +} + +func emptyObserved() *ObservedState { + return &ObservedState{ + ProjectName: "myproject", + Containers: map[string][]ObservedContainer{}, + Networks: map[string][]ObservedNetwork{}, + Volumes: map[string][]ObservedVolume{}, + } +} + +func observedServiceContainer(service string, number int, state container.ContainerState, hash string) ObservedContainer { + name := "myproject-" + service + "-" + strconv.Itoa(number) + return ObservedContainer{ + ID: name + "-id", + Name: name, + State: state, + ConfigHash: hash, + Number: number, + Summary: container.Summary{ + ID: name + "-id", + Names: []string{"/" + name}, + State: state, + // the reconciler reads these back from the raw summary (e.g. + // nextContainerNumber): keep them consistent with the typed fields + Labels: map[string]string{ + api.ServiceLabel: service, + api.ContainerNumberLabel: strconv.Itoa(number), + api.ConfigHashLabel: hash, + }, + }, + } +} + +// A fresh up plans Create and Start as one DAG: each start depends on its +// own create, and the service_started dependency is a plain edge — the +// dependent's start waits for the dependency's chain end, no wait node. +func TestPlanStart_FreshUpWithStartedDependency(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Services: types.Services{ + "db": {Name: "db"}, + "web": { + Name: "web", + DependsOn: types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionStarted, Required: true}, + }, + }, + }, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + assert.Equal(t, plan.String(), strings.TrimSpace(` +[] -> #1 service:db:1, CreateContainer, no existing container +[1] -> #2 service:web:1, CreateContainer, no existing container +[1] -> #3 service:db:1, StartContainer, start [start:db:1] {start} +[2,3] -> #4 service:web:1, StartContainer, start [start:web:1] {start} +`)+"\n") +} + +// A condition other than service_started materializes as one wait node per +// (service, condition), shared by every dependent; health is re-observed at +// execution time, the plan only encodes what to wait for. +func TestPlanStart_HealthyConditionDeduplicated(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Services: types.Services{ + "db": {Name: "db"}, + "web": { + Name: "web", + DependsOn: types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }, + }, + "worker": { + Name: "worker", + DependsOn: types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }, + }, + }, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + assert.Equal(t, plan.String(), strings.TrimSpace(` +[] -> #1 service:db:1, CreateContainer, no existing container +[1] -> #2 service:web:1, CreateContainer, no existing container +[1] -> #3 service:worker:1, CreateContainer, no existing container +[1] -> #4 service:db:1, StartContainer, start [start:db:1] {start} +[4] -> #5 wait:db:service_healthy, WaitCondition, depends_on condition {start} +[2,5] -> #6 service:web:1, StartContainer, start [start:web:1] {start} +[3,5] -> #7 service:worker:1, StartContainer, start [start:worker:1] {start} +`)+"\n") +} + +// pre_start runs once per service before the first replica start, only when +// no replica was running at observation; replicas start sequentially, each +// chain link (start, then post_start when declared) gating the next. +func TestPlanStart_HooksAndReplicaChain(t *testing.T) { + two := 2 + project := &types.Project{ + Name: "myproject", + Services: types.Services{ + "app": { + Name: "app", + PreStart: []types.ServiceHook{{}}, + PostStart: []types.ServiceHook{{}}, + Scale: &two, + }, + }, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + assert.Equal(t, plan.String(), strings.TrimSpace(` +[] -> #1 service:app:1, CreateContainer, no existing container +[] -> #2 service:app:2, CreateContainer, no existing container +[1] -> #3 service:app:1, RunPreStart, pre_start hooks [start:app:1] {start} +[3] -> #4 service:app:1, StartContainer, start [start:app:1] {start} +[4] -> #5 service:app:1, RunPostStart, post_start hooks [start:app:1] {start} +[2,5] -> #6 service:app:2, StartContainer, start [start:app:2] {start} +[6] -> #7 service:app:2, RunPostStart, post_start hooks [start:app:2] {start} +`)+"\n") +} + +// With a replica already running and untouched by the plan, pre_start is +// gated off and the running replica gets no node — only the non-running one +// starts, the imperative isNotRunning role expressed in the plan. +func TestPlanStart_RunningReplicaGatesPreStart(t *testing.T) { + two := 2 + service := types.ServiceConfig{ + Name: "app", + PreStart: []types.ServiceHook{{}}, + Scale: &two, + } + project := &types.Project{ + Name: "myproject", + Services: types.Services{"app": service}, + } + hash, err := serviceHashWithResolvedRefs(service, nil) + assert.NilError(t, err) + observed := emptyObserved() + observed.Containers["app"] = []ObservedContainer{ + observedServiceContainer("app", 1, container.StateRunning, hash), + observedServiceContainer("app", 2, container.StateExited, hash), + } + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + assert.Equal(t, plan.String(), strings.TrimSpace(` +[] -> #1 service:app:2, StartContainer, start [start:app:2] {start} +`)+"\n") +} + +// A recreated replica's start-phase node resolves its container from the +// recreate chain's create node — not the rename node registered in +// containerNodes, whose execution stores no result — and orders after the +// chain's end. +func TestPlanStart_RecreatedReplicaTargetsCreateNode(t *testing.T) { + service := types.ServiceConfig{Name: "app"} + project := &types.Project{ + Name: "myproject", + Services: types.Services{"app": service}, + } + observed := emptyObserved() + observed.Containers["app"] = []ObservedContainer{ + observedServiceContainer("app", 1, container.StateRunning, "stale-hash"), + } + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + var create, rename, start *PlanNode + for _, n := range plan.Nodes { + switch { + case n.Operation.Type == OpCreateContainer: + create = n + case n.Operation.Type == OpRenameContainer: + rename = n + case n.Operation.Type == OpStartContainer && n.Phase == PhaseStart: + start = n + } + } + if create == nil || rename == nil || start == nil { + t.Fatalf("plan misses expected nodes (create=%v rename=%v start=%v):\n%s", create, rename, start, plan) + } + assert.Equal(t, start.Operation.CreateNodeID, create.ID) + assert.Assert(t, start.Operation.Container == nil) + assert.Assert(t, slices.Contains(start.DependsOn, rename)) +} + +// An exceptional-state container (paused, dead, ...) gets its bare +// create-phase restart and NOTHING in the start phase: the imperative engine +// finds it running when the start phase looks, so it neither re-starts nor +// injects — and its being running gates pre_start for the whole service. +func TestPlanStart_ExceptionalStateReplicaCountsAsRunning(t *testing.T) { + service := types.ServiceConfig{ + Name: "app", + PreStart: []types.ServiceHook{{Command: []string{"echo", "hi"}}}, + } + project := &types.Project{ + Name: "myproject", + Services: types.Services{"app": service}, + } + hash, err := serviceHashWithResolvedRefs(service, nil) + assert.NilError(t, err) + observed := emptyObserved() + observed.Containers["app"] = []ObservedContainer{ + observedServiceContainer("app", 1, container.StatePaused, hash), + } + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + var createPhase *PlanNode + for _, n := range plan.Nodes { + if n.Phase == PhaseStart { + t.Fatalf("no start-phase node expected for a create-phase-restarted replica, got %s:\n%s", n.Operation.Type, plan) + } + if n.Operation.Type == OpStartContainer { + createPhase = n + } + } + assert.Assert(t, createPhase != nil, "the historical bare restart must remain:\n%s", plan) +} + +// Dependency conditions are re-verified even when nothing has to start — the +// imperative engine calls waitDependencies for every visited service before +// looking at what to start, so an up with everything running still fails on +// an unhealthy required dependency. The dependent's visit end (the wait) also +// orders whoever depends on IT via service_started. +func TestPlanStart_AllRunningStillWaitsOnConditions(t *testing.T) { + db := types.ServiceConfig{Name: "db"} + app := types.ServiceConfig{ + Name: "app", + DependsOn: types.DependsOnConfig{"db": {Condition: types.ServiceConditionHealthy, Required: true}}, + } + web := types.ServiceConfig{ + Name: "web", + DependsOn: types.DependsOnConfig{"app": {Condition: types.ServiceConditionStarted, Required: true}}, + } + project := &types.Project{ + Name: "myproject", + Services: types.Services{"db": db, "app": app, "web": web}, + } + observed := emptyObserved() + for _, svc := range []types.ServiceConfig{db, app, web} { + hash, err := serviceHashWithResolvedRefs(svc, nil) + assert.NilError(t, err) + observed.Containers[svc.Name] = []ObservedContainer{ + observedServiceContainer(svc.Name, 1, container.StateRunning, hash), + } + } + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + var wait *PlanNode + for _, n := range plan.Nodes { + if n.Operation.Type == OpWaitCondition { + wait = n + } + } + if wait == nil { + t.Fatalf("expected the db:service_healthy wait despite everything running:\n%s", plan) + } + assert.Equal(t, wait.Operation.Name, "db") + assert.Equal(t, wait.Operation.Condition, types.ServiceConditionHealthy) + assert.Assert(t, !wait.Operation.BestEffort) +} + +// Imperative parity (startService): under scope Start, a scale>0 service +// with no container at all fails the plan the way `compose start` fails. +func TestPlanStart_NoContainerToStartFails(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Services: types.Services{"app": types.ServiceConfig{Name: "app"}}, + } + + _, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeStart), noPrompt) + assert.ErrorContains(t, err, `service "app" has no container to start`) +} + +// A replica condemned by scale-down must never receive a start-phase node: +// the imperative engine only starts what survives the convergence — and the +// condemned replica does not count as running for the pre_start gating. +func TestPlanStart_ScaleDownTargetIsNeverStarted(t *testing.T) { + one := 1 + service := types.ServiceConfig{ + Name: "app", + Deploy: &types.DeployConfig{Replicas: &one}, + } + project := &types.Project{ + Name: "myproject", + Services: types.Services{"app": service}, + } + hash, err := serviceHashWithResolvedRefs(service, nil) + assert.NilError(t, err) + observed := emptyObserved() + observed.Containers["app"] = []ObservedContainer{ + observedServiceContainer("app", 1, container.StateRunning, hash), + // the excess replica is exited: without the condemned filter it + // would fall through to the "observed, not running" start path + observedServiceContainer("app", 2, container.StateExited, hash), + } + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + for _, n := range plan.Nodes { + if n.Phase == PhaseStart { + t.Fatalf("no start-phase node expected (replica 1 runs, replica 2 is condemned), got %s %s:\n%s", n.Operation.Type, n.Operation.ResourceID, plan) + } + if n.Operation.Type == OpRemoveContainer { + assert.Equal(t, n.Operation.ResourceID, "service:app:2") + } + } +} + +// A running container the create phase stops WITHOUT recreating (network +// recreate) is down when the start phase looks: the plan restarts it — +// ordered after its last reconnect — and it does not gate pre_start, exactly +// like the imperative engine restarting it from its second snapshot. +func TestStartPhaseReplicas_StoppedByPlanRestarts(t *testing.T) { + stop := &PlanNode{ID: 1, Operation: Operation{Type: OpStopContainer}} + reconnect := &PlanNode{ID: 2, Operation: Operation{Type: OpConnectNetwork}} + observed := emptyObserved() + hash, err := serviceHashWithResolvedRefs(types.ServiceConfig{Name: "app"}, nil) + assert.NilError(t, err) + oc := observedServiceContainer("app", 1, container.StateRunning, hash) + observed.Containers["app"] = []ObservedContainer{oc} + + r := &reconciler{ + observed: observed, + containerNodes: map[string]map[int]*PlanNode{}, + removedByPlan: map[string]bool{}, + stoppedByPlan: map[string]*PlanNode{oc.ID: stop}, + connectNodes: map[string][]*PlanNode{oc.ID: {reconnect}}, + } + + replicas, anyRunning := r.startPhaseReplicas(types.ServiceConfig{Name: "app"}) + assert.Assert(t, !anyRunning, "a stopped-by-plan container must not gate pre_start") + if len(replicas) != 1 { + t.Fatalf("expected the stopped-by-plan container to be restarted, got %d replicas", len(replicas)) + } + assert.Equal(t, replicas[0].after, reconnect, "the restart orders after the last reconnect") + assert.Assert(t, replicas[0].container != nil) +} + +// plannedReplica leaves the target unresolved on an unexpected registration, +// so execution fails with a clean error instead of a plan-time panic. +func TestPlannedReplicaUnexpectedNodeLeavesTargetUnresolved(t *testing.T) { + node := &PlanNode{ID: 7, Operation: Operation{Type: OpConnectNetwork}} + rep := plannedReplica("app", 1, node) + assert.Equal(t, rep.resID, "service:app:1") + assert.Equal(t, rep.createNodeID, 0) + assert.Assert(t, rep.container == nil) + assert.Equal(t, rep.after, node) +} + +// Replica start order is numeric, not lexicographic: with 10+ replicas, +// replica 2 starts before replica 10. +func TestPlanStart_ReplicaOrderIsNumeric(t *testing.T) { + eleven := 11 + project := &types.Project{ + Name: "myproject", + Services: types.Services{ + "app": {Name: "app", Scale: &eleven}, + }, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + var order []string + for _, n := range plan.Nodes { + if n.Operation.Type == OpStartContainer { + order = append(order, n.Operation.ResourceID) + } + } + expected := make([]string, 0, 11) + for i := 1; i <= 11; i++ { + expected = append(expected, "service:app:"+strconv.Itoa(i)) + } + assert.DeepEqual(t, order, expected) +} + +// Scope Start plans only the start phase over observed containers — the +// future `compose start`: no convergence, exited containers start in +// dependency order, running ones are left alone. +func TestPlanStart_StartOnlyScope(t *testing.T) { + db := types.ServiceConfig{Name: "db"} + web := types.ServiceConfig{ + Name: "web", + DependsOn: types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionStarted, Required: true}, + }, + } + project := &types.Project{ + Name: "myproject", + Services: types.Services{"db": db, "web": web}, + } + dbHash, err := serviceHashWithResolvedRefs(db, nil) + assert.NilError(t, err) + webHash, err := serviceHashWithResolvedRefs(web, nil) + assert.NilError(t, err) + observed := emptyObserved() + observed.Containers["db"] = []ObservedContainer{observedServiceContainer("db", 1, container.StateExited, dbHash)} + observed.Containers["web"] = []ObservedContainer{observedServiceContainer("web", 1, container.StateCreated, webHash)} + + plan, err := reconcile(t.Context(), project, observed, startScopeOptions(ScopeStart), noPrompt) + assert.NilError(t, err) + + assert.Equal(t, plan.String(), strings.TrimSpace(` +[] -> #1 service:db:1, StartContainer, start [start:db:1] {start} +[1] -> #2 service:web:1, StartContainer, start [start:web:1] {start} +`)+"\n") +} + +// An optional (required: false) condition marks the shared wait node +// best-effort — a missing dependency is skipped, not fatal; one required +// dependent upgrades the node for everyone. +func TestPlanStart_OptionalConditionIsBestEffort(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Services: types.Services{ + "db": {Name: "db"}, + "web": { + Name: "web", + DependsOn: types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: false}, + }, + }, + }, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + + var wait *PlanNode + for _, n := range plan.Nodes { + if n.Operation.Type == OpWaitCondition { + wait = n + } + } + assert.Assert(t, wait != nil) + assert.Assert(t, wait.Operation.BestEffort) + + // a second dependent requiring the same condition upgrades the node + project.Services["worker"] = types.ServiceConfig{ + Name: "worker", + DependsOn: types.DependsOnConfig{ + "db": {Condition: types.ServiceConditionHealthy, Required: true}, + }, + } + plan, err = reconcile(t.Context(), project, emptyObserved(), startScopeOptions(ScopeCreateStart), noPrompt) + assert.NilError(t, err) + wait = nil + for _, n := range plan.Nodes { + if n.Operation.Type == OpWaitCondition { + wait = n + } + } + assert.Assert(t, wait != nil) + assert.Assert(t, !wait.Operation.BestEffort) +} + +// The default scope keeps yesterday's plans byte-identical: no start-phase +// node ever appears unless a caller opts in. +func TestPlanStart_DefaultScopeIsInert(t *testing.T) { + project := &types.Project{ + Name: "myproject", + Services: types.Services{"app": {Name: "app"}}, + } + + plan, err := reconcile(t.Context(), project, emptyObserved(), defaultReconcileOptions(), noPrompt) + assert.NilError(t, err) + + for _, n := range plan.Nodes { + assert.Assert(t, n.Phase == PhaseCreate) + } + assert.Equal(t, plan.String(), "[] -> #1 service:app:1, CreateContainer, no existing container\n") +}