Skip to content

Commit b86915b

Browse files
authored
Delete waves, status hints (#77)
* [INCOMPATIBLE CHANGE] add delete waves This commit adds delete waves, which can be controlled through the new annotation <operator-name>/delete-order; by default, all objects have a delete order of zero, and therefore are deleted in one wave. Note that delete waves are independent of apply waves. This commit brings some incompatible changes: 1. The annotation <operator-name>/order was renamed to <operator-name>/apply-order; the according constant in pkg/types was renamed as well. 2. The InventoryItem type was enhanced with new fields; consumers therefore *must* regenerate their CRDs. * exlude purge order from inventory for now * add new annotation <operator-name>/status-hint this can be used to tweak the status detection (done by kstatus); currently there are two options available that can be passed as value of this annotation (comma-separated): 'has-observed-generation', and 'has-ready-condition', which tells kstatus that the object does have an observedGeneration field in the status, resp. a condition of type Ready, even if it is not (yet) present in the status. * bump controller-tools * update generated artifacts * update docs
1 parent 97834ce commit b86915b

File tree

16 files changed

+650
-302
lines changed

16 files changed

+650
-302
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ $(LOCALBIN):
2727
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
2828

2929
## Tool Versions
30-
CONTROLLER_TOOLS_VERSION ?= v0.9.2
30+
CONTROLLER_TOOLS_VERSION ?= v0.14.0
3131

3232
.PHONY: controller-gen
3333
controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.

go.mod

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
module github.com/sap/component-operator-runtime
22

3-
go 1.21.7
3+
go 1.22.1
44

55
require (
66
github.com/Masterminds/sprig/v3 v3.2.3
77
github.com/hashicorp/go-multierror v1.1.1
8+
github.com/iancoleman/strcase v0.3.0
89
github.com/pkg/errors v0.9.1
9-
github.com/sap/go-generics v0.2.0
10+
github.com/sap/go-generics v0.2.3
1011
github.com/spf13/pflag v1.0.5
12+
golang.org/x/time v0.5.0
1113
k8s.io/api v0.29.2
1214
k8s.io/apiextensions-apiserver v0.29.2
1315
k8s.io/apimachinery v0.29.2
@@ -71,7 +73,6 @@ require (
7173
golang.org/x/sys v0.17.0 // indirect
7274
golang.org/x/term v0.17.0 // indirect
7375
golang.org/x/text v0.14.0 // indirect
74-
golang.org/x/time v0.5.0 // indirect
7576
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
7677
google.golang.org/appengine v1.6.8 // indirect
7778
google.golang.org/protobuf v1.32.0 // indirect

go.sum

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
6868
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
6969
github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
7070
github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
71+
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
72+
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
7173
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
7274
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
7375
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
@@ -98,8 +100,8 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
98100
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
99101
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
100102
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
101-
github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
102-
github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
103+
github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM=
104+
github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
103105
github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
104106
github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0=
105107
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -116,8 +118,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
116118
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
117119
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
118120
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
119-
github.com/sap/go-generics v0.2.0 h1:uXjK6eZDj4XFe52KiMfX7YsHJ+YyOrhUgohe1hNT/78=
120-
github.com/sap/go-generics v0.2.0/go.mod h1:LPjEUR4matw9C7GZdHYMExVN8+LeNK5LmrL24JKr8eg=
121+
github.com/sap/go-generics v0.2.3 h1:cEY63YaVIqvOu2347drCilMvdgM1p2we2QwY4k/Nas0=
122+
github.com/sap/go-generics v0.2.3/go.mod h1:eBhccCEzOiM5dn1W2kupUMOAm4uS9CfKHzQsDlZHQzc=
121123
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
122124
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
123125
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

internal/backoff/backoff.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"sync"
1010
"time"
1111

12+
"golang.org/x/time/rate"
1213
"k8s.io/client-go/util/workqueue"
1314
)
1415

@@ -21,7 +22,14 @@ type Backoff struct {
2122
func NewBackoff(maxDelay time.Duration) *Backoff {
2223
return &Backoff{
2324
activities: make(map[any]any),
24-
limiter: workqueue.NewItemExponentialFailureRateLimiter(20*time.Millisecond, maxDelay),
25+
// resulting per-item backoff is the maximum of a 300-times-20ms-then-maxDelay per-item limiter,
26+
// and an overall 10-per-second-burst-20 bucket limiter;
27+
// as a consequence, we have up to 20 almost immediate retries, then a phase of 10 retries per seconnd
28+
// for approximately 30s, and then slow retries at the rate given by maxDelay
29+
limiter: workqueue.NewMaxOfRateLimiter(
30+
workqueue.NewItemFastSlowRateLimiter(20*time.Millisecond, maxDelay, 300),
31+
&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 20)},
32+
),
2533
}
2634
}
2735

internal/kstatus/analyzer.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and component-operator-runtime contributors
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package kstatus
7+
8+
import (
9+
"strings"
10+
11+
"github.com/iancoleman/strcase"
12+
13+
batchv1 "k8s.io/api/batch/v1"
14+
corev1 "k8s.io/api/core/v1"
15+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
16+
"k8s.io/apimachinery/pkg/runtime/schema"
17+
kstatus "sigs.k8s.io/cli-utils/pkg/kstatus/status"
18+
19+
"github.com/sap/component-operator-runtime/pkg/types"
20+
)
21+
22+
const conditionTypeReady = "Ready"
23+
24+
type statusAnalyzer struct {
25+
reconcilerName string
26+
}
27+
28+
func NewStatusAnalyzer(reconcilerName string) StatusAnalyzer {
29+
return &statusAnalyzer{
30+
reconcilerName: reconcilerName,
31+
}
32+
}
33+
34+
func (s *statusAnalyzer) ComputeStatus(object *unstructured.Unstructured) (Status, error) {
35+
if hint, ok := object.GetAnnotations()[s.reconcilerName+"/"+types.AnnotationKeySuffixStatusHint]; ok {
36+
object = object.DeepCopy()
37+
38+
for _, hint := range strings.Split(hint, ",") {
39+
switch strcase.ToKebab(hint) {
40+
case types.StatusHintHasObservedGeneration:
41+
_, found, err := unstructured.NestedInt64(object.Object, "status", "observedGeneration")
42+
if err != nil {
43+
return UnknownStatus, err
44+
}
45+
if !found {
46+
if err := unstructured.SetNestedField(object.Object, -1, "status", "observedGeneration"); err != nil {
47+
return UnknownStatus, err
48+
}
49+
}
50+
case types.StatusHintHasReadyCondition:
51+
foundReadyCondition := false
52+
conditions, found, err := unstructured.NestedSlice(object.Object, "status", "conditions")
53+
if err != nil {
54+
return UnknownStatus, err
55+
}
56+
if !found {
57+
conditions = make([]any, 0)
58+
}
59+
for _, condition := range conditions {
60+
if condition, ok := condition.(map[string]any); ok {
61+
condType, found, err := unstructured.NestedString(condition, "type")
62+
if err != nil {
63+
return UnknownStatus, err
64+
}
65+
if found && condType == conditionTypeReady {
66+
foundReadyCondition = true
67+
break
68+
}
69+
}
70+
}
71+
if !foundReadyCondition {
72+
conditions = append(conditions, map[string]any{
73+
"type": conditionTypeReady,
74+
"status": string(corev1.ConditionUnknown),
75+
})
76+
if err := unstructured.SetNestedSlice(object.Object, conditions, "status", "conditions"); err != nil {
77+
return UnknownStatus, err
78+
}
79+
}
80+
}
81+
}
82+
}
83+
84+
res, err := kstatus.Compute(object)
85+
if err != nil {
86+
return UnknownStatus, err
87+
}
88+
89+
switch object.GroupVersionKind() {
90+
case schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}:
91+
// other than kstatus we want to consider jobs as InProgress if its pods are still running, resp. did not (yet) finish successfully
92+
if res.Status == kstatus.CurrentStatus {
93+
done := false
94+
objc, err := kstatus.GetObjectWithConditions(object.UnstructuredContent())
95+
if err != nil {
96+
return UnknownStatus, err
97+
}
98+
for _, cond := range objc.Status.Conditions {
99+
if cond.Type == string(batchv1.JobComplete) && cond.Status == corev1.ConditionTrue {
100+
done = true
101+
break
102+
}
103+
if cond.Type == string(batchv1.JobFailed) && cond.Status == corev1.ConditionTrue {
104+
done = true
105+
break
106+
}
107+
}
108+
if !done {
109+
res.Status = kstatus.InProgressStatus
110+
}
111+
}
112+
}
113+
114+
return Status(res.Status), nil
115+
}

internal/kstatus/status.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and component-operator-runtime contributors
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package kstatus
7+
8+
func (s Status) String() string {
9+
return string(s)
10+
}

internal/kstatus/types.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and component-operator-runtime contributors
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package kstatus
7+
8+
import (
9+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
10+
kstatus "sigs.k8s.io/cli-utils/pkg/kstatus/status"
11+
)
12+
13+
// TODO: the StatusAnalyzer interface should be public.
14+
15+
// The StatusAnalyzer interface models types which allow to extract a kstatus-compatible status from an object.
16+
type StatusAnalyzer interface {
17+
ComputeStatus(object *unstructured.Unstructured) (Status, error)
18+
}
19+
20+
type Status kstatus.Status
21+
22+
const (
23+
InProgressStatus Status = Status(kstatus.InProgressStatus)
24+
FailedStatus Status = Status(kstatus.FailedStatus)
25+
CurrentStatus Status = Status(kstatus.CurrentStatus)
26+
TerminatingStatus Status = Status(kstatus.TerminatingStatus)
27+
NotFoundStatus Status = Status(kstatus.NotFoundStatus)
28+
UnknownStatus Status = Status(kstatus.UnknownStatus)
29+
)

pkg/component/reconcile.go

Lines changed: 9 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232

3333
"github.com/sap/component-operator-runtime/internal/backoff"
3434
"github.com/sap/component-operator-runtime/internal/cluster"
35+
"github.com/sap/component-operator-runtime/internal/kstatus"
3536
"github.com/sap/component-operator-runtime/pkg/manifests"
3637
"github.com/sap/component-operator-runtime/pkg/types"
3738
)
@@ -86,86 +87,13 @@ type ReconcilerOptions struct {
8687
SchemeBuilder types.SchemeBuilder
8788
}
8889

89-
// AdoptionPolicy defines how the reconciler reacts if a dependent object exists but has no or a different owner.
90-
type AdoptionPolicy string
91-
92-
const (
93-
// Fail if the dependent object exists but has no or a different owner.
94-
AdoptionPolicyNever AdoptionPolicy = "Never"
95-
// Adopt existing dependent objects if they have no owner set.
96-
AdoptionPolicyIfUnowned AdoptionPolicy = "IfUnowned"
97-
// Adopt existing dependent objects, even if they have a conflicting owner.
98-
AdoptionPolicyAlways AdoptionPolicy = "Always"
99-
)
100-
101-
var adoptionPolicyByAnnotation = map[string]AdoptionPolicy{
102-
types.AdoptionPolicyNever: AdoptionPolicyNever,
103-
types.AdoptionPolicyIfUnowned: AdoptionPolicyIfUnowned,
104-
types.AdoptionPolicyAlways: AdoptionPolicyAlways,
105-
}
106-
107-
// ReconcilePolicy defines when the reconciler will reconcile the dependent object.
108-
type ReconcilePolicy string
109-
110-
const (
111-
// Reconcile the dependent object if its manifest, as produced by the generator, changes.
112-
ReconcilePolicyOnObjectChange ReconcilePolicy = "OnObjectChange"
113-
// Reconcile the dependent object if its manifest, as produced by the generator, changes, or if the owning
114-
// component changes (identified by a change of its metadata.generation).
115-
ReconcilePolicyOnObjectOrComponentChange ReconcilePolicy = "OnObjectOrComponentChange"
116-
// Reconcile the dependent object only once; afterwards it will never be touched again by the reconciler.
117-
ReconcilePolicyOnce ReconcilePolicy = "Once"
118-
)
119-
120-
var reconcilePolicyByAnnotation = map[string]ReconcilePolicy{
121-
types.ReconcilePolicyOnObjectChange: ReconcilePolicyOnObjectChange,
122-
types.ReconcilePolicyOnObjectOrComponentChange: ReconcilePolicyOnObjectOrComponentChange,
123-
types.ReconcilePolicyOnce: ReconcilePolicyOnce,
124-
}
125-
126-
// UpdatePolicy defines how the reconciler will update dependent objects.
127-
type UpdatePolicy string
128-
129-
const (
130-
// Recreate (that is: delete and create) existing dependent objects.
131-
UpdatePolicyRecreate UpdatePolicy = "Recreate"
132-
// Replace existing dependent objects.
133-
UpdatePolicyReplace UpdatePolicy = "Replace"
134-
// Use server side apply to update existing dependents.
135-
UpdatePolicySsaMerge UpdatePolicy = "SsaMerge"
136-
// Use server side apply to update existing dependents and, in addition, reclaim fields owned by certain
137-
// field owners, such as kubectl or helm.
138-
UpdatePolicySsaOverride UpdatePolicy = "SsaOverride"
139-
)
140-
141-
var updatePolicyByAnnotation = map[string]UpdatePolicy{
142-
types.UpdatePolicyRecreate: UpdatePolicyRecreate,
143-
types.UpdatePolicyReplace: UpdatePolicyReplace,
144-
types.UpdatePolicySsaMerge: UpdatePolicySsaMerge,
145-
types.UpdatePolicySsaOverride: UpdatePolicySsaOverride,
146-
}
147-
148-
// DeletePolicy defines how the reconciler will delete dependent objects.
149-
type DeletePolicy string
150-
151-
const (
152-
// Delete dependent objects.
153-
DeletePolicyDelete DeletePolicy = "Delete"
154-
// Orphan dependent objects.
155-
DeletePolicyOrphan DeletePolicy = "Orphan"
156-
)
157-
158-
var deletePolicyByAnnotation = map[string]DeletePolicy{
159-
types.DeletePolicyDelete: DeletePolicyDelete,
160-
types.DeletePolicyOrphan: DeletePolicyOrphan,
161-
}
162-
16390
// Reconciler provides the implementation of controller-runtime's Reconciler interface, for a given Component type T.
16491
type Reconciler[T Component] struct {
16592
name string
16693
id string
16794
client cluster.Client
16895
resourceGenerator manifests.Generator
96+
statusAnalyzer kstatus.StatusAnalyzer
16997
options ReconcilerOptions
17098
clients *cluster.ClientFactory
17199
backoff *backoff.Backoff
@@ -196,6 +124,7 @@ func NewReconciler[T Component](name string, resourceGenerator manifests.Generat
196124
return &Reconciler[T]{
197125
name: name,
198126
resourceGenerator: resourceGenerator,
127+
statusAnalyzer: kstatus.NewStatusAnalyzer(name),
199128
options: options,
200129
backoff: backoff.NewBackoff(10 * time.Second),
201130
postReadHooks: []HookFunc[T]{resolveReferences[T]},
@@ -249,6 +178,11 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result
249178
// always attempt to update the status
250179
skipStatusUpdate := false
251180
defer func() {
181+
if r := recover(); r != nil {
182+
log.Error(fmt.Errorf("panic occurred during reconcile"), "panic", r)
183+
// re-panic in order skip the remaining steps
184+
panic(r)
185+
}
252186
log.V(1).Info("reconcile done", "withError", err != nil, "requeue", result.Requeue || result.RequeueAfter > 0, "requeueAfter", result.RequeueAfter.String())
253187
if status.State == StateReady || err != nil {
254188
r.backoff.Forget(req)
@@ -314,7 +248,7 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result
314248
if err != nil {
315249
return ctrl.Result{}, errors.Wrap(err, "error getting client for component")
316250
}
317-
target := newReconcileTarget[T](r.name, r.id, targetClient, r.resourceGenerator, *r.options.CreateMissingNamespaces, *r.options.AdoptionPolicy, *r.options.UpdatePolicy)
251+
target := newReconcileTarget[T](r.name, r.id, targetClient, r.resourceGenerator, r.statusAnalyzer, *r.options.CreateMissingNamespaces, *r.options.AdoptionPolicy, *r.options.UpdatePolicy)
318252
hookCtx := newContext(ctx).WithClient(targetClient)
319253

320254
// do the reconciliation

0 commit comments

Comments
 (0)