Skip to content

Commit acc4b4f

Browse files
chattonchatton
authored
Move Wait functions into separate package (#551)
1 parent 46a8240 commit acc4b4f

File tree

3 files changed

+151
-140
lines changed

3 files changed

+151
-140
lines changed

‎test/e2e/e2eutil.go‎

Lines changed: 0 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,15 @@ import (
44
"context"
55
"fmt"
66
"reflect"
7-
"testing"
8-
"time"
97

108
"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar"
119

12-
"github.com/pkg/errors"
13-
1410
mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
15-
"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset"
16-
appsv1 "k8s.io/api/apps/v1"
1711
corev1 "k8s.io/api/core/v1"
1812
rbacv1 "k8s.io/api/rbac/v1"
1913
apiErrors "k8s.io/apimachinery/pkg/api/errors"
2014
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2115
"k8s.io/apimachinery/pkg/types"
22-
"k8s.io/apimachinery/pkg/util/wait"
23-
"sigs.k8s.io/controller-runtime/pkg/client"
2416
k8sClient "sigs.k8s.io/controller-runtime/pkg/client"
2517
)
2618

@@ -47,127 +39,6 @@ func UpdateMongoDBResource(original *mdbv1.MongoDBCommunity, updateFunc func(*md
4739
return TestClient.Update(context.TODO(), original)
4840
}
4941

50-
// WaitForConfigMapToExist waits until a ConfigMap of the given name exists
51-
// using the provided retryInterval and timeout
52-
func WaitForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) {
53-
cm := corev1.ConfigMap{}
54-
return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, OperatorNamespace)
55-
}
56-
57-
// WaitForSecretToExist waits until a Secret of the given name exists
58-
// using the provided retryInterval and timeout
59-
func WaitForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) {
60-
s := corev1.Secret{}
61-
return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace)
62-
}
63-
64-
// WaitForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase
65-
func WaitForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error {
66-
return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool {
67-
t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase)
68-
return db.Status.Phase == phase
69-
})
70-
}
71-
72-
// waitForMongoDBCondition polls and waits for a given condition to be true
73-
func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error {
74-
mdbNew := mdbv1.MongoDBCommunity{}
75-
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
76-
err = TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew)
77-
if err != nil {
78-
return false, err
79-
}
80-
ready := condition(mdbNew)
81-
return ready, nil
82-
})
83-
}
84-
85-
// WaitForStatefulSetToExist waits until a StatefulSet of the given name exists
86-
// using the provided retryInterval and timeout
87-
func WaitForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) {
88-
sts := appsv1.StatefulSet{}
89-
return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace)
90-
}
91-
92-
// WaitForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name
93-
// have reached the ready status
94-
func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error {
95-
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
96-
return sts.Spec.UpdateStrategy.Type == strategy
97-
})
98-
}
99-
100-
// WaitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name
101-
// have reached the ready status
102-
func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
103-
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
104-
return statefulset.IsReady(sts, mdb.Spec.Members)
105-
})
106-
}
107-
108-
// waitForStatefulSetCondition waits until all replicas of the StatefulSet with the given name
109-
// is not ready.
110-
func WaitForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
111-
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
112-
return !statefulset.IsReady(sts, mdb.Spec.Members)
113-
})
114-
}
115-
116-
// WaitForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct
117-
// and does not account for the updated replicas
118-
func WaitForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
119-
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
120-
return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas
121-
})
122-
}
123-
124-
func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error {
125-
_, err := WaitForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace)
126-
if err != nil {
127-
return errors.Errorf("error waiting for stateful set to be created: %s", err)
128-
}
129-
130-
sts := appsv1.StatefulSet{}
131-
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
132-
err = TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts)
133-
if err != nil {
134-
return false, err
135-
}
136-
t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n",
137-
mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration)
138-
ready := condition(sts)
139-
return ready, nil
140-
})
141-
}
142-
143-
func WaitForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error {
144-
return wait.Poll(time.Second*3, timeout, func() (done bool, err error) {
145-
err = TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod)
146-
if err != nil {
147-
return false, err
148-
}
149-
for _, status := range pod.Status.ContainerStatuses {
150-
t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready)
151-
if status.Name == containerName && status.Ready == isReady {
152-
return true, nil
153-
}
154-
}
155-
return false, nil
156-
})
157-
}
158-
159-
// waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists
160-
// using the provided retryInterval and timeout provided.
161-
func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error {
162-
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
163-
err = TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj)
164-
if err != nil {
165-
return false, client.IgnoreNotFound(err)
166-
}
167-
return true, nil
168-
})
169-
}
170-
17142
func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) {
17243
mongodbNamespace := namespace
17344
if mongodbNamespace == "" {

‎test/e2e/mongodbtests/mongodbtests.go‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7-
"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container"
87
"testing"
98
"time"
109

10+
"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container"
11+
"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait"
12+
1113
mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
1214
"github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram"
1315
"github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig"
@@ -46,7 +48,7 @@ func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
4648
// failure threshold being high
4749
func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
4850
return func(t *testing.T) {
49-
err := e2eutil.WaitForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45)
51+
err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45)
5052
if err != nil {
5153
t.Fatal(err)
5254
}
@@ -58,7 +60,7 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi
5860
// reaches the running state
5961
func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) {
6062
return func(t *testing.T) {
61-
err := e2eutil.WaitForStatefulSetToBeReady(t, mdb, interval, timeout)
63+
err := wait.ForStatefulSetToBeReady(t, mdb, interval, timeout)
6264
if err != nil {
6365
t.Fatal(err)
6466
}
@@ -69,7 +71,7 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, tim
6971
// statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state.
7072
func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) {
7173
return func(t *testing.T) {
72-
err := e2eutil.WaitForStatefulSetToBeUnready(t, mdb, interval, timeout)
74+
err := wait.ForStatefulSetToBeUnready(t, mdb, interval, timeout)
7375
if err != nil {
7476
t.Fatal(err)
7577
}
@@ -136,7 +138,7 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO
136138
// resource has the correct Update Strategy
137139
func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) {
138140
return func(t *testing.T) {
139-
err := e2eutil.WaitForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8)
141+
err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8)
140142
if err != nil {
141143
t.Fatal(err)
142144
}
@@ -147,7 +149,7 @@ func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.S
147149
// MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase
148150
func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
149151
return func(t *testing.T) {
150-
err := e2eutil.WaitForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12)
152+
err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12)
151153
if err != nil {
152154
t.Fatal(err)
153155
}
@@ -158,7 +160,7 @@ func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T)
158160
// MongoDBReachesFailed ensure the MongoDB resource reaches the Failed phase.
159161
func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
160162
return func(t *testing.T) {
161-
err := e2eutil.WaitForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5)
163+
err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5)
162164
if err != nil {
163165
t.Fatal(err)
164166
}
@@ -168,7 +170,7 @@ func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
168170

169171
func AutomationConfigSecretExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
170172
return func(t *testing.T) {
171-
s, err := e2eutil.WaitForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace)
173+
s, err := wait.ForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace)
172174
assert.NoError(t, err)
173175

174176
t.Logf("Secret %s/%s was successfully created", mdb.AutomationConfigSecretName(), mdb.Namespace)
@@ -218,7 +220,7 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu
218220
func GetConnectionStringSecret(mdb mdbv1.MongoDBCommunity, user scram.User) corev1.Secret {
219221
secret := corev1.Secret{}
220222
secretNamespacedName := types.NamespacedName{Name: user.GetConnectionStringSecretName(mdb), Namespace: mdb.Namespace}
221-
e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret)
223+
_ = e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret)
222224
return secret
223225
}
224226

@@ -353,15 +355,15 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerN
353355
func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) {
354356
return func(t *testing.T) {
355357
pod := podFromMongoDBCommunity(mdb, podNum)
356-
assert.NoError(t, e2eutil.WaitForPodReadiness(t, false, containerName, time.Minute*10, pod))
358+
assert.NoError(t, wait.ForPodReadiness(t, false, containerName, time.Minute*10, pod))
357359
}
358360
}
359361

360362
// PodContainerBecomesReady waits until the container with 'containerName' in the pod #podNum becomes ready.
361363
func PodContainerBecomesReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) {
362364
return func(t *testing.T) {
363365
pod := podFromMongoDBCommunity(mdb, podNum)
364-
assert.NoError(t, e2eutil.WaitForPodReadiness(t, true, containerName, time.Minute*3, pod))
366+
assert.NoError(t, wait.ForPodReadiness(t, true, containerName, time.Minute*3, pod))
365367
}
366368
}
367369

‎test/e2e/util/wait/wait.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package wait
2+
3+
import (
4+
"context"
5+
"testing"
6+
"time"
7+
8+
mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
9+
"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset"
10+
e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
11+
"github.com/pkg/errors"
12+
appsv1 "k8s.io/api/apps/v1"
13+
corev1 "k8s.io/api/core/v1"
14+
"k8s.io/apimachinery/pkg/types"
15+
"k8s.io/apimachinery/pkg/util/wait"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
)
18+
19+
// ForConfigMapToExist waits until a ConfigMap of the given name exists
20+
// using the provided retryInterval and timeout
21+
func ForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) {
22+
cm := corev1.ConfigMap{}
23+
return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, e2eutil.OperatorNamespace)
24+
}
25+
26+
// ForSecretToExist waits until a Secret of the given name exists
27+
// using the provided retryInterval and timeout
28+
func ForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) {
29+
s := corev1.Secret{}
30+
return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace)
31+
}
32+
33+
// ForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase
34+
func ForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error {
35+
return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool {
36+
t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase)
37+
return db.Status.Phase == phase
38+
})
39+
}
40+
41+
// waitForMongoDBCondition polls and waits for a given condition to be true
42+
func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error {
43+
mdbNew := mdbv1.MongoDBCommunity{}
44+
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
45+
err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew)
46+
if err != nil {
47+
return false, err
48+
}
49+
ready := condition(mdbNew)
50+
return ready, nil
51+
})
52+
}
53+
54+
// ForStatefulSetToExist waits until a StatefulSet of the given name exists
55+
// using the provided retryInterval and timeout
56+
func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) {
57+
sts := appsv1.StatefulSet{}
58+
return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace)
59+
}
60+
61+
// ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name
62+
// have reached the ready status
63+
func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error {
64+
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
65+
return sts.Spec.UpdateStrategy.Type == strategy
66+
})
67+
}
68+
69+
// ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name
70+
// have reached the ready status
71+
func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
72+
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
73+
return statefulset.IsReady(sts, mdb.Spec.Members)
74+
})
75+
}
76+
77+
// ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name
78+
// is not ready.
79+
func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
80+
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
81+
return !statefulset.IsReady(sts, mdb.Spec.Members)
82+
})
83+
}
84+
85+
// ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct
86+
// and does not account for the updated replicas
87+
func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
88+
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
89+
return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas
90+
})
91+
}
92+
93+
func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error {
94+
_, err := ForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace)
95+
if err != nil {
96+
return errors.Errorf("error waiting for stateful set to be created: %s", err)
97+
}
98+
99+
sts := appsv1.StatefulSet{}
100+
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
101+
err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts)
102+
if err != nil {
103+
return false, err
104+
}
105+
t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n",
106+
mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration)
107+
ready := condition(sts)
108+
return ready, nil
109+
})
110+
}
111+
112+
func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error {
113+
return wait.Poll(time.Second*3, timeout, func() (done bool, err error) {
114+
err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod)
115+
if err != nil {
116+
return false, err
117+
}
118+
for _, status := range pod.Status.ContainerStatuses {
119+
t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready)
120+
if status.Name == containerName && status.Ready == isReady {
121+
return true, nil
122+
}
123+
}
124+
return false, nil
125+
})
126+
}
127+
128+
// waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists
129+
// using the provided retryInterval and timeout provided.
130+
func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error {
131+
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
132+
err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj)
133+
if err != nil {
134+
return false, client.IgnoreNotFound(err)
135+
}
136+
return true, nil
137+
})
138+
}

0 commit comments

Comments
 (0)