clm: add retries while applying/deleting in case of ephemeral errors (#187)

Author: cbarbian-sap

clm/cmd/apply.go

@@ -8,6 +8,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -111,18 +112,27 @@ func newApplyCmd() *cobra.Command {
 				}
 			}()
 
+			const maxErrCount = 15
+			errCount := 0
+
 			for {
 				release.State = component.StateProcessing
 				ok, err := reconciler.Apply(context.TODO(), &release.Inventory, objects, namespace, ownerId, release.Revision)
 				if err != nil {
-					return err
-				}
-				if ok {
-					release.State = component.StateReady
-					break
-				}
-				if err := releaseClient.Update(context.TODO(), release); err != nil {
-					return err
+					if !isEphmeralError(err) || errCount >= maxErrCount {
+						return err
+					}
+					errCount++
+					fmt.Fprintf(os.Stderr, "Error: %s (retrying %d/%d)\n", err, errCount, maxErrCount)
+				} else {
+					errCount = 0
+					if ok {
+						release.State = component.StateReady
+						break
+					}
+					if err := releaseClient.Update(context.TODO(), release); err != nil {
+						return err
+					}
 				}
 				select {
 				case <-time.After(backoff.Next()):

clm/cmd/delete.go

@@ -8,6 +8,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -88,17 +89,26 @@ func newDeleteCmd() *cobra.Command {
 				}
 			}()
 
+			const maxErrCount = 15
+			errCount := 0
+
 			for {
 				release.State = component.StateDeleting
 				ok, err := reconciler.Delete(context.TODO(), &release.Inventory)
 				if err != nil {
-					return err
-				}
-				if ok {
-					break
-				}
-				if err := releaseClient.Update(context.TODO(), release); err != nil {
-					return err
+					if !isEphmeralError(err) || errCount >= maxErrCount {
+						return err
+					}
+					errCount++
+					fmt.Fprintf(os.Stderr, "Error: %s (retrying %d/%d)\n", err, errCount, maxErrCount)
+				} else {
+					errCount = 0
+					if ok {
+						break
+					}
+					if err := releaseClient.Update(context.TODO(), release); err != nil {
+						return err
+					}
 				}
 				select {
 				case <-time.After(backoff.Next()):

clm/cmd/util.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/sap/go-generics/slices"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -58,6 +59,13 @@ func getClient(kubeConfigPath string) (cluster.Client, error) {
 	return cluster.NewClientFor(config, scheme, fullName)
 }
 
+func isEphmeralError(err error) bool {
+	if apierrors.IsConflict(err) {
+		return true
+	}
+	return false
+}
+
 func formatTimestamp(t time.Time) string {
 	d := time.Now().Sub(t)
 	if d > 24*time.Hour {