diff --git a/build.sh b/build.sh index 999036e9..e071d1a8 100755 --- a/build.sh +++ b/build.sh @@ -52,6 +52,46 @@ function e2e() { make e2e_test } +## e2e_gotest - Runs only the e2e Go test suite against the already deployed cluster. +function e2e_gotest() { + make e2e_test_run_gotest +} + +## e2e_gotest_single - Runs a single e2e test by regex pattern. +function e2e_gotest_single() { + USE_GKE_E2E_AUTH_PLUGIN=True \ + TEST_INFRA_JSON="$PWD/bin/testinfra.json" \ + go test --count=1 -v -race ./tests/... -run "$1" +} + +## e2e_logs - Fetches logs from the operator controller manager in the e2e cluster. +function e2e_logs() { + USE_GKE_E2E_AUTH_PLUGIN=True bin/kubectl --kubeconfig=bin/e2e-kubeconfig.yaml logs -n cloud-sql-proxy-operator-system deployment/cloud-sql-proxy-operator-controller-manager --tail=200 "$@" +} + +## e2e_status - Checks pod and service status in the e2e clusters. +function e2e_status() { + echo "=== Public E2E Cluster ===" + USE_GKE_E2E_AUTH_PLUGIN=True bin/kubectl --kubeconfig=bin/e2e-kubeconfig.yaml get pods,svc,crd -A + echo "=== Private E2E Cluster ===" + USE_GKE_E2E_AUTH_PLUGIN=True bin/kubectl --kubeconfig=bin/e2e-private-kubeconfig.yaml get pods,svc,crd -A +} + +## kubectl_cmd - Runs kubectl on the public e2e cluster. +function kubectl_cmd() { + USE_GKE_E2E_AUTH_PLUGIN=True bin/kubectl --kubeconfig=bin/e2e-kubeconfig.yaml "$@" +} + +## kubectl_private_cmd - Runs kubectl on the private e2e cluster. +function kubectl_private_cmd() { + USE_GKE_E2E_AUTH_PLUGIN=True bin/kubectl --kubeconfig=bin/e2e-private-kubeconfig.yaml "$@" +} + +## e2e_deploy - Deploys CRD and operator to both e2e clusters. +function e2e_deploy() { + make e2e_build_deploy +} + function get_golang_tool() { name="$1" github_repo="$2" diff --git a/internal/api/v1/authproxyworkload_test.go b/internal/api/v1/authproxyworkload_test.go index 12ff49f1..d1fb1633 100644 --- a/internal/api/v1/authproxyworkload_test.go +++ b/internal/api/v1/authproxyworkload_test.go @@ -16,11 +16,18 @@ package v1_test import ( "context" + "fmt" "testing" cloudsqlapi "github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/api/v1" + admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) func ptr[T int | int32 | int64 | string | bool](i T) *T { @@ -596,3 +603,239 @@ func printFieldErrors(t *testing.T, err error) { } } } + +type fakeSARClient struct { + client.Client + allowedRules func(spec authorizationv1.SubjectAccessReviewSpec) bool +} + +func (f *fakeSARClient) Create(_ context.Context, obj client.Object, _ ...client.CreateOption) error { + sar, ok := obj.(*authorizationv1.SubjectAccessReview) + if !ok { + return fmt.Errorf("unexpected object type %T", obj) + } + if f.allowedRules != nil { + sar.Status.Allowed = f.allowedRules(sar.Spec) + } else { + sar.Status.Allowed = true + } + return nil +} + +func TestAuthProxyWorkload_ValidateAuthorization(t *testing.T) { + tests := []struct { + desc string + spec cloudsqlapi.AuthProxyWorkloadSpec + allowedRules func(spec authorizationv1.SubjectAccessReviewSpec) bool + wantAllowed bool + }{ + { + desc: "Allowed: User has pod delete and deployment update/patch", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "Deployment", + Name: "my-deployment", + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(_ authorizationv1.SubjectAccessReviewSpec) bool { + return true + }, + wantAllowed: true, + }, + { + desc: "Denied: User lacks pod delete permission", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "Deployment", + Name: "my-deployment", + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(spec authorizationv1.SubjectAccessReviewSpec) bool { + if spec.ResourceAttributes != nil && spec.ResourceAttributes.Resource == "pods" && spec.ResourceAttributes.Verb == "delete" { + return false + } + return true + }, + wantAllowed: false, + }, + { + desc: "Denied: User lacks deployment update permission", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "Deployment", + Name: "my-deployment", + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(spec authorizationv1.SubjectAccessReviewSpec) bool { + if spec.ResourceAttributes != nil && spec.ResourceAttributes.Resource == "deployments" { + return false + } + return true + }, + wantAllowed: false, + }, + { + desc: "Allowed: User has wildcard permission for label selector", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "StatefulSet", + Selector: &v1.LabelSelector{ + MatchLabels: map[string]string{"app": "db"}, + }, + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(_ authorizationv1.SubjectAccessReviewSpec) bool { + return true + }, + wantAllowed: true, + }, + { + desc: "Denied: User lacks wildcard permission for label selector", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "StatefulSet", + Selector: &v1.LabelSelector{ + MatchLabels: map[string]string{"app": "db"}, + }, + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(spec authorizationv1.SubjectAccessReviewSpec) bool { + if spec.ResourceAttributes != nil && spec.ResourceAttributes.Resource == "statefulsets" { + return false + } + return true + }, + wantAllowed: false, + }, + { + desc: "Allowed: User has container override permission when specifying custom container", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "Deployment", + Name: "my-deployment", + }, + AuthProxyContainer: &cloudsqlapi.AuthProxyContainerSpec{ + Container: &corev1.Container{ + Name: "custom-proxy", + Image: "custom-image:v1", + }, + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(_ authorizationv1.SubjectAccessReviewSpec) bool { + return true + }, + wantAllowed: true, + }, + { + desc: "Denied: User lacks container override permission when specifying custom container", + spec: cloudsqlapi.AuthProxyWorkloadSpec{ + Workload: cloudsqlapi.WorkloadSelectorSpec{ + Kind: "Deployment", + Name: "my-deployment", + }, + AuthProxyContainer: &cloudsqlapi.AuthProxyContainerSpec{ + Container: &corev1.Container{ + Name: "custom-proxy", + Image: "custom-image:v1", + }, + }, + Instances: []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db1", + PortEnvName: "DB_PORT", + }}, + }, + allowedRules: func(spec authorizationv1.SubjectAccessReviewSpec) bool { + if spec.ResourceAttributes != nil && spec.ResourceAttributes.Subresource == "containeroverride" { + return false + } + return true + }, + wantAllowed: false, + }, + } + + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + p := &cloudsqlapi.AuthProxyWorkload{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-authproxy", + Namespace: "test-ns", + }, + Spec: tc.spec, + } + + req := admissionRequest("alice") + ctx := admission.NewContextWithRequest(context.Background(), req) + + validator := &cloudsqlapi.AuthProxyWorkloadValidator{ + Client: &fakeSARClient{ + allowedRules: tc.allowedRules, + }, + } + + _, err := validator.ValidateCreate(ctx, p) + if tc.wantAllowed && err != nil { + t.Fatalf("expected allowed, got error: %v", err) + } + if !tc.wantAllowed && err == nil { + t.Fatalf("expected forbidden error, got nil") + } + + // Also verify ValidateUpdate with modified spec + oldObj := p.DeepCopy() + oldObj.Spec.Instances = []cloudsqlapi.InstanceSpec{{ + ConnectionString: "proj:region:db2", + PortEnvName: "DB_PORT", + }} + _, err = validator.ValidateUpdate(ctx, oldObj, p) + if tc.wantAllowed && err != nil { + t.Fatalf("expected update allowed, got error: %v", err) + } + if !tc.wantAllowed && err == nil { + t.Fatalf("expected update forbidden error, got nil") + } + + // Verify ValidateUpdate with unchanged spec (e.g. controller adding finalizers) is always allowed + _, err = validator.ValidateUpdate(ctx, p, p) + if err != nil { + t.Fatalf("expected update with unchanged spec to be allowed, got: %v", err) + } + }) + } +} + +func admissionRequest(user string) admission.Request { + return admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + UserInfo: authenticationv1.UserInfo{ + Username: user, + UID: "uid-" + user, + Groups: []string{"system:authenticated"}, + }, + }, + } +} diff --git a/internal/api/v1/authproxyworkload_webhook.go b/internal/api/v1/authproxyworkload_webhook.go index 50d5d933..33caad02 100644 --- a/internal/api/v1/authproxyworkload_webhook.go +++ b/internal/api/v1/authproxyworkload_webhook.go @@ -21,8 +21,8 @@ import ( "reflect" "cloud.google.com/go/cloudsqlconn/instance" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/validation" @@ -30,7 +30,9 @@ import ( apivalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) // log is for logging in this package. @@ -39,11 +41,12 @@ var authproxyworkloadlog = logf.Log.WithName("authproxyworkload-resource") func (r *AuthProxyWorkload) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr, r). WithDefaulter(&AuthProxyWorkloadDefaulter{}). - WithValidator(&AuthProxyWorkloadValidator{}). + WithValidator(&AuthProxyWorkloadValidator{Client: mgr.GetClient()}). Complete() } +// +kubebuilder:object:generate=false // +kubebuilder:webhook:path=/mutate-cloudsql-cloud-google-com-v1-authproxyworkload,mutating=true,failurePolicy=fail,sideEffects=None,groups=cloudsql.cloud.google.com,resources=authproxyworkloads,verbs=create;update,versions=v1,name=mauthproxyworkload.kb.io,admissionReviewVersions=v1 type AuthProxyWorkloadDefaulter struct { } @@ -58,12 +61,15 @@ func (*AuthProxyWorkloadDefaulter) Default(_ context.Context, r *AuthProxyWorklo return nil } +// +kubebuilder:object:generate=false +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create // +kubebuilder:webhook:path=/validate-cloudsql-cloud-google-com-v1-authproxyworkload,mutating=false,failurePolicy=fail,sideEffects=None,groups=cloudsql.cloud.google.com,resources=authproxyworkloads,verbs=create;update,versions=v1,name=vauthproxyworkload.kb.io,admissionReviewVersions=v1 type AuthProxyWorkloadValidator struct { + Client client.Client } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (*AuthProxyWorkloadValidator) ValidateCreate(_ context.Context, r *AuthProxyWorkload) (warnings admission.Warnings, err error) { +func (v *AuthProxyWorkloadValidator) ValidateCreate(ctx context.Context, r *AuthProxyWorkload) (warnings admission.Warnings, err error) { allErrs := r.validate() if len(allErrs) > 0 { return nil, apierrors.NewInvalid( @@ -72,12 +78,15 @@ func (*AuthProxyWorkloadValidator) ValidateCreate(_ context.Context, r *AuthProx Kind: "AuthProxyWorkload"}, r.Name, allErrs) } + if err := v.validateAuthorization(ctx, r); err != nil { + return nil, err + } return nil, nil } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (*AuthProxyWorkloadValidator) ValidateUpdate(_ context.Context, old, newObj *AuthProxyWorkload) (warnings admission.Warnings, err error) { +func (v *AuthProxyWorkloadValidator) ValidateUpdate(ctx context.Context, old, newObj *AuthProxyWorkload) (warnings admission.Warnings, err error) { allErrs := newObj.validate() allErrs = append(allErrs, newObj.validateUpdateFrom(old)...) if len(allErrs) > 0 { @@ -87,14 +96,187 @@ func (*AuthProxyWorkloadValidator) ValidateUpdate(_ context.Context, old, newObj Kind: "AuthProxyWorkload"}, newObj.Name, allErrs) } + if !reflect.DeepEqual(old.Spec, newObj.Spec) { + if err := v.validateAuthorization(ctx, newObj); err != nil { + return nil, err + } + } return nil, nil } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (*AuthProxyWorkloadValidator) ValidateDelete(_ context.Context, _ *AuthProxyWorkload) (admission.Warnings, error) { +func (v *AuthProxyWorkloadValidator) ValidateDelete(_ context.Context, _ *AuthProxyWorkload) (admission.Warnings, error) { return nil, nil } +func (v *AuthProxyWorkloadValidator) validateAuthorization(ctx context.Context, r *AuthProxyWorkload) error { + if v.Client == nil { + return nil + } + req, err := admission.RequestFromContext(ctx) + if err != nil || req.UserInfo.Username == "" { + return nil + } + + // 1. Pod delete check: User must be authorized to delete pods in the target namespace. + sarPod := &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: req.UserInfo.Username, + UID: req.UserInfo.UID, + Groups: req.UserInfo.Groups, + Extra: convertExtra(req.UserInfo.Extra), + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: r.Namespace, + Verb: "delete", + Group: "", + Resource: "pods", + }, + }, + } + if err := v.Client.Create(ctx, sarPod); err != nil { + return apierrors.NewInternalError(fmt.Errorf("failed to check authorization for deleting pods: %w", err)) + } + if !sarPod.Status.Allowed { + return apierrors.NewForbidden( + schema.GroupResource{Group: GroupVersion.Group, Resource: "authproxyworkloads"}, + r.Name, + fmt.Errorf("user %q is not authorized to delete pods in namespace %q", req.UserInfo.Username, r.Namespace), + ) + } + + // 2. Workload update and patch checks: User must be authorized to update and patch the targeted workload. + group, resource, err := groupResourceForKind(r.Spec.Workload.Kind) + if err != nil { + return apierrors.NewInvalid( + schema.GroupKind{Group: GroupVersion.Group, Kind: "AuthProxyWorkload"}, + r.Name, + field.ErrorList{field.Invalid(field.NewPath("spec", "workload", "kind"), r.Spec.Workload.Kind, err.Error())}, + ) + } + + if r.Spec.Workload.Name != "" { + for _, verb := range []string{"update", "patch"} { + sar := &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: req.UserInfo.Username, + UID: req.UserInfo.UID, + Groups: req.UserInfo.Groups, + Extra: convertExtra(req.UserInfo.Extra), + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: r.Namespace, + Verb: verb, + Group: group, + Resource: resource, + Name: r.Spec.Workload.Name, + }, + }, + } + if err := v.Client.Create(ctx, sar); err != nil { + return apierrors.NewInternalError(fmt.Errorf("failed to check authorization for %s on %s %s: %w", verb, resource, r.Spec.Workload.Name, err)) + } + if !sar.Status.Allowed { + return apierrors.NewForbidden( + schema.GroupResource{Group: GroupVersion.Group, Resource: "authproxyworkloads"}, + r.Name, + fmt.Errorf("user %q is not authorized to %s %s %q in namespace %q", req.UserInfo.Username, verb, resource, r.Spec.Workload.Name, r.Namespace), + ) + } + } + } else if r.Spec.Workload.Selector != nil { + for _, verb := range []string{"update", "patch"} { + sar := &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: req.UserInfo.Username, + UID: req.UserInfo.UID, + Groups: req.UserInfo.Groups, + Extra: convertExtra(req.UserInfo.Extra), + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: r.Namespace, + Verb: verb, + Group: group, + Resource: resource, + }, + }, + } + if err := v.Client.Create(ctx, sar); err != nil { + return apierrors.NewInternalError(fmt.Errorf("failed to check authorization for %s on %s: %w", verb, resource, err)) + } + if !sar.Status.Allowed { + return apierrors.NewForbidden( + schema.GroupResource{Group: GroupVersion.Group, Resource: "authproxyworkloads"}, + r.Name, + fmt.Errorf("user %q is not authorized to %s %s by label selector in namespace %q (requires namespace-wide update and patch permissions on %s)", req.UserInfo.Username, verb, resource, r.Namespace, resource), + ) + } + } + } + + // 3. Custom container override check + if r.Spec.AuthProxyContainer != nil && r.Spec.AuthProxyContainer.Container != nil { + sarOverride := &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: req.UserInfo.Username, + UID: req.UserInfo.UID, + Groups: req.UserInfo.Groups, + Extra: convertExtra(req.UserInfo.Extra), + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: r.Namespace, + Verb: "create", + Group: GroupVersion.Group, + Resource: "authproxyworkloads", + Subresource: "containeroverride", + Name: r.Name, + }, + }, + } + if err := v.Client.Create(ctx, sarOverride); err != nil { + return apierrors.NewInternalError(fmt.Errorf("failed to check authorization for container override: %w", err)) + } + if !sarOverride.Status.Allowed { + return apierrors.NewForbidden( + schema.GroupResource{Group: GroupVersion.Group, Resource: "authproxyworkloads"}, + r.Name, + fmt.Errorf("user %q is not authorized to specify custom container override (requires containeroverride permission on authproxyworkloads)", req.UserInfo.Username), + ) + } + } + + return nil +} + +func convertExtra(in map[string]authenticationv1.ExtraValue) map[string]authorizationv1.ExtraValue { + if in == nil { + return nil + } + out := make(map[string]authorizationv1.ExtraValue, len(in)) + for k, v := range in { + out[k] = authorizationv1.ExtraValue(v) + } + return out +} + +func groupResourceForKind(kindArg string) (string, string, error) { + _, gk := schema.ParseKindArg(kindArg) + switch gk.Kind { + case "Deployment": + return "apps", "deployments", nil + case "StatefulSet": + return "apps", "statefulsets", nil + case "DaemonSet": + return "apps", "daemonsets", nil + case "ReplicaSet": + return "apps", "replicasets", nil + case "Job": + return "batch", "jobs", nil + case "CronJob": + return "batch", "cronjobs", nil + case "Pod": + return "", "pods", nil + default: + return "", "", fmt.Errorf("unsupported kind %q", kindArg) + } +} + func (r *AuthProxyWorkload) validate() field.ErrorList { var allErrs field.ErrorList diff --git a/internal/controller/authproxyworkload_controller.go b/internal/controller/authproxyworkload_controller.go index 51b74c6b..ddf8788a 100644 --- a/internal/controller/authproxyworkload_controller.go +++ b/internal/controller/authproxyworkload_controller.go @@ -99,6 +99,7 @@ func (r *AuthProxyWorkloadReconciler) SetupWithManager(mgr ctrl.Manager) error { //+kubebuilder:rbac:groups=apps,resources=deployments;statefulsets;daemonsets;replicasets,verbs=update;patch //+kubebuilder:rbac:groups=apps,resources=*,verbs=get;list;watch +//+kubebuilder:rbac:groups=batch,resources=jobs;cronjobs,verbs=update;patch //+kubebuilder:rbac:groups=batch,resources=*,verbs=get;list;watch //+kubebuilder:rbac:groups="",resources=*,verbs=get;list;watch @@ -107,6 +108,7 @@ func (r *AuthProxyWorkloadReconciler) SetupWithManager(mgr ctrl.Manager) error { //+kubebuilder:rbac:groups=cloudsql.cloud.google.com,resources=authproxyworkloads/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create // Reconcile updates the state of the cluster so that AuthProxyWorkload instances // have their configuration reflected correctly on workload PodSpec configuration. diff --git a/internal/testhelpers/resources.go b/internal/testhelpers/resources.go index 1141196a..a1775c97 100644 --- a/internal/testhelpers/resources.go +++ b/internal/testhelpers/resources.go @@ -362,7 +362,7 @@ func (cc *TestCaseClient) CreateWorkload(ctx context.Context, o client.Object) e func (cc *TestCaseClient) GetAuthProxyWorkloadAfterReconcile(ctx context.Context, key types.NamespacedName) (*cloudsqlapi.AuthProxyWorkload, error) { createdPodmod := &cloudsqlapi.AuthProxyWorkload{} // We'll need to retry getting this newly created resource, given that creation may not immediately happen. - err := RetryUntilSuccess(6, DefaultRetryInterval, func() error { + err := RetryUntilSuccess(24, DefaultRetryInterval, func() error { err := cc.Client.Get(ctx, key, createdPodmod) if err != nil { return err @@ -521,7 +521,7 @@ func (cc *TestCaseClient) ExpectContainerCount(ctx context.Context, key types.Na got int deployment = &appsv1.Deployment{} ) - err := RetryUntilSuccess(6, DefaultRetryInterval, func() error { + err := RetryUntilSuccess(24, DefaultRetryInterval, func() error { err := cc.Client.Get(ctx, key, deployment) if err != nil { return err