K8SPG-555: fix updating CA secrets to 2.5.0 (#906)

* K8SPG-555: fix updating CA secrets to 2.5.0

https://perconadev.atlassian.net/browse/K8SPG-555

* fix

* add validation for custom certificates

* improve validation

* break into multiple lines

* fix

* small fix

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: pooknull

internal/controller/postgrescluster/pki.go

@@ -21,6 +21,8 @@ import (
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/percona/percona-postgresql-operator/internal/naming"
@@ -69,8 +71,22 @@ func (r *Reconciler) reconcileRootCertificate(
 		}
 	}
 
-	err := errors.WithStack(client.IgnoreNotFound(
-		r.Client.Get(ctx, client.ObjectKeyFromObject(existing), existing)))
+	err := errors.WithStack(
+		r.Client.Get(ctx, client.ObjectKeyFromObject(existing), existing))
+	// K8SPG-555: we need to check ca certificate from old operator versions
+	// TODO: remove when 2.4.0 will become unsupported
+	if k8serrors.IsNotFound(err) {
+		nn := client.ObjectKeyFromObject(existing)
+		nn.Name = naming.RootCertSecret
+		err = errors.WithStack(
+			r.Client.Get(ctx, nn, existing))
+		if err == nil {
+			existing.Name = naming.RootCertSecret
+		}
+	}
+	if k8serrors.IsNotFound(err) {
+		err = nil
+	}
 
 	root := &pki.RootCertificateAuthority{}
 
@@ -81,17 +97,21 @@ func (r *Reconciler) reconcileRootCertificate(
 		_ = root.Certificate.UnmarshalText(existing.Data[certificateKey])
 		_ = root.PrivateKey.UnmarshalText(existing.Data[privateKey])
 
+		if cluster.Spec.CustomRootCATLSSecret != nil {
+			return root, err
+		}
+
 		if !pki.RootIsValid(root) {
 			root, err = pki.NewRootCertificateAuthority()
 			err = errors.WithStack(err)
 		}
 	}
-	if cluster.Spec.CustomRootCATLSSecret != nil {
-		return root, err
-	}
 
 	intent := &corev1.Secret{
-		ObjectMeta: naming.PostgresRootCASecret(cluster),
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      existing.Name,
+			Namespace: existing.Namespace,
+		},
 	}
 	intent.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret"))
 	intent.Data = make(map[string][]byte)

internal/naming/names.go

@@ -90,9 +90,11 @@ const (
 )
 
 const (
-	// K8SPG-555: use PostgresRootCASecret instead.
+	// Deprecated: K8SPG-555: use PostgresRootCASecret instead.
+	// Currently it's used to update certificates from older operator version
 	// RootCertSecret is the default root certificate secret name
-	// RootCertSecret = "pgo-root-cacert" /* #nosec */
+	// TODO: remove when 2.4.0 will become unsupported
+	RootCertSecret = "pgo-root-cacert" /* #nosec */
 
 	// ClusterCertSecret is the default cluster leaf certificate secret name
 	ClusterCertSecret = "%s-cluster-cert" /* #nosec */

percona/controller/pgcluster/controller.go

@@ -213,6 +213,10 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R
 		return reconcile.Result{}, nil
 	}
 
+	if err := r.reconcileTLS(ctx, cr); err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "reconcile TLS")
+	}
+
 	if err := r.reconcileExternalWatchers(ctx, cr); err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "start external watchers")
 	}
@@ -282,6 +286,131 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R
 	return ctrl.Result{}, nil
 }
 
+func (r *PGClusterReconciler) reconcileTLS(ctx context.Context, cr *v2.PerconaPGCluster) error {
+	checkSecretProjection := func(p *corev1.SecretProjection, requiredPaths ...string) error {
+		if p == nil {
+			return nil
+		}
+
+		if p.Name == "" {
+			return errors.New("secret name is not specified")
+		}
+
+		secret := new(corev1.Secret)
+		nn := types.NamespacedName{Name: p.Name, Namespace: cr.Namespace}
+		if err := r.Client.Get(ctx, nn, secret); err != nil {
+			return errors.Wrapf(err, "failed to get secret %s", nn.Name)
+		}
+
+		pathMap := make(map[string]struct{})
+		for _, item := range p.Items {
+			if _, ok := secret.Data[item.Key]; !ok {
+				return errors.Errorf("key %s doesn't exist in secret %s", item.Key, secret.Name)
+			}
+			pathMap[item.Path] = struct{}{}
+		}
+
+		for _, path := range requiredPaths {
+			if _, ok := pathMap[path]; !ok {
+				if _, ok := secret.Data[path]; !ok {
+					return errors.Errorf("required path %s was not found both in secret %s and in the .items section", path, secret.Name)
+				}
+			}
+		}
+
+		return nil
+	}
+
+	if err := checkSecretProjection(cr.Spec.Secrets.CustomRootCATLSSecret, "root.crt", "root.key"); err != nil {
+		return errors.Wrap(err, "failed to validate .spec.customRootCATLSSecret")
+	}
+
+	certPaths := []string{"tls.key", "tls.crt"}
+	if cr.Spec.Secrets.CustomRootCATLSSecret == nil {
+		certPaths = append(certPaths, "ca.crt")
+	}
+	if err := checkSecretProjection(cr.Spec.Secrets.CustomTLSSecret, certPaths...); err != nil {
+		return errors.Wrap(err, "failed to validate .spec.customTLSSecret")
+	}
+	if err := checkSecretProjection(cr.Spec.Secrets.CustomReplicationClientTLSSecret, certPaths...); err != nil {
+		return errors.Wrap(err, "failed to validate .spec.customReplicationTLSSecret")
+	}
+
+	if cr.Spec.Secrets.CustomRootCATLSSecret != nil {
+		return nil
+	}
+
+	if err := r.reconcileOldCACert(ctx, cr); err != nil {
+		return errors.Wrap(err, "reconcile old CA")
+	}
+
+	return nil
+}
+
+func (r *PGClusterReconciler) reconcileOldCACert(ctx context.Context, cr *v2.PerconaPGCluster) error {
+	oldCASecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      naming.RootCertSecret,
+			Namespace: cr.Namespace,
+		},
+	}
+	err := r.Client.Get(ctx, client.ObjectKeyFromObject(oldCASecret), oldCASecret)
+	if client.IgnoreNotFound(err) != nil {
+		return errors.Wrap(err, "failed to get old ca secret")
+	}
+
+	if cr.CompareVersion("2.5.0") < 0 {
+		if k8serrors.IsNotFound(err) {
+			// K8SPG-555: We should create an empty secret with old name, so that crunchy part can populate it
+			// instead of creating secrets unique to the cluster
+			// TODO: remove when 2.4.0 will become unsupported
+			if err := r.Client.Create(ctx, oldCASecret); err != nil {
+				return errors.Wrap(err, "failed to create ca secret")
+			}
+		}
+		return nil
+	}
+	if k8serrors.IsNotFound(err) {
+		return nil
+	}
+
+	// K8SPG-555: Previously we used a single CA secret for all clusters in a namespace.
+	// We should copy the contents of the old CA secret, if it exists, to the new one, which is unique for each cluster.
+	// TODO: remove when 2.4.0 will become unsupported
+	newCASecret := &corev1.Secret{
+		ObjectMeta: naming.PostgresRootCASecret(
+			&v1beta1.PostgresCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      cr.Name,
+					Namespace: cr.Namespace,
+				},
+			}),
+	}
+	err = r.Client.Get(ctx, client.ObjectKeyFromObject(newCASecret), new(corev1.Secret))
+	if client.IgnoreNotFound(err) != nil {
+		return errors.Wrap(err, "failed to get new ca secret")
+	}
+
+	if k8serrors.IsNotFound(err) {
+		err := r.Client.Get(ctx, types.NamespacedName{
+			Name:      cr.Name,
+			Namespace: cr.Namespace,
+		}, new(v1beta1.PostgresCluster))
+		if client.IgnoreNotFound(err) != nil {
+			return errors.Wrap(err, "failed to get crunchy cluster")
+		}
+		// If the cluster is new, we should not copy the old CA secret.
+		// We should create an empty secret instead, so that crunchy part can populate it.
+		if !k8serrors.IsNotFound(err) {
+			newCASecret.Data = oldCASecret.Data
+		}
+		if err := r.Client.Create(ctx, newCASecret); err != nil {
+			return errors.Wrap(err, "failed to create updated CA secret")
+		}
+	}
+	return nil
+}
+
 func (r *PGClusterReconciler) reconcilePMM(ctx context.Context, cr *v2.PerconaPGCluster) error {
 	if !cr.PMMEnabled() {
 		return nil

percona/controller/pgcluster/finalizer.go

@@ -82,6 +82,7 @@ func (r *PGClusterReconciler) deleteTLSSecrets(ctx context.Context, cr *v2.Perco
 		naming.ClusterPGBouncer(crunchyCluster),
 	}
 	if cr.Spec.Secrets.CustomRootCATLSSecret == nil {
+		secretsMeta = append(secretsMeta, metav1.ObjectMeta{Namespace: crunchyCluster.Namespace, Name: naming.RootCertSecret})
 		secretsMeta = append(secretsMeta, naming.PostgresRootCASecret(crunchyCluster))
 	}
 	if cr.Spec.Secrets.CustomTLSSecret == nil {