CLOUDP-87698: perform check for readiness probe init container already existing (#431)

Author: chatton

README.md

@@ -2,6 +2,9 @@
 
 <img align="right" src="https://mongodb-kubernetes-operator.s3.amazonaws.com/img/Leaf-Forest%402x.png">
 
+###v0.6.0 has introduced breaking changes. If you are upgrading from a previous version, follow the upgrade instructions outlined [in the release notes](https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v0.6.0)
+
+
 This is a [Kubernetes Operator](https://coreos.com/operators/) which deploys MongoDB Community into Kubernetes clusters.
 
 If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes).

controllers/construct/mongodbstatefulset.go

@@ -25,7 +25,7 @@ const (
 	MongodbName = "mongod"
 
 	versionUpgradeHookName         = "mongod-posthook"
-	readinessProbeContainerName    = "mongodb-agent-readinessprobe"
+	ReadinessProbeContainerName    = "mongodb-agent-readinessprobe"
 	dataVolumeName                 = "data-volume"
 	logVolumeName                  = "logs-volume"
 	readinessProbePath             = "/opt/scripts/readinessprobe"
@@ -169,7 +169,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe
 				podtemplatespec.WithContainer(AgentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), mongodbAgentVolumeMounts)),
 				podtemplatespec.WithContainer(MongodbName, mongodbContainer(mdb.GetMongoDBVersion(), mongodVolumeMounts)),
 				podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})),
-				podtemplatespec.WithInitContainer(readinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount})),
+				podtemplatespec.WithInitContainer(ReadinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount})),
 			),
 		))
 }
@@ -261,7 +261,7 @@ func logsPvc() persistentvolumeclaim.Modification {
 // this container will copy the readiness probe binary into the /opt/scripts directory.
 func readinessProbeInit(volumeMount []corev1.VolumeMount) container.Modification {
 	return container.Apply(
-		container.WithName(readinessProbeContainerName),
+		container.WithName(ReadinessProbeContainerName),
 		container.WithCommand([]string{"cp", "/probes/readinessprobe", "/opt/scripts/readinessprobe"}),
 		container.WithImage(os.Getenv(ReadinessProbeImageEnv)),
 		container.WithImagePullPolicy(corev1.PullAlways),

controllers/replica_set_controller.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container"
+
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions"
 
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/agent"
@@ -313,6 +315,11 @@ func (r *ReplicaSetReconciler) deployAutomationConfig(mdb mdbv1.MongoDBCommunity
 		return true, nil
 	}
 
+	if isPreReadinessInitContainerStatefulSet(sts) {
+		r.log.Debugf("The existing StatefulSet did not have the readiness probe init container, skipping pod annotation check.")
+		return true, nil
+	}
+
 	r.log.Debugf("Waiting for agents to reach version %d", ac.Version)
 	// Note: we pass in the expected number of replicas this reconciliation as we scale members one at a time. If we were
 	// to pass in the final member count, we would be waiting for agents that do not exist yet to be ready.
@@ -556,3 +563,9 @@ func getDomain(service, namespace, clusterName string) string {
 	}
 	return fmt.Sprintf("%s.%s.svc.%s", service, namespace, clusterName)
 }
+
+// isPreReadinessInitContainerStatefulSet determines if the existing StatefulSet has been configured with the readiness probe init container.
+// if this is not the case, then we should ensure to skip past the annotation check otherwise the pods will remain in pending state forever.
+func isPreReadinessInitContainerStatefulSet(sts appsv1.StatefulSet) bool {
+	return container.GetByName(construct.ReadinessProbeContainerName, sts.Spec.Template.Spec.InitContainers) == nil
+}

dev_notes/RELEASE_NOTES.md

@@ -1,10 +1,34 @@
 *(Please use the [release template](release-notes-template.md) as the template for this document)*
 <!-- Next release -->
-# MongoDB Kubernetes Operator 0.5.3
+# MongoDB Kubernetes Operator 0.6.0
 ## Kubernetes Operator
+
+* Breaking Changes
+  * A new VolumeClaimTemplate has been added `logs-volume`. When you deploy the operator, if there is an existing StatefulSet the operator will attempt to perform an invalid update. The existing StatefulSet must be deleted before upgrading the operator.
+  
+  * The user of the mongod and mongodb-agent containers has changed. This means that there will be permissions
+    issues when upgrading from an earlier version of the operator. In order to update the permissions in the volume, you can use an init container.
+
+* Upgrade instructions
+
+  Remove the current operator deployment
+  -  `kubectl delete deployment <operator-deployment>`
+  Delete the existing StatefulSet for the MongoDBCommunity resource
+  Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. 
+  -   `kubectl delete statefulset <mdb-resource-name>`
+  Install the new operator
+  - follow the regular [installation instruction](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/install-upgrade.md)
+  Patch the StatefulSet once it has been created. This will add an init container that will update the permissions of the existing volume.
+  - `kubectl patch statefulset <sts-name> --type='json' --patch '[ {"op":"add","path":"/spec/template/spec/initContainers/-", "value": { "name": "change-data-dir-permissions", "image": "busybox", "command": [ "chown", "-R", "2000", "/data" ], "securityContext": { "runAsNonRoot": false, "runAsUser": 0, "runAsGroup":0 }, "volumeMounts": [ { "mountPath": "/data", "name" : "data-volume" } ] } } ]'`
+   
 * Bug fixes
   * Fixes an issue that prevented the agents from reaching goal state when upgrading minor version of MongoDB.
 
+ ## Updated Image Tags
+ * mongodb-kubernetes-operator:0.6.0
+ * mongodb-agent:0.29.0.6830-1
+
+
 <!-- Past Releases -->
 # MongoDB Kubernetes Operator 0.5.2
 ## Kubernetes Operator