Headless mode readiness probe (#192)

Author: chatton

.dockerignore

@@ -1,7 +1,6 @@
 .github
 .idea
 agent
-build/_output
 zz_*
 vendor/
 scripts/

README.md

@@ -111,7 +111,7 @@ If you want to deploy the operator on OpenShift you will have to provide the env
 
 See [here](deploy/crds/mongodb.com_v1_mongodb_openshift_cr.yaml) for an example of how to provide the required configuration for a MongoDB ReplicaSet.
 
-See [here](deploy/operator_openshift.yaml) for an example of how to configure the Operator deployment.
+See [here](deploy/openshift/operator_openshift.yaml) for an example of how to configure the Operator deployment.
 
 #### Example
 

agent/Dockerfile

@@ -21,7 +21,7 @@ RUN mkdir -p agent \
     && rm -r mongodb-mms-automation-agent-*
 
 RUN mkdir -p /var/lib/mongodb-mms-automation/probes/ \
-    && curl --retry 3 https://readinessprobe-test.s3-us-west-1.amazonaws.com/readiness -o /var/lib/mongodb-mms-automation/probes/readinessprobe \
+    && curl --retry 3 https://readinessprobe.s3-us-west-1.amazonaws.com/readiness -o /var/lib/mongodb-mms-automation/probes/readinessprobe \
     && chmod +x /var/lib/mongodb-mms-automation/probes/readinessprobe \
     && mkdir -p /var/log/mongodb-mms-automation/ \
     && chmod -R +wr /var/log/mongodb-mms-automation/ \

build/Dockerfile

@@ -4,6 +4,14 @@ ENV OPERATOR=/usr/local/bin/mongodb-kubernetes-operator \
     USER_UID=1001 \
     USER_NAME=mongodb-kubernetes-operator
 
+RUN apt-get update  && \
+    apt-get install -y curl
+
+ENV manifest_version=4.4
+RUN mkdir -p /content/ \
+        && chmod -R +r /content/ \
+        && curl --fail --retry 3 -o /usr/local/version_manifest.json "https://opsmanager.mongodb.com/static/version_manifest/${manifest_version}.json"
+
 # install operator binary
 COPY build/_output/bin/mongodb-kubernetes-operator ${OPERATOR}
 

deploy/operator_openshift.yaml renamed to deploy/openshift/operator_openshift.yaml

@@ -47,7 +47,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDB, sts *app
 	assert.Equal(t, mdb.Namespace, sts.Namespace)
 	assert.Equal(t, appsv1.RollingUpdateStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type)
 	assert.Equal(t, operatorServiceAccountName, sts.Spec.Template.Spec.ServiceAccountName)
-	assert.Len(t, sts.Spec.Template.Spec.Containers[0].Env, 1)
+	assert.Len(t, sts.Spec.Template.Spec.Containers[0].Env, 4)
 	assert.Len(t, sts.Spec.Template.Spec.Containers[1].Env, 1)
 
 	agentContainer := sts.Spec.Template.Spec.Containers[0]

pkg/controller/mongodb/build_statefulset_test.go

@@ -63,15 +63,18 @@ const (
 	mongodbImageEnv              = "MONGODB_IMAGE"
 	mongodbRepoUrl               = "MONGODB_REPO_URL"
 	mongodbToolsVersionEnv       = "MONGODB_TOOLS_VERSION"
+	headlessAgentEnv             = "HEADLESS_AGENT"
+	podNamespaceEnv              = "POD_NAMESPACE"
+	automationConfigEnv          = "AUTOMATION_CONFIG_MAP"
 
-	AutomationConfigKey            = "automation-config"
+	AutomationConfigKey            = "cluster-config.json"
 	agentName                      = "mongodb-agent"
 	mongodbName                    = "mongod"
 	versionUpgradeHookName         = "mongod-posthook"
 	dataVolumeName                 = "data-volume"
 	versionManifestFilePath        = "/usr/local/version_manifest.json"
 	readinessProbePath             = "/var/lib/mongodb-mms-automation/probes/readinessprobe"
-	clusterFilePath                = "/var/lib/automation/config/automation-config"
+	clusterFilePath                = "/var/lib/automation/config/cluster-config.json"
 	operatorServiceAccountName     = "mongodb-kubernetes-operator"
 	agentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json"
 
@@ -185,7 +188,7 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R
 	if err := r.ensureAutomationConfig(mdb); err != nil {
 		return status.Update(r.client.Status(), &mdb,
 			statusOptions().
-				withMessage(Error, fmt.Sprintf("error creating automation config config map: %s", err)).
+				withMessage(Error, fmt.Sprintf("error creating automation config secret: %s", err)).
 				withFailedPhase(),
 		)
 	}
@@ -249,6 +252,7 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R
 	if !ready {
 		return status.Update(r.client.Status(), &mdb,
 			statusOptions().
+				withMembers(int(currentSts.Status.ReadyReplicas)).
 				withMessage(Info, fmt.Sprintf("StatefulSet %s/%s is not yet ready, retrying in 10 seconds", mdb.Namespace, mdb.Name)).
 				withPendingPhase(10),
 		)
@@ -356,7 +360,7 @@ func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, existingSta
 	//some issues with nil/empty maps not being compared correctly otherwise
 	areEqual := bytes.Equal(stsCopyBytes, stsBytes)
 
-	isReady := statefulset.IsReady(*existingStatefulSet, mdb.Spec.Members)
+	isReady := statefulset.IsReady(*existingStatefulSet, mdb.ReplicasThisReconciliation())
 	if existingStatefulSet.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType && !isReady {
 		r.log.Info("StatefulSet has left ready state, version upgrade in progress")
 		annotations := map[string]string{
@@ -492,6 +496,7 @@ func buildService(mdb mdbv1.MongoDB) corev1.Service {
 		SetServiceType(corev1.ServiceTypeClusterIP).
 		SetClusterIP("None").
 		SetPort(27017).
+		SetPublishNotReadyAddresses(true).
 		Build()
 }
 
@@ -519,17 +524,17 @@ func (r ReplicaSetReconciler) buildAutomationConfigSecret(mdb mdbv1.MongoDB) (co
 
 	authModification, err := scram.EnsureScram(r.client, mdb.ScramCredentialsNamespacedName(), mdb)
 	if err != nil {
-		return corev1.Secret{}, err
+		return corev1.Secret{}, errors.Errorf("could not ensure scram credentials: %s", err)
 	}
 
 	tlsModification, err := getTLSConfigModification(r.client, mdb)
 	if err != nil {
-		return corev1.Secret{}, err
+		return corev1.Secret{}, errors.Errorf("could not configure TLS modification: %s", err)
 	}
 
 	currentAC, err := getCurrentAutomationConfig(r.client, mdb)
 	if err != nil {
-		return corev1.Secret{}, err
+		return corev1.Secret{}, errors.Errorf("could not read existing automation config: %s", err)
 	}
 
 	ac, err := buildAutomationConfig(
@@ -540,11 +545,11 @@ func (r ReplicaSetReconciler) buildAutomationConfigSecret(mdb mdbv1.MongoDB) (co
 		tlsModification,
 	)
 	if err != nil {
-		return corev1.Secret{}, err
+		return corev1.Secret{}, fmt.Errorf("could not build automation config: %s", err)
 	}
 	acBytes, err := json.Marshal(ac)
 	if err != nil {
-		return corev1.Secret{}, err
+		return corev1.Secret{}, fmt.Errorf("could not marshal automation config: %s", err)
 	}
 
 	return secret.Builder().
@@ -590,7 +595,7 @@ func isChangingVersion(mdb mdbv1.MongoDB) bool {
 	return false
 }
 
-func mongodbAgentContainer(volumeMounts []corev1.VolumeMount) container.Modification {
+func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount) container.Modification {
 	return container.Apply(
 		container.WithName(agentName),
 		container.WithImage(os.Getenv(agentImageEnv)),
@@ -609,6 +614,23 @@ func mongodbAgentContainer(volumeMounts []corev1.VolumeMount) container.Modifica
 		},
 		),
 		container.WithEnvs(
+			corev1.EnvVar{
+				Name:  headlessAgentEnv,
+				Value: "true",
+			},
+			corev1.EnvVar{
+				Name: podNamespaceEnv,
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						APIVersion: "v1",
+						FieldPath:  "metadata.namespace",
+					},
+				},
+			},
+			corev1.EnvVar{
+				Name:  automationConfigEnv,
+				Value: automationConfigSecretName,
+			},
 			corev1.EnvVar{
 				Name:  agentHealthStatusFilePathEnv,
 				Value: agentHealthStatusFilePathValue,
@@ -705,7 +727,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific
 				podtemplatespec.WithVolume(hooksVolume),
 				podtemplatespec.WithVolume(automationConfigVolume),
 				podtemplatespec.WithServiceAccount(operatorServiceAccountName),
-				podtemplatespec.WithContainer(agentName, mongodbAgentContainer([]corev1.VolumeMount{agentHealthStatusVolumeMount, automationConfigVolumeMount, dataVolume})),
+				podtemplatespec.WithContainer(agentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), []corev1.VolumeMount{agentHealthStatusVolumeMount, automationConfigVolumeMount, dataVolume})),
 				podtemplatespec.WithContainer(mongodbName, mongodbContainer(mdb.Spec.Version, []corev1.VolumeMount{mongodHealthStatusVolumeMount, dataVolume, hooksVolumeMount})),
 				podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})),
 				buildTLSPodSpecModification(mdb),

pkg/controller/mongodb/replica_set_controller.go

@@ -432,6 +432,8 @@ func TestReplicaSet_IsScaledDown_OneMember_AtATime_WhenItAlreadyExists(t *testin
 	assert.Equal(t, true, res.Requeue)
 	assert.Equal(t, 4, mdb.Status.Members)
 
+	makeStatefulSetReady(t, mgr.GetClient(), mdb)
+
 	res, err = r.Reconcile(reconcile.Request{NamespacedName: mdb.NamespacedName()})
 
 	assert.NoError(t, err)
@@ -472,6 +474,8 @@ func TestReplicaSet_IsScaledUp_OneMember_AtATime_WhenItAlreadyExists(t *testing.
 	assert.Equal(t, true, res.Requeue)
 	assert.Equal(t, 4, mdb.Status.Members)
 
+	makeStatefulSetReady(t, mgr.GetClient(), mdb)
+
 	res, err = r.Reconcile(reconcile.Request{NamespacedName: mdb.NamespacedName()})
 
 	assert.NoError(t, err)
@@ -522,7 +526,7 @@ func TestReplicaSet_IsScaledUpToDesiredMembers_WhenFirstCreated(t *testing.T) {
 
 func TestOpenshift_Configuration(t *testing.T) {
 	sts := performReconciliationAndGetStatefulSet(t, "openshift_mdb.yaml")
-	assert.Equal(t, "MANAGED_SECURITY_CONTEXT", sts.Spec.Template.Spec.Containers[0].Env[1].Name)
+	assert.Equal(t, "MANAGED_SECURITY_CONTEXT", sts.Spec.Template.Spec.Containers[0].Env[3].Name)
 	assert.Equal(t, "MANAGED_SECURITY_CONTEXT", sts.Spec.Template.Spec.Containers[1].Env[1].Name)
 }
 
@@ -631,8 +635,8 @@ func makeStatefulSetReady(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDB) {
 	sts := appsv1.StatefulSet{}
 	err := c.Get(context.TODO(), mdb.NamespacedName(), &sts)
 	assert.NoError(t, err)
-	sts.Status.ReadyReplicas = int32(mdb.Spec.Members)
-	sts.Status.UpdatedReplicas = int32(mdb.Spec.Members)
+	sts.Status.ReadyReplicas = int32(mdb.ReplicasThisReconciliation())
+	sts.Status.UpdatedReplicas = int32(mdb.ReplicasThisReconciliation())
 	err = c.Update(context.TODO(), &sts)
 	assert.NoError(t, err)
 }

pkg/controller/mongodb/replicaset_controller_test.go

@@ -94,7 +94,9 @@ def dump_automation_configs(namespace: str) -> None:
         return
     for mdb in mongodb_resources:
         name = mdb["metadata"]["name"]
-        dump_secret_keys_namespaced(namespace, ["automation-config"], f"{name}-config")
+        dump_secret_keys_namespaced(
+            namespace, ["cluster-config.json"], f"{name}-config"
+        )
 
 
 def dump_all(namespace: str) -> None:

scripts/dev/dump_diagnostic.py

@@ -7,14 +7,12 @@
     load_yaml_from_file,
 )
 import k8s_conditions
-import k8s_request_data
 import dump_diagnostic
 from dockerutil import build_and_push_image
 from typing import Dict
 from dev_config import load_config, DevConfig
 from kubernetes import client, config
 import argparse
-import time
 import os
 import sys
 import yaml
@@ -78,11 +76,12 @@ def _prepare_testrunner_environment(config_file: str) -> None:
     )
 
 
-def create_kube_config() -> None:
+def create_kube_config(config_file: str) -> None:
     """Replicates the local kubeconfig file (pointed at by KUBECONFIG),
     as a ConfigMap."""
     corev1 = client.CoreV1Api()
     print("Creating kube-config ConfigMap")
+    dev_config = load_config(config_file)
 
     svc = corev1.read_namespaced_service("kubernetes", "default")
 
@@ -104,7 +103,7 @@ def create_kube_config() -> None:
     )
 
     k8s_conditions.ignore_if_already_exists(
-        lambda: corev1.create_namespaced_config_map("default", config_map)
+        lambda: corev1.create_namespaced_config_map(dev_config.namespace, config_map)
     )
 
 
@@ -318,7 +317,7 @@ def main() -> int:
     config.load_kube_config()
 
     dev_config = load_config(args.config_file)
-    create_kube_config()
+    create_kube_config(args.config_file)
 
     try:
         build_and_push_images(args, dev_config)