Increase Stability of FCV tests (#550)

Author: chatton

api/v1/mongodbcommunity_types.go

@@ -339,7 +339,7 @@ type Authentication struct {
 	// +optional
 	// +kubebuilder:default:=true
 	// +nullable
-	IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers"`
+	IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=SCRAM

test/e2e/feature_compatibility_version/feature_compatibility_version_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait"
+
 	. "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"
 
 	e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
@@ -49,7 +51,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) {
 	t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) {
 		defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
 		t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6"))
-		t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb))
+		t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
 	})
 
 	t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds())
@@ -59,7 +61,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) {
 	t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) {
 		defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
 		t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6"))
-		t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb))
+		t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
 	})
 
 	t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", tester.HasFCV("4.0", 3))

test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait"
+
 	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
@@ -51,7 +53,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) {
 	t.Run("MongoDB is reachable", func(t *testing.T) {
 		defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
 		t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6"))
-		t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb))
+		t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
 		t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds())
 	})
 
@@ -64,7 +66,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) {
 			})
 			assert.NoError(t, err)
 		})
-		t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetBecomesReady(&mdb))
+		t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
 		t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
 	})
 	t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.2", tester.HasFCV("4.2", 3))

test/e2e/mongodbtests/mongodbtests.go

@@ -33,22 +33,32 @@ func SkipTestIfLocal(t *testing.T, msg string, f func(t *testing.T)) {
 
 // StatefulSetBecomesReady ensures that the underlying stateful set
 // reaches the running state.
-func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
-	return statefulSetIsReady(mdb, time.Second*15, time.Minute*12)
+func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
+	defaultOpts := []wait.Configuration{
+		wait.RetryInterval(time.Second * 15),
+		wait.Timeout(time.Minute * 15),
+	}
+	defaultOpts = append(defaultOpts, opts...)
+	return statefulSetIsReady(mdb, defaultOpts...)
 }
 
 // StatefulSetBecomesUnready ensures the underlying stateful set reaches
 // the unready state.
-func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
-	return statefulSetIsNotReady(mdb, time.Second*15, time.Minute*12)
+func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
+	defaultOpts := []wait.Configuration{
+		wait.RetryInterval(time.Second * 15),
+		wait.Timeout(time.Minute * 15),
+	}
+	defaultOpts = append(defaultOpts, opts...)
+	return statefulSetIsNotReady(mdb, defaultOpts...)
 }
 
 // StatefulSetIsReadyAfterScaleDown ensures that a replica set is scaled down correctly
 // note: scaling down takes considerably longer than scaling up due the readiness probe
 // failure threshold being high
 func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
 	return func(t *testing.T) {
-		err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45)
+		err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, wait.RetryInterval(time.Second*60), wait.Timeout(time.Minute*45))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -58,9 +68,9 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi
 
 // StatefulSetIsReady ensures that the underlying stateful set
 // reaches the running state
-func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) {
+func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
 	return func(t *testing.T) {
-		err := wait.ForStatefulSetToBeReady(t, mdb, interval, timeout)
+		err := wait.ForStatefulSetToBeReady(t, mdb, opts...)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -69,9 +79,9 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, tim
 }
 
 // statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state.
-func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) {
+func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
 	return func(t *testing.T) {
-		err := wait.ForStatefulSetToBeUnready(t, mdb, interval, timeout)
+		err := wait.ForStatefulSetToBeUnready(t, mdb, opts...)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -138,7 +148,7 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO
 // resource has the correct Update Strategy
 func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) {
 	return func(t *testing.T) {
-		err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8)
+		err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, wait.RetryInterval(time.Second*15), wait.Timeout(time.Minute*8))
 		if err != nil {
 			t.Fatal(err)
 		}

test/e2e/util/wait/wait.go

@@ -60,44 +60,48 @@ func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration,
 
 // ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name
 // have reached the ready status
-func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error {
-	return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
+func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, opts ...Configuration) error {
+	options := newOptions(opts...)
+	return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
 		return sts.Spec.UpdateStrategy.Type == strategy
 	})
 }
 
 // ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name
 // have reached the ready status
-func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
-	return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
+func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error {
+	options := newOptions(opts...)
+	return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
 		return statefulset.IsReady(sts, mdb.Spec.Members)
 	})
 }
 
 // ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name
 // is not ready.
-func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
-	return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
+func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error {
+	options := newOptions(opts...)
+	return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
 		return !statefulset.IsReady(sts, mdb.Spec.Members)
 	})
 }
 
 // ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct
 // and does not account for the updated replicas
-func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
-	return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
+func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error {
+	options := newOptions(opts...)
+	return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
 		return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas
 	})
 }
 
-func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error {
-	_, err := ForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace)
+func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error {
+	_, err := ForStatefulSetToExist(mdb.Name, waitOpts.RetryInterval, waitOpts.Timeout, mdb.Namespace)
 	if err != nil {
 		return errors.Errorf("error waiting for stateful set to be created: %s", err)
 	}
 
 	sts := appsv1.StatefulSet{}
-	return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
+	return wait.Poll(waitOpts.RetryInterval, waitOpts.Timeout, func() (done bool, err error) {
 		err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts)
 		if err != nil {
 			return false, err

test/e2e/util/wait/wait_options.go

@@ -0,0 +1,41 @@
+package wait
+
+import "time"
+
+type Configuration func(*Options)
+
+// Options holds values which can be configured when waiting for specific confitions.
+type Options struct {
+	RetryInterval time.Duration
+	Timeout       time.Duration
+}
+
+// RetryInterval specifies the RetryInterval
+func RetryInterval(retryInterval time.Duration) Configuration {
+	return func(options *Options) {
+		options.RetryInterval = retryInterval
+	}
+}
+
+// Timeout specifies the Timeout
+func Timeout(timeout time.Duration) Configuration {
+	return func(options *Options) {
+		options.Timeout = timeout
+	}
+}
+
+// newOptions returns an Options that has been configured with default values.
+func newOptions(fns ...Configuration) Options {
+	defaults := defaultStatefulSetReadinessOptions()
+	for _, fn := range fns {
+		fn(&defaults)
+	}
+	return defaults
+}
+
+func defaultStatefulSetReadinessOptions() Options {
+	return Options{
+		RetryInterval: time.Second * 15,
+		Timeout:       time.Minute * 12,
+	}
+}