From dfbb41c08b6ed83b9d7deeba8d5802a42851433c Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Sun, 15 Nov 2026 15:07:39 +0100 Subject: [PATCH] Fix flaky drift-detection-manager group tests: wait for the cache The specs of "clusters deleted while they are in a group" wrote the group ConfigMaps and read them back right away through testEnv.Client, which is served from a cache. A stale read made removeStaleGroupMembers miss the deleted cluster, or the final check see the old members: Expected ["a-9meju91b8i", "a-9t2csubehc"] to consist of ["a-9meju91b8i"] Wait with Eventually for the groups to be visible before removing stale members, and for the result afterwards. Removing stale members is retried inside the Eventually when the Deployment survives. Co-Authored-By: Claude Sonnet 5.5 --- controllers/resourcesummary_group_test.go | 68 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/controllers/resourcesummary_group_test.go b/controllers/resourcesummary_group_test.go index 6439df20..de73f202 100644 --- a/controllers/resourcesummary_group_test.go +++ b/controllers/resourcesummary_group_test.go @@ -331,8 +331,12 @@ var _ = Describe("drift-detection-manager serving a group of clusters", func() { } removeStaleMembers := func(groupName string) { - deployment, err := getGroupDeployment(groupName) - Expect(err).To(BeNil()) + var deployment *appsv1.Deployment + Eventually(func() error { + var err error + deployment, err = getGroupDeployment(groupName) + return err + }, timeout, pollingInterval).Should(BeNil()) controllers.RemoveStaleGroupMembers(context.TODO(), testEnv.Client, testEnv.Config, deployment, logger) } @@ -346,6 +350,35 @@ var _ = Describe("drift-detection-manager serving a group of clusters", func() { return names } + // testEnv.Client is served from a cache, so what has just been written may not be visible yet: + // the helpers below wait for the cache instead of reading it once. + + // waitForMembers waits until the clusters in the groups of the namespace are exactly expected. + waitForMembers := func(clusterNamespace string, expected ...string) { + Eventually(func() []string { return memberNames(clusterNamespace) }, + timeout, pollingInterval).Should(ConsistOf(expected)) + } + + // waitForNoGroups waits until the namespace has no group left. + waitForNoGroups := func(clusterNamespace string) { + Eventually(func() int { return len(groupsOf(clusterNamespace)) }, + timeout, pollingInterval).Should(BeZero()) + } + + // firstGroupName waits for the namespace to have a group and returns the name of the first one. + firstGroupName := func(clusterNamespace string) string { + var groupName string + Eventually(func() bool { + groups := groupsOf(clusterNamespace) + if len(groups) == 0 { + return false + } + groupName = groups[0].Name + return true + }, timeout, pollingInterval).Should(BeTrue()) + return groupName + } + It("a cluster that does not exist anymore leaves its group, the others stay", func() { controllers.SetAgentClustersPerInstance(3) @@ -355,13 +388,20 @@ var _ = Describe("drift-detection-manager serving a group of clusters", func() { deployInGroup(clusterNamespace, existing) deployInGroup(clusterNamespace, deleted) - groupName := groupsOf(clusterNamespace)[0].Name - - removeStaleMembers(groupName) - - Expect(memberNames(clusterNamespace)).To(ConsistOf(existing)) - _, err := getGroupDeployment(groupName) - Expect(err).To(BeNil()) + // removeStaleMembers lists the groups too: both clusters must be visible before it runs + waitForMembers(clusterNamespace, existing, deleted) + groupName := firstGroupName(clusterNamespace) + + // Removing stale members can be repeated: if the groups it lists are not up to date it + // does nothing, and the next attempt sees them. + Eventually(func() []string { + removeStaleMembers(groupName) + return memberNames(clusterNamespace) + }, timeout, pollingInterval).Should(ConsistOf(existing)) + Eventually(func() error { + _, err := getGroupDeployment(groupName) + return err + }, timeout, pollingInterval).Should(BeNil()) }) It("the Deployment is removed with the last cluster of the group", func() { @@ -372,11 +412,12 @@ var _ = Describe("drift-detection-manager serving a group of clusters", func() { deployInGroup(clusterNamespace, deleted1) deployInGroup(clusterNamespace, deleted2) - groupName := groupsOf(clusterNamespace)[0].Name + waitForMembers(clusterNamespace, deleted1, deleted2) + groupName := firstGroupName(clusterNamespace) removeStaleMembers(groupName) - Expect(groupsOf(clusterNamespace)).To(BeEmpty()) + waitForNoGroups(clusterNamespace) Eventually(func() bool { _, err := getGroupDeployment(groupName) return apierrors.IsNotFound(err) @@ -390,7 +431,8 @@ var _ = Describe("drift-detection-manager serving a group of clusters", func() { deleted := randomString() deployInGroup(clusterNamespace, deleted) - groupName := groupsOf(clusterNamespace)[0].Name + waitForMembers(clusterNamespace, deleted) + groupName := firstGroupName(clusterNamespace) resourceSummary := createResourceSummary(clusterNamespace, deleted) // The ResourceSummary is deleted, and the cluster is not removed yet: the @@ -406,7 +448,7 @@ var _ = Describe("drift-detection-manager serving a group of clusters", func() { }, timeout, pollingInterval).Should(BeTrue()) removeStaleMembers(groupName) - Expect(groupsOf(clusterNamespace)).To(BeEmpty()) + waitForNoGroups(clusterNamespace) }) }) })