diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index 1f03fb625..fa92786d2 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -186,8 +186,8 @@ type UpdateRunSpec struct { // The resource snapshot index of the selected resources to be updated across clusters. // The index represents a group of resource snapshots that includes all the resources a ResourcePlacement selected. - // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="resourceSnapshotIndex is immutable" + // +kubebuilder:validation:Optional ResourceSnapshotIndex string `json:"resourceSnapshotIndex"` // The name of the update strategy that specifies the stages and the sequence @@ -374,6 +374,11 @@ type UpdateRunStatus struct { // +kubebuilder:validation:Optional PolicyObservedClusterCount int `json:"policyObservedClusterCount,omitempty"` + // ResourceSnapshotIndexUsed records the resource snapshot index that the update run is based on. + // The index represents the same resource snapshots as specified in the spec field, or the latest. + // +kubbebuilder:validation:Optional + ResourceSnapshotIndexUsed string `json:"resourceSnapshotIndexUsed,omitempty"` + // ApplyStrategy is the apply strategy that the stagedUpdateRun is using. // It is the same as the apply strategy in the CRP when the staged update run starts. // The apply strategy is not updated during the update run even if it changes in the CRP. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml index bf75d613d..a2ddf03ec 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml @@ -1200,7 +1200,6 @@ spec: type: string required: - placementName - - resourceSnapshotIndex - stagedRolloutStrategyName type: object x-kubernetes-validations: @@ -1895,6 +1894,11 @@ spec: All clusters involved in the update run are selected from the list of clusters scheduled by the CRP according to the current policy. type: string + resourceSnapshotIndexUsed: + description: |- + ResourceSnapshotIndexUsed records the resource snapshot index that the update run is based on. + The index represents the same resource snapshots as specified in the spec field, or the latest. + type: string stagedUpdateStrategySnapshot: description: |- UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml index 10fe5f738..0db56cd16 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml @@ -120,7 +120,6 @@ spec: type: string required: - placementName - - resourceSnapshotIndex - stagedRolloutStrategyName type: object x-kubernetes-validations: @@ -815,6 +814,11 @@ spec: All clusters involved in the update run are selected from the list of clusters scheduled by the CRP according to the current policy. type: string + resourceSnapshotIndexUsed: + description: |- + ResourceSnapshotIndexUsed records the resource snapshot index that the update run is based on. + The index represents the same resource snapshots as specified in the spec field, or the latest. + type: string stagedUpdateStrategySnapshot: description: |- UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index c69e34f11..87f5eaf4b 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -96,7 +96,7 @@ func (r *Reconciler) executeUpdatingStage( updateRunSpec := updateRun.GetUpdateRunSpec() updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] // The parse error is ignored because the initialization should have caught it. - resourceIndex, _ := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) + resourceIndex, _ := strconv.Atoi(updateRunStatus.ResourceSnapshotIndexUsed) resourceSnapshotName := fmt.Sprintf(placementv1beta1.ResourceSnapshotNameFmt, updateRunSpec.PlacementName, resourceIndex) updateRunRef := klog.KObj(updateRun) // Create the map of the toBeUpdatedBindings. diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 9b3e06833..980283497 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -160,7 +160,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -521,7 +521,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -680,7 +680,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -774,7 +774,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -883,7 +883,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1014,7 +1014,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1106,7 +1106,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1163,7 +1163,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index 45a4ba48a..f8ff0682e 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -471,30 +471,10 @@ func validateAfterStageTask(tasks []placementv1beta1.StageTask) error { func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey types.NamespacedName, updateRun placementv1beta1.UpdateRunObj) error { updateRunRef := klog.KObj(updateRun) updateRunSpec := updateRun.GetUpdateRunSpec() - placementName := placementKey.Name - snapshotIndex, err := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) - if err != nil || snapshotIndex < 0 { - err := controller.NewUserError(fmt.Errorf("invalid resource snapshot index `%s` provided, expected an integer >= 0", updateRunSpec.ResourceSnapshotIndex)) - klog.ErrorS(err, "Failed to parse the resource snapshot index", "updateRun", updateRunRef) - // no more retries here. - return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) - } - - resourceSnapshotList, err := controller.ListAllResourceSnapshotWithAnIndex(ctx, r.Client, updateRunSpec.ResourceSnapshotIndex, placementName, placementKey.Namespace) + resourceSnapshotObjs, err := r.getResourceSnapshotObjs(ctx, placementKey, updateRun) if err != nil { - klog.ErrorS(err, "Failed to list the resourceSnapshots associated with the placement", - "placement", placementKey, "resourceSnapshotIndex", snapshotIndex, "updateRun", updateRunRef) - // err can be retried. - return controller.NewAPIServerError(true, err) - } - - resourceSnapshotObjs := resourceSnapshotList.GetResourceSnapshotObjs() - if len(resourceSnapshotObjs) == 0 { - err := controller.NewUserError(fmt.Errorf("no resourceSnapshots with index `%d` found for placement `%s`", snapshotIndex, placementKey)) - klog.ErrorS(err, "No specified resourceSnapshots found", "updateRun", updateRunRef) - // no more retries here. - return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + return err } // Look for the master resourceSnapshot. @@ -509,12 +489,18 @@ func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey t // No masterResourceSnapshot found. if masterResourceSnapshot == nil { - err := controller.NewUnexpectedBehaviorError(fmt.Errorf("no master resourceSnapshot found for placement `%s` with index `%d`", placementKey, snapshotIndex)) + err := controller.NewUnexpectedBehaviorError(fmt.Errorf("no master resourceSnapshot found for placement %s", placementKey)) klog.ErrorS(err, "Failed to find master resourceSnapshot", "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } - klog.V(2).InfoS("Found master resourceSnapshot", "placement", placementKey, "index", snapshotIndex, "updateRun", updateRunRef) + + klog.V(2).InfoS("Found master resourceSnapshot", "placement", placementKey, "masterResourceSnapshot", masterResourceSnapshot.GetName(), "updateRun", updateRunRef) + + // Record the resource snapshot index used. + updateRunStatus := updateRun.GetUpdateRunStatus() + updateRunStatus.ResourceSnapshotIndexUsed = masterResourceSnapshot.GetLabels()[placementv1beta1.ResourceIndexLabel] + updateRun.SetUpdateRunStatus(*updateRunStatus) resourceSnapshotRef := klog.KObj(masterResourceSnapshot) // Fetch all the matching overrides. @@ -526,7 +512,6 @@ func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey t } // Pick the overrides associated with each target cluster. - updateRunStatus := updateRun.GetUpdateRunStatus() for _, stageStatus := range updateRunStatus.StagesStatus { for i := range stageStatus.Clusters { clusterStatus := &stageStatus.Clusters[i] @@ -543,6 +528,58 @@ func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey t return nil } +// getResourceSnapshotObjs retrieves the list of resource snapshot objects from the specified ResourceSnapshotIndex. +// If ResourceSnapshotIndex is unspecified, it returns the list of latest resource snapshots. +func (r *Reconciler) getResourceSnapshotObjs(ctx context.Context, placementKey types.NamespacedName, updateRun placementv1beta1.UpdateRunObj) ([]placementv1beta1.ResourceSnapshotObj, error) { + updateRunRef := klog.KObj(updateRun) + updateRunSpec := updateRun.GetUpdateRunSpec() + var resourceSnapshotObjs []placementv1beta1.ResourceSnapshotObj + if updateRunSpec.ResourceSnapshotIndex != "" { + snapshotIndex, err := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) + if err != nil || snapshotIndex < 0 { + err := controller.NewUserError(fmt.Errorf("invalid resource snapshot index `%s` provided, expected an integer >= 0", updateRunSpec.ResourceSnapshotIndex)) + klog.ErrorS(err, "Failed to parse the resource snapshot index", "updateRun", updateRunRef) + // no more retries here. + return nil, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + } + + resourceSnapshotList, err := controller.ListAllResourceSnapshotWithAnIndex(ctx, r.Client, updateRunSpec.ResourceSnapshotIndex, placementKey.Name, placementKey.Namespace) + if err != nil { + klog.ErrorS(err, "Failed to list the resourceSnapshots associated with the placement", + "placement", placementKey, "resourceSnapshotIndex", snapshotIndex, "updateRun", updateRunRef) + // list err can be retried. + return nil, controller.NewAPIServerError(true, err) + } + + resourceSnapshotObjs = resourceSnapshotList.GetResourceSnapshotObjs() + if len(resourceSnapshotObjs) == 0 { + err := controller.NewUserError(fmt.Errorf("no resourceSnapshots with index `%d` found for placement `%s`", snapshotIndex, placementKey)) + klog.ErrorS(err, "No specified resourceSnapshots found", "updateRun", updateRunRef) + // no more retries here. + return resourceSnapshotObjs, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + } + return resourceSnapshotObjs, nil + } + + klog.V(2).InfoS("No resource snapshot index specified, fetching latest resource snapshots", "placement", placementKey, "updateRun", updateRunRef) + latestResourceSnapshots, err := controller.ListLatestResourceSnapshots(ctx, r.Client, placementKey) + if err != nil { + klog.ErrorS(err, "Failed to list the latest resourceSnapshots associated with the placement", + "placement", placementKey, "updateRun", updateRunRef) + // list err can be retried. + return nil, controller.NewAPIServerError(true, err) + } + + resourceSnapshotObjs = latestResourceSnapshots.GetResourceSnapshotObjs() + if len(resourceSnapshotObjs) == 0 { + err := fmt.Errorf("no latest resourceSnapshots found for placement `%s`. This might be a transient state, need retry", placementKey) + klog.ErrorS(err, "No latest resourceSnapshots found for placement. This might be transient, need retry", "placement", placementKey, "updateRun", updateRunRef) + // retryable error. + return resourceSnapshotObjs, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + } + return resourceSnapshotObjs, nil +} + // recordInitializationSucceeded records the successful initialization condition in the UpdateRun status. func (r *Reconciler) recordInitializationSucceeded(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { updateRunStatus := updateRun.GetUpdateRunStatus() diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 8bfab157a..f70103b6b 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -57,7 +57,7 @@ var _ = Describe("Updaterun initialization tests", func() { var resourceBindings []*placementv1beta1.ClusterResourceBinding var targetClusters []*clusterv1beta1.MemberCluster var unscheduledClusters []*clusterv1beta1.MemberCluster - var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot + var resourceSnapshot, resourceSnapshot2, resourceSnapshot3 *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot BeforeEach(func() { @@ -75,6 +75,8 @@ var _ = Describe("Updaterun initialization tests", func() { resourceBindings, targetClusters, unscheduledClusters = generateTestClusterResourceBindingsAndClusters(1) policySnapshot = generateTestClusterSchedulingPolicySnapshot(1, len(targetClusters)) resourceSnapshot = generateTestClusterResourceSnapshot() + resourceSnapshot2 = generateTestClusterResourceSnapshot() + resourceSnapshot3 = generateTestClusterResourceSnapshot() // Set smaller wait time for testing stageUpdatingWaitTime = time.Second * 3 @@ -113,9 +115,13 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Delete(ctx, updateStrategy)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) updateStrategy = nil - By("Deleting the clusterResourceSnapshot") + By("Deleting the clusterResourceSnapshots") Expect(k8sClient.Delete(ctx, resourceSnapshot)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) resourceSnapshot = nil + Expect(k8sClient.Delete(ctx, resourceSnapshot2)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + resourceSnapshot2 = nil + Expect(k8sClient.Delete(ctx, resourceSnapshot3)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + resourceSnapshot3 = nil By("Deleting the clusterResourceOverride") Expect(k8sClient.Delete(ctx, clusterResourceOverride)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) @@ -654,7 +660,8 @@ var _ = Describe("Updaterun initialization tests", func() { return err } - want := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + // no resource snapshot created in this test + want := generateSucceededInitializationStatus(crp, updateRun, "", policySnapshot, updateStrategy, clusterResourceOverride) // No clusters should be selected in the first stage. want.StagesStatus[0].Clusters = []placementv1beta1.ClusterUpdatingStatus{} // All clusters should be selected in the second stage and sorted by name. @@ -696,7 +703,8 @@ var _ = Describe("Updaterun initialization tests", func() { return err } - want := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + // no resource snapshot created in this test + want := generateSucceededInitializationStatus(crp, updateRun, "", policySnapshot, updateStrategy, clusterResourceOverride) for i := range want.StagesStatus[0].Clusters { // Remove the CROs, as they are not added in this test. want.StagesStatus[0].Clusters[i].ClusterResourceOverrideSnapshots = nil @@ -783,6 +791,18 @@ var _ = Describe("Updaterun initialization tests", func() { validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) }) + It("Should fail to initialize if the specified resource snapshot is not found when no resource index specified - no resourceSnapshots at all", func() { + By("Creating a new clusterStagedUpdateRun without specifying resourceSnapshotIndex") + updateRun.Spec.ResourceSnapshotIndex = "" + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization failed") + validateFailedInitCondition(ctx, updateRun, "no latest resourceSnapshots found for placement") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) + }) + It("Should fail to initialize if the specified resource snapshot is not found - no CRP label found", func() { By("Creating a new resource snapshot associated with another CRP") resourceSnapshot.Labels[placementv1beta1.PlacementTrackingLabel] = "not-exist-crp" @@ -828,6 +848,29 @@ var _ = Describe("Updaterun initialization tests", func() { validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) }) + It("Should select latest resource snapshot in the status when no resource index defined", func() { + By("Creating a new resource snapshot") + Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) + + By("Creating a new cluster resource override") + Expect(k8sClient.Create(ctx, clusterResourceOverride)).To(Succeed()) + + By("Creating a new clusterStagedUpdateRun without specifying resourceSnapshotIndex") + updateRun.Spec.ResourceSnapshotIndex = "" + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the clusterStagedUpdateRun stats") + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) + want := generateExecutionStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") + + By("Validating the clusterStagedUpdateRun initialized consistently") + validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + }) + It("Should put related ClusterResourceOverrides in the status", func() { By("Creating a new resource snapshot") Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) @@ -839,7 +882,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the clusterStagedUpdateRun stats") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) want := generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") @@ -850,6 +893,72 @@ var _ = Describe("Updaterun initialization tests", func() { validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) }) }) + + It("Should pick latest master resource snapshot if multiple snapshots", func() { + By("Creating a new clusterResourcePlacement") + Expect(k8sClient.Create(ctx, crp)).To(Succeed()) + + By("Creating scheduling policy snapshot") + Expect(k8sClient.Create(ctx, policySnapshot)).To(Succeed()) + + By("Setting the latest policy snapshot condition as fully scheduled") + meta.SetStatusCondition(&policySnapshot.Status.Conditions, metav1.Condition{ + Type: string(placementv1beta1.PolicySnapshotScheduled), + Status: metav1.ConditionTrue, + ObservedGeneration: policySnapshot.Generation, + Reason: "scheduled", + }) + Expect(k8sClient.Status().Update(ctx, policySnapshot)).Should(Succeed(), "failed to update the policy snapshot condition") + + By("Creating the member clusters") + for _, cluster := range targetClusters { + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + } + for _, cluster := range unscheduledClusters { + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + } + + By("Creating a bunch of ClusterResourceBindings") + for _, binding := range resourceBindings { + Expect(k8sClient.Create(ctx, binding)).To(Succeed()) + } + + By("Creating a clusterStagedUpdateStrategy") + Expect(k8sClient.Create(ctx, updateStrategy)).To(Succeed()) + + By("Creating a new resource snapshot") + resourceSnapshot.Labels[placementv1beta1.IsLatestSnapshotLabel] = "false" + Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) + + By("Creating a another new resource snapshot") + resourceSnapshot2.Name = testCRPName + "-1-snapshot" + resourceSnapshot2.Labels[placementv1beta1.IsLatestSnapshotLabel] = "false" + resourceSnapshot2.Labels[placementv1beta1.ResourceIndexLabel] = "1" + Expect(k8sClient.Create(ctx, resourceSnapshot2)).To(Succeed()) + + By("Creating a latest master resource snapshot") + resourceSnapshot3.Name = testCRPName + "-2-snapshot" + resourceSnapshot3.Labels[placementv1beta1.ResourceIndexLabel] = "2" + Expect(k8sClient.Create(ctx, resourceSnapshot3)).To(Succeed()) + + By("Creating a new cluster resource override") + Expect(k8sClient.Create(ctx, clusterResourceOverride)).To(Succeed()) + + By("Creating a new clusterStagedUpdateRun without specifying resourceSnapshotIndex") + updateRun.Spec.ResourceSnapshotIndex = "" + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the clusterStagedUpdateRun status") + initialized := generateSucceededInitializationStatus(crp, updateRun, "2", policySnapshot, updateStrategy, clusterResourceOverride) + want := generateExecutionStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") + + By("Validating the clusterStagedUpdateRun initialized consistently") + validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + }) }) func validateFailedInitCondition(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun, message string) { @@ -872,6 +981,7 @@ func validateFailedInitCondition(ctx context.Context, updateRun *placementv1beta func generateSucceededInitializationStatus( crp *placementv1beta1.ClusterResourcePlacement, updateRun *placementv1beta1.ClusterStagedUpdateRun, + resourceSnapshotIndex string, policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy, clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot, @@ -879,6 +989,7 @@ func generateSucceededInitializationStatus( status := &placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], PolicyObservedClusterCount: 10, + ResourceSnapshotIndexUsed: resourceSnapshotIndex, ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), UpdateStrategySnapshot: &updateStrategy.Spec, StagesStatus: []placementv1beta1.StageUpdatingStatus{ @@ -933,12 +1044,14 @@ func generateSucceededInitializationStatus( func generateSucceededInitializationStatusForSmallClusters( crp *placementv1beta1.ClusterResourcePlacement, updateRun *placementv1beta1.ClusterStagedUpdateRun, + resourceSnapshotIndex string, policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy, ) *placementv1beta1.UpdateRunStatus { status := &placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], PolicyObservedClusterCount: 3, + ResourceSnapshotIndexUsed: resourceSnapshotIndex, ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), UpdateStrategySnapshot: &updateStrategy.Spec, StagesStatus: []placementv1beta1.StageUpdatingStatus{ diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index de1cd71e8..bcc255473 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -110,7 +110,7 @@ var _ = Describe("UpdateRun validation tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -442,7 +442,7 @@ var _ = Describe("UpdateRun validation tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 889f1abe3..344faf6f4 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -2092,6 +2092,7 @@ func updateRunSucceedConditions(generation int64) []metav1.Condition { func clusterStagedUpdateRunStatusSucceededActual( updateRunName string, + wantResourceIndex string, wantPolicyIndex string, wantClusterCount int, wantApplyStrategy *placementv1beta1.ApplyStrategy, @@ -2109,6 +2110,7 @@ func clusterStagedUpdateRunStatusSucceededActual( wantStatus := placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, PolicyObservedClusterCount: wantClusterCount, ApplyStrategy: wantApplyStrategy.DeepCopy(), UpdateStrategySnapshot: wantStrategySpec, @@ -2126,7 +2128,7 @@ func clusterStagedUpdateRunStatusSucceededActual( func stagedUpdateRunStatusSucceededActual( updateRunName, namespace string, - wantPolicyIndex string, + wantResourceIndex, wantPolicyIndex string, wantClusterCount int, wantApplyStrategy *placementv1beta1.ApplyStrategy, wantStrategySpec *placementv1beta1.UpdateStrategySpec, @@ -2143,6 +2145,7 @@ func stagedUpdateRunStatusSucceededActual( wantStatus := placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, PolicyObservedClusterCount: wantClusterCount, ApplyStrategy: wantApplyStrategy.DeepCopy(), UpdateStrategySnapshot: wantStrategySpec, diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index 9880c2eef..b11fe9ccd 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -149,7 +149,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -214,7 +214,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -252,7 +252,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -348,7 +348,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -398,7 +398,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -445,7 +445,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the cluster staged update run successfully", func() { // need to go through two stages - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -537,7 +537,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-3 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -586,7 +586,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -632,7 +632,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should remove resources on member-cluster-1 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -804,7 +804,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -904,7 +904,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) }) @@ -1015,7 +1015,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) // Verify complete rollout - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) // Verify new configmap is on all member clusters @@ -1141,7 +1141,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should complete the staged update run after approval", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1214,7 +1214,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1340,7 +1340,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) By("Validating staged update run has succeeded") - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) By("Validating CRP status as completed") @@ -1392,7 +1392,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the second staged update run and complete the CRP", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], @@ -1440,7 +1440,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1483,7 +1483,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, @@ -1522,7 +1522,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index b5a5428b1..133b7d9a8 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -142,7 +142,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to all the members and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -207,7 +207,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -245,7 +245,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -339,7 +339,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -389,7 +389,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -436,7 +436,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the staged update run successfully", func() { // need to go through two stages - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -526,7 +526,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-3 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -575,7 +575,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -621,7 +621,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should remove resources on member-cluster-1 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -766,7 +766,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -860,7 +860,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) }) @@ -969,7 +969,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) // Verify complete rollout. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) // Verify new configmap is on all member clusters. @@ -1045,7 +1045,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters)