From f4b4d0fe187694d8f9749f41e4781a950813349e Mon Sep 17 00:00:00 2001 From: Francesco Giudici Date: Fri, 6 Nov 2026 08:48:53 +0100 Subject: [PATCH] Fix upgrade E2E coverage for pre-existing extensions Follow one ClusterExtension through standard OLM installation, the OLM upgrade, and a subsequent bundle update. Separate scenarios with shared once-only installers previously updated a new extension created after the upgrade and missed failures migrating the original extension. Record live resource identities and assert the experimental CE to COD to COS ownership handoff, migration revision, normal rollout revisions, and current-generation status. Retain Helm-backed standard upgrade coverage, refresh runtime feature gates, and reject stale leader leases. Use small generated bundles and leave migration runtime changes for a separate fix. The experimental upgrade cases should expose the existing ownership conflict until that fix lands. Co-authored-by: Codex Signed-off-by: Francesco Giudici --- test/e2e/steps/hooks.go | 32 +- test/e2e/steps/steps.go | 18 +- test/e2e/steps/upgrade_steps.go | 102 +++++-- test/e2e/steps/upgrade_storage_steps.go | 288 ++++++++++++++++++ test/e2e/steps/upgrade_storage_steps_test.go | 40 +++ test/upgrade-e2e/README.md | 41 +++ .../features/operator-upgrade.feature | 28 +- test/upgrade-e2e/upgrade_test.go | 4 + 8 files changed, 504 insertions(+), 49 deletions(-) create mode 100644 test/e2e/steps/upgrade_storage_steps.go create mode 100644 test/e2e/steps/upgrade_storage_steps_test.go diff --git a/test/e2e/steps/hooks.go b/test/e2e/steps/hooks.go index fb5980c726..274a67e621 100644 --- a/test/e2e/steps/hooks.go +++ b/test/e2e/steps/hooks.go @@ -7,7 +7,6 @@ import ( "fmt" "os/exec" "path/filepath" - "regexp" "strconv" "strings" @@ -16,6 +15,7 @@ import ( "github.com/spf13/pflag" "golang.org/x/sync/errgroup" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/component-base/featuregate" @@ -59,6 +59,9 @@ type scenarioContext struct { extensionObjects []client.Object revisionSecrets []resource resourceUIDs map[string]types.UID + upgradeObjects []*unstructured.Unstructured + upgradePods []corev1.Pod + upgradeStorageUIDs map[string]types.UID proxy *recordingProxy catalogAddr string catalogCleanup func() @@ -159,10 +162,17 @@ func BeforeSuite() { featureGates[catalogdHAFeature] = true } + if err := refreshOLMConfiguration(); err != nil { + logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)") + } +} + +// refreshOLMConfiguration must also run after installing or upgrading OLM: +// extension resource discovery depends on the deployed runtime's feature gates. +func refreshOLMConfiguration() error { olm, catalogdDep, err := detectOLMDeployments() if err != nil { - logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)") - return + return err } olmNamespace = olm.Namespace componentNamespaces["operator-controller"] = olmNamespace @@ -185,22 +195,26 @@ func BeforeSuite() { } } - featureGatePattern := regexp.MustCompile(`--feature-gates=([[:alnum:]]+)=(true|false)`) + featureGates[features.BoxcutterRuntime] = false for _, c := range olm.Spec.Template.Spec.Containers { if c.Name == "manager" { for _, arg := range c.Args { - if matches := featureGatePattern.FindStringSubmatch(arg); matches != nil { - v, err := strconv.ParseBool(matches[2]) - if err != nil { - panic(fmt.Errorf("failed to parse feature gate %q=%q: %v", matches[1], matches[2], err)) + if gates, found := strings.CutPrefix(arg, "--feature-gates="); found { + for _, gate := range strings.Split(gates, ",") { + name, value, _ := strings.Cut(gate, "=") + v, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("failed to parse feature gate %q: %w", gate, err) + } + featureGates[featuregate.Feature(name)] = v } - featureGates[featuregate.Feature(matches[1])] = v } } } } logger.Info(fmt.Sprintf("Enabled feature gates: %v", featureGates)) + return nil } func CheckFeatureTags(ctx context.Context, sc *godog.Scenario) (context.Context, error) { diff --git a/test/e2e/steps/steps.go b/test/e2e/steps/steps.go index 273ced10a1..2235bac0a6 100644 --- a/test/e2e/steps/steps.go +++ b/test/e2e/steps/steps.go @@ -236,6 +236,11 @@ func RegisterSteps(sc *godog.ScenarioContext) { // Upgrade-specific steps sc.Step(`^(?i)the latest stable OLM release is installed$`, LatestStableOLMReleaseIsInstalled) sc.Step(`^(?i)OLM is upgraded$`, OLMIsUpgraded) + sc.Step(`^(?i)Helm-backed extension resource identities are remembered$`, RememberHelmExtensionIdentities) + sc.Step(`^(?i)pre-existing ClusterExtension storage has transitioned after OLM upgrade$`, PreExistingExtensionStorageTransitioned) + sc.Step(`^(?i)updated ClusterExtension storage is healthy$`, UpdatedExtensionStorageIsHealthy) + sc.Step(`^(?i)extension resources keep their pre-upgrade identities$`, ExtensionKeepsPreUpgradeIdentities) + sc.Step(`^(?i)retained extension resources keep their pre-upgrade identities$`, RetainedExtensionKeepsPreUpgradeIdentities) sc.Step(`^(?i)(catalogd|operator-controller) is ready to reconcile resources$`, ComponentIsReadyToReconcile) sc.Step(`^(?i)all (ClusterCatalog|ClusterExtension) resources are reconciled$`, allResourcesAreReconciled) sc.Step(`^(?i)ClusterExtension is reconciled$`, ClusterExtensionIsReconciled) @@ -2555,11 +2560,18 @@ func helmReleaseFromSecret(secret *corev1.Secret) (*release.Release, error) { return nil, err } - var rel release.Release - if err = json.Unmarshal(releaseJsonBytes, &rel); err != nil { + // The chunked storage driver wraps release.Release because Helm excludes + // Labels from its JSON representation. Restore those labels explicitly so + // storage assertions can verify the persisted bundle metadata. + var wrapped struct { + release.Release + Labels map[string]string `json:"labels"` + } + if err = json.Unmarshal(releaseJsonBytes, &wrapped); err != nil { return nil, err } - return &rel, nil + wrapped.Release.Labels = wrapped.Labels + return &wrapped.Release, nil } // collectHelmReleaseObjects returns a slice of client.Object containing the manifests in rel diff --git a/test/e2e/steps/upgrade_steps.go b/test/e2e/steps/upgrade_steps.go index ad7fdda720..c2c4194705 100644 --- a/test/e2e/steps/upgrade_steps.go +++ b/test/e2e/steps/upgrade_steps.go @@ -8,42 +8,48 @@ import ( "os" "os/exec" "strings" - "sync" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" catalogdv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/operator-controller/features" ) +// waitForUpgradeCheck retains the last failed assertion, including controller +// conditions, so migration failures report their cause rather than just a timeout. +func waitForUpgradeCheck(ctx context.Context, check func() error) error { + var lastErr error + err := wait.PollUntilContextTimeout(ctx, tick, timeout, true, func(context.Context) (bool, error) { + lastErr = check() + return lastErr == nil, nil + }) + if err != nil { + return fmt.Errorf("upgrade assertion timed out: %w: %v", err, lastErr) + } + return nil +} + // leaseNames maps component labels to their leader election lease names. var leaseNames = map[string]string{ "catalogd": "catalogd-operator-lock", "operator-controller": "9c4404e7.operatorframework.io", } -var ( - installOperator = sync.OnceValue(func() error { - err := runInstallScript("RELEASE_INSTALL") - if err != nil { - return err - } - olm, _, err := detectOLMDeployments() - if err != nil { - return err - } - olmNamespace = olm.Namespace - return nil - }) - upgradeOperator = sync.OnceValue(func() error { - return runInstallScript("RELEASE_UPGRADE") - }) -) - // LatestStableOLMReleaseIsInstalled downloads and executes the latest stable OLM release install script. -// Uses sync.Once to ensure the install only happens once across multiple scenarios. func LatestStableOLMReleaseIsInstalled(_ context.Context) error { - return installOperator() + if err := runInstallScript("RELEASE_INSTALL"); err != nil { + return err + } + if err := refreshOLMConfiguration(); err != nil { + return err + } + if featureGates[features.BoxcutterRuntime] { + return fmt.Errorf("upgrade coverage requires an initial standard OLM installation") + } + return nil } func runInstallScript(envVar string) error { @@ -72,9 +78,11 @@ func runInstallScript(envVar string) error { } // OLMIsUpgraded applies the locally built OLM manifest to upgrade OLM. -// Uses sync.Once to ensure the upgrade only happens once across multiple scenarios. -func OLMIsUpgraded(ctx context.Context) error { - return upgradeOperator() +func OLMIsUpgraded(_ context.Context) error { + if err := runInstallScript("RELEASE_UPGRADE"); err != nil { + return err + } + return refreshOLMConfiguration() } // ComponentIsReadyToReconcile waits for the named component's deployment to be fully rolled out, @@ -105,17 +113,55 @@ func ComponentIsReadyToReconcile(ctx context.Context, component string) error { } // Leader election can take up to LeaseDuration (137s) + RetryPeriod (26s) ≈ 163s in the worst case - waitFor(ctx, func() bool { + if err := waitForUpgradeCheck(ctx, func() error { output, err := k8sClient(ctx, "get", "lease", leaseName, "-n", ns, "-o", "jsonpath={.spec.holderIdentity}") if err != nil || output == "" { - return false + return fmt.Errorf("waiting for %s leader lease: %s (%v)", component, output, err) } // Extract pod name from lease holder identity (format: _) podName := strings.Split(output, "_")[0] + // A lease left by the previous controller is insufficient. Require its + // holder to belong to the fully rolled-out deployment's current ReplicaSet. + podJSON, err := k8sClient(ctx, "get", "pod", podName, "-n", ns, "-o", "json") + if err != nil { + return fmt.Errorf("reading %s leader pod: %w", component, err) + } + var pod corev1.Pod + if err := json.Unmarshal([]byte(podJSON), &pod); err != nil { + return err + } + if !podReady(&pod) || metav1.GetControllerOf(&pod) == nil { + return fmt.Errorf("%s leader pod %s is not ready or has no controller", component, podName) + } + rsJSON, err := k8sClient(ctx, "get", "replicaset", metav1.GetControllerOf(&pod).Name, "-n", ns, "-o", "json") + if err != nil { + return err + } + depJSON, err := k8sClient(ctx, "get", "deployment", depName, "-n", ns, "-o", "json") + if err != nil { + return err + } + var rs appsv1.ReplicaSet + var dep appsv1.Deployment + if err := json.Unmarshal([]byte(rsJSON), &rs); err != nil { + return err + } + if err := json.Unmarshal([]byte(depJSON), &dep); err != nil { + return err + } + owner := metav1.GetControllerOf(&rs) + revision := dep.Annotations["deployment.kubernetes.io/revision"] + if owner == nil || owner.Kind != "Deployment" || owner.UID != dep.UID || + metav1.GetControllerOf(&pod).Kind != "ReplicaSet" || metav1.GetControllerOf(&pod).UID != rs.UID || + !generationObserved(dep.Generation, dep.Status.ObservedGeneration) || revision == "" || rs.Annotations["deployment.kubernetes.io/revision"] != revision { + return fmt.Errorf("%s lease holder %s is not from deployment revision %s", component, podName, revision) + } sc.leaderPods[component] = podName - return true - }) + return nil + }); err != nil { + return err + } logger.V(1).Info("Component is ready to reconcile", "component", component, "leaderPod", sc.leaderPods[component]) return nil diff --git a/test/e2e/steps/upgrade_storage_steps.go b/test/e2e/steps/upgrade_storage_steps.go new file mode 100644 index 0000000000..493453c283 --- /dev/null +++ b/test/e2e/steps/upgrade_storage_steps.go @@ -0,0 +1,288 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/operator-controller/features" + "github.com/operator-framework/operator-controller/internal/shared/labels" +) + +// RememberHelmExtensionIdentities reads live objects, since Helm's stored +// manifests do not contain the UIDs needed to detect deletion and recreation. +func RememberHelmExtensionIdentities(ctx context.Context) error { + sc := scenarioCtx(ctx) + if featureGates[features.BoxcutterRuntime] { + return fmt.Errorf("expected Helm-backed storage before OLM upgrade") + } + objects, err := listHelmReleaseResources(sc.clusterExtensionName) + if err != nil { + return err + } + if len(objects) == 0 { + return fmt.Errorf("helm release has no managed resources") + } + for _, obj := range objects { + live, err := getResource(obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName(), obj.GetNamespace()) + if err != nil { + return err + } + if live.GetUID() == "" { + return fmt.Errorf("resource %s has no live UID", client.ObjectKeyFromObject(live)) + } + sc.upgradeObjects = append(sc.upgradeObjects, live) + } + // Pods are not bundle manifests. Preserve their identities through migration, + // but allow a normal pod rollout when the bundle version changes afterward. + return waitForUpgradeCheck(ctx, func() error { + out, err := k8sClient(ctx, "get", "pods", "-n", sc.namespace, "-l", "app="+sc.id, "-o", "json") + if err != nil { + return err + } + var pods corev1.PodList + if err := json.Unmarshal([]byte(out), &pods); err != nil { + return err + } + if len(pods.Items) == 0 { + return fmt.Errorf("no fixture workload pods found before OLM upgrade") + } + for _, pod := range pods.Items { + if !podReady(&pod) { + return fmt.Errorf("fixture pod %s is not ready before OLM upgrade", pod.Name) + } + } + sc.upgradePods = pods.Items + return nil + }) +} + +func podReady(pod *corev1.Pod) bool { + if pod.DeletionTimestamp != nil { + return false + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func ExtensionKeepsPreUpgradeIdentities(ctx context.Context) error { + if err := checkExtensionIdentities(ctx, false); err != nil { + return err + } + for _, before := range scenarioCtx(ctx).upgradePods { + out, err := k8sClient(ctx, "get", "pod", before.Name, "-n", before.Namespace, "-o", "json") + if err != nil { + return err + } + var after corev1.Pod + if err := json.Unmarshal([]byte(out), &after); err != nil { + return err + } + if after.UID != before.UID || !podReady(&after) { + return fmt.Errorf("workload pod %s was replaced or is no longer ready after OLM upgrade", before.Name) + } + } + return nil +} + +func RetainedExtensionKeepsPreUpgradeIdentities(ctx context.Context) error { + return checkExtensionIdentities(ctx, true) +} + +func checkExtensionIdentities(ctx context.Context, afterUpdate bool) error { + sc := scenarioCtx(ctx) + if len(sc.upgradeObjects) == 0 { + return fmt.Errorf("no pre-upgrade extension resource identities remembered") + } + retained := map[string]bool{} + if afterUpdate { + objects, err := listExtensionResources(sc.clusterExtensionName) + if err != nil { + return err + } + for _, obj := range objects { + retained[fmt.Sprintf("%s/%s", obj.GetObjectKind().GroupVersionKind().Kind, client.ObjectKeyFromObject(obj))] = true + } + } + for _, before := range sc.upgradeObjects { + // Rendered RBAC names incorporate the CSV version. Those resources are + // legitimately replaced by an extension update; all other fixture names + // are stable and must retain their UIDs, including the Deployment and CRD. + if afterUpdate && before.GroupVersionKind().Group == "rbac.authorization.k8s.io" && + !retained[fmt.Sprintf("%s/%s", before.GetKind(), client.ObjectKeyFromObject(before))] { + continue + } + after, err := getResource(before.GetKind(), before.GetName(), before.GetNamespace()) + if err != nil { + return err + } + if after.GetUID() != before.GetUID() { + return fmt.Errorf("%s %s was replaced: UID changed from %s to %s", before.GetKind(), + client.ObjectKeyFromObject(before), before.GetUID(), after.GetUID()) + } + } + return nil +} + +// PreExistingExtensionStorageTransitioned expects a complete CE -> COD -> COS +// handoff for experimental OLM. Standard upgrades must keep using Helm storage. +func PreExistingExtensionStorageTransitioned(ctx context.Context) error { + return checkUpgradedExtensionStorage(ctx, 2, "1.0.0") +} + +func UpdatedExtensionStorageIsHealthy(ctx context.Context) error { + return checkUpgradedExtensionStorage(ctx, 3, "1.0.1") +} + +func checkUpgradedExtensionStorage(ctx context.Context, revision int64, version string) error { + sc := scenarioCtx(ctx) + if !featureGates[features.BoxcutterRuntime] { + // Do not request experimental APIs on standard installations. Reading + // the live Helm release also verifies that updating the same CE still + // writes Helm storage, rather than accepting only retained CE status. + secret, err := helmReleaseSecretForExtension(sc.clusterExtensionName) + if err != nil { + return err + } + if secret == nil { + return fmt.Errorf("no deployed Helm release for %s", sc.clusterExtensionName) + } + rel, err := helmReleaseFromSecret(secret) + if err != nil { + return err + } + if rel.Labels[labels.BundleVersionKey] != version { + return fmt.Errorf("helm release bundle version = %q, want %q", rel.Labels[labels.BundleVersionKey], version) + } + return nil + } + + return waitForUpgradeCheck(ctx, func() error { + ext, err := getResource("clusterextension", sc.clusterExtensionName, "") + if err != nil { + return err + } + obj, err := getResource("clusterobjectdeployment", sc.clusterExtensionName, "") + if err != nil { + return err + } + var cod ocv1.ClusterObjectDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &cod); err != nil { + return err + } + if err := requireController(&cod, ext); err != nil { + return err + } + if revision == 3 && sc.upgradeStorageUIDs["cod"] != cod.UID { + return fmt.Errorf("COD %s UID changed during extension update", cod.Name) + } + if !conditionMatches(currentCondition(cod.Generation, cod.Status.Conditions, ocv1.ClusterObjectDeploymentTypeAvailable), metav1.ConditionTrue, nil, nil) || + !conditionMatches(currentCondition(cod.Generation, cod.Status.Conditions, ocv1.ClusterObjectDeploymentTypeProgressing), metav1.ConditionFalse, ptr.To(ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed), nil) { + return fmt.Errorf("COD %s has not completed migration/update: conditions=%+v", cod.Name, cod.Status.Conditions) + } + if cod.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + return fmt.Errorf("COD template collision protection = %s, want Prevent", cod.Spec.Template.Spec.CollisionProtection) + } + revisions, err := listClusterObjectSetsInGroup(ctx, sc.clusterExtensionName) + if err != nil { + return err + } + if int64(len(revisions)) != revision { + return fmt.Errorf("group %s has %d revisions, want %d", sc.clusterExtensionName, len(revisions), revision) + } + seen := map[int64]bool{} + var active *ocv1.ClusterObjectSet + for i := range revisions { + rev := &revisions[i] + if rev.Spec.Group != cod.Name || rev.Spec.Revision < 1 || rev.Spec.Revision > revision || seen[rev.Spec.Revision] { + return fmt.Errorf("unexpected group/revision for COS %s: %s/%d", rev.Name, rev.Spec.Group, rev.Spec.Revision) + } + seen[rev.Spec.Revision] = true + if err := requireController(rev, &cod); err != nil { + return err + } + if revision == 3 && rev.Spec.Revision < revision && sc.upgradeStorageUIDs[rev.Name] != rev.UID { + return fmt.Errorf("COS %s UID changed during extension update", rev.Name) + } + protection := ocv1.CollisionProtectionPrevent + if rev.Spec.Revision == 1 { + protection = ocv1.CollisionProtectionNone + if rev.Labels[labels.MigratedFromHelmKey] != "true" || rev.Annotations[labels.BundleVersionKey] != "1.0.0" { + return fmt.Errorf("COS %s is not the migrated 1.0.0 Helm release", rev.Name) + } + } + if rev.Spec.CollisionProtection != protection || rev.Status.CompletedAt.IsZero() { + return fmt.Errorf("COS %s: protection=%s completedAt=%s, want %s and completed rollout", rev.Name, + rev.Spec.CollisionProtection, rev.Status.CompletedAt, protection) + } + if rev.Spec.Revision == revision { + active = rev + if rev.Spec.LifecycleState != ocv1.ClusterObjectSetLifecycleStateActive || + !conditionMatches(currentCondition(rev.Generation, rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady), metav1.ConditionTrue, nil, nil) || + rev.Annotations[labels.BundleVersionKey] != version { + return fmt.Errorf("COS %s is not active/ready at bundle version %s: conditions=%+v", rev.Name, version, rev.Status.Conditions) + } + } else if rev.Spec.LifecycleState != ocv1.ClusterObjectSetLifecycleStateArchived || + !conditionMatches(currentCondition(rev.Generation, rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady), metav1.ConditionFalse, ptr.To(ocv1.ClusterObjectSetReasonArchived), nil) { + return fmt.Errorf("COS %s is not archived: conditions=%+v", rev.Name, rev.Status.Conditions) + } + } + if active == nil { + return fmt.Errorf("no active revision %d", revision) + } + objects, err := listExtensionRevisionResources(sc.clusterExtensionName) + if err != nil { + return err + } + if len(objects) == 0 { + return fmt.Errorf("active revision %s contains no objects", active.Name) + } + for _, desired := range objects { + live, err := getResource(desired.GetObjectKind().GroupVersionKind().Kind, desired.GetName(), desired.GetNamespace()) + if err != nil { + return err + } + if err := requireController(live, active); err != nil { + return err + } + } + if revision == 2 { + sc.upgradeStorageUIDs = map[string]types.UID{"cod": cod.UID} + for _, rev := range revisions { + sc.upgradeStorageUIDs[rev.Name] = rev.UID + } + } + return nil + }) +} + +func requireController(object, controller client.Object) error { + owner := metav1.GetControllerOf(object) + gvk := controller.GetObjectKind().GroupVersionKind() + // Typed list items may omit TypeMeta. The controller identity is still + // unambiguous from the API type and its live name/UID. + switch controller.(type) { + case *ocv1.ClusterObjectDeployment: + gvk = ocv1.GroupVersion.WithKind(ocv1.ClusterObjectDeploymentKind) + case *ocv1.ClusterObjectSet: + gvk = ocv1.GroupVersion.WithKind(ocv1.ClusterObjectSetKind) + } + if owner == nil || owner.APIVersion != gvk.GroupVersion().String() || owner.Kind != gvk.Kind || + owner.Name != controller.GetName() || owner.UID != controller.GetUID() { + return fmt.Errorf("%s %s controller=%+v, want %s %s (UID %s)", object.GetObjectKind().GroupVersionKind().Kind, + object.GetName(), owner, gvk.Kind, controller.GetName(), controller.GetUID()) + } + return nil +} diff --git a/test/e2e/steps/upgrade_storage_steps_test.go b/test/e2e/steps/upgrade_storage_steps_test.go new file mode 100644 index 0000000000..4c10769d5d --- /dev/null +++ b/test/e2e/steps/upgrade_storage_steps_test.go @@ -0,0 +1,40 @@ +package steps + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "helm.sh/helm/v3/pkg/release" + corev1 "k8s.io/api/core/v1" + + "github.com/operator-framework/operator-controller/internal/shared/labels" +) + +func TestHelmReleaseFromSecretRestoresStoredBundleLabels(t *testing.T) { + // Match the chunked driver's wrapper: release.Release.Labels itself has + // json:"-", so decoding directly into release.Release silently loses it. + stored := struct { + release.Release + Labels map[string]string `json:"labels"` + }{ + Release: release.Release{Name: "extension", Manifest: "bundle manifests", Version: 2}, + Labels: map[string]string{labels.BundleVersionKey: "1.0.1"}, + } + data, err := json.Marshal(stored) + require.NoError(t, err) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err = writer.Write(data) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + rel, err := helmReleaseFromSecret(&corev1.Secret{Data: map[string][]byte{"chunk": compressed.Bytes()}}) + require.NoError(t, err) + require.Equal(t, "extension", rel.Name) + require.Equal(t, "bundle manifests", rel.Manifest) + require.Equal(t, 2, rel.Version) + require.Equal(t, "1.0.1", rel.Labels[labels.BundleVersionKey]) +} diff --git a/test/upgrade-e2e/README.md b/test/upgrade-e2e/README.md index d2edf87cbe..9fab846cb6 100644 --- a/test/upgrade-e2e/README.md +++ b/test/upgrade-e2e/README.md @@ -2,6 +2,31 @@ End-to-end tests that verify OLM upgrades don't break pre-existing ClusterCatalogs and ClusterExtensions. +One scenario follows the same catalog and ClusterExtension through the entire upgrade. +It installs a small generated bundle at version 1.0.0 on standard OLM and records the +extension's UID, live managed resource UIDs, and ready workload pod UIDs before upgrading OLM. +After the upgrade it checks catalog reconciliation and the extension's storage: + +- **Standard destination:** the existing extension remains Helm-backed. +- **Experimental destination:** the extension controls a ClusterObjectDeployment (COD), + which controls migrated revision 1 with collision protection `None` and normal + revision 2 with `Prevent`. Revision 1 must have completed and be archived; + revision 2 must be active and ready, with its managed resources under its control. + COD availability and revision conditions must reflect their current generations. + +The scenario verifies the original extension, every original managed resource, and +workload pods retain their UIDs through migration. It then updates that same extension +to 1.0.1. Experimental OLM must produce ready revision 3, archive the earlier revisions, +and preserve the COD and earlier revision UIDs. Standard OLM must update the Helm +release. Stable bundle resource names must keep their UIDs; versioned RBAC resources +and workload pods may be replaced during this extension update. + +Feature gates and component namespaces are refreshed from the deployed configuration +after installation and upgrade. Leader leases must point to pods belonging to the +current deployment revision, so stale leases cannot stand in for upgraded controllers. +The scenario starts on standard OLM; experimental-to-experimental upgrades are outside +this coverage. The fixtures use ordinary etcd limits and strict CRD upgrade safety. + These tests use the [Godog](https://github.com/cucumber/godog) BDD framework with shared step definitions from `test/e2e/steps/`. The upgrade-specific steps live in `test/e2e/steps/upgrade_steps.go`. See [test/e2e/README.md](../e2e/README.md) for more information on the Godog framework and writing tests. @@ -31,6 +56,22 @@ The following Makefile targets run upgrade tests for different upgrade paths: | `make test-st2ex-e2e` | Switchover from a locally built **standard** install to a locally built **experimental** install | Each target handles kind cluster creation, image building/loading, test execution, and cleanup. +The targets recreate their named clusters. To preserve an existing investigation cluster, +use a separately named disposable cluster and an explicit kubeconfig with the lower-level runner: + +```bash +RELEASE_INSTALL=https://github.com/operator-framework/operator-controller/releases/download/v1.12.0/install.sh \ +RELEASE_UPGRADE="$PWD/experimental-install.sh" \ +ROOT_DIR="$PWD" KIND_CLUSTER_NAME=upgrade-coverage-test \ +KUBECONFIG="$PWD/.kubeconfig/upgrade-coverage-test.kubeconfig" \ +go test -count=1 -v ./test/upgrade-e2e/upgrade_test.go +``` + +First generate the local install scripts and load the locally built images into that cluster. +Set `RELEASE_INSTALL` to the local `standard-install.sh` for local standard-to-experimental +coverage, or `RELEASE_UPGRADE` to it for released standard-to-local standard coverage. +On failure the scenario leaves its resources for inspection. Migration assertions retain +the last observed error and COD conditions, including ownership conflicts, in timeout output. ## Environment Variables diff --git a/test/upgrade-e2e/features/operator-upgrade.feature b/test/upgrade-e2e/features/operator-upgrade.feature index 5a310c57c7..8540556756 100644 --- a/test/upgrade-e2e/features/operator-upgrade.feature +++ b/test/upgrade-e2e/features/operator-upgrade.feature @@ -4,14 +4,14 @@ Feature: Operator upgrade verification pre-existing ClusterCatalogs and ClusterExtensions continue to function and can be updated. - Background: + Scenario: Pre-existing catalog and extension survive OLM upgrade and extension update Given the latest stable OLM release is installed And an image registry is available And a catalog "test" with packages: | package | version | channel | replaces | contents | | test | 1.0.0 | beta | | CRD, Deployment, ConfigMap | | test | 1.0.1 | beta | 1.0.0 | CRD, Deployment, ConfigMap | - And ServiceAccount "olm-sa" with permissions to install extensions is available in "${TEST_NAMESPACE}" namespace + And namespace "${TEST_NAMESPACE}" is available And ClusterExtension is applied """ apiVersion: olm.operatorframework.io/v1 @@ -20,8 +20,6 @@ Feature: Operator upgrade verification name: ${NAME} spec: namespace: ${TEST_NAMESPACE} - serviceAccount: - name: olm-sa source: sourceType: Catalog catalog: @@ -32,17 +30,29 @@ Feature: Operator upgrade verification version: 1.0.0 """ And ClusterExtension is available - And OLM is upgraded + And ClusterExtension is rolled out + And bundle "${PACKAGE:test}.1.0.0" is installed in version "1.0.0" + And resource "clusterextension/${NAME}" UID is remembered + And Helm-backed extension resource identities are remembered + When OLM is upgraded And catalogd is ready to reconcile resources And operator-controller is ready to reconcile resources - Scenario: ClusterCatalog continues unpacking after OLM upgrade - When catalog "test" is reconciled + And catalog "test" is reconciled Then catalog "test" reports Progressing as True with Reason Succeeded And catalog "test" reports Serving as True with Reason Available - Scenario: ClusterExtension remains functional after OLM upgrade - Given ClusterExtension is reconciled + And pre-existing ClusterExtension storage has transitioned after OLM upgrade + And ClusterExtension is available + And ClusterExtension is rolled out + And bundle "${PACKAGE:test}.1.0.0" is installed in version "1.0.0" + And resource "clusterextension/${NAME}" keeps its UID + And extension resources keep their pre-upgrade identities When ClusterExtension version is updated to "1.0.1" Then ClusterExtension is available + And ClusterExtension is rolled out And bundle "${PACKAGE:test}.1.0.1" is installed in version "1.0.1" + And deployment "test-operator-${SCENARIO_ID}" rollout is complete + And updated ClusterExtension storage is healthy + And resource "clusterextension/${NAME}" keeps its UID + And retained extension resources keep their pre-upgrade identities diff --git a/test/upgrade-e2e/upgrade_test.go b/test/upgrade-e2e/upgrade_test.go index 5991dd81ae..117cb60894 100644 --- a/test/upgrade-e2e/upgrade_test.go +++ b/test/upgrade-e2e/upgrade_test.go @@ -30,8 +30,12 @@ func TestMain(m *testing.M) { opts.Paths = pflag.Args() sc := godog.TestSuite{ + TestSuiteInitializer: func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, ScenarioInitializer: func(sc *godog.ScenarioContext) { sc.Before(steps.CreateScenarioContext) + sc.After(steps.ScenarioCleanup) steps.RegisterSteps(sc) }, Options: &opts,