diff --git a/test/e2e/steps/hooks.go b/test/e2e/steps/hooks.go index fb5980c726..274a67e621 100644 --- a/test/e2e/steps/hooks.go +++ b/test/e2e/steps/hooks.go @@ -7,7 +7,6 @@ import ( "fmt" "os/exec" "path/filepath" - "regexp" "strconv" "strings" @@ -16,6 +15,7 @@ import ( "github.com/spf13/pflag" "golang.org/x/sync/errgroup" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/component-base/featuregate" @@ -59,6 +59,9 @@ type scenarioContext struct { extensionObjects []client.Object revisionSecrets []resource resourceUIDs map[string]types.UID + upgradeObjects []*unstructured.Unstructured + upgradePods []corev1.Pod + upgradeStorageUIDs map[string]types.UID proxy *recordingProxy catalogAddr string catalogCleanup func() @@ -159,10 +162,17 @@ func BeforeSuite() { featureGates[catalogdHAFeature] = true } + if err := refreshOLMConfiguration(); err != nil { + logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)") + } +} + +// refreshOLMConfiguration must also run after installing or upgrading OLM: +// extension resource discovery depends on the deployed runtime's feature gates. +func refreshOLMConfiguration() error { olm, catalogdDep, err := detectOLMDeployments() if err != nil { - logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)") - return + return err } olmNamespace = olm.Namespace componentNamespaces["operator-controller"] = olmNamespace @@ -185,22 +195,26 @@ func BeforeSuite() { } } - featureGatePattern := regexp.MustCompile(`--feature-gates=([[:alnum:]]+)=(true|false)`) + featureGates[features.BoxcutterRuntime] = false for _, c := range olm.Spec.Template.Spec.Containers { if c.Name == "manager" { for _, arg := range c.Args { - if matches := featureGatePattern.FindStringSubmatch(arg); matches != nil { - v, err := strconv.ParseBool(matches[2]) - if err != nil { - panic(fmt.Errorf("failed to parse feature gate %q=%q: %v", matches[1], matches[2], err)) + if gates, found := strings.CutPrefix(arg, "--feature-gates="); found { + for _, gate := range strings.Split(gates, ",") { + name, value, _ := strings.Cut(gate, "=") + v, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("failed to parse feature gate %q: %w", gate, err) + } + featureGates[featuregate.Feature(name)] = v } - featureGates[featuregate.Feature(matches[1])] = v } } } } logger.Info(fmt.Sprintf("Enabled feature gates: %v", featureGates)) + return nil } func CheckFeatureTags(ctx context.Context, sc *godog.Scenario) (context.Context, error) { diff --git a/test/e2e/steps/steps.go b/test/e2e/steps/steps.go index 273ced10a1..2235bac0a6 100644 --- a/test/e2e/steps/steps.go +++ b/test/e2e/steps/steps.go @@ -236,6 +236,11 @@ func RegisterSteps(sc *godog.ScenarioContext) { // Upgrade-specific steps sc.Step(`^(?i)the latest stable OLM release is installed$`, LatestStableOLMReleaseIsInstalled) sc.Step(`^(?i)OLM is upgraded$`, OLMIsUpgraded) + sc.Step(`^(?i)Helm-backed extension resource identities are remembered$`, RememberHelmExtensionIdentities) + sc.Step(`^(?i)pre-existing ClusterExtension storage has transitioned after OLM upgrade$`, PreExistingExtensionStorageTransitioned) + sc.Step(`^(?i)updated ClusterExtension storage is healthy$`, UpdatedExtensionStorageIsHealthy) + sc.Step(`^(?i)extension resources keep their pre-upgrade identities$`, ExtensionKeepsPreUpgradeIdentities) + sc.Step(`^(?i)retained extension resources keep their pre-upgrade identities$`, RetainedExtensionKeepsPreUpgradeIdentities) sc.Step(`^(?i)(catalogd|operator-controller) is ready to reconcile resources$`, ComponentIsReadyToReconcile) sc.Step(`^(?i)all (ClusterCatalog|ClusterExtension) resources are reconciled$`, allResourcesAreReconciled) sc.Step(`^(?i)ClusterExtension is reconciled$`, ClusterExtensionIsReconciled) @@ -2555,11 +2560,18 @@ func helmReleaseFromSecret(secret *corev1.Secret) (*release.Release, error) { return nil, err } - var rel release.Release - if err = json.Unmarshal(releaseJsonBytes, &rel); err != nil { + // The chunked storage driver wraps release.Release because Helm excludes + // Labels from its JSON representation. Restore those labels explicitly so + // storage assertions can verify the persisted bundle metadata. + var wrapped struct { + release.Release + Labels map[string]string `json:"labels"` + } + if err = json.Unmarshal(releaseJsonBytes, &wrapped); err != nil { return nil, err } - return &rel, nil + wrapped.Release.Labels = wrapped.Labels + return &wrapped.Release, nil } // collectHelmReleaseObjects returns a slice of client.Object containing the manifests in rel diff --git a/test/e2e/steps/upgrade_steps.go b/test/e2e/steps/upgrade_steps.go index ad7fdda720..c2c4194705 100644 --- a/test/e2e/steps/upgrade_steps.go +++ b/test/e2e/steps/upgrade_steps.go @@ -8,42 +8,48 @@ import ( "os" "os/exec" "strings" - "sync" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" catalogdv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/operator-controller/features" ) +// waitForUpgradeCheck retains the last failed assertion, including controller +// conditions, so migration failures report their cause rather than just a timeout. +func waitForUpgradeCheck(ctx context.Context, check func() error) error { + var lastErr error + err := wait.PollUntilContextTimeout(ctx, tick, timeout, true, func(context.Context) (bool, error) { + lastErr = check() + return lastErr == nil, nil + }) + if err != nil { + return fmt.Errorf("upgrade assertion timed out: %w: %v", err, lastErr) + } + return nil +} + // leaseNames maps component labels to their leader election lease names. var leaseNames = map[string]string{ "catalogd": "catalogd-operator-lock", "operator-controller": "9c4404e7.operatorframework.io", } -var ( - installOperator = sync.OnceValue(func() error { - err := runInstallScript("RELEASE_INSTALL") - if err != nil { - return err - } - olm, _, err := detectOLMDeployments() - if err != nil { - return err - } - olmNamespace = olm.Namespace - return nil - }) - upgradeOperator = sync.OnceValue(func() error { - return runInstallScript("RELEASE_UPGRADE") - }) -) - // LatestStableOLMReleaseIsInstalled downloads and executes the latest stable OLM release install script. -// Uses sync.Once to ensure the install only happens once across multiple scenarios. func LatestStableOLMReleaseIsInstalled(_ context.Context) error { - return installOperator() + if err := runInstallScript("RELEASE_INSTALL"); err != nil { + return err + } + if err := refreshOLMConfiguration(); err != nil { + return err + } + if featureGates[features.BoxcutterRuntime] { + return fmt.Errorf("upgrade coverage requires an initial standard OLM installation") + } + return nil } func runInstallScript(envVar string) error { @@ -72,9 +78,11 @@ func runInstallScript(envVar string) error { } // OLMIsUpgraded applies the locally built OLM manifest to upgrade OLM. -// Uses sync.Once to ensure the upgrade only happens once across multiple scenarios. -func OLMIsUpgraded(ctx context.Context) error { - return upgradeOperator() +func OLMIsUpgraded(_ context.Context) error { + if err := runInstallScript("RELEASE_UPGRADE"); err != nil { + return err + } + return refreshOLMConfiguration() } // ComponentIsReadyToReconcile waits for the named component's deployment to be fully rolled out, @@ -105,17 +113,55 @@ func ComponentIsReadyToReconcile(ctx context.Context, component string) error { } // Leader election can take up to LeaseDuration (137s) + RetryPeriod (26s) ≈ 163s in the worst case - waitFor(ctx, func() bool { + if err := waitForUpgradeCheck(ctx, func() error { output, err := k8sClient(ctx, "get", "lease", leaseName, "-n", ns, "-o", "jsonpath={.spec.holderIdentity}") if err != nil || output == "" { - return false + return fmt.Errorf("waiting for %s leader lease: %s (%v)", component, output, err) } // Extract pod name from lease holder identity (format: _) podName := strings.Split(output, "_")[0] + // A lease left by the previous controller is insufficient. Require its + // holder to belong to the fully rolled-out deployment's current ReplicaSet. + podJSON, err := k8sClient(ctx, "get", "pod", podName, "-n", ns, "-o", "json") + if err != nil { + return fmt.Errorf("reading %s leader pod: %w", component, err) + } + var pod corev1.Pod + if err := json.Unmarshal([]byte(podJSON), &pod); err != nil { + return err + } + if !podReady(&pod) || metav1.GetControllerOf(&pod) == nil { + return fmt.Errorf("%s leader pod %s is not ready or has no controller", component, podName) + } + rsJSON, err := k8sClient(ctx, "get", "replicaset", metav1.GetControllerOf(&pod).Name, "-n", ns, "-o", "json") + if err != nil { + return err + } + depJSON, err := k8sClient(ctx, "get", "deployment", depName, "-n", ns, "-o", "json") + if err != nil { + return err + } + var rs appsv1.ReplicaSet + var dep appsv1.Deployment + if err := json.Unmarshal([]byte(rsJSON), &rs); err != nil { + return err + } + if err := json.Unmarshal([]byte(depJSON), &dep); err != nil { + return err + } + owner := metav1.GetControllerOf(&rs) + revision := dep.Annotations["deployment.kubernetes.io/revision"] + if owner == nil || owner.Kind != "Deployment" || owner.UID != dep.UID || + metav1.GetControllerOf(&pod).Kind != "ReplicaSet" || metav1.GetControllerOf(&pod).UID != rs.UID || + !generationObserved(dep.Generation, dep.Status.ObservedGeneration) || revision == "" || rs.Annotations["deployment.kubernetes.io/revision"] != revision { + return fmt.Errorf("%s lease holder %s is not from deployment revision %s", component, podName, revision) + } sc.leaderPods[component] = podName - return true - }) + return nil + }); err != nil { + return err + } logger.V(1).Info("Component is ready to reconcile", "component", component, "leaderPod", sc.leaderPods[component]) return nil diff --git a/test/e2e/steps/upgrade_storage_steps.go b/test/e2e/steps/upgrade_storage_steps.go new file mode 100644 index 0000000000..493453c283 --- /dev/null +++ b/test/e2e/steps/upgrade_storage_steps.go @@ -0,0 +1,288 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/operator-controller/features" + "github.com/operator-framework/operator-controller/internal/shared/labels" +) + +// RememberHelmExtensionIdentities reads live objects, since Helm's stored +// manifests do not contain the UIDs needed to detect deletion and recreation. +func RememberHelmExtensionIdentities(ctx context.Context) error { + sc := scenarioCtx(ctx) + if featureGates[features.BoxcutterRuntime] { + return fmt.Errorf("expected Helm-backed storage before OLM upgrade") + } + objects, err := listHelmReleaseResources(sc.clusterExtensionName) + if err != nil { + return err + } + if len(objects) == 0 { + return fmt.Errorf("helm release has no managed resources") + } + for _, obj := range objects { + live, err := getResource(obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName(), obj.GetNamespace()) + if err != nil { + return err + } + if live.GetUID() == "" { + return fmt.Errorf("resource %s has no live UID", client.ObjectKeyFromObject(live)) + } + sc.upgradeObjects = append(sc.upgradeObjects, live) + } + // Pods are not bundle manifests. Preserve their identities through migration, + // but allow a normal pod rollout when the bundle version changes afterward. + return waitForUpgradeCheck(ctx, func() error { + out, err := k8sClient(ctx, "get", "pods", "-n", sc.namespace, "-l", "app="+sc.id, "-o", "json") + if err != nil { + return err + } + var pods corev1.PodList + if err := json.Unmarshal([]byte(out), &pods); err != nil { + return err + } + if len(pods.Items) == 0 { + return fmt.Errorf("no fixture workload pods found before OLM upgrade") + } + for _, pod := range pods.Items { + if !podReady(&pod) { + return fmt.Errorf("fixture pod %s is not ready before OLM upgrade", pod.Name) + } + } + sc.upgradePods = pods.Items + return nil + }) +} + +func podReady(pod *corev1.Pod) bool { + if pod.DeletionTimestamp != nil { + return false + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func ExtensionKeepsPreUpgradeIdentities(ctx context.Context) error { + if err := checkExtensionIdentities(ctx, false); err != nil { + return err + } + for _, before := range scenarioCtx(ctx).upgradePods { + out, err := k8sClient(ctx, "get", "pod", before.Name, "-n", before.Namespace, "-o", "json") + if err != nil { + return err + } + var after corev1.Pod + if err := json.Unmarshal([]byte(out), &after); err != nil { + return err + } + if after.UID != before.UID || !podReady(&after) { + return fmt.Errorf("workload pod %s was replaced or is no longer ready after OLM upgrade", before.Name) + } + } + return nil +} + +func RetainedExtensionKeepsPreUpgradeIdentities(ctx context.Context) error { + return checkExtensionIdentities(ctx, true) +} + +func checkExtensionIdentities(ctx context.Context, afterUpdate bool) error { + sc := scenarioCtx(ctx) + if len(sc.upgradeObjects) == 0 { + return fmt.Errorf("no pre-upgrade extension resource identities remembered") + } + retained := map[string]bool{} + if afterUpdate { + objects, err := listExtensionResources(sc.clusterExtensionName) + if err != nil { + return err + } + for _, obj := range objects { + retained[fmt.Sprintf("%s/%s", obj.GetObjectKind().GroupVersionKind().Kind, client.ObjectKeyFromObject(obj))] = true + } + } + for _, before := range sc.upgradeObjects { + // Rendered RBAC names incorporate the CSV version. Those resources are + // legitimately replaced by an extension update; all other fixture names + // are stable and must retain their UIDs, including the Deployment and CRD. + if afterUpdate && before.GroupVersionKind().Group == "rbac.authorization.k8s.io" && + !retained[fmt.Sprintf("%s/%s", before.GetKind(), client.ObjectKeyFromObject(before))] { + continue + } + after, err := getResource(before.GetKind(), before.GetName(), before.GetNamespace()) + if err != nil { + return err + } + if after.GetUID() != before.GetUID() { + return fmt.Errorf("%s %s was replaced: UID changed from %s to %s", before.GetKind(), + client.ObjectKeyFromObject(before), before.GetUID(), after.GetUID()) + } + } + return nil +} + +// PreExistingExtensionStorageTransitioned expects a complete CE -> COD -> COS +// handoff for experimental OLM. Standard upgrades must keep using Helm storage. +func PreExistingExtensionStorageTransitioned(ctx context.Context) error { + return checkUpgradedExtensionStorage(ctx, 2, "1.0.0") +} + +func UpdatedExtensionStorageIsHealthy(ctx context.Context) error { + return checkUpgradedExtensionStorage(ctx, 3, "1.0.1") +} + +func checkUpgradedExtensionStorage(ctx context.Context, revision int64, version string) error { + sc := scenarioCtx(ctx) + if !featureGates[features.BoxcutterRuntime] { + // Do not request experimental APIs on standard installations. Reading + // the live Helm release also verifies that updating the same CE still + // writes Helm storage, rather than accepting only retained CE status. + secret, err := helmReleaseSecretForExtension(sc.clusterExtensionName) + if err != nil { + return err + } + if secret == nil { + return fmt.Errorf("no deployed Helm release for %s", sc.clusterExtensionName) + } + rel, err := helmReleaseFromSecret(secret) + if err != nil { + return err + } + if rel.Labels[labels.BundleVersionKey] != version { + return fmt.Errorf("helm release bundle version = %q, want %q", rel.Labels[labels.BundleVersionKey], version) + } + return nil + } + + return waitForUpgradeCheck(ctx, func() error { + ext, err := getResource("clusterextension", sc.clusterExtensionName, "") + if err != nil { + return err + } + obj, err := getResource("clusterobjectdeployment", sc.clusterExtensionName, "") + if err != nil { + return err + } + var cod ocv1.ClusterObjectDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &cod); err != nil { + return err + } + if err := requireController(&cod, ext); err != nil { + return err + } + if revision == 3 && sc.upgradeStorageUIDs["cod"] != cod.UID { + return fmt.Errorf("COD %s UID changed during extension update", cod.Name) + } + if !conditionMatches(currentCondition(cod.Generation, cod.Status.Conditions, ocv1.ClusterObjectDeploymentTypeAvailable), metav1.ConditionTrue, nil, nil) || + !conditionMatches(currentCondition(cod.Generation, cod.Status.Conditions, ocv1.ClusterObjectDeploymentTypeProgressing), metav1.ConditionFalse, ptr.To(ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed), nil) { + return fmt.Errorf("COD %s has not completed migration/update: conditions=%+v", cod.Name, cod.Status.Conditions) + } + if cod.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + return fmt.Errorf("COD template collision protection = %s, want Prevent", cod.Spec.Template.Spec.CollisionProtection) + } + revisions, err := listClusterObjectSetsInGroup(ctx, sc.clusterExtensionName) + if err != nil { + return err + } + if int64(len(revisions)) != revision { + return fmt.Errorf("group %s has %d revisions, want %d", sc.clusterExtensionName, len(revisions), revision) + } + seen := map[int64]bool{} + var active *ocv1.ClusterObjectSet + for i := range revisions { + rev := &revisions[i] + if rev.Spec.Group != cod.Name || rev.Spec.Revision < 1 || rev.Spec.Revision > revision || seen[rev.Spec.Revision] { + return fmt.Errorf("unexpected group/revision for COS %s: %s/%d", rev.Name, rev.Spec.Group, rev.Spec.Revision) + } + seen[rev.Spec.Revision] = true + if err := requireController(rev, &cod); err != nil { + return err + } + if revision == 3 && rev.Spec.Revision < revision && sc.upgradeStorageUIDs[rev.Name] != rev.UID { + return fmt.Errorf("COS %s UID changed during extension update", rev.Name) + } + protection := ocv1.CollisionProtectionPrevent + if rev.Spec.Revision == 1 { + protection = ocv1.CollisionProtectionNone + if rev.Labels[labels.MigratedFromHelmKey] != "true" || rev.Annotations[labels.BundleVersionKey] != "1.0.0" { + return fmt.Errorf("COS %s is not the migrated 1.0.0 Helm release", rev.Name) + } + } + if rev.Spec.CollisionProtection != protection || rev.Status.CompletedAt.IsZero() { + return fmt.Errorf("COS %s: protection=%s completedAt=%s, want %s and completed rollout", rev.Name, + rev.Spec.CollisionProtection, rev.Status.CompletedAt, protection) + } + if rev.Spec.Revision == revision { + active = rev + if rev.Spec.LifecycleState != ocv1.ClusterObjectSetLifecycleStateActive || + !conditionMatches(currentCondition(rev.Generation, rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady), metav1.ConditionTrue, nil, nil) || + rev.Annotations[labels.BundleVersionKey] != version { + return fmt.Errorf("COS %s is not active/ready at bundle version %s: conditions=%+v", rev.Name, version, rev.Status.Conditions) + } + } else if rev.Spec.LifecycleState != ocv1.ClusterObjectSetLifecycleStateArchived || + !conditionMatches(currentCondition(rev.Generation, rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady), metav1.ConditionFalse, ptr.To(ocv1.ClusterObjectSetReasonArchived), nil) { + return fmt.Errorf("COS %s is not archived: conditions=%+v", rev.Name, rev.Status.Conditions) + } + } + if active == nil { + return fmt.Errorf("no active revision %d", revision) + } + objects, err := listExtensionRevisionResources(sc.clusterExtensionName) + if err != nil { + return err + } + if len(objects) == 0 { + return fmt.Errorf("active revision %s contains no objects", active.Name) + } + for _, desired := range objects { + live, err := getResource(desired.GetObjectKind().GroupVersionKind().Kind, desired.GetName(), desired.GetNamespace()) + if err != nil { + return err + } + if err := requireController(live, active); err != nil { + return err + } + } + if revision == 2 { + sc.upgradeStorageUIDs = map[string]types.UID{"cod": cod.UID} + for _, rev := range revisions { + sc.upgradeStorageUIDs[rev.Name] = rev.UID + } + } + return nil + }) +} + +func requireController(object, controller client.Object) error { + owner := metav1.GetControllerOf(object) + gvk := controller.GetObjectKind().GroupVersionKind() + // Typed list items may omit TypeMeta. The controller identity is still + // unambiguous from the API type and its live name/UID. + switch controller.(type) { + case *ocv1.ClusterObjectDeployment: + gvk = ocv1.GroupVersion.WithKind(ocv1.ClusterObjectDeploymentKind) + case *ocv1.ClusterObjectSet: + gvk = ocv1.GroupVersion.WithKind(ocv1.ClusterObjectSetKind) + } + if owner == nil || owner.APIVersion != gvk.GroupVersion().String() || owner.Kind != gvk.Kind || + owner.Name != controller.GetName() || owner.UID != controller.GetUID() { + return fmt.Errorf("%s %s controller=%+v, want %s %s (UID %s)", object.GetObjectKind().GroupVersionKind().Kind, + object.GetName(), owner, gvk.Kind, controller.GetName(), controller.GetUID()) + } + return nil +} diff --git a/test/e2e/steps/upgrade_storage_steps_test.go b/test/e2e/steps/upgrade_storage_steps_test.go new file mode 100644 index 0000000000..4c10769d5d --- /dev/null +++ b/test/e2e/steps/upgrade_storage_steps_test.go @@ -0,0 +1,40 @@ +package steps + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "helm.sh/helm/v3/pkg/release" + corev1 "k8s.io/api/core/v1" + + "github.com/operator-framework/operator-controller/internal/shared/labels" +) + +func TestHelmReleaseFromSecretRestoresStoredBundleLabels(t *testing.T) { + // Match the chunked driver's wrapper: release.Release.Labels itself has + // json:"-", so decoding directly into release.Release silently loses it. + stored := struct { + release.Release + Labels map[string]string `json:"labels"` + }{ + Release: release.Release{Name: "extension", Manifest: "bundle manifests", Version: 2}, + Labels: map[string]string{labels.BundleVersionKey: "1.0.1"}, + } + data, err := json.Marshal(stored) + require.NoError(t, err) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err = writer.Write(data) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + rel, err := helmReleaseFromSecret(&corev1.Secret{Data: map[string][]byte{"chunk": compressed.Bytes()}}) + require.NoError(t, err) + require.Equal(t, "extension", rel.Name) + require.Equal(t, "bundle manifests", rel.Manifest) + require.Equal(t, 2, rel.Version) + require.Equal(t, "1.0.1", rel.Labels[labels.BundleVersionKey]) +} diff --git a/test/upgrade-e2e/README.md b/test/upgrade-e2e/README.md index d2edf87cbe..9fab846cb6 100644 --- a/test/upgrade-e2e/README.md +++ b/test/upgrade-e2e/README.md @@ -2,6 +2,31 @@ End-to-end tests that verify OLM upgrades don't break pre-existing ClusterCatalogs and ClusterExtensions. +One scenario follows the same catalog and ClusterExtension through the entire upgrade. +It installs a small generated bundle at version 1.0.0 on standard OLM and records the +extension's UID, live managed resource UIDs, and ready workload pod UIDs before upgrading OLM. +After the upgrade it checks catalog reconciliation and the extension's storage: + +- **Standard destination:** the existing extension remains Helm-backed. +- **Experimental destination:** the extension controls a ClusterObjectDeployment (COD), + which controls migrated revision 1 with collision protection `None` and normal + revision 2 with `Prevent`. Revision 1 must have completed and be archived; + revision 2 must be active and ready, with its managed resources under its control. + COD availability and revision conditions must reflect their current generations. + +The scenario verifies the original extension, every original managed resource, and +workload pods retain their UIDs through migration. It then updates that same extension +to 1.0.1. Experimental OLM must produce ready revision 3, archive the earlier revisions, +and preserve the COD and earlier revision UIDs. Standard OLM must update the Helm +release. Stable bundle resource names must keep their UIDs; versioned RBAC resources +and workload pods may be replaced during this extension update. + +Feature gates and component namespaces are refreshed from the deployed configuration +after installation and upgrade. Leader leases must point to pods belonging to the +current deployment revision, so stale leases cannot stand in for upgraded controllers. +The scenario starts on standard OLM; experimental-to-experimental upgrades are outside +this coverage. The fixtures use ordinary etcd limits and strict CRD upgrade safety. + These tests use the [Godog](https://github.com/cucumber/godog) BDD framework with shared step definitions from `test/e2e/steps/`. The upgrade-specific steps live in `test/e2e/steps/upgrade_steps.go`. See [test/e2e/README.md](../e2e/README.md) for more information on the Godog framework and writing tests. @@ -31,6 +56,22 @@ The following Makefile targets run upgrade tests for different upgrade paths: | `make test-st2ex-e2e` | Switchover from a locally built **standard** install to a locally built **experimental** install | Each target handles kind cluster creation, image building/loading, test execution, and cleanup. +The targets recreate their named clusters. To preserve an existing investigation cluster, +use a separately named disposable cluster and an explicit kubeconfig with the lower-level runner: + +```bash +RELEASE_INSTALL=https://github.com/operator-framework/operator-controller/releases/download/v1.12.0/install.sh \ +RELEASE_UPGRADE="$PWD/experimental-install.sh" \ +ROOT_DIR="$PWD" KIND_CLUSTER_NAME=upgrade-coverage-test \ +KUBECONFIG="$PWD/.kubeconfig/upgrade-coverage-test.kubeconfig" \ +go test -count=1 -v ./test/upgrade-e2e/upgrade_test.go +``` + +First generate the local install scripts and load the locally built images into that cluster. +Set `RELEASE_INSTALL` to the local `standard-install.sh` for local standard-to-experimental +coverage, or `RELEASE_UPGRADE` to it for released standard-to-local standard coverage. +On failure the scenario leaves its resources for inspection. Migration assertions retain +the last observed error and COD conditions, including ownership conflicts, in timeout output. ## Environment Variables diff --git a/test/upgrade-e2e/features/operator-upgrade.feature b/test/upgrade-e2e/features/operator-upgrade.feature index 5a310c57c7..8540556756 100644 --- a/test/upgrade-e2e/features/operator-upgrade.feature +++ b/test/upgrade-e2e/features/operator-upgrade.feature @@ -4,14 +4,14 @@ Feature: Operator upgrade verification pre-existing ClusterCatalogs and ClusterExtensions continue to function and can be updated. - Background: + Scenario: Pre-existing catalog and extension survive OLM upgrade and extension update Given the latest stable OLM release is installed And an image registry is available And a catalog "test" with packages: | package | version | channel | replaces | contents | | test | 1.0.0 | beta | | CRD, Deployment, ConfigMap | | test | 1.0.1 | beta | 1.0.0 | CRD, Deployment, ConfigMap | - And ServiceAccount "olm-sa" with permissions to install extensions is available in "${TEST_NAMESPACE}" namespace + And namespace "${TEST_NAMESPACE}" is available And ClusterExtension is applied """ apiVersion: olm.operatorframework.io/v1 @@ -20,8 +20,6 @@ Feature: Operator upgrade verification name: ${NAME} spec: namespace: ${TEST_NAMESPACE} - serviceAccount: - name: olm-sa source: sourceType: Catalog catalog: @@ -32,17 +30,29 @@ Feature: Operator upgrade verification version: 1.0.0 """ And ClusterExtension is available - And OLM is upgraded + And ClusterExtension is rolled out + And bundle "${PACKAGE:test}.1.0.0" is installed in version "1.0.0" + And resource "clusterextension/${NAME}" UID is remembered + And Helm-backed extension resource identities are remembered + When OLM is upgraded And catalogd is ready to reconcile resources And operator-controller is ready to reconcile resources - Scenario: ClusterCatalog continues unpacking after OLM upgrade - When catalog "test" is reconciled + And catalog "test" is reconciled Then catalog "test" reports Progressing as True with Reason Succeeded And catalog "test" reports Serving as True with Reason Available - Scenario: ClusterExtension remains functional after OLM upgrade - Given ClusterExtension is reconciled + And pre-existing ClusterExtension storage has transitioned after OLM upgrade + And ClusterExtension is available + And ClusterExtension is rolled out + And bundle "${PACKAGE:test}.1.0.0" is installed in version "1.0.0" + And resource "clusterextension/${NAME}" keeps its UID + And extension resources keep their pre-upgrade identities When ClusterExtension version is updated to "1.0.1" Then ClusterExtension is available + And ClusterExtension is rolled out And bundle "${PACKAGE:test}.1.0.1" is installed in version "1.0.1" + And deployment "test-operator-${SCENARIO_ID}" rollout is complete + And updated ClusterExtension storage is healthy + And resource "clusterextension/${NAME}" keeps its UID + And retained extension resources keep their pre-upgrade identities diff --git a/test/upgrade-e2e/upgrade_test.go b/test/upgrade-e2e/upgrade_test.go index 5991dd81ae..117cb60894 100644 --- a/test/upgrade-e2e/upgrade_test.go +++ b/test/upgrade-e2e/upgrade_test.go @@ -30,8 +30,12 @@ func TestMain(m *testing.M) { opts.Paths = pflag.Args() sc := godog.TestSuite{ + TestSuiteInitializer: func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, ScenarioInitializer: func(sc *godog.ScenarioContext) { sc.Before(steps.CreateScenarioContext) + sc.After(steps.ScenarioCleanup) steps.RegisterSteps(sc) }, Options: &opts,