From 7409854956fd13b5907505da8bbe664596050e6b Mon Sep 17 00:00:00 2001 From: Federico Bonfigli Date: Wed, 17 Jun 2026 17:31:47 +0200 Subject: [PATCH 1/3] Update AWS CCM e2e test module version Updates the AWS CCM e2e tests module version to the latest version which includes changes like the addition of e2e tests for the BYO Security Group for AWS Network Load Balancer feat. --- openshift-tests/ccm-aws-tests/go.mod | 2 +- openshift-tests/ccm-aws-tests/go.sum | 4 +- .../tests/e2e/cloudconfig.go | 386 ++++++ .../tests/e2e/loadbalancer.go | 1049 ++++++++++++----- .../ccm-aws-tests/vendor/modules.txt | 2 +- 5 files changed, 1138 insertions(+), 305 deletions(-) create mode 100644 openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go diff --git a/openshift-tests/ccm-aws-tests/go.mod b/openshift-tests/ccm-aws-tests/go.mod index bc14fcc74..3fe96a468 100644 --- a/openshift-tests/ccm-aws-tests/go.mod +++ b/openshift-tests/ccm-aws-tests/go.mod @@ -17,7 +17,7 @@ require ( k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 k8s.io/client-go v0.36.0 - k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c + k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a k8s.io/kubernetes v1.36.0 k8s.io/pod-security-admission v0.36.0 ) diff --git a/openshift-tests/ccm-aws-tests/go.sum b/openshift-tests/ccm-aws-tests/go.sum index d6e3ecfa1..66e331587 100644 --- a/openshift-tests/ccm-aws-tests/go.sum +++ b/openshift-tests/ccm-aws-tests/go.sum @@ -275,8 +275,8 @@ k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= k8s.io/cloud-provider v0.36.0 h1:PtiHsId1lBJixCbl5T+gUzbgOYAPschYj8tEAxxe0Ts= k8s.io/cloud-provider v0.36.0/go.mod h1:y/3sksoC0taJZR0PcAAYUqVyD6Jzu2X0lD4yCEPXPuI= -k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c h1:mXxJGVHCgE0diY1xgPg1mlmGuDhj2o8KzehT+YcOED8= -k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c/go.mod h1:BVLb2sMjVqNAN4t7bADRqtpdVA7brv8LcNas/kdKUJo= +k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a h1:3vIWSg8oFCQUal5G8Xj8m2hM7VrSJyqxaZkX1b6ICbM= +k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a/go.mod h1:vu0ofZopcm0LmGHdelw1LaP7oZdFZjgQ6k2fN+/MO5g= k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= k8s.io/component-helpers v0.36.0 h1:KznLAOD7oPxjaeheW4SOQijz9UtMO8Nvp89+lR8FYks= diff --git a/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go new file mode 100644 index 000000000..fcb3d07fa --- /dev/null +++ b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go @@ -0,0 +1,386 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + // ccmDaemonSetName is the name of the CCM DaemonSet in kube-system namespace. + ccmDaemonSetName = "aws-cloud-controller-manager" + // ccmNamespace is the namespace where CCM daemonset is deployed by kops. + ccmNamespace = "kube-system" + // ccmLabelSelector is the label selector to identify CCM pods. + ccmLabelSelector = "k8s-app=aws-cloud-controller-manager" + // tempConfigMapName is the name used for the injected config map with the modified cloud config. + tempConfigMapName = "aws-cloud-config-e2e" + // defaultRestartTimeout is the default timeout for waiting for CCM pods to restart and become ready after config changes. + defaultRestartTimeout = 3 * time.Minute + // pollInterval is the interval for polling pod readiness after CCM restart. + pollInterval = 5 * time.Second +) + +// cloudConfigManager helps to change the CCM cloud configuration for e2e tests. +// It is engineered to work with kops-provisioned clusters that run CI tests. +// It modifies the CCM cloud config at runtime by creating a temporary ConfigMap and +// patching the DaemonSet to use it instead of the original hostPath volume. +type cloudConfigManager struct { + // State tracking for restoration + originalVolume *v1.Volume + originalVolumeMount *v1.VolumeMount + configMapKey string + + // restartTimeout defines for how long to wait for CCM pods to become ready + // after configuration changes. + restartTimeout time.Duration +} + +// cloudConfigManagerOption is a functional option for configuring ccmCloudConfigManager. +type cloudConfigManagerOption func(*cloudConfigManager) + +// withRestartTimeout sets the timeout for waiting for CCM pods to become ready after configuration changes. +func withRestartTimeout(timeout time.Duration) cloudConfigManagerOption { + return func(m *cloudConfigManager) { + m.restartTimeout = timeout + } +} + +// newCloudConfigManager creates a new CCM cloud config manager with the provided options. +func newCloudConfigManager(opts ...cloudConfigManagerOption) *cloudConfigManager { + m := &cloudConfigManager{ + restartTimeout: defaultRestartTimeout, + } + + // Apply options + for _, opt := range opts { + opt(m) + } + + return m +} + +// setCloudConfig modifies the CCM cloud configuration with the provided content, storing the original +// configuration state for later restoration. +// +// Steps: +// - Finds the cloud-config path from CCM args +// - Creates a temporary ConfigMap with the desired config +// - Patches the CCM DaemonSet to use the ConfigMap instead of hostPath +// - Adds subPath to volumeMount to mount ConfigMap key as a file +// - Restarts CCM pods and waits for them to become ready +// +// Parameters: +// - ctx: Context for the operation +// - cs: Kubernetes clientset +// - configContent: The cloud config file content to set. +// +// Returns error if any step fails or if CCM is not using hostPath. +func (m *cloudConfigManager) setCloudConfig(ctx context.Context, cs clientset.Interface, configContent string) error { + framework.Logf("=== Setting CCM cloud configuration ===") + + // Get CCM DaemonSet + ds, err := cs.AppsV1().DaemonSets(ccmNamespace).Get(ctx, ccmDaemonSetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get CCM DaemonSet: %w", err) + } + + // Find cloud-config file path from CCM container args + if len(ds.Spec.Template.Spec.Containers) == 0 { + return fmt.Errorf("CCM DaemonSet has no containers") + } + cloudConfigPath := getCloudConfigPath(ds.Spec.Template.Spec.Containers[0]) + + // Check if CCM actually uses a cloud config. If not, return an error as it's an unsupported scenario. + if cloudConfigPath == "" { + return fmt.Errorf("CCM does not use --cloud-config flag") + } + framework.Logf("Cloud config path from CCM args: %s", cloudConfigPath) + + // Find the volume and volumeMount for cloud config + _, volumeIdx, err := m.findCloudConfigVolume(ds, cloudConfigPath) + if err != nil { + return err + } + + // Verify it's hostPath-based + if m.originalVolume.HostPath == nil { + return fmt.Errorf("CCM cloud config is not hostPath-based (only hostPath configs are supported)") + } + framework.Logf("Current config mount: HostPath=%s", m.originalVolume.HostPath.Path) + + // Create ConfigMap and patch DaemonSet. This also stores a copy of the original volume for later restoration. + if err := m.createConfigMapAndPatchDaemonSet(ctx, cs, ds, volumeIdx, configContent); err != nil { + return err + } + + // Restart CCM pods and wait for them to be ready + return restartCCMPods(ctx, cs, m.restartTimeout) +} + +// restoreCloudConfig restores the original CCM cloud configuration as saved by setCloudConfig. +// This restores the hostPath volume created by kops and deletes the temporary ConfigMap injected. +func (m *cloudConfigManager) restoreCloudConfig(ctx context.Context, cs clientset.Interface) error { + framework.Logf("=== Restoring original CCM configuration ===") + + ds, err := cs.AppsV1().DaemonSets(ccmNamespace).Get(ctx, ccmDaemonSetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get DaemonSet: %w", err) + } + + if m.originalVolume == nil { + return fmt.Errorf("no original volume to restore") + } + + // Restore original volume + for i, vol := range ds.Spec.Template.Spec.Volumes { + if vol.Name == m.originalVolume.Name { + ds.Spec.Template.Spec.Volumes[i] = *m.originalVolume + framework.Logf("Restored original volume: %s", m.originalVolume.Name) + break + } + } + + // Restore original volumeMount (remove subPath) + if m.originalVolumeMount != nil { + for i, container := range ds.Spec.Template.Spec.Containers { + for j, mount := range container.VolumeMounts { + if mount.Name == m.originalVolume.Name { + ds.Spec.Template.Spec.Containers[i].VolumeMounts[j] = *m.originalVolumeMount + framework.Logf("Restored original volumeMount") + break + } + } + } + } + + // Update DaemonSet + _, err = cs.AppsV1().DaemonSets(ccmNamespace).Update(ctx, ds, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to restore DaemonSet: %w", err) + } + framework.Logf("Restored DaemonSet to use hostPath") + + // Delete temporary ConfigMap + err = cs.CoreV1().ConfigMaps(ccmNamespace).Delete(ctx, tempConfigMapName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete ConfigMap: %w", err) + } + framework.Logf("Deleted temporary ConfigMap %s", tempConfigMapName) + + // Restart CCM pods to load original config + return restartCCMPods(ctx, cs, m.restartTimeout) +} + +// getCloudConfigPath extracts the cloud config path from CCM container arguments. +// Returns empty string if --cloud-config flag is not found. +func getCloudConfigPath(container v1.Container) string { + for i, arg := range container.Args { + if arg == "--cloud-config" && i+1 < len(container.Args) { + return container.Args[i+1] + } else if strings.HasPrefix(arg, "--cloud-config=") { + return strings.TrimPrefix(arg, "--cloud-config=") + } + } + + return "" // No cloud-config flag found +} + +// findCloudConfigVolume locates the volume and volumeMount for the cloud config in the CCM DaemonSet. +// It returns the volume name, volume index, and ConfigMap key. +func (m *cloudConfigManager) findCloudConfigVolume(ds *appsv1.DaemonSet, cloudConfigPath string) (volumeName string, volumeIdx int, err error) { + if len(ds.Spec.Template.Spec.Containers) == 0 { + return "", -1, fmt.Errorf("CCM DaemonSet has no containers") + } + container := ds.Spec.Template.Spec.Containers[0] + + // Find volumeMount that matches the cloud config path + for _, mount := range container.VolumeMounts { + if mount.MountPath == cloudConfigPath { + volumeName = mount.Name + // If SubPath is set, that's the ConfigMap key; otherwise use basename of path + if mount.SubPath != "" { + m.configMapKey = mount.SubPath + } else { + m.configMapKey = filepath.Base(cloudConfigPath) + } + framework.Logf("Found cloud config volumeMount: name=%s, mountPath=%s, subPath=%s, key=%s", + volumeName, mount.MountPath, mount.SubPath, m.configMapKey) + break + } + } + if volumeName == "" { + return "", -1, fmt.Errorf("cloud config volumeMount not found for path %s", cloudConfigPath) + } + + // Find the volume by name + for i, vol := range ds.Spec.Template.Spec.Volumes { + if vol.Name == volumeName { + volumeIdx = i + m.originalVolume = vol.DeepCopy() + framework.Logf("Found cloud config volume: name=%s, type=%s", vol.Name, getVolumeType(vol)) + return volumeName, volumeIdx, nil + } + } + + return "", -1, fmt.Errorf("cloud config volume not found for name %s", volumeName) +} + +// createConfigMapAndPatchDaemonSet creates a new ConfigMap with cloud config and patches +// the CCM DaemonSet to use it instead of hostPath. +func (m *cloudConfigManager) createConfigMapAndPatchDaemonSet(ctx context.Context, cs clientset.Interface, ds *appsv1.DaemonSet, volumeIdx int, configContent string) error { + framework.Logf("Creating temporary ConfigMap and patching DaemonSet") + + // Create ConfigMap with the detected key + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: tempConfigMapName, + Namespace: ccmNamespace, + }, + Data: map[string]string{ + m.configMapKey: configContent, + }, + } + _, err := cs.CoreV1().ConfigMaps(ccmNamespace).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + // ConfigMap already exists, update it + framework.Logf("ConfigMap %s already exists, updating...", tempConfigMapName) + _, err = cs.CoreV1().ConfigMaps(ccmNamespace).Update(ctx, cm, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update existing ConfigMap: %w", err) + } + framework.Logf("Updated ConfigMap %s with key '%s'", tempConfigMapName, m.configMapKey) + } else { + return fmt.Errorf("failed to create ConfigMap: %w", err) + } + } else { + framework.Logf("Created ConfigMap %s with key '%s'", tempConfigMapName, m.configMapKey) + } + + // Patch DaemonSet volume to use ConfigMap + ds.Spec.Template.Spec.Volumes[volumeIdx] = v1.Volume{ + Name: m.originalVolume.Name, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: tempConfigMapName, + }, + }, + }, + } + + // Find and patch volumeMount to use subPath + // ConfigMap mounts as directory, hostPath as file + // Without subPath, /etc/kubernetes/cloud.config becomes a directory + for i, container := range ds.Spec.Template.Spec.Containers { + for j, mount := range container.VolumeMounts { + if mount.Name == m.originalVolume.Name { + // Save original volumeMount for rollback + m.originalVolumeMount = mount.DeepCopy() + // Add subPath to mount ConfigMap key as file + ds.Spec.Template.Spec.Containers[i].VolumeMounts[j].SubPath = m.configMapKey + framework.Logf("Added subPath=%s to volumeMount", m.configMapKey) + break + } + } + } + + _, err = cs.AppsV1().DaemonSets(ccmNamespace).Update(ctx, ds, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update DaemonSet: %w", err) + } + framework.Logf("Patched DaemonSet to use ConfigMap") + + return nil +} + +// restartCCMPods restarts all CCM pods and waits for them to become ready. +func restartCCMPods(ctx context.Context, cs clientset.Interface, timeout time.Duration) error { + framework.Logf("Restarting CCM pods") + + ccmPods, err := cs.CoreV1().Pods(ccmNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: ccmLabelSelector, + }) + if err != nil { + return fmt.Errorf("failed to list CCM pods: %w", err) + } + + for _, pod := range ccmPods.Items { + err = cs.CoreV1().Pods(ccmNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + // Ignore NotFound errors - pod may already be deleting due to DaemonSet update + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete pod %s: %w", pod.Name, err) + } + } + + // Wait for new pods to be Running AND Ready + framework.Logf("Waiting for CCM pods to become ready (timeout: %v)", timeout) + err = wait.PollUntilContextTimeout(ctx, pollInterval, timeout, true, func(ctx context.Context) (bool, error) { + pods, err := cs.CoreV1().Pods(ccmNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: ccmLabelSelector, + }) + if err != nil || len(pods.Items) == 0 { + return false, nil + } + for _, pod := range pods.Items { + if pod.Status.Phase != v1.PodRunning { + return false, nil + } + // Check container ready status + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == v1.PodReady && condition.Status == v1.ConditionTrue { + ready = true + break + } + } + if !ready { + return false, nil + } + } + return true, nil + }) + if err != nil { + return fmt.Errorf("CCM pods did not become ready within %v: %w", timeout, err) + } + framework.Logf("CCM restarted successfully") + + return nil +} + +// getVolumeType returns a string describing the type of volume. +func getVolumeType(vol v1.Volume) string { + if vol.ConfigMap != nil { + return "ConfigMap" + } else if vol.HostPath != nil { + return "HostPath" + } + return "Unknown" +} diff --git a/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go index 217706061..ffd4ee169 100644 --- a/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go +++ b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go @@ -36,6 +36,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" ) @@ -45,6 +47,7 @@ const ( annotationLBInternal = "service.beta.kubernetes.io/aws-load-balancer-internal" annotationLBTargetNodeLabels = "service.beta.kubernetes.io/aws-load-balancer-target-node-labels" annotationLBTargetGroupAttributes = "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes" + annotationLBSecurityGroups = "service.beta.kubernetes.io/aws-load-balancer-security-groups" ) var ( @@ -56,354 +59,647 @@ var ( } ) -// loadbalancer tests -var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() { - f := framework.NewDefaultFramework("cloud-provider-aws") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged +// loadBalancerTestCases defines the structure for different load balancer test scenarios, +// including configuration, hooks for dynamic behavior, and test verification controls. +type loadBalancerTestCases struct { + // Overall test case configuration. + name string + resourceSuffix string + extraAnnotations map[string]string + listenerCount int + + // Hooks + // HookPostServiceConfig hook runs after the service manifest is created, and before the service is created. + hookPostServiceConfig func(cfg *e2eTestConfig) + // HookPostServiceCreate hook runs after the test is run. + hookPostServiceCreate func(cfg *e2eTestConfig) + // HookPreTest hook runs before the test is run. + hookPreTest func(cfg *e2eTestConfig) + + // Flags to override default test behavior. + overrideTestRunInClusterReachableHTTP bool + requireAffinity bool + + // Test verification + skipTestFailure bool +} - var ( - cs clientset.Interface - ns *v1.Namespace - ) +// runTestCase is a helper function that executes a load balancer test case +func runTestCase(ctx context.Context, tc loadBalancerTestCases, cs clientset.Interface, ns *v1.Namespace) { + By("setting up test environment and discovering worker nodes") + e2e := newE2eTestConfig(cs) + e2e.discoverClusterWorkerNode() + framework.Logf("[SETUP] Test case: %s", tc.name) + framework.Logf("[SETUP] Worker nodes discovered: %d nodes, selector: %s, sample node: %s", e2e.nodeCount, e2e.nodeSelector, e2e.nodeSingleSample) - BeforeEach(func() { - cs = f.ClientSet - ns = f.Namespace - }) + loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs) + framework.Logf("[CONFIG] AWS load balancer timeout: %s", loadBalancerCreateTimeout) - AfterEach(func() { - // After each test + By("building service configuration with annotations") + + serviceName := "lbconfig-test" + if len(tc.resourceSuffix) > 0 { + serviceName = serviceName + "-" + tc.resourceSuffix + } + framework.Logf("[CONFIG] Service name: %s, namespace: %s", serviceName, ns.Name) + e2e.LBJig = e2eservice.NewTestJig(cs, ns.Name, serviceName) + + // Hook annotations to support dynamic config + e2e.svc = e2e.buildService(tc.listenerCount, tc.extraAnnotations) + framework.Logf("[CONFIG] Service ports: %d, extra annotations: %v", len(e2e.svc.Spec.Ports), tc.extraAnnotations) + + if tc.hookPostServiceConfig != nil { + By("executing hook post-service-config: applying service configuration") + framework.Logf("[HOOK] Executing post-service-config hook") + tc.hookPostServiceConfig(e2e) + framework.Logf("[HOOK] Final service annotations: %v", e2e.svc.Annotations) + } + + By("creating LoadBalancer service in Kubernetes") + if _, err := e2e.LBJig.Client.CoreV1().Services(e2e.LBJig.Namespace).Create(context.TODO(), e2e.svc, metav1.CreateOptions{}); err != nil { + framework.ExpectNoError(fmt.Errorf("failed to create LoadBalancer Service %q: %v", e2e.svc.Name, err)) + } + framework.Logf("[K8S] LoadBalancer service created successfully") + + By("waiting for AWS load balancer provisioning") + var err error + e2e.svc, err = e2e.LBJig.WaitForLoadBalancer(ctx, loadBalancerCreateTimeout) + // Collect comprehensive debugging information when LoadBalancer provisioning fails + if err != nil { + serviceName := e2e.LBJig.Name + if e2e.svc != nil { + serviceName = e2e.svc.Name + } + framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) + framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) + + // Ensure we have detailed debugging information before failing + framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + + // Fail the test immediately to prevent further execution + framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + } + framework.Logf("[AWS] Load balancer provisioned successfully") + + By("creating backend server pods") + _, err = e2e.LBJig.Run(ctx, e2e.buildDeployment(tc.requireAffinity)) + if err != nil { + serviceName := e2e.LBJig.Name + if e2e.svc != nil { + serviceName = e2e.svc.Name + } + framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) + framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) + + // Ensure we have detailed debugging information before failing + framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + + // Fail the test immediately to prevent further execution + framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + } + + framework.Logf("[K8S] Backend pods created, affinity required: %t", tc.requireAffinity) + + if tc.hookPostServiceCreate != nil { + By("executing hook post-service-create: applying service configuration") + tc.hookPostServiceCreate(e2e) + } + + By("collecting service and load balancer information") + if e2e.svc == nil { + framework.Logf("=== Service Validation Error Debug Information ===") + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of Service Validation Error Debug Information ===") + framework.Failf("Service is nil after LoadBalancer provisioning for service %s", e2e.LBJig.Name) + } + if len(e2e.svc.Spec.Ports) == 0 { + framework.Logf("=== Service Ports Error Debug Information ===") + framework.Logf("Service spec: %+v", e2e.svc.Spec) + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of Service Ports Error Debug Information ===") + framework.Failf("No ports found in service spec for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } + if len(e2e.svc.Status.LoadBalancer.Ingress) == 0 { + framework.Logf("=== LoadBalancer Ingress Error Debug Information ===") + framework.Logf("Service status: %+v", e2e.svc.Status) + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of LoadBalancer Ingress Error Debug Information ===") + framework.Failf("No ingress found in LoadBalancer status for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } + + svcPort := int(e2e.svc.Spec.Ports[0].Port) + ingressAddress := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) + framework.Logf("[LB-INFO] Ingress address: %s, port: %d", ingressAddress, svcPort) + + if ingressAddress == "" { + framework.Logf("=== Empty Ingress Address Debug Information ===") + framework.Logf("LoadBalancer ingress[0]: %+v", e2e.svc.Status.LoadBalancer.Ingress[0]) + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of Empty Ingress Address Debug Information ===") + framework.Failf("LoadBalancer ingress address is empty for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } + + if tc.hookPreTest != nil { + By("executing pre-test hook") + tc.hookPreTest(e2e) + } + + // overrideTestRunInClusterReachableHTTP changes the default test function to run the client in the cluster. + if tc.overrideTestRunInClusterReachableHTTP { + By("testing HTTP connectivity for internal load balancer") + framework.Logf("[TEST] Running internal connectivity test from node: %s", e2e.nodeSingleSample) + err := inClusterTestReachableHTTP(cs, ns.Name, e2e.nodeSingleSample, ingressAddress, svcPort) + if err != nil && tc.skipTestFailure { + Skip(err.Error()) + } + framework.ExpectNoError(err) + } else { + By("testing HTTP connectivity for external/internet-facing load balancer") + framework.Logf("[TEST] Running external connectivity test to %s:%d", ingressAddress, svcPort) + e2eservice.TestReachableHTTP(ctx, ingressAddress, svcPort, e2eservice.LoadBalancerLagTimeoutAWS) + } + framework.Logf("[TEST] HTTP connectivity test completed successfully") + + // Update the service to cluster IP + By("cleaning up: converting service to ClusterIP") + _, err = e2e.LBJig.UpdateService(ctx, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP }) + framework.ExpectNoError(err) + + // Wait for the load balancer to be destroyed asynchronously + By("cleaning up: waiting for load balancer destruction") + framework.Logf("[CLEANUP] Waiting for load balancer destruction") + _, err = e2e.LBJig.WaitForLoadBalancerDestroy(ctx, ingressAddress, svcPort, loadBalancerCreateTimeout) + framework.ExpectNoError(err) + framework.Logf("[CLEANUP] Load balancer destroyed successfully") +} - type loadBalancerTestCases struct { - // Overall test case configuration. - name string - resourceSuffix string - extraAnnotations map[string]string - listenerCount int - - // Hooks - // HookPostServiceConfig hook runs after the service manifest is created, and before the service is created. - hookPostServiceConfig func(cfg *e2eTestConfig) - // HookPostServiceCreate hook runs after the test is run. - hookPostServiceCreate func(cfg *e2eTestConfig) - // HookPreTest hook runs before the test is run. - hookPreTest func(cfg *e2eTestConfig) - - // Flags to override default test behavior. - overrideTestRunInClusterReachableHTTP bool - requireAffinity bool - - // Test verification - skipTestFailure bool - } - cases := []loadBalancerTestCases{ - { - name: "CLB should be reachable with default configurations", - resourceSuffix: "", - extraAnnotations: map[string]string{}, +// clbTests: Tests related to CLB load balancers +var clbTests = []loadBalancerTestCases{ + { + name: "CLB should be reachable with default configurations", + resourceSuffix: "", + extraAnnotations: map[string]string{}, + }, + // Hairpining traffic test for CLB. + { + name: "CLB internal should be reachable with hairpinning traffic", + resourceSuffix: "hp-clb-int", + extraAnnotations: map[string]string{ + annotationLBInternal: "true", }, - { - name: "NLB should be reachable with default configurations", - resourceSuffix: "nlb", - extraAnnotations: map[string]string{annotationLBType: "nlb"}, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) + if cfg.svc.Annotations == nil { + cfg.svc.Annotations = map[string]string{} + } + cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) }, - { - name: "NLB should be reachable with target-node-labels", - resourceSuffix: "sg-nd", - extraAnnotations: map[string]string{annotationLBType: "nlb"}, - hookPostServiceConfig: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-config patching service annotations to test node label selector") - if cfg.svc.Annotations == nil { - cfg.svc.Annotations = map[string]string{} - } - cfg.svc.Annotations[annotationLBTargetNodeLabels] = cfg.nodeSelector - }, - hookPostServiceCreate: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-create to validate the number of targets in the load balancer selected") - if len(cfg.svc.Status.LoadBalancer.Ingress) == 0 { - framework.Failf("No ingress found in LoadBalancer status for service %s/%s", cfg.svc.Namespace, cfg.svc.Name) - } - lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname - framework.ExpectNoError(getLBTargetCount(cfg.ctx, lbDNS, cfg.nodeCount), "AWS LB target count validation failed") - }, + overrideTestRunInClusterReachableHTTP: true, + requireAffinity: true, + }, +} + +// nlbTests: Generic NLB tests that test NLB functionality and are not specific to a security group configuration layout +var nlbTests = []loadBalancerTestCases{ + { + name: "NLB should be reachable with default configurations", + resourceSuffix: "nlb", + extraAnnotations: map[string]string{annotationLBType: "nlb"}, + }, + { + name: "NLB should be reachable with target-node-labels", + resourceSuffix: "sg-nd", + extraAnnotations: map[string]string{annotationLBType: "nlb"}, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config patching service annotations to test node label selector") + if cfg.svc.Annotations == nil { + cfg.svc.Annotations = map[string]string{} + } + cfg.svc.Annotations[annotationLBTargetNodeLabels] = cfg.nodeSelector }, - // Hairpining traffic test for CLB. - { - name: "CLB internal should be reachable with hairpinning traffic", - resourceSuffix: "hp-clb-int", - extraAnnotations: map[string]string{ - annotationLBInternal: "true", - }, - hookPostServiceConfig: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) - if cfg.svc.Annotations == nil { - cfg.svc.Annotations = map[string]string{} - } - cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) - }, - overrideTestRunInClusterReachableHTTP: true, - requireAffinity: true, + hookPostServiceCreate: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-create to validate the number of targets in the load balancer selected") + if len(cfg.svc.Status.LoadBalancer.Ingress) == 0 { + framework.Failf("No ingress found in LoadBalancer status for service %s/%s", cfg.svc.Namespace, cfg.svc.Name) + } + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + framework.ExpectNoError(getLBTargetCount(cfg.ctx, lbDNS, cfg.nodeCount), "AWS LB target count validation failed") }, - // Hairpining traffic test for NLB. - // The target type instance (default) sets the preserve client IP attribute to true, - // the NLB target group attributes are set to preserve_client_ip.enabled=false to allow hairpining traffic. - // The test also validates the target group attributes are set correctly to AWS resource. - { - name: "NLB internal should be reachable with hairpinning traffic", - resourceSuffix: "hp-nlb-int", - extraAnnotations: map[string]string{ - annotationLBType: "nlb", - annotationLBInternal: "true", - annotationLBTargetGroupAttributes: "preserve_client_ip.enabled=false", - }, - listenerCount: 1, - overrideTestRunInClusterReachableHTTP: true, - requireAffinity: true, - hookPostServiceConfig: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) - if cfg.svc.Annotations == nil { - cfg.svc.Annotations = map[string]string{} - } - cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) - }, - hookPreTest: func(e2e *e2eTestConfig) { - framework.Logf("running hook pre-test: verify target group attributes are set correctly to AWS resource") + }, + // Hairpining traffic test for NLB. + // The target type instance (default) sets the preserve client IP attribute to true, + // the NLB target group attributes are set to preserve_client_ip.enabled=false to allow hairpining traffic. + // The test also validates the target group attributes are set correctly to AWS resource. + { + name: "NLB internal should be reachable with hairpinning traffic", + resourceSuffix: "hp-nlb-int", + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + annotationLBInternal: "true", + annotationLBTargetGroupAttributes: "preserve_client_ip.enabled=false", + }, + listenerCount: 1, + overrideTestRunInClusterReachableHTTP: true, + requireAffinity: true, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) + if cfg.svc.Annotations == nil { + cfg.svc.Annotations = map[string]string{} + } + cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) + }, + hookPreTest: func(e2e *e2eTestConfig) { + framework.Logf("running hook pre-test: verify target group attributes are set correctly to AWS resource") - if e2e.svc.Status.LoadBalancer.Ingress[0].Hostname == "" && e2e.svc.Status.LoadBalancer.Ingress[0].IP == "" { - framework.Failf("LoadBalancer ingress is empty (no hostname or IP) for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + if e2e.svc.Status.LoadBalancer.Ingress[0].Hostname == "" && e2e.svc.Status.LoadBalancer.Ingress[0].IP == "" { + framework.Failf("LoadBalancer ingress is empty (no hostname or IP) for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } - hostAddr := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) - framework.Logf("Load balancer's ingress address: %s", hostAddr) + hostAddr := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) + framework.Logf("Load balancer's ingress address: %s", hostAddr) - if hostAddr == "" { - framework.Failf("Unable to get LoadBalancer ingress address for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + if hostAddr == "" { + framework.Failf("Unable to get LoadBalancer ingress address for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } - elbClient, err := getAWSClientLoadBalancer(e2e.ctx) - framework.ExpectNoError(err, "failed to create AWS ELB client") + elbClient, err := getAWSClientLoadBalancer(e2e.ctx) + framework.ExpectNoError(err, "failed to create AWS ELB client") - // DescribeLoadBalancers API doesn't support filtering by DNS name directly - // Use AWS SDK paginator to search through all load balancers - foundLB, err := getAWSLoadBalancerFromDNSName(e2e.ctx, elbClient, hostAddr) - framework.ExpectNoError(err, "failed to find load balancer with DNS name %s", hostAddr) - if foundLB == nil { - framework.Failf("Found load balancer is nil for DNS name %s", hostAddr) - } + // DescribeLoadBalancers API doesn't support filtering by DNS name directly + // Use AWS SDK paginator to search through all load balancers + foundLB, err := getAWSLoadBalancerFromDNSName(e2e.ctx, elbClient, hostAddr) + framework.ExpectNoError(err, "failed to find load balancer with DNS name %s", hostAddr) + if foundLB == nil { + framework.Failf("Found load balancer is nil for DNS name %s", hostAddr) + } - lbARN := aws.ToString(foundLB.LoadBalancerArn) - if lbARN == "" { - framework.Failf("Load balancer ARN is empty for DNS name %s", hostAddr) - } - framework.Logf("Found load balancer: %s with ARN: %s", aws.ToString(foundLB.LoadBalancerName), lbARN) - - // lookup target group ARN from load balancer ARN - targetGroups, err := elbClient.DescribeTargetGroups(e2e.ctx, &elbv2.DescribeTargetGroupsInput{ - LoadBalancerArn: aws.String(lbARN), - }) - framework.ExpectNoError(err, "failed to describe target groups") - gomega.Expect(len(targetGroups.TargetGroups)).To(gomega.Equal(1)) - - targetGroupAttributes, err := elbClient.DescribeTargetGroupAttributes(e2e.ctx, &elbv2.DescribeTargetGroupAttributesInput{ - TargetGroupArn: aws.String(aws.ToString(targetGroups.TargetGroups[0].TargetGroupArn)), - }) - framework.ExpectNoError(err, "failed to describe target group attributes") - - // verify if the target group attributes are set correctly - - annotationToDict := map[string]string{} - for _, v := range strings.Split(e2e.svc.Annotations[annotationLBTargetGroupAttributes], ",") { - parts := strings.Split(v, "=") - annotationToDict[parts[0]] = parts[1] - } - framework.Logf("TG attribute Annotation to dict: %v", annotationToDict) + lbARN := aws.ToString(foundLB.LoadBalancerArn) + if lbARN == "" { + framework.Failf("Load balancer ARN is empty for DNS name %s", hostAddr) + } + framework.Logf("Found load balancer: %s with ARN: %s", aws.ToString(foundLB.LoadBalancerName), lbARN) - framework.Logf("=== All Target Group Attributes from AWS ===") - for _, attr := range targetGroupAttributes.Attributes { - framework.Logf(" %s=%s", aws.ToString(attr.Key), aws.ToString(attr.Value)) - } + // lookup target group ARN from load balancer ARN + targetGroups, err := elbClient.DescribeTargetGroups(e2e.ctx, &elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: aws.String(lbARN), + }) + framework.ExpectNoError(err, "failed to describe target groups") + gomega.Expect(len(targetGroups.TargetGroups)).To(gomega.Equal(1)) - framework.Logf("=== Expected Target Group Attributes from Annotation ===") - for key, value := range annotationToDict { - framework.Logf(" %s=%s", key, value) - } + targetGroupAttributes, err := elbClient.DescribeTargetGroupAttributes(e2e.ctx, &elbv2.DescribeTargetGroupAttributesInput{ + TargetGroupArn: aws.String(aws.ToString(targetGroups.TargetGroups[0].TargetGroupArn)), + }) + framework.ExpectNoError(err, "failed to describe target group attributes") + + // verify if the target group attributes are set correctly + + annotationToDict := map[string]string{} + for _, v := range strings.Split(e2e.svc.Annotations[annotationLBTargetGroupAttributes], ",") { + parts := strings.Split(v, "=") + annotationToDict[parts[0]] = parts[1] + } + framework.Logf("TG attribute Annotation to dict: %v", annotationToDict) + + framework.Logf("=== All Target Group Attributes from AWS ===") + for _, attr := range targetGroupAttributes.Attributes { + framework.Logf(" %s=%s", aws.ToString(attr.Key), aws.ToString(attr.Value)) + } + + framework.Logf("=== Expected Target Group Attributes from Annotation ===") + for key, value := range annotationToDict { + framework.Logf(" %s=%s", key, value) + } - // Check if our expected attributes are present and match - framework.Logf("=== Verifying Target Group Attributes ===") - for _, attr := range targetGroupAttributes.Attributes { - if expectedValue, ok := annotationToDict[aws.ToString(attr.Key)]; ok { - actualValue := aws.ToString(attr.Value) - framework.Logf("Checking attribute: %s", aws.ToString(attr.Key)) - framework.Logf(" Expected: %s", expectedValue) - framework.Logf(" Actual: %s", actualValue) - - if actualValue != expectedValue { - framework.Failf("Target group attribute mismatch for %s: expected %s, got %s", aws.ToString(attr.Key), expectedValue, actualValue) - } else { - framework.Logf("✓ Target group attribute %s matches expected value %s", aws.ToString(attr.Key), expectedValue) - } + // Check if our expected attributes are present and match + framework.Logf("=== Verifying Target Group Attributes ===") + for _, attr := range targetGroupAttributes.Attributes { + if expectedValue, ok := annotationToDict[aws.ToString(attr.Key)]; ok { + actualValue := aws.ToString(attr.Value) + framework.Logf("Checking attribute: %s", aws.ToString(attr.Key)) + framework.Logf(" Expected: %s", expectedValue) + framework.Logf(" Actual: %s", actualValue) + + if actualValue != expectedValue { + framework.Failf("Target group attribute mismatch for %s: expected %s, got %s", aws.ToString(attr.Key), expectedValue, actualValue) + } else { + framework.Logf("✓ Target group attribute %s matches expected value %s", aws.ToString(attr.Key), expectedValue) } } - }, + } }, - } + }, +} + +// managedSgModeNLBTests: NLB tests related to managed/BYO security groups, requiring NLBSecurityGroupMode=Managed in the cloud config +var managedSgModeNLBTests = []loadBalancerTestCases{ + // Test creation of NLB with Bring Your Own (BYO) Security Group from the start. + // This test validates: + // - NLB can be created with BYO SG annotation from the start + // - NLB has ONLY the BYO SG attached (no managed SG) + // - NLB is reachable with BYO SG + { + name: "NLB created with BYO Security Group should have BYO Security Group associated", + resourceSuffix: "nlb-byo", + listenerCount: 1, + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + }, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("Creating BYO security group for NLB") - serviceNameBase := "lbconfig-test" - for _, tc := range cases { - It(tc.name, func(ctx context.Context) { - By("setting up test environment and discovering worker nodes") - e2e := newE2eTestConfig(cs) - e2e.discoverClusterWorkerNode() - framework.Logf("[SETUP] Test case: %s", tc.name) - framework.Logf("[SETUP] Worker nodes discovered: %d nodes, selector: %s, sample node: %s", e2e.nodeCount, e2e.nodeSelector, e2e.nodeSingleSample) + securityGroupName := cfg.svc.Namespace + "-" + cfg.svc.Name + "-nlb-byo-sg" + var err error + cfg.byoSecurityGroupID, err = createSecurityGroup(cfg.ctx, cfg, securityGroupName, fmt.Sprintf("BYO Security Group for NLB e2e test service %s/%s", cfg.svc.Namespace, cfg.svc.Name)) + framework.ExpectNoError(err, "Failed to create BYO security group") + framework.Logf("Created BYO security group: %s", cfg.byoSecurityGroupID) + + DeferCleanup(func(ctx context.Context) { + if cfg.byoSecurityGroupID != "" { + framework.Logf("Cleaning up BYO security group: %s", cfg.byoSecurityGroupID) + framework.Logf("Waiting for ENIs to be detached from security group...") + gomega.Eventually(ctx, func() error { + return deleteSecurityGroup(ctx, cfg.byoSecurityGroupID) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed(), "Failed to delete BYO security group") + framework.Logf("✓ Deleted BYO security group: %s", cfg.byoSecurityGroupID) + } + }) - loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs) - framework.Logf("[CONFIG] AWS load balancer timeout: %s", loadBalancerCreateTimeout) + framework.ExpectNoError(authorizeSecurityGroupToPorts(cfg.ctx, cfg.byoSecurityGroupID, cfg.svc.Spec.Ports), "Failed to authorize BYO security group to service ports") - By("building service configuration with annotations") - serviceName := serviceNameBase - if len(tc.resourceSuffix) > 0 { - serviceName = serviceName + "-" + tc.resourceSuffix - } - framework.Logf("[CONFIG] Service name: %s, namespace: %s", serviceName, ns.Name) - e2e.LBJig = e2eservice.NewTestJig(cs, ns.Name, serviceName) - - // Hook annotations to support dynamic config - e2e.svc = e2e.buildService(tc.listenerCount, tc.extraAnnotations) - framework.Logf("[CONFIG] Service ports: %d, extra annotations: %v", len(e2e.svc.Spec.Ports), tc.extraAnnotations) - - if tc.hookPostServiceConfig != nil { - By("executing hook post-service-config: applying service configuration") - framework.Logf("[HOOK] Executing post-service-config hook") - tc.hookPostServiceConfig(e2e) - framework.Logf("[HOOK] Final service annotations: %v", e2e.svc.Annotations) + cfg.svc.Annotations[annotationLBSecurityGroups] = cfg.byoSecurityGroupID + framework.Logf("Added BYO SG annotation: %s=%s", annotationLBSecurityGroups, cfg.byoSecurityGroupID) + }, + hookPreTest: func(cfg *e2eTestConfig) { + framework.Logf("Verifying NLB has only BYO SG attached") + + if len(cfg.svc.Status.LoadBalancer.Ingress) == 0 { + framework.Failf("No ingress found in LoadBalancer status for service %s/%s", cfg.svc.Namespace, cfg.svc.Name) } + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + framework.Logf("NLB DNS: %s", lbDNS) + + attachedSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + framework.ExpectNoError(err, "Failed to get NLB security groups") + framework.Logf("NLB %s has security groups: %+v", lbDNS, attachedSGs) - By("creating LoadBalancer service in Kubernetes") - if _, err := e2e.LBJig.Client.CoreV1().Services(e2e.LBJig.Namespace).Create(context.TODO(), e2e.svc, metav1.CreateOptions{}); err != nil { - framework.ExpectNoError(fmt.Errorf("failed to create LoadBalancer Service %q: %v", e2e.svc.Name, err)) + if len(attachedSGs) != 1 { + framework.Failf("Expected NLB to have exactly 1 security group (BYO SG), got %d: %v", len(attachedSGs), attachedSGs) } - framework.Logf("[K8S] LoadBalancer service created successfully") + if attachedSGs[0] != cfg.byoSecurityGroupID { + framework.Failf("Expected NLB to have BYO SG %q, got %q", cfg.byoSecurityGroupID, attachedSGs[0]) + } + framework.Logf("✓ Verified: NLB has only the BYO SG attached: %s", cfg.byoSecurityGroupID) + }, + }, + // Test transition of NLB from Managed Security Group to Bring Your Own (BYO) Security Group (SG) + // This test validates: + // - NLB created with managed (cluster-owned) SG + // - After annotation is added to use BYO SG the NLB should transition to + // have BYO SG attached and managed SG deleted + // - NLB is reachable + { + name: "NLB should transition from Managed SG to BYO Security Group", + resourceSuffix: "nlb-m2b", + listenerCount: 1, + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + }, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("Creating BYO security group for later transition") - By("waiting for AWS load balancer provisioning") + securityGroupName := cfg.svc.Namespace + "-" + cfg.svc.Name + "-nlb-byo-sg" var err error - e2e.svc, err = e2e.LBJig.WaitForLoadBalancer(ctx, loadBalancerCreateTimeout) - // Collect comprehensive debugging information when LoadBalancer provisioning fails - if err != nil { - serviceName := e2e.LBJig.Name - if e2e.svc != nil { - serviceName = e2e.svc.Name + cfg.byoSecurityGroupID, err = createSecurityGroup(cfg.ctx, cfg, securityGroupName, fmt.Sprintf("BYO Security Group for NLB e2e test service %s/%s", cfg.svc.Namespace, cfg.svc.Name)) + framework.ExpectNoError(err, "Failed to create BYO security group") + framework.Logf("Created BYO security group: %s", cfg.byoSecurityGroupID) + + DeferCleanup(func(ctx context.Context) { + if cfg.byoSecurityGroupID != "" { + framework.Logf("Cleaning up BYO security group: %s", cfg.byoSecurityGroupID) + framework.Logf("Waiting for ENIs to be detached from security group...") + gomega.Eventually(ctx, func() error { + return deleteSecurityGroup(ctx, cfg.byoSecurityGroupID) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed(), "Failed to delete BYO security group after waiting for ENI detachment") + framework.Logf("✓ Deleted BYO security group: %s", cfg.byoSecurityGroupID) } - framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) - framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) - - // Ensure we have detailed debugging information before failing - framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + }) - // Fail the test immediately to prevent further execution - framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + framework.ExpectNoError(authorizeSecurityGroupToPorts(cfg.ctx, cfg.byoSecurityGroupID, cfg.svc.Spec.Ports), "Failed to authorize BYO security group to service ports") + }, + hookPreTest: func(cfg *e2eTestConfig) { + framework.Logf("Transitioning from managed SG to BYO SG") + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + + // Step 1: Verify NLB has managed SG + framework.Logf("Step 1: Verifying NLB has managed security group") + managedSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + framework.ExpectNoError(err, "Failed to get load balancer security groups") + framework.Logf("NLB %s has security groups: %+v", lbDNS, managedSGs) + + if len(managedSGs) != 1 { + framework.Failf("Expected NLB to have 1 managed security group, got %d", len(managedSGs)) } - framework.Logf("[AWS] Load balancer provisioned successfully") + managedSGID := managedSGs[0] + framework.Logf("Managed SG ID: %s", managedSGID) + + // Step 2: Add BYO SG annotation + framework.Logf("Step 2: Adding BYO SG annotation to service") + cfg.svc.Annotations[annotationLBSecurityGroups] = cfg.byoSecurityGroupID + newSvc, err := cfg.kubeClient.CoreV1().Services(cfg.LBJig.Namespace).Update(cfg.ctx, cfg.svc, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "Failed to update Kubernetes Service with BYO SG annotation") + cfg.svc = newSvc + framework.Logf("Updated service with BYO SG annotation: %s=%s", annotationLBSecurityGroups, cfg.byoSecurityGroupID) + + // Step 3: Wait for controller to process the update and verify NLB now has BYO SG + framework.Logf("Step 3: Waiting for controller to update NLB security groups and verifying BYO SG is attached") + gomega.Eventually(cfg.ctx, func() ([]string, error) { + return getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + }, 2*time.Minute, 5*time.Second).Should(gomega.And( + gomega.HaveLen(1), + gomega.ContainElement(cfg.byoSecurityGroupID), + ), "NLB should have exactly 1 security group (BYO SG) after annotation update") + framework.Logf("✓ Verified: NLB has only the BYO SG attached: %s", cfg.byoSecurityGroupID) + + // Verify managed SG was deleted + framework.Logf("Verifying managed SG %s was deleted", managedSGID) + gomega.Eventually(cfg.ctx, func() error { + _, err := getSecurityGroup(cfg.ctx, managedSGID) + return err + }, 2*time.Minute, 5*time.Second).Should(gomega.And( + gomega.HaveOccurred(), + gomega.MatchError(gomega.ContainSubstring("InvalidGroup.NotFound")), + ), "Managed SG should be deleted after transition to BYO SG") + framework.Logf("✓ Verified: Managed SG %s was deleted", managedSGID) + }, + }, + // Test transition of NLB from BYO Security Group to managed Security Group (SG) + // This test validates: + // - NLB created with BYO SG + // - After annotation for BYO SG is removed the NLB should transition to + // have a new managed SG attached and the BYO SG deassociated (but not deleted) + // - NLB is reachable + { + name: "NLB should transition from BYO SG to Managed Security Group", + resourceSuffix: "nlb-b2m", + listenerCount: 1, + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + }, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config to create BYO security group before service creation") - By("creating backend server pods") - _, err = e2e.LBJig.Run(ctx, e2e.buildDeployment(tc.requireAffinity)) - if err != nil { - serviceName := e2e.LBJig.Name - if e2e.svc != nil { - serviceName = e2e.svc.Name + // Create BYO security group + securityGroupName := cfg.svc.Namespace + "-" + cfg.svc.Name + "-nlb-byo-sg" + var err error + cfg.byoSecurityGroupID, err = createSecurityGroup(cfg.ctx, cfg, securityGroupName, fmt.Sprintf("BYO Security Group for NLB e2e test service %s/%s", cfg.svc.Namespace, cfg.svc.Name)) + framework.ExpectNoError(err, "Failed to create BYO security group") + framework.Logf("Created BYO security group: %s", cfg.byoSecurityGroupID) + + DeferCleanup(func(ctx context.Context) { + if cfg.byoSecurityGroupID != "" { + framework.Logf("Cleaning up BYO security group: %s", cfg.byoSecurityGroupID) + framework.Logf("Waiting for ENIs to be detached from security group...") + gomega.Eventually(ctx, func() error { + return deleteSecurityGroup(ctx, cfg.byoSecurityGroupID) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed(), "Failed to delete BYO security group after waiting for ENI detachment") + framework.Logf("✓ Deleted BYO security group: %s", cfg.byoSecurityGroupID) } - framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) - framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) + }) - // Ensure we have detailed debugging information before failing - framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + framework.ExpectNoError(authorizeSecurityGroupToPorts(cfg.ctx, cfg.byoSecurityGroupID, cfg.svc.Spec.Ports), "Failed to authorize BYO security group to service ports") - // Fail the test immediately to prevent further execution - framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + cfg.svc.Annotations[annotationLBSecurityGroups] = cfg.byoSecurityGroupID + framework.Logf("Added BYO SG annotation: %s=%s", annotationLBSecurityGroups, cfg.byoSecurityGroupID) + }, + hookPreTest: func(cfg *e2eTestConfig) { + framework.Logf("running hook pre-test to transition from BYO SG to managed SG") + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + + // Step 1: Verify NLB has BYO SG + framework.Logf("Step 1: Verifying NLB has BYO security group") + byoSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + framework.ExpectNoError(err, "Failed to get load balancer security groups") + framework.Logf("NLB %s has security groups: %+v", lbDNS, byoSGs) + + if len(byoSGs) != 1 { + framework.Failf("Expected NLB to have exactly 1 security group (BYO SG), got %d: %v", len(byoSGs), byoSGs) + } + if byoSGs[0] != cfg.byoSecurityGroupID { + framework.Failf("Expected NLB to have BYO SG %q, got %q", cfg.byoSecurityGroupID, byoSGs[0]) } + framework.Logf("✓ Verified: NLB has BYO SG attached: %s", cfg.byoSecurityGroupID) + + // Step 2: Remove BYO SG annotation + framework.Logf("Step 2: Removing BYO SG annotation from service") + delete(cfg.svc.Annotations, annotationLBSecurityGroups) + newSvc, err := cfg.kubeClient.CoreV1().Services(cfg.LBJig.Namespace).Update(cfg.ctx, cfg.svc, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "Failed to update Kubernetes Service to remove BYO SG annotation") + cfg.svc = newSvc + framework.Logf("Removed BYO SG annotation from service") + + // Step 3: Wait for controller to process the update and verify NLB has new managed SG + framework.Logf("Step 3: Waiting for controller to update NLB security groups and verifying new managed SG is attached") + err = wait.PollUntilContextTimeout(cfg.ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + managedSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + if err != nil { + framework.Logf("Failed to get load balancer security groups: %v", err) + return false, nil + } + if len(managedSGs) == 0 { + framework.Logf("No security groups attached yet") + return false, nil + } + if managedSGs[0] == cfg.byoSecurityGroupID { + framework.Logf("Still has BYO SG attached") + return false, nil + } + framework.Logf("NLB %s has security groups after removing BYO annotation: %+v", lbDNS, managedSGs) + return true, nil + }) + framework.ExpectNoError(err, "Failed waiting for NLB to get new managed SG") + + // Verify BYO SG still exists but is not attached + framework.Logf("Verifying BYO SG %s still exists", cfg.byoSecurityGroupID) + byoSG, err := getSecurityGroup(cfg.ctx, cfg.byoSecurityGroupID) + framework.ExpectNoError(err, "Failed to get BYO security group") + gomega.Expect(byoSG).ToNot(gomega.BeNil(), "BYO SG should still exist after transition to managed SG") + framework.Logf("✓ Verified: BYO SG %s still exists and was not deleted", cfg.byoSecurityGroupID) + }, + }, +} - framework.Logf("[K8S] Backend pods created, affinity required: %t", tc.requireAffinity) +// execute load balancer tests +var _ = Describe("[cloud-provider-aws-e2e]", func() { + f := framework.NewDefaultFramework("cloud-provider-aws") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - if tc.hookPostServiceCreate != nil { - By("executing hook post-service-create: applying service configuration") - tc.hookPostServiceCreate(e2e) - } + var ( + cs clientset.Interface + ns *v1.Namespace + ) - By("collecting service and load balancer information") - if e2e.svc == nil { - framework.Logf("=== Service Validation Error Debug Information ===") - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of Service Validation Error Debug Information ===") - framework.Failf("Service is nil after LoadBalancer provisioning for service %s", e2e.LBJig.Name) - } - if len(e2e.svc.Spec.Ports) == 0 { - framework.Logf("=== Service Ports Error Debug Information ===") - framework.Logf("Service spec: %+v", e2e.svc.Spec) - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of Service Ports Error Debug Information ===") - framework.Failf("No ports found in service spec for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } - if len(e2e.svc.Status.LoadBalancer.Ingress) == 0 { - framework.Logf("=== LoadBalancer Ingress Error Debug Information ===") - framework.Logf("Service status: %+v", e2e.svc.Status) - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of LoadBalancer Ingress Error Debug Information ===") - framework.Failf("No ingress found in LoadBalancer status for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + BeforeEach(func() { + cs = f.ClientSet + ns = f.Namespace + }) - svcPort := int(e2e.svc.Spec.Ports[0].Port) - ingressAddress := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) - framework.Logf("[LB-INFO] Ingress address: %s, port: %d", ingressAddress, svcPort) + AfterEach(func() { + // After each test + }) - if ingressAddress == "" { - framework.Logf("=== Empty Ingress Address Debug Information ===") - framework.Logf("LoadBalancer ingress[0]: %+v", e2e.svc.Status.LoadBalancer.Ingress[0]) - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of Empty Ingress Address Debug Information ===") - framework.Failf("LoadBalancer ingress address is empty for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + // run all load balancer tests with the default configuration + Context("loadbalancer", func() { + // CLB tests + for _, tc := range clbTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) + }) + } - if tc.hookPreTest != nil { - By("executing pre-test hook") - tc.hookPreTest(e2e) - } + // Generic NLB tests + for _, tc := range nlbTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) + }) + } - // overrideTestRunInClusterReachableHTTP changes the default test function to run the client in the cluster. - if tc.overrideTestRunInClusterReachableHTTP { - By("testing HTTP connectivity for internal load balancer") - framework.Logf("[TEST] Running internal connectivity test from node: %s", e2e.nodeSingleSample) - err := inClusterTestReachableHTTP(cs, ns.Name, e2e.nodeSingleSample, ingressAddress, svcPort) - if err != nil && tc.skipTestFailure { - Skip(err.Error()) - } - framework.ExpectNoError(err) - } else { - By("testing HTTP connectivity for external/internet-facing load balancer") - framework.Logf("[TEST] Running external connectivity test to %s:%d", ingressAddress, svcPort) - e2eservice.TestReachableHTTP(ctx, ingressAddress, svcPort, e2eservice.LoadBalancerLagTimeoutAWS) - } - framework.Logf("[TEST] HTTP connectivity test completed successfully") + // NLBSecurityGroupMode=Managed specific tests + for _, tc := range managedSgModeNLBTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) + }) + } + }) + + // Run relevant NLB tests with NLBSecurityGroupMode disabled via cloud config override + Context("[nlb-security-group-mode-disabled] loadbalancer", Serial, Ordered, func() { + cloudConfigMgr := newCloudConfigManager(withRestartTimeout(3 * time.Minute)) - // Update the service to cluster IP - By("cleaning up: converting service to ClusterIP") - _, err = e2e.LBJig.UpdateService(ctx, func(s *v1.Service) { - s.Spec.Type = v1.ServiceTypeClusterIP + BeforeAll(func(ctx context.Context) { + // Disable NLB managed security group mode by setting an empty config (default CCM configuration) + err := cloudConfigMgr.setCloudConfig(ctx, cs, "") + framework.ExpectNoError(err, "Failed to disable NLB managed security group mode") + }) + + // Only run nlb tests not tied to NLBSecurityGroupMode=Managed + for _, tc := range nlbTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) }) - framework.ExpectNoError(err) - - // Wait for the load balancer to be destroyed asynchronously - By("cleaning up: waiting for load balancer destruction") - framework.Logf("[CLEANUP] Waiting for load balancer destruction") - _, err = e2e.LBJig.WaitForLoadBalancerDestroy(ctx, ingressAddress, svcPort, loadBalancerCreateTimeout) - framework.ExpectNoError(err) - framework.Logf("[CLEANUP] Load balancer destroyed successfully") + } + + AfterAll(func(ctx context.Context) { + // Restore original CCM configuration + err := cloudConfigMgr.restoreCloudConfig(ctx, cs) + framework.ExpectNoError(err, "Failed to restore original CCM configuration") }) - } + }) }) type e2eTestConfig struct { @@ -416,6 +712,7 @@ type e2eTestConfig struct { cfgPodProtocol v1.Protocol cfgDefaultAnnotations map[string]string LBJig *e2eservice.TestJig + byoSecurityGroupID string // service instance svc *v1.Service @@ -998,3 +1295,153 @@ func gatherEventosOnFailure(ctx context.Context, cs clientset.Interface, namespa gatherControllerLogs(ctx, cs, namespace, resourceName) gatherServiceStatus(ctx, cs, namespace, resourceName) } + +// getAWSClientEC2 creates an EC2 client for AWS operations +func getAWSClientEC2(ctx context.Context) (*ec2.Client, error) { + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRetryer(func() aws.Retryer { + return retry.AddWithMaxAttempts(retry.NewStandard(), 10) + }), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + return ec2.NewFromConfig(cfg), nil +} + +// createSecurityGroup creates a security group for testing +func createSecurityGroup(ctx context.Context, cfg *e2eTestConfig, name, description string) (string, error) { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return "", err + } + + // Get VPC ID from any node in the cluster + nodes, err := cfg.kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return "", fmt.Errorf("failed to list nodes: %w", err) + } + if len(nodes.Items) == 0 { + return "", fmt.Errorf("no nodes found in cluster") + } + + // Extract VPC ID from node's provider ID + // Provider ID format: aws:///us-west-2a/i-0123456789abcdef0 + providerID := nodes.Items[0].Spec.ProviderID + if providerID == "" { + return "", fmt.Errorf("node %s has no provider ID", nodes.Items[0].Name) + } + + // Get instance details to find VPC ID + instanceID := providerID[strings.LastIndex(providerID, "/")+1:] + describeResult, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + }) + if err != nil { + return "", fmt.Errorf("failed to describe instance: %w", err) + } + if len(describeResult.Reservations) == 0 || len(describeResult.Reservations[0].Instances) == 0 { + return "", fmt.Errorf("instance not found: %s", instanceID) + } + + vpcID := describeResult.Reservations[0].Instances[0].VpcId + if vpcID == nil { + return "", fmt.Errorf("instance has no VPC ID") + } + + // Create security group + result, err := ec2Client.CreateSecurityGroup(ctx, &ec2.CreateSecurityGroupInput{ + GroupName: aws.String(name), + Description: aws.String(description), + VpcId: vpcID, + }) + if err != nil { + return "", fmt.Errorf("failed to create security group: %w", err) + } + + return aws.ToString(result.GroupId), nil +} + +// authorizeSecurityGroupToPorts adds ingress rules for the given service ports +func authorizeSecurityGroupToPorts(ctx context.Context, securityGroupID string, ports []v1.ServicePort) error { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return err + } + + permissions := make([]ec2types.IpPermission, 0, len(ports)) + for _, port := range ports { + protocol := strings.ToLower(string(port.Protocol)) + permissions = append(permissions, ec2types.IpPermission{ + IpProtocol: aws.String(protocol), + FromPort: aws.Int32(port.Port), + ToPort: aws.Int32(port.Port), + IpRanges: []ec2types.IpRange{ + {CidrIp: aws.String("0.0.0.0/0")}, + }, + }) + } + + _, err = ec2Client.AuthorizeSecurityGroupIngress(ctx, &ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(securityGroupID), + IpPermissions: permissions, + }) + if err != nil { + return fmt.Errorf("failed to authorize security group ingress: %w", err) + } + + return nil +} + +// getLoadBalancerSecurityGroups retrieves security groups attached to a load balancer +func getLoadBalancerSecurityGroups(ctx context.Context, lbDNS string) ([]string, error) { + elbClient, err := getAWSClientLoadBalancer(ctx) + if err != nil { + return nil, err + } + + lb, err := getAWSLoadBalancerFromDNSName(ctx, elbClient, lbDNS) + if err != nil { + return nil, err + } + + return lb.SecurityGroups, nil +} + +// deleteSecurityGroup deletes a security group +func deleteSecurityGroup(ctx context.Context, securityGroupID string) error { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return err + } + + _, err = ec2Client.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(securityGroupID), + }) + if err != nil { + return fmt.Errorf("failed to delete security group %s: %w", securityGroupID, err) + } + + return nil +} + +// getSecurityGroup retrieves a security group by ID +func getSecurityGroup(ctx context.Context, securityGroupID string) (*ec2types.SecurityGroup, error) { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return nil, err + } + + result, err := ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + GroupIds: []string{securityGroupID}, + }) + if err != nil { + return nil, fmt.Errorf("failed to describe security group: %w", err) + } + + if len(result.SecurityGroups) == 0 { + return nil, fmt.Errorf("security group not found: %s", securityGroupID) + } + + return &result.SecurityGroups[0], nil +} diff --git a/openshift-tests/ccm-aws-tests/vendor/modules.txt b/openshift-tests/ccm-aws-tests/vendor/modules.txt index c10bb0464..59daa403b 100644 --- a/openshift-tests/ccm-aws-tests/vendor/modules.txt +++ b/openshift-tests/ccm-aws-tests/vendor/modules.txt @@ -1141,7 +1141,7 @@ k8s.io/client-go/util/workqueue # k8s.io/cloud-provider v0.35.1 => k8s.io/cloud-provider v0.36.0 ## explicit; go 1.26.0 k8s.io/cloud-provider/service/helpers -# k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c +# k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a ## explicit; go 1.26.0 k8s.io/cloud-provider-aws/tests/e2e # k8s.io/component-base v0.36.0 From 91909330a5543ed92d83ee1d0a767834f711baa8 Mon Sep 17 00:00:00 2001 From: Federico Bonfigli Date: Fri, 19 Jun 2026 14:13:33 +0200 Subject: [PATCH 2/3] Fix failing unit test on CI Attempts to fix a failing unit test that passes locally but not on CI by wrapping it into an Eventually block --- pkg/controllers/cloud_config_sync_controller_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/cloud_config_sync_controller_test.go b/pkg/controllers/cloud_config_sync_controller_test.go index a9e370ef9..b785424b3 100644 --- a/pkg/controllers/cloud_config_sync_controller_test.go +++ b/pkg/controllers/cloud_config_sync_controller_test.go @@ -604,9 +604,11 @@ var _ = Describe("Cloud config sync reconciler", func() { Expect(cl.Status().Update(ctx, infraResource.DeepCopy())).To(Succeed()) _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{}) Expect(err).To(BeNil()) - allCMs := &corev1.ConfigMapList{} - Expect(cl.List(ctx, allCMs, &client.ListOptions{Namespace: targetNamespaceName})).To(Succeed()) - Expect(len(allCMs.Items)).To(BeZero()) + Eventually(func() int { + allCMs := &corev1.ConfigMapList{} + Expect(cl.List(ctx, allCMs, &client.ListOptions{Namespace: targetNamespaceName})).To(Succeed()) + return len(allCMs.Items) + }).Should(BeZero()) }) }) From 9f8e0098697540e1b3068618232d716460a54468 Mon Sep 17 00:00:00 2001 From: Federico Bonfigli Date: Fri, 19 Jun 2026 14:25:20 +0200 Subject: [PATCH 3/3] Add explicit timeout for Eventually statement in unit test --- pkg/controllers/cloud_config_sync_controller_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cloud_config_sync_controller_test.go b/pkg/controllers/cloud_config_sync_controller_test.go index b785424b3..3a1933676 100644 --- a/pkg/controllers/cloud_config_sync_controller_test.go +++ b/pkg/controllers/cloud_config_sync_controller_test.go @@ -608,7 +608,7 @@ var _ = Describe("Cloud config sync reconciler", func() { allCMs := &corev1.ConfigMapList{} Expect(cl.List(ctx, allCMs, &client.ListOptions{Namespace: targetNamespaceName})).To(Succeed()) return len(allCMs.Items) - }).Should(BeZero()) + }, 2*time.Minute, 5*time.Second).Should(BeZero()) }) })