From 095945d33715a54f8f0320264a2d7d17c3ec03ad Mon Sep 17 00:00:00 2001 From: Rohit Patil Date: Wed, 16 Sep 2026 20:04:00 +0530 Subject: [PATCH 1/2] e2e automation network policy --- test/e2e/network_policy.go | 363 ++++++++++++++++++ test/e2e/network_policy_helpers.go | 584 +++++++++++++++++++++++++++++ test/e2e/operator.go | 45 +++ test/e2e/operator_test.go | 5 +- 4 files changed, 996 insertions(+), 1 deletion(-) create mode 100644 test/e2e/network_policy.go create mode 100644 test/e2e/network_policy_helpers.go diff --git a/test/e2e/network_policy.go b/test/e2e/network_policy.go new file mode 100644 index 000000000..8b68be615 --- /dev/null +++ b/test/e2e/network_policy.go @@ -0,0 +1,363 @@ +package e2e + +// NetworkPolicy e2e coverage for the operand policy +// allow-all-egress-and-metrics-ingress-operand. Specs are Ginkgo/OTE only, +// adapted to CLI Manager namespaces, labels, and ports: +// +// ns: openshift-cli-manager-operator (has openshift.io/cluster-monitoring=true) +// pod: app=openshift-cli-manager (leader via lease cli-manager-lock) +// ports: metrics 60000, plugin 9449 (ingress-restricted, not JobSet unrestricted 9443), health 8443 +// +// 1/2 policy created with expected selectors/ports/egress/policyTypes/ownerRef +// 3 custom unmanaged NetworkPolicy is not pruned +// 3.1-3.5 metrics ingress on 60000 (monitoring allowed, default/wrong-port denied; +// same-ns ALLOWED because this operator ns has cluster-monitoring) +// 3.6-3.7, 3.10 plugin ingress on 9449 (ingress ns allowed, others denied) +// 3.8-3.9 host-network allowed; health port 8443 denied from monitoring +// 4.1-4.4 unrestricted egress (DNS, API, internet, prometheus) +// 5 ServiceMonitor and metrics/plugin services +// 7.1-7.6 reconciliation after port/selector/policyTypes/ns-selector/empty-ingress/delete +// 8.1-8.2 unlabeled pods are not selected (no default-deny) + +import ( + "context" + "testing" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8sclient "k8s.io/client-go/kubernetes" + + "github.com/openshift/cli-manager-operator/pkg/operator/operatorclient" +) + +var _ = g.Describe("[Operator][Serial] CLI Manager NetworkPolicy", g.Ordered, func() { + var ( + ctx context.Context + cancelFnc context.CancelFunc + kubeClient *k8sclient.Clientset + ) + + g.BeforeAll(func() { + g.By("Setting up the CLI Manager operator") + var err error + ctx, cancelFnc, kubeClient, err = setupOperator(g.GinkgoTB()) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for the operand NetworkPolicy and leader pod") + waitForOperandNetworkPolicy(g.GinkgoTB(), ctx, kubeClient) + waitForLeaderOperandPod(g.GinkgoTB(), ctx, kubeClient) + }) + + g.AfterAll(func() { + teardownOperator() + if cancelFnc != nil { + cancelFnc() + } + }) + + g.It("should ensure operand NetworkPolicy is defined [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandNetworkPolicyDefined(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should enforce operand metrics ingress on port 60000 [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandMetricsIngress(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should enforce operand plugin download ingress on port 9449 [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandPluginIngress(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should block operand health ingress and allow host-network [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandHealthAndHostNetworkIngress(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should allow operand egress connectivity [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandEgress(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should configure ServiceMonitor for operand metrics [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandServiceMonitor(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should not apply operand NetworkPolicy to unlabeled pods [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testUnlabeledPodEgress(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should preserve unmanaged custom NetworkPolicies [NetworkPolicy] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testCustomNetworkPolicyPreserved(g.GinkgoTB(), ctx, kubeClient) + }) + + g.It("should restore operand NetworkPolicy after delete or mutation [NetworkPolicy][Timeout:30m][Disruptive] [Suite:openshift/cli-manager-operator/operator/serial]", func() { + testOperandNetworkPolicyReconciliation(g.GinkgoTB(), ctx, kubeClient) + }) +}) + +func testOperandNetworkPolicyDefined(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + t.Logf("=== Validating %s ===", operandNetworkPolicyName) + + policy := GetNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName) + t.Logf(" - Policy found: %s/%s", policy.Namespace, policy.Name) + + RequirePodSelectorLabel(t, policy, operandAppLabelKey, operatorclient.OperandName) + t.Logf(" - PodSelector: %s=%s", operandAppLabelKey, operatorclient.OperandName) + + RequireIngressPort(t, policy, corev1.ProtocolTCP, metricsPort) + RequireIngressFromNamespace(t, policy, metricsPort, openshiftMonitoringNamespace) + RequireIngressFromNamespace(t, policy, metricsPort, openshiftUWMNamespace) + RequireIngressFromNamespaceLabel(t, policy, metricsPort, clusterMonitoringLabel, "true") + t.Logf(" - Ingress: TCP/%d from monitoring namespaces", metricsPort) + + RequireIngressPort(t, policy, corev1.ProtocolTCP, pluginPort) + RequireIngressFromPolicyGroup(t, policy, pluginPort, ingressPolicyGroupKey) + t.Logf(" - Ingress: TCP/%d from %s", pluginPort, ingressPolicyGroupKey) + + if HasPortInIngress(policy.Spec.Ingress, corev1.ProtocolTCP, healthPort) { + t.Fatalf("%s/%s: port %d must not be allowed by ingress rules", policy.Namespace, policy.Name, healthPort) + } + t.Logf(" - Ingress: TCP/%d is not allowed (health/serving)", healthPort) + + RequireUnrestrictedEgress(t, policy) + t.Logf(" - Egress: unrestricted [{}]") + + o.Expect(policy.Spec.PolicyTypes).To(o.ContainElement(networkingv1.PolicyTypeIngress)) + o.Expect(policy.Spec.PolicyTypes).To(o.ContainElement(networkingv1.PolicyTypeEgress)) + + RequireOwnerReference(t, policy, "operator.openshift.io/v1", "CliManager", operatorclient.OperatorConfigName) + t.Logf(" - OwnerReference: CliManager/%s", operatorclient.OperatorConfigName) + + t.Logf("=== operand NetworkPolicy validated ===") +} + +func testOperandMetricsIngress(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + leader := waitForLeaderOperandPod(t, ctx, kubeClient) + leaderIPs := PodIPs(leader) + testLabels := map[string]string{"test": "cli-manager-netpol"} + + t.Logf("=== Testing metrics ingress on port %d to leader %s ===", metricsPort, leader.Name) + + t.Logf("3.1 Allowed — Ingress from %s on port %d", openshiftMonitoringNamespace, metricsPort) + ExpectConnectivity(ctx, t, kubeClient, openshiftMonitoringNamespace, testLabels, leaderIPs, metricsPort, true) + + if namespaceExists(ctx, kubeClient, openshiftUWMNamespace) { + t.Logf("3.2 Allowed — Ingress from %s on port %d", openshiftUWMNamespace, metricsPort) + ExpectConnectivity(ctx, t, kubeClient, openshiftUWMNamespace, testLabels, leaderIPs, metricsPort, true) + } else { + t.Logf("3.2 Skipping %s; namespace not present", openshiftUWMNamespace) + } + + t.Logf("3.3 Blocked — Ingress from default namespace on port %d", metricsPort) + ExpectConnectivity(ctx, t, kubeClient, "default", testLabels, leaderIPs, metricsPort, false) + + t.Logf("3.4 Blocked — Ingress from %s on unused port %d", openshiftMonitoringNamespace, unusedPort) + ExpectConnectivity(ctx, t, kubeClient, openshiftMonitoringNamespace, testLabels, leaderIPs, unusedPort, false) + + t.Logf("3.5 Allowed — Ingress from operator namespace on port %d (cluster-monitoring label)", metricsPort) + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, testLabels, leaderIPs, metricsPort, true) + + t.Logf("=== metrics ingress verified ===") +} + +func testOperandPluginIngress(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + leader := waitForLeaderOperandPod(t, ctx, kubeClient) + leaderIPs := PodIPs(leader) + testLabels := map[string]string{"test": "cli-manager-netpol"} + + t.Logf("=== Testing plugin download ingress on port %d to leader %s ===", pluginPort, leader.Name) + + t.Logf("3.6 Allowed — Ingress from %s on port %d", openshiftIngressNamespace, pluginPort) + ExpectConnectivity(ctx, t, kubeClient, openshiftIngressNamespace, testLabels, leaderIPs, pluginPort, true) + + t.Logf("3.7 Blocked — Ingress from default namespace on port %d", pluginPort) + ExpectConnectivity(ctx, t, kubeClient, "default", testLabels, leaderIPs, pluginPort, false) + + t.Logf("3.10 Blocked — Ingress from operator namespace on port %d (no ingress policy-group label)", pluginPort) + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, testLabels, leaderIPs, pluginPort, false) + + t.Logf("=== plugin download ingress verified ===") +} + +func testOperandHealthAndHostNetworkIngress(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + leader := waitForLeaderOperandPod(t, ctx, kubeClient) + leaderIPs := PodIPs(leader) + testLabels := map[string]string{"test": "cli-manager-netpol"} + + t.Logf("=== Testing health ingress and host-network bypass ===") + + t.Logf("3.9 Blocked — Ingress from %s on port %d", openshiftMonitoringNamespace, healthPort) + ExpectConnectivity(ctx, t, kubeClient, openshiftMonitoringNamespace, testLabels, leaderIPs, healthPort, false) + + t.Logf("3.8 Allowed — Host network (kubelet path) to port %d", metricsPort) + ExpectHostNetworkConnectivity(ctx, t, kubeClient, openshiftMonitoringNamespace, leader.Spec.NodeName, leaderIPs, metricsPort, true) + + t.Logf("=== health and host-network ingress verified ===") +} + +func testOperandEgress(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + clientLabels := operandClientLabels() + + t.Logf("=== Testing operand egress (unrestricted [{}]) ===") + + dnsSvc, err := kubeClient.CoreV1().Services(openshiftDNSNamespace).Get(ctx, "dns-default", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should get dns-default service") + t.Logf("4.1 Allowed — DNS resolution via %s:53", dnsSvc.Spec.ClusterIP) + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, clientLabels, ServiceClusterIPs(dnsSvc), 53, true) + + kubeSvc, err := kubeClient.CoreV1().Services("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should get kubernetes service") + t.Logf("4.2 Allowed — API server connectivity on 443") + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, clientLabels, ServiceClusterIPs(kubeSvc), 443, true) + + t.Logf("4.3 Allowed — External internet (1.1.1.1:443)") + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, clientLabels, []string{"1.1.1.1"}, 443, true) + + promSvc, err := kubeClient.CoreV1().Services(openshiftMonitoringNamespace).Get(ctx, prometheusK8sServiceName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should get prometheus-k8s service") + promPort := int32(9091) + for _, p := range promSvc.Spec.Ports { + if p.Name == "web" || p.Port == 9091 { + promPort = p.Port + break + } + } + t.Logf("4.4 Allowed — Cross-namespace egress to prometheus-k8s:%d", promPort) + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, clientLabels, ServiceClusterIPs(promSvc), promPort, true) + + t.Logf("=== operand egress verified ===") +} + +func testOperandServiceMonitor(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + t.Logf("=== Validating ServiceMonitor and metrics service ===") + + dynamicClient := GetApiDynamicClient() + gvr := schema.GroupVersionResource{Group: "monitoring.coreos.com", Version: "v1", Resource: "servicemonitors"} + obj, err := dynamicClient.Resource(gvr).Namespace(operatorclient.OperatorNamespace).Get(ctx, operandServiceMonitorName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "ServiceMonitor %s should exist", operandServiceMonitorName) + + labels, found, err := unstructured.NestedStringMap(obj.Object, "spec", "selector", "matchLabels") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue(), "ServiceMonitor should have a selector") + o.Expect(labels[operandAppLabelKey]).To(o.Equal(operatorclient.OperandName)) + + endpoints, found, err := unstructured.NestedSlice(obj.Object, "spec", "endpoints") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue()) + o.Expect(endpoints).NotTo(o.BeEmpty()) + endpoint, ok := endpoints[0].(map[string]any) + o.Expect(ok).To(o.BeTrue(), "ServiceMonitor endpoint should be an object") + + port, found, err := unstructured.NestedString(endpoint, "port") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue()) + o.Expect(port).To(o.Equal(operandMetricsPortName)) + + scheme, found, err := unstructured.NestedString(endpoint, "scheme") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue()) + o.Expect(scheme).To(o.Equal("https")) + t.Logf(" - ServiceMonitor port=%s scheme=%s selector=%s=%s", port, scheme, operandAppLabelKey, labels[operandAppLabelKey]) + + metricsSvc, err := kubeClient.CoreV1().Services(operatorclient.OperatorNamespace).Get(ctx, operandMetricsServiceName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "metrics service should exist") + o.Expect(metricsSvc.Spec.ClusterIP).To(o.Equal(corev1.ClusterIPNone), "metrics service should be headless") + o.Expect(metricsSvc.Spec.Ports).NotTo(o.BeEmpty()) + o.Expect(metricsSvc.Spec.Ports[0].Port).To(o.Equal(metricsPort)) + t.Logf(" - Service %s is headless on port %d", operandMetricsServiceName, metricsPort) + + pluginSvc, err := kubeClient.CoreV1().Services(operatorclient.OperatorNamespace).Get(ctx, operandPluginServiceName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "plugin service should exist") + o.Expect(pluginSvc.Spec.Ports).NotTo(o.BeEmpty()) + o.Expect(pluginSvc.Spec.Ports[0].Port).To(o.Equal(pluginPort)) + t.Logf(" - Service %s on port %d", operandPluginServiceName, pluginPort) + + t.Logf("=== ServiceMonitor configuration verified ===") +} + +func testUnlabeledPodEgress(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + dnsSvc, err := kubeClient.CoreV1().Services(openshiftDNSNamespace).Get(ctx, "dns-default", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + dnsIPs := ServiceClusterIPs(dnsSvc) + + t.Logf("=== Testing unlabeled vs labeled pod egress ===") + + t.Logf("8.1 Unlabeled pod can reach DNS (no default-deny policy)") + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, map[string]string{"test": "unlabeled"}, dnsIPs, 53, true) + + t.Logf("8.2 Labeled operand pod can reach DNS via unrestricted egress") + ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, operandClientLabels(), dnsIPs, 53, true) + + t.Logf("=== unlabeled/labeled egress verified ===") +} + +func testCustomNetworkPolicyPreserved(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + const customName = "test-custom-np" + t.Logf("=== Creating unmanaged NetworkPolicy %s/%s ===", operatorclient.OperatorNamespace, customName) + + custom := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: customName, + Namespace: operatorclient.OperatorNamespace, + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"test": "true"}, + }, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test-app"}}}, + }, + }, + }, + }, + } + _, err := kubeClient.NetworkingV1().NetworkPolicies(operatorclient.OperatorNamespace).Create(ctx, custom, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should create unmanaged custom NetworkPolicy") + defer func() { + _ = kubeClient.NetworkingV1().NetworkPolicies(operatorclient.OperatorNamespace).Delete(ctx, customName, metav1.DeleteOptions{}) + }() + + AssertUnmanagedNetworkPolicyPreserved(t, ctx, kubeClient, operatorclient.OperatorNamespace, customName, 30*time.Second) + t.Logf("=== unmanaged custom NetworkPolicy preserved ===") +} + +func testOperandNetworkPolicyReconciliation(t testing.TB, ctx context.Context, kubeClient k8sclient.Interface) { + t.Helper() + t.Logf("=== Testing operand NetworkPolicy reconciliation ===") + + t.Logf("7.1 Mutate ingress port and wait for revert") + MutatePortAndRestoreNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName, networkPolicyReconcileTimeout) + + t.Logf("7.2 Mutate pod selector and wait for revert") + MutateAndRestoreNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName, networkPolicyReconcileTimeout) + + t.Logf("7.4 Mutate policyTypes and wait for revert") + MutatePolicyTypesAndRestoreNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName, networkPolicyReconcileTimeout) + + t.Logf("7.5 Mutate monitoring namespaceSelector and wait for revert") + MutateNamespaceSelectorAndRestoreNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName, networkPolicyReconcileTimeout) + + t.Logf("7.6 Clear ingress rules and wait for restore") + MutateEmptyIngressAndRestoreNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName, networkPolicyReconcileTimeout) + + t.Logf("7.3 Delete policy and wait for recreate") + expected := GetNetworkPolicy(t, ctx, kubeClient, operatorclient.OperatorNamespace, operandNetworkPolicyName) + RestoreNetworkPolicy(t, ctx, kubeClient, expected, networkPolicyReconcileTimeout) + + LogNetworkPolicyEvents(t, ctx, kubeClient, []string{operatorclient.OperatorNamespace}, operandNetworkPolicyName) + t.Logf("=== operand NetworkPolicy reconciliation verified ===") +} diff --git a/test/e2e/network_policy_helpers.go b/test/e2e/network_policy_helpers.go new file mode 100644 index 000000000..8ced04a74 --- /dev/null +++ b/test/e2e/network_policy_helpers.go @@ -0,0 +1,584 @@ +package e2e + +import ( + "context" + "fmt" + "net" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + "github.com/openshift/cli-manager-operator/pkg/operator/operatorclient" +) + +const ( + defaultAgnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.45" + + operandNetworkPolicyName = "allow-all-egress-and-metrics-ingress-operand" + operandAppLabelKey = "app" + operandLeaseName = "cli-manager-lock" + + metricsPort int32 = 60000 + pluginPort int32 = 9449 + healthPort int32 = 8443 + unusedPort int32 = 8080 + + clusterMonitoringLabel = "openshift.io/cluster-monitoring" + ingressPolicyGroupKey = "policy-group.network.openshift.io/ingress" + + openshiftMonitoringNamespace = "openshift-monitoring" + openshiftUWMNamespace = "openshift-user-workload-monitoring" + openshiftIngressNamespace = "openshift-ingress" + openshiftDNSNamespace = "openshift-dns" + prometheusK8sServiceName = "prometheus-k8s" + operandMetricsServiceName = "openshift-cli-manager-metrics" + operandPluginServiceName = "openshift-cli-manager" + operandServiceMonitorName = "openshift-cli-manager" + operandMetricsPortName = "cli-manager-metrics-port" + connectivityTimeout = 2 * time.Minute + networkPolicyReconcileTimeout = 10 * time.Minute +) + +// IsIPv6 returns true if the given IP string is an IPv6 address. +func IsIPv6(ip string) bool { + return net.ParseIP(ip) != nil && strings.Contains(ip, ":") +} + +// FormatIPPort formats an IP:port pair, using brackets for IPv6 addresses. +func FormatIPPort(ip string, port int32) string { + if IsIPv6(ip) { + return fmt.Sprintf("[%s]:%d", ip, port) + } + return fmt.Sprintf("%s:%d", ip, port) +} + +// PodIPs returns all IP addresses assigned to a pod (dual-stack aware). +func PodIPs(pod *corev1.Pod) []string { + var ips []string + for _, podIP := range pod.Status.PodIPs { + if podIP.IP != "" { + ips = append(ips, podIP.IP) + } + } + if len(ips) == 0 && pod.Status.PodIP != "" { + ips = append(ips, pod.Status.PodIP) + } + return ips +} + +// ServiceClusterIPs returns all ClusterIPs for a service (dual-stack aware). +func ServiceClusterIPs(svc *corev1.Service) []string { + if len(svc.Spec.ClusterIPs) > 0 { + return svc.Spec.ClusterIPs + } + if svc.Spec.ClusterIP != "" && svc.Spec.ClusterIP != corev1.ClusterIPNone { + return []string{svc.Spec.ClusterIP} + } + return nil +} + +func boolptr(value bool) *bool { + return &value +} + +func int64ptr(value int64) *int64 { + return &value +} + +func namespaceExists(ctx context.Context, client kubernetes.Interface, namespace string) bool { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + return err == nil +} + +func runConnectivityCheck(ctx context.Context, kubeClient kubernetes.Interface, namespace string, labels map[string]string, serverIP string, port int32, hostNetwork bool, nodeName string) (bool, error) { + name := fmt.Sprintf("np-client-%s", rand.String(5)) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + HostNetwork: hostNetwork, + NodeName: nodeName, + Tolerations: []corev1.Toleration{ + {Operator: corev1.TolerationOpExists}, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{ + { + Name: "connect", + Image: defaultAgnhostImage, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolptr(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + }, + Command: []string{"/agnhost"}, + Args: []string{ + "connect", + "--protocol=tcp", + "--timeout=5s", + FormatIPPort(serverIP, port), + }, + }, + }, + }, + } + if hostNetwork { + pod.Spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + } + + _, err := kubeClient.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return false, err + } + defer func() { + _ = kubeClient.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + }() + + if err := WaitForPodCompletion(ctx, kubeClient, namespace, name); err != nil { + return false, err + } + completed, err := kubeClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if len(completed.Status.ContainerStatuses) == 0 { + return false, fmt.Errorf("no container status recorded for pod %s", name) + } + terminated := completed.Status.ContainerStatuses[0].State.Terminated + if terminated == nil { + return false, fmt.Errorf("container in pod %s has not terminated", name) + } + return terminated.ExitCode == 0, nil +} + +// ExpectConnectivity checks connectivity from a pod in the given namespace +// (with clientLabels) to each serverIP on the specified port. +func ExpectConnectivity(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace string, clientLabels map[string]string, serverIPs []string, port int32, shouldSucceed bool) { + t.Helper() + for _, ip := range serverIPs { + family := "IPv4" + if IsIPv6(ip) { + family = "IPv6" + } + t.Logf("checking %s connectivity %s -> %s expected=%t", family, namespace, FormatIPPort(ip, port), shouldSucceed) + if err := pollConnectivity(ctx, kubeClient, namespace, clientLabels, ip, port, shouldSucceed, false, "", connectivityTimeout); err != nil { + t.Fatalf("connectivity check failed for %s %s -> %s (expected %t): %v", family, namespace, FormatIPPort(ip, port), shouldSucceed, err) + } + } +} + +// ExpectHostNetworkConnectivity checks connectivity from a host-network pod on +// the given node. Kubelet / host-network traffic bypasses NetworkPolicy. +func ExpectHostNetworkConnectivity(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace, nodeName string, serverIPs []string, port int32, shouldSucceed bool) { + t.Helper() + for _, ip := range serverIPs { + family := "IPv4" + if IsIPv6(ip) { + family = "IPv6" + } + t.Logf("checking %s host-network connectivity node=%s -> %s expected=%t", family, nodeName, FormatIPPort(ip, port), shouldSucceed) + if err := pollConnectivity(ctx, kubeClient, namespace, nil, ip, port, shouldSucceed, true, nodeName, connectivityTimeout); err != nil { + t.Fatalf("host-network connectivity check failed for %s node=%s -> %s (expected %t): %v", family, nodeName, FormatIPPort(ip, port), shouldSucceed, err) + } + } +} + +func pollConnectivity(ctx context.Context, kubeClient kubernetes.Interface, namespace string, clientLabels map[string]string, serverIP string, port int32, shouldSucceed, hostNetwork bool, nodeName string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(_ context.Context) (bool, error) { + succeeded, err := runConnectivityCheck(ctx, kubeClient, namespace, clientLabels, serverIP, port, hostNetwork, nodeName) + if err != nil { + return false, nil + } + return succeeded == shouldSucceed, nil + }) +} + +// WaitForPodCompletion waits up to 2 minutes for a pod to reach Succeeded or Failed. +func WaitForPodCompletion(ctx context.Context, kubeClient kubernetes.Interface, namespace, name string) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := kubeClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed, nil + }) +} + +// HasPort returns true if the given list of NetworkPolicy ports includes a port +// matching the specified protocol and port number. +func HasPort(ports []networkingv1.NetworkPolicyPort, protocol corev1.Protocol, port int32) bool { + for _, p := range ports { + if p.Protocol != nil && *p.Protocol != protocol { + continue + } + if p.Port == nil || p.Port.IntValue() == int(port) { + return true + } + } + return false +} + +// HasPortInIngress returns true if any ingress rule contains the specified protocol/port. +func HasPortInIngress(rules []networkingv1.NetworkPolicyIngressRule, protocol corev1.Protocol, port int32) bool { + for _, rule := range rules { + if HasPort(rule.Ports, protocol, port) { + return true + } + } + return false +} + +// HasIngressFromNamespace returns true if any ingress rule allows traffic from +// the specified namespace on the given port (TCP) via kubernetes.io/metadata.name. +func HasIngressFromNamespace(rules []networkingv1.NetworkPolicyIngressRule, port int32, namespace string) bool { + return HasIngressFromNamespaceLabel(rules, port, "kubernetes.io/metadata.name", namespace) +} + +// HasIngressFromNamespaceLabel returns true if any ingress rule allows traffic +// from namespaces with the given label on the specified port. +func HasIngressFromNamespaceLabel(rules []networkingv1.NetworkPolicyIngressRule, port int32, key, value string) bool { + for _, rule := range rules { + if !HasPort(rule.Ports, corev1.ProtocolTCP, port) { + continue + } + for _, peer := range rule.From { + if peer.NamespaceSelector == nil || peer.NamespaceSelector.MatchLabels == nil { + continue + } + if actual, ok := peer.NamespaceSelector.MatchLabels[key]; ok && actual == value { + return true + } + } + } + return false +} + +// HasIngressFromPolicyGroup returns true if any ingress rule allows traffic +// from namespaces with the given policy-group label key on the specified port. +func HasIngressFromPolicyGroup(rules []networkingv1.NetworkPolicyIngressRule, port int32, policyGroupLabelKey string) bool { + for _, rule := range rules { + if !HasPort(rule.Ports, corev1.ProtocolTCP, port) { + continue + } + for _, peer := range rule.From { + if peer.NamespaceSelector == nil || peer.NamespaceSelector.MatchLabels == nil { + continue + } + if _, ok := peer.NamespaceSelector.MatchLabels[policyGroupLabelKey]; ok { + return true + } + } + } + return false +} + +// GetNetworkPolicy fetches a NetworkPolicy by namespace and name. +func GetNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string) *networkingv1.NetworkPolicy { + t.Helper() + policy, err := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get NetworkPolicy %s/%s: %v", namespace, name, err) + } + return policy +} + +// RequirePodSelectorLabel asserts that the policy's podSelector contains the given key=value label. +func RequirePodSelectorLabel(t testing.TB, policy *networkingv1.NetworkPolicy, key, value string) { + t.Helper() + actual, ok := policy.Spec.PodSelector.MatchLabels[key] + if !ok || actual != value { + t.Fatalf("%s/%s: expected podSelector %s=%s, got %v", policy.Namespace, policy.Name, key, value, policy.Spec.PodSelector.MatchLabels) + } +} + +// RequireOwnerReference asserts that the policy is owned by the given API object. +func RequireOwnerReference(t testing.TB, policy *networkingv1.NetworkPolicy, apiVersion, kind, name string) { + t.Helper() + for _, ref := range policy.OwnerReferences { + if ref.APIVersion == apiVersion && ref.Kind == kind && ref.Name == name { + return + } + } + t.Fatalf("%s/%s: expected ownerReference %s %s/%s, got %v", policy.Namespace, policy.Name, apiVersion, kind, name, policy.OwnerReferences) +} + +// RequireIngressPort asserts that the policy has an ingress rule with the specified protocol and port. +func RequireIngressPort(t testing.TB, policy *networkingv1.NetworkPolicy, protocol corev1.Protocol, port int32) { + t.Helper() + if !HasPortInIngress(policy.Spec.Ingress, protocol, port) { + t.Fatalf("%s/%s: expected ingress port %s/%d", policy.Namespace, policy.Name, protocol, port) + } +} + +// RequireUnrestrictedEgress asserts that the policy has at least one egress rule +// with no port and no destination restrictions. +func RequireUnrestrictedEgress(t testing.TB, policy *networkingv1.NetworkPolicy) { + t.Helper() + if len(policy.Spec.Egress) == 0 { + t.Fatalf("%s/%s: expected at least one egress rule, got none", policy.Namespace, policy.Name) + } + for _, rule := range policy.Spec.Egress { + if len(rule.Ports) == 0 && len(rule.To) == 0 { + return + } + } + t.Fatalf("%s/%s: no unrestricted egress rule [{}] found among %d rules", policy.Namespace, policy.Name, len(policy.Spec.Egress)) +} + +// RequireIngressFromNamespace asserts that the policy allows ingress from the specified namespace on the given port. +func RequireIngressFromNamespace(t testing.TB, policy *networkingv1.NetworkPolicy, port int32, namespace string) { + t.Helper() + if !HasIngressFromNamespace(policy.Spec.Ingress, port, namespace) { + t.Fatalf("%s/%s: expected ingress from namespace %s on port %d", policy.Namespace, policy.Name, namespace, port) + } +} + +// RequireIngressFromNamespaceLabel asserts that the policy allows ingress from +// namespaces with the given label on the specified port. +func RequireIngressFromNamespaceLabel(t testing.TB, policy *networkingv1.NetworkPolicy, port int32, key, value string) { + t.Helper() + if !HasIngressFromNamespaceLabel(policy.Spec.Ingress, port, key, value) { + t.Fatalf("%s/%s: expected ingress from namespaces with %s=%s on port %d", policy.Namespace, policy.Name, key, value, port) + } +} + +// RequireIngressFromPolicyGroup asserts that the policy allows ingress from +// namespaces with the given policy-group label on the specified port. +func RequireIngressFromPolicyGroup(t testing.TB, policy *networkingv1.NetworkPolicy, port int32, policyGroupLabelKey string) { + t.Helper() + if !HasIngressFromPolicyGroup(policy.Spec.Ingress, port, policyGroupLabelKey) { + t.Fatalf("%s/%s: expected ingress from policy-group %s on port %d", policy.Namespace, policy.Name, policyGroupLabelKey, port) + } +} + +// RestoreNetworkPolicy deletes the given network policy and waits for the +// operator to recreate it with the expected spec. +func RestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, expected *networkingv1.NetworkPolicy, timeout time.Duration) { + t.Helper() + namespace := expected.Namespace + name := expected.Name + t.Logf("deleting NetworkPolicy %s/%s and waiting for restoration", namespace, name) + if err := client.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + t.Fatalf("failed to delete NetworkPolicy %s/%s: %v", namespace, name, err) + } + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + current, err := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return apiequality.Semantic.DeepEqual(expected.Spec, current.Spec), nil + }) + if err != nil { + t.Fatalf("timed out waiting for NetworkPolicy %s/%s spec to be restored", namespace, name) + } + t.Logf("NetworkPolicy %s/%s spec restored after delete", namespace, name) +} + +// MutateAndRestoreNetworkPolicy patches the policy's podSelector with a +// spurious label, then waits for the operator to reconcile it back. +func MutateAndRestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + patch := []byte(`{"spec":{"podSelector":{"matchLabels":{"np-reconcile":"mutated"}}}}`) + mutateAndRestoreNetworkPolicy(t, ctx, client, namespace, name, types.MergePatchType, patch, timeout, "podSelector override") +} + +// MutatePortAndRestoreNetworkPolicy changes the metrics ingress port and waits +// for the operator to restore the original spec. +func MutatePortAndRestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + patch := []byte(`[{"op":"replace","path":"/spec/ingress/0/ports/0/port","value":9999}]`) + mutateAndRestoreNetworkPolicy(t, ctx, client, namespace, name, types.JSONPatchType, patch, timeout, "ingress port override") +} + +// MutatePolicyTypesAndRestoreNetworkPolicy drops Egress from policyTypes and +// waits for the operator to restore Ingress+Egress. +func MutatePolicyTypesAndRestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + patch := []byte(`[{"op":"replace","path":"/spec/policyTypes","value":["Ingress"]}]`) + mutateAndRestoreNetworkPolicy(t, ctx, client, namespace, name, types.JSONPatchType, patch, timeout, "policyTypes override") +} + +// MutateNamespaceSelectorAndRestoreNetworkPolicy flips the monitoring +// namespaceSelector and waits for the operator to restore it. +func MutateNamespaceSelectorAndRestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + patch := []byte(`[{"op":"replace","path":"/spec/ingress/0/from/0/namespaceSelector/matchLabels/openshift.io~1cluster-monitoring","value":"false"}]`) + mutateAndRestoreNetworkPolicy(t, ctx, client, namespace, name, types.JSONPatchType, patch, timeout, "namespaceSelector override") +} + +// MutateEmptyIngressAndRestoreNetworkPolicy clears all ingress rules and waits +// for the operator to restore them. +func MutateEmptyIngressAndRestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + patch := []byte(`[{"op":"replace","path":"/spec/ingress","value":[]}]`) + mutateAndRestoreNetworkPolicy(t, ctx, client, namespace, name, types.JSONPatchType, patch, timeout, "empty ingress override") +} + +// AssertUnmanagedNetworkPolicyPreserved waits through a reconcile window and +// fails if the operator deletes a custom NetworkPolicy it does not own. +func AssertUnmanagedNetworkPolicyPreserved(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + t.Logf("waiting %s to confirm unmanaged NetworkPolicy %s/%s is not pruned", timeout, namespace, name) + pollErr := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + _, getErr := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if k8serrors.IsNotFound(getErr) { + return false, fmt.Errorf("operator deleted unmanaged NetworkPolicy %s/%s", namespace, name) + } + if getErr != nil { + return false, nil + } + return false, nil + }) + _, getErr := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if k8serrors.IsNotFound(getErr) { + t.Fatalf("unmanaged NetworkPolicy %s/%s was deleted by the operator", namespace, name) + } + if getErr != nil { + t.Fatalf("failed to get unmanaged NetworkPolicy %s/%s: %v", namespace, name, getErr) + } + if pollErr != nil && !wait.Interrupted(pollErr) { + t.Fatalf("unmanaged NetworkPolicy %s/%s was not preserved: %v", namespace, name, pollErr) + } + t.Logf("unmanaged NetworkPolicy %s/%s still exists", namespace, name) +} + +func mutateAndRestoreNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface, namespace, name string, patchType types.PatchType, patch []byte, timeout time.Duration, description string) { + t.Helper() + original := GetNetworkPolicy(t, ctx, client, namespace, name) + t.Logf("mutating NetworkPolicy %s/%s (%s) and waiting for reconciliation", namespace, name, description) + _, err := client.NetworkingV1().NetworkPolicies(namespace).Patch(ctx, name, patchType, patch, metav1.PatchOptions{}) + if err != nil { + t.Fatalf("failed to patch NetworkPolicy %s/%s: %v", namespace, name, err) + } + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + current, getErr := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if getErr != nil { + return false, nil + } + return apiequality.Semantic.DeepEqual(original.Spec, current.Spec), nil + }) + if err != nil { + t.Fatalf("timed out waiting for NetworkPolicy %s/%s spec to be restored after %s", namespace, name, description) + } + t.Logf("NetworkPolicy %s/%s spec restored after %s", namespace, name, description) +} + +// LogNetworkPolicyEvents searches for NetworkPolicy-related events (best-effort). +func LogNetworkPolicyEvents(t testing.TB, ctx context.Context, client kubernetes.Interface, namespaces []string, policyName string) { + t.Helper() + found := false + _ = wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { + for _, namespace := range namespaces { + eventList, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Logf("unable to list events in %s: %v", namespace, err) + continue + } + for _, event := range eventList.Items { + isNPEvent := strings.HasPrefix(event.Reason, "NetworkPolicy") || + event.InvolvedObject.Kind == "NetworkPolicy" || + (policyName != "" && strings.Contains(event.Message, policyName)) + if isNPEvent { + t.Logf("event in %s: type=%s reason=%s involvedObject=%s/%s message=%q", + namespace, event.Type, event.Reason, + event.InvolvedObject.Kind, event.InvolvedObject.Name, + event.Message) + found = true + } + } + } + if found { + return true, nil + } + return false, nil + }) + if !found { + t.Logf("no NetworkPolicy events observed for %s (best-effort)", policyName) + } +} + +func waitForOperandNetworkPolicy(t testing.TB, ctx context.Context, client kubernetes.Interface) { + t.Helper() + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := client.NetworkingV1().NetworkPolicies(operatorclient.OperatorNamespace).Get(ctx, operandNetworkPolicyName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, nil + } + return true, nil + }) + if err != nil { + t.Fatalf("timed out waiting for NetworkPolicy %s/%s", operatorclient.OperatorNamespace, operandNetworkPolicyName) + } +} + +func waitForLeaderOperandPod(t testing.TB, ctx context.Context, client kubernetes.Interface) *corev1.Pod { + t.Helper() + var leader *corev1.Pod + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := getLeaderOperandPod(ctx, client) + if err != nil { + t.Logf("waiting for leader operand pod: %v", err) + return false, nil + } + if len(PodIPs(pod)) == 0 { + t.Logf("leader operand pod %s has no IPs yet", pod.Name) + return false, nil + } + leader = pod + return true, nil + }) + if err != nil { + t.Fatalf("timed out waiting for leader operand pod: %v", err) + } + t.Logf("leader operand pod %s ips=%v node=%s", leader.Name, PodIPs(leader), leader.Spec.NodeName) + return leader +} + +func getLeaderOperandPod(ctx context.Context, client kubernetes.Interface) (*corev1.Pod, error) { + lease, err := client.CoordinationV1().Leases(operatorclient.OperatorNamespace).Get(ctx, operandLeaseName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get lease %s/%s: %w", operatorclient.OperatorNamespace, operandLeaseName, err) + } + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" { + return nil, fmt.Errorf("lease %s/%s has no holderIdentity", operatorclient.OperatorNamespace, operandLeaseName) + } + + holder := *lease.Spec.HolderIdentity + podName := holder + if i := strings.LastIndex(holder, "_"); i > 0 { + podName = holder[:i] + } + + pod, err := client.CoreV1().Pods(operatorclient.OperatorNamespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get leader pod %s from holderIdentity %s: %w", podName, holder, err) + } + return pod, nil +} + +func operandClientLabels() map[string]string { + return map[string]string{operandAppLabelKey: operatorclient.OperandName} +} diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 4c5347448..02bae1499 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -16,6 +16,7 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apiruntime "k8s.io/apimachinery/pkg/runtime" @@ -54,6 +55,7 @@ var _ = g.Describe("[Operator][Serial] CLI Manager Operator", g.Ordered, func() }) g.AfterAll(func() { + teardownOperator() if cancelFnc != nil { cancelFnc() } @@ -335,6 +337,49 @@ func setupOperator(t testing.TB) (context.Context, context.CancelFunc, *k8sclien return ctx, cancelFnc, kubeClient, nil } +// teardownOperator removes resources created by setupOperator. It runs from +// AfterAll, so a single-spec run still cleans the cluster, and a full suite +// waits until every spec in the Ordered container has finished. +func teardownOperator() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + kubeClient := GetKubeClient() + cliManagerClient := GetCLIManagerClient() + + klog.Infof("Tearing down CLI Manager operator resources") + + err := cliManagerClient.ClimanagersV1().CliManagers(operatorclient.OperatorNamespace).Delete(ctx, operatorclient.OperatorConfigName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("failed to delete CliManager CR: %v", err) + } + + err = kubeClient.CoreV1().Namespaces().Delete(ctx, operatorclient.OperatorNamespace, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("failed to delete namespace %s: %v", operatorclient.OperatorNamespace, err) + } + + waitErr := wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + _, getErr := kubeClient.CoreV1().Namespaces().Get(ctx, operatorclient.OperatorNamespace, metav1.GetOptions{}) + if apierrors.IsNotFound(getErr) { + return true, nil + } + return false, nil + }) + if waitErr != nil { + klog.Warningf("timed out waiting for namespace %s deletion: %v", operatorclient.OperatorNamespace, waitErr) + } + + if err := kubeClient.RbacV1().ClusterRoleBindings().Delete(ctx, "openshift-cli-manager-operator", metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("failed to delete ClusterRoleBinding: %v", err) + } + if err := kubeClient.RbacV1().ClusterRoles().Delete(ctx, "openshift-cli-manager-operator", metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("failed to delete ClusterRole: %v", err) + } + + klog.Infof("CLI Manager operator teardown complete") +} + // installKrew downloads and installs krew. func installKrew(t testing.TB) { tmpDir, err := os.MkdirTemp("", "krew-install") diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go index 5a3424cf4..fcf522d44 100644 --- a/test/e2e/operator_test.go +++ b/test/e2e/operator_test.go @@ -45,7 +45,10 @@ func TestExtended(t *testing.T) { if err != nil { t.Fatalf("Failed to setup operator: %v", err) } - defer cancelFnc() + defer func() { + teardownOperator() + cancelFnc() + }() t.Run("CLI Manager functionality", func(t *testing.T) { testCLIManager(t, ctx, kubeClient) From 7fa77fded19686b5db7236def9b48968fb20392a Mon Sep 17 00:00:00 2001 From: Rohit Patil Date: Thu, 17 Sep 2026 15:31:41 +0530 Subject: [PATCH 2/2] Fix e2e ingress test to handle deny-all policies in openshift-ingress on OCP 5.x+ The openshift-ingress namespace on OCP 5.x+ has deny-all network policies that block non-router pods, causing test step 3.6 to fail. Use a temporary namespace with the ingress policy-group label instead, and add a conditional test (3.11) that verifies the deny-all behavior when the policy is present. --- test/e2e/network_policy.go | 19 ++++++++++++++-- test/e2e/network_policy_helpers.go | 36 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/test/e2e/network_policy.go b/test/e2e/network_policy.go index 8b68be615..24edf8584 100644 --- a/test/e2e/network_policy.go +++ b/test/e2e/network_policy.go @@ -171,10 +171,16 @@ func testOperandPluginIngress(t testing.TB, ctx context.Context, kubeClient k8sc leaderIPs := PodIPs(leader) testLabels := map[string]string{"test": "cli-manager-netpol"} + // Use a temp namespace with the ingress policy-group label instead of + // openshift-ingress directly — that namespace has deny-all policies + // blocking non-router pods on OCP 5.x+. + ingressNS, cleanup := createTempIngressNamespace(t, ctx, kubeClient) + defer cleanup() + t.Logf("=== Testing plugin download ingress on port %d to leader %s ===", pluginPort, leader.Name) - t.Logf("3.6 Allowed — Ingress from %s on port %d", openshiftIngressNamespace, pluginPort) - ExpectConnectivity(ctx, t, kubeClient, openshiftIngressNamespace, testLabels, leaderIPs, pluginPort, true) + t.Logf("3.6 Allowed — Ingress from %s (policy-group ingress label) on port %d", ingressNS, pluginPort) + ExpectConnectivity(ctx, t, kubeClient, ingressNS, testLabels, leaderIPs, pluginPort, true) t.Logf("3.7 Blocked — Ingress from default namespace on port %d", pluginPort) ExpectConnectivity(ctx, t, kubeClient, "default", testLabels, leaderIPs, pluginPort, false) @@ -182,6 +188,15 @@ func testOperandPluginIngress(t testing.TB, ctx context.Context, kubeClient k8sc t.Logf("3.10 Blocked — Ingress from operator namespace on port %d (no ingress policy-group label)", pluginPort) ExpectConnectivity(ctx, t, kubeClient, operatorclient.OperatorNamespace, testLabels, leaderIPs, pluginPort, false) + if ingressNamespaceHasDenyAll(ctx, kubeClient) { + t.Logf("3.11 Blocked — Non-router pod in %s blocked by %s deny-all on port %d", + openshiftIngressNamespace, openshiftIngressDenyAllPolicy, pluginPort) + ExpectConnectivity(ctx, t, kubeClient, openshiftIngressNamespace, testLabels, leaderIPs, pluginPort, false) + } else { + t.Logf("3.11 Skipping — %s/%s not present, non-router pods can egress", + openshiftIngressNamespace, openshiftIngressDenyAllPolicy) + } + t.Logf("=== plugin download ingress verified ===") } diff --git a/test/e2e/network_policy_helpers.go b/test/e2e/network_policy_helpers.go index 8ced04a74..a2316c3a8 100644 --- a/test/e2e/network_policy_helpers.go +++ b/test/e2e/network_policy_helpers.go @@ -39,6 +39,7 @@ const ( openshiftMonitoringNamespace = "openshift-monitoring" openshiftUWMNamespace = "openshift-user-workload-monitoring" openshiftIngressNamespace = "openshift-ingress" + openshiftIngressDenyAllPolicy = "openshift-ingress-deny-all" openshiftDNSNamespace = "openshift-dns" prometheusK8sServiceName = "prometheus-k8s" operandMetricsServiceName = "openshift-cli-manager-metrics" @@ -582,3 +583,38 @@ func getLeaderOperandPod(ctx context.Context, client kubernetes.Interface) (*cor func operandClientLabels() map[string]string { return map[string]string{operandAppLabelKey: operatorclient.OperandName} } + +func ingressNamespaceHasDenyAll(ctx context.Context, client kubernetes.Interface) bool { + _, err := client.NetworkingV1().NetworkPolicies(openshiftIngressNamespace).Get(ctx, openshiftIngressDenyAllPolicy, metav1.GetOptions{}) + return err == nil +} + +// createTempIngressNamespace creates a temporary namespace with the +// policy-group.network.openshift.io/ingress label so connectivity tests +// don't depend on openshift-ingress, which may have deny-all policies +// blocking non-router pods (OCP 5.x+). +func createTempIngressNamespace(t testing.TB, ctx context.Context, client kubernetes.Interface) (string, func()) { + t.Helper() + name := fmt.Sprintf("np-ingress-test-%s", rand.String(5)) + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + ingressPolicyGroupKey: "", + "pod-security.kubernetes.io/enforce": "restricted", + "pod-security.kubernetes.io/enforce-version": "latest", + }, + }, + } + _, err := client.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create temp ingress namespace %s: %v", name, err) + } + t.Logf("created temp namespace %s with label %s", name, ingressPolicyGroupKey) + cleanup := func() { + if delErr := client.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}); delErr != nil { + t.Logf("failed to delete temp namespace %s: %v", name, delErr) + } + } + return name, cleanup +}