diff --git a/test/extended/networking/external_gateway.go b/test/extended/networking/external_gateway.go index b0f29685d6ac..7af9fdef62ee 100644 --- a/test/extended/networking/external_gateway.go +++ b/test/extended/networking/external_gateway.go @@ -1,63 +1,211 @@ package networking import ( + "context" + "fmt" + "regexp" + "time" + g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" exutil "github.com/openshift/origin/test/extended/util" e2e "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" admissionapi "k8s.io/pod-security-admission/api" ) var _ = g.Describe("[sig-network] external gateway address", func() { - oc := exutil.NewCLIWithPodSecurityLevel("ns-global", admissionapi.LevelPrivileged) + oc := exutil.NewCLIWithoutNamespace("ns-global") InOVNKubernetesContext(func() { f := oc.KubeFramework() + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged g.It("should match the address family of the pod", func() { + labelKey, labelValue := "test", "external-gateway" + labels := map[string]string{ + labelKey: labelValue, + } + + // Expected error message for APB policy sync failure + errorLog := "Failed to sync APB policy %s.*gateway specified for namespace %s.*%s" + + // Returns true if the ovnkube-controller logs contain the expected error message + checkLogs := func(ovnkubePodInfo ovnKubePodInfo, regex *regexp.Regexp) (bool, error) { + logs, err := e2epod.GetPodLogs(context.TODO(), f.ClientSet, ovnNamespace, ovnkubePodInfo.podName, ovnkubePodInfo.containerName) + if err != nil { + return false, err + } + return regex.MatchString(logs), nil + } + + g.By("creating namespace with external-gateway label") + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, labels) + expectNoError(err) + f.Namespace = ns + + g.By("determining cluster pod IP address family") podIPFamily := GetIPFamilyForCluster(f) o.Expect(podIPFamily).NotTo(o.Equal(Unknown)) + // Set external gateway address into an IPv6 address and make sure // pod ip address matches with IPv6 address family. - setNamespaceExternalGateway(f, "fd00:10:244:2::6") - podIPs, err := createPod(f.ClientSet, f.Namespace.Name, "test-ipv6-pod") + apbPolicyNameIPv6 := "static-egress-route-ipv6" + g.By(fmt.Sprintf("applying IPv6 AdminPolicyBasedExternalRoute %s with gateway fd00:10:244:2::6", apbPolicyNameIPv6)) + setNamespaceExternalGateway(apbPolicyNameIPv6, []string{"fd00:10:244:2::6"}, labelKey, labelValue) + + podNameIPv6 := "test-ipv6-pod" + g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameIPv6, f.Namespace.Name)) + pod, err := createPod(f.ClientSet, f.Namespace.Name, podNameIPv6) + expectNoError(err) + podIPs := pod.Status.PodIPs e2e.Logf("pod IPs are %v after setting external gw with IPv6 address", podIPs) + + g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName)) + ovnkubePodInfo, err := ovnkubePod(oc, pod.Spec.NodeName) + expectNoError(err) + + regexIPv6, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameIPv6, f.Namespace.Name, podNameIPv6)) + expectNoError(err) switch podIPFamily { - case DualStack: - expectNoError(err) - o.Expect(getIPFamily(podIPs)).To(o.Equal(DualStack)) + case DualStack, IPv6: + g.By(fmt.Sprintf("verifying ovnkube-node logs do not report APB sync failure for IPv6 gateway on %s cluster", podIPFamily)) + o.Consistently(func() bool { + found, err := checkLogs(ovnkubePodInfo, regexIPv6) + if err != nil { + e2e.Logf("Error checking logs: %v", err) + return true + } + return found + }). + WithPolling(20 * time.Second). + WithTimeout(2 * time.Minute). + Should(o.BeFalse()) case IPv4: // This is an expected failure when pod network in IPv4 address family // whereas external gateway is set with IPv6 address - expectError(err) - case IPv6: - expectNoError(err) - o.Expect(getIPFamily(podIPs)).To(o.Equal(IPv6)) + g.By("verifying ovnkube-node logs report APB sync failure for mismatched IPv6 gateway on IPv4 cluster") + o.Eventually(func() bool { + found, err := checkLogs(ovnkubePodInfo, regexIPv6) + if err != nil { + e2e.Logf("Error checking logs: %v", err) + return false + } + return found + }). + WithPolling(20 * time.Second). + WithTimeout(2 * time.Minute). + Should(o.BeTrue()) } + + g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameIPv6)) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameIPv6, "--ignore-not-found").Execute() + expectNoError(err) + + g.By(fmt.Sprintf("deleting pod %s", podNameIPv6)) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute() + expectNoError(err) + // Set external gateway address into an IPv4 address and make sure // pod ip address matches with IPv4 address family. - setNamespaceExternalGateway(f, "10.10.10.1") - podIPs, err = createPod(f.ClientSet, f.Namespace.Name, "test-ipv4-pod") + apbPolicyNameIPv4 := "static-egress-route-ipv4" + g.By(fmt.Sprintf("applying IPv4 AdminPolicyBasedExternalRoute %s with gateway 10.10.10.1", apbPolicyNameIPv4)) + setNamespaceExternalGateway(apbPolicyNameIPv4, []string{"10.10.10.1"}, labelKey, labelValue) + + podNameIPv4 := "test-ipv4-pod" + g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameIPv4, f.Namespace.Name)) + pod, err = createPod(f.ClientSet, f.Namespace.Name, podNameIPv4) + expectNoError(err) + podIPs = pod.Status.PodIPs e2e.Logf("pod IPs are %v after setting external gw with IPv4 address", podIPs) + + g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName)) + ovnkubePodInfo, err = ovnkubePod(oc, pod.Spec.NodeName) + expectNoError(err) + + regexIPv4, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameIPv4, f.Namespace.Name, podNameIPv4)) + expectNoError(err) switch podIPFamily { - case DualStack: - expectNoError(err) - o.Expect(getIPFamily(podIPs)).To(o.Equal(DualStack)) - case IPv4: - expectNoError(err) - o.Expect(getIPFamily(podIPs)).To(o.Equal(IPv4)) + case DualStack, IPv4: + g.By(fmt.Sprintf("verifying ovnkube-node logs do not report APB sync failure for IPv4 gateway on %s cluster", podIPFamily)) + o.Consistently(func() bool { + found, err := checkLogs(ovnkubePodInfo, regexIPv4) + if err != nil { + e2e.Logf("Error checking logs: %v", err) + return true + } + return found + }). + WithPolling(20 * time.Second). + WithTimeout(2 * time.Minute). + Should(o.BeFalse()) case IPv6: // This is an expected failure when pod network in IPv6 address family // whereas external gateway is set with IPv4 address - expectError(err) + g.By("verifying ovnkube-node logs report APB sync failure for mismatched IPv4 gateway on IPv6 cluster") + o.Eventually(func() bool { + found, err := checkLogs(ovnkubePodInfo, regexIPv4) + if err != nil { + e2e.Logf("Error checking logs: %v", err) + return false + } + return found + }). + WithPolling(20 * time.Second). + WithTimeout(2 * time.Minute). + Should(o.BeTrue()) } + + g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameIPv4)) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameIPv4, "--ignore-not-found").Execute() + expectNoError(err) + + g.By(fmt.Sprintf("deleting pod %s", podNameIPv4)) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute() + expectNoError(err) + // Set external gateway address supporting Dual Stack and make sure // pod ip address(es) match with desired address family. - setNamespaceExternalGateway(f, "10.10.10.1,fd00:10:244:2::6") - podIPs, err = createPod(f.ClientSet, f.Namespace.Name, "test-dual-stack-pod") - o.Expect(err).NotTo(o.HaveOccurred()) + apbPolicyNameDualStack := "static-egress-route-dual-stack" + g.By(fmt.Sprintf("applying dual-stack AdminPolicyBasedExternalRoute %s with gateways 10.10.10.1 and fd00:10:244:2::6", apbPolicyNameDualStack)) + setNamespaceExternalGateway(apbPolicyNameDualStack, []string{"10.10.10.1", "fd00:10:244:2::6"}, labelKey, labelValue) + + podNameDualStack := "test-dual-stack-pod" + g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameDualStack, f.Namespace.Name)) + pod, err = createPod(f.ClientSet, f.Namespace.Name, podNameDualStack) + expectNoError(err) + podIPs = pod.Status.PodIPs e2e.Logf("pod IPs are %v after setting external gw with Dual Stack address", podIPs) + + g.By("verifying pod IP address family matches cluster") o.Expect(getIPFamily(podIPs)).To(o.Equal(podIPFamily)) + + g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName)) + ovnkubePodInfo, err = ovnkubePod(oc, pod.Spec.NodeName) + expectNoError(err) + + regexDualStack, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameDualStack, f.Namespace.Name, podNameDualStack)) + expectNoError(err) + g.By("verifying ovnkube-node logs do not report APB sync failure for dual-stack gateway") + o.Consistently(func() bool { + found, err := checkLogs(ovnkubePodInfo, regexDualStack) + if err != nil { + e2e.Logf("Error checking logs: %v", err) + return true + } + return found + }). + WithPolling(20 * time.Second). + WithTimeout(2 * time.Minute). + Should(o.BeFalse()) + + g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameDualStack)) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameDualStack, "--ignore-not-found").Execute() + expectNoError(err) + + g.By(fmt.Sprintf("deleting pod %s", podNameDualStack)) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute() + expectNoError(err) }) }) }) diff --git a/test/extended/networking/util.go b/test/extended/networking/util.go index 795f4e4fbed3..c00473cb1c39 100644 --- a/test/extended/networking/util.go +++ b/test/extended/networking/util.go @@ -1,6 +1,7 @@ package networking import ( + "bytes" "context" "encoding/json" "errors" @@ -11,6 +12,7 @@ import ( "path/filepath" "strconv" "strings" + "text/template" "time" netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" @@ -317,18 +319,54 @@ func modifyNetworkConfig(configClient configv1client.Interface, autoAssignCIDRs, expectNoError(kubeAPIServerRollout.Err()) } -func setNamespaceExternalGateway(f *e2e.Framework, gatewayIP string) { - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - ns, err := f.ClientSet.CoreV1().Namespaces().Get(context.Background(), f.Namespace.Name, metav1.GetOptions{}) - expectNoError(err) - if ns.Annotations == nil { - ns.Annotations = make(map[string]string) - } - ns.Annotations["k8s.ovn.org/routing-external-gws"] = gatewayIP - _, err = f.ClientSet.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{}) - return err +const adminPolicyBasedExternalRouteTemplate = ` +apiVersion: k8s.ovn.org/v1 +kind: AdminPolicyBasedExternalRoute +metadata: + name: {{ .Name }} +spec: + from: + namespaceSelector: + matchLabels: + {{ .LabelKey | quote }}: {{ .LabelValue | quote }} + nextHops: + static: +{{- range .IPs }} + - ip: {{ . | quote }} +{{- end }}` + +type adminPolicyBasedExternalRouteParams struct { + Name string + LabelKey string + LabelValue string + IPs []string +} + +func renderAdminPolicyBasedExternalRoute(params adminPolicyBasedExternalRouteParams) (string, error) { + tmpl, err := template.New("adminPolicyBasedExternalRoute").Funcs(template.FuncMap{ + "quote": func(s string) string { + return fmt.Sprintf("%q", s) + }, + }).Parse(adminPolicyBasedExternalRouteTemplate) + if err != nil { + return "", err + } + var manifest bytes.Buffer + if err := tmpl.Execute(&manifest, params); err != nil { + return "", err + } + return manifest.String(), nil +} + +func setNamespaceExternalGateway(apbPolicyName string, gatewayIPs []string, labelKey string, labelValue string) { + manifest, err := renderAdminPolicyBasedExternalRoute(adminPolicyBasedExternalRouteParams{ + Name: apbPolicyName, + LabelKey: labelKey, + LabelValue: labelValue, + IPs: gatewayIPs, }) expectNoError(err) + expectNoError(applyManifest("", manifest)) } // findAppropriateNodes tries to find a source and destination for a type of node connectivity @@ -464,9 +502,14 @@ func networkAttachmentDefinitionClient(config *rest.Config) (dynamic.Namespaceab } func GetIPFamilyForCluster(f *e2e.Framework) IPFamily { - podIPs, err := createPod(f.ClientSet, f.Namespace.Name, "test-ip-family-pod") + pod, err := createPod(f.ClientSet, f.Namespace.Name, "test-ip-family-pod") + expectNoError(err) + podIPFamily := getIPFamily(pod.Status.PodIPs) + // Delete the temporary pod + err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) expectNoError(err) - return getIPFamily(podIPs) + // Return the IP family of the pod + return podIPFamily } func getIPFamily(podIPs []corev1.PodIP) IPFamily { @@ -493,21 +536,20 @@ func getIPFamily(podIPs []corev1.PodIP) IPFamily { } } -func createPod(client k8sclient.Interface, ns, generateName string) ([]corev1.PodIP, error) { +func createPod(client k8sclient.Interface, ns, generateName string) (*corev1.Pod, error) { pod := e2epod.NewAgnhostPod(ns, "", nil, nil, nil) pod.ObjectMeta.GenerateName = generateName execPod, err := client.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) expectNoError(err, "failed to create new pod in namespace: %s", ns) - var podIPs []corev1.PodIP + var retrievedPod *corev1.Pod err = wait.PollImmediate(poll, 2*time.Minute, func() (bool, error) { - retrievedPod, err := client.CoreV1().Pods(execPod.Namespace).Get(context.TODO(), execPod.Name, metav1.GetOptions{}) + retrievedPod, err = client.CoreV1().Pods(execPod.Namespace).Get(context.TODO(), execPod.Name, metav1.GetOptions{}) if err != nil { return false, err } - podIPs = retrievedPod.Status.PodIPs return retrievedPod.Status.Phase == corev1.PodRunning, nil }) - return podIPs, err + return retrievedPod, err } // SubnetIPs enumerates all IP addresses in an IP subnet (starting with the provided IP address and including the broadcast address).