Skip to content

[release-4.21] OCPBUGS-90523: Use AdminPolicyBasedExternalRoute CR for external gateway test #31316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
openshift-cherrypick-robot wants to merge 1 commit into
base: release-4.21
Choose a base branch
from
+229 −39
Open

[release-4.21] OCPBUGS-90523: Use AdminPolicyBasedExternalRoute CR for external gateway test #31316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 170 additions & 22 deletions test/extended/networking/external_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,63 +1,211 @@
package networking

import (
"context"
"fmt"
"regexp"
"time"

g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"
exutil "github.com/openshift/origin/test/extended/util"
e2e "k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
admissionapi "k8s.io/pod-security-admission/api"
)

var _ = g.Describe("[sig-network] external gateway address", func() {
oc := exutil.NewCLIWithPodSecurityLevel("ns-global", admissionapi.LevelPrivileged)
oc := exutil.NewCLIWithoutNamespace("ns-global")

InOVNKubernetesContext(func() {
f := oc.KubeFramework()
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged

g.It("should match the address family of the pod", func() {
labelKey, labelValue := "test", "external-gateway"
labels := map[string]string{
labelKey: labelValue,
}

// Expected error message for APB policy sync failure
errorLog := "Failed to sync APB policy %s.*gateway specified for namespace %s.*%s"

// Returns true if the ovnkube-controller logs contain the expected error message
checkLogs := func(ovnkubePodInfo ovnKubePodInfo, regex *regexp.Regexp) (bool, error) {
logs, err := e2epod.GetPodLogs(context.TODO(), f.ClientSet, ovnNamespace, ovnkubePodInfo.podName, ovnkubePodInfo.containerName)
if err != nil {
return false, err
}
return regex.MatchString(logs), nil
}

g.By("creating namespace with external-gateway label")
ns, err := f.CreateNamespace(context.TODO(), f.BaseName, labels)
expectNoError(err)
f.Namespace = ns

g.By("determining cluster pod IP address family")
podIPFamily := GetIPFamilyForCluster(f)
o.Expect(podIPFamily).NotTo(o.Equal(Unknown))

// Set external gateway address into an IPv6 address and make sure
// pod ip address matches with IPv6 address family.
setNamespaceExternalGateway(f, "fd00:10:244:2::6")
podIPs, err := createPod(f.ClientSet, f.Namespace.Name, "test-ipv6-pod")
apbPolicyNameIPv6 := "static-egress-route-ipv6"
g.By(fmt.Sprintf("applying IPv6 AdminPolicyBasedExternalRoute %s with gateway fd00:10:244:2::6", apbPolicyNameIPv6))
setNamespaceExternalGateway(apbPolicyNameIPv6, []string{"fd00:10:244:2::6"}, labelKey, labelValue)

podNameIPv6 := "test-ipv6-pod"
g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameIPv6, f.Namespace.Name))
pod, err := createPod(f.ClientSet, f.Namespace.Name, podNameIPv6)
expectNoError(err)
podIPs := pod.Status.PodIPs
e2e.Logf("pod IPs are %v after setting external gw with IPv6 address", podIPs)

g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName))
ovnkubePodInfo, err := ovnkubePod(oc, pod.Spec.NodeName)
expectNoError(err)

regexIPv6, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameIPv6, f.Namespace.Name, podNameIPv6))
expectNoError(err)
switch podIPFamily {
case DualStack:
expectNoError(err)
o.Expect(getIPFamily(podIPs)).To(o.Equal(DualStack))
case DualStack, IPv6:
g.By(fmt.Sprintf("verifying ovnkube-node logs do not report APB sync failure for IPv6 gateway on %s cluster", podIPFamily))
o.Consistently(func() bool {
found, err := checkLogs(ovnkubePodInfo, regexIPv6)
if err != nil {
e2e.Logf("Error checking logs: %v", err)
return true
}
return found
}).
WithPolling(20 * time.Second).
WithTimeout(2 * time.Minute).
Should(o.BeFalse())
case IPv4:
// This is an expected failure when pod network in IPv4 address family
// whereas external gateway is set with IPv6 address
expectError(err)
case IPv6:
expectNoError(err)
o.Expect(getIPFamily(podIPs)).To(o.Equal(IPv6))
g.By("verifying ovnkube-node logs report APB sync failure for mismatched IPv6 gateway on IPv4 cluster")
o.Eventually(func() bool {
found, err := checkLogs(ovnkubePodInfo, regexIPv6)
if err != nil {
e2e.Logf("Error checking logs: %v", err)
return false
}
return found
}).
WithPolling(20 * time.Second).
WithTimeout(2 * time.Minute).
Should(o.BeTrue())
}

g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameIPv6))
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameIPv6, "--ignore-not-found").Execute()
expectNoError(err)

g.By(fmt.Sprintf("deleting pod %s", podNameIPv6))
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute()
expectNoError(err)

// Set external gateway address into an IPv4 address and make sure
// pod ip address matches with IPv4 address family.
setNamespaceExternalGateway(f, "10.10.10.1")
podIPs, err = createPod(f.ClientSet, f.Namespace.Name, "test-ipv4-pod")
apbPolicyNameIPv4 := "static-egress-route-ipv4"
g.By(fmt.Sprintf("applying IPv4 AdminPolicyBasedExternalRoute %s with gateway 10.10.10.1", apbPolicyNameIPv4))
setNamespaceExternalGateway(apbPolicyNameIPv4, []string{"10.10.10.1"}, labelKey, labelValue)

podNameIPv4 := "test-ipv4-pod"
g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameIPv4, f.Namespace.Name))
pod, err = createPod(f.ClientSet, f.Namespace.Name, podNameIPv4)
expectNoError(err)
podIPs = pod.Status.PodIPs
e2e.Logf("pod IPs are %v after setting external gw with IPv4 address", podIPs)

g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName))
ovnkubePodInfo, err = ovnkubePod(oc, pod.Spec.NodeName)
expectNoError(err)

regexIPv4, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameIPv4, f.Namespace.Name, podNameIPv4))
expectNoError(err)
switch podIPFamily {
case DualStack:
expectNoError(err)
o.Expect(getIPFamily(podIPs)).To(o.Equal(DualStack))
case IPv4:
expectNoError(err)
o.Expect(getIPFamily(podIPs)).To(o.Equal(IPv4))
case DualStack, IPv4:
g.By(fmt.Sprintf("verifying ovnkube-node logs do not report APB sync failure for IPv4 gateway on %s cluster", podIPFamily))
o.Consistently(func() bool {
found, err := checkLogs(ovnkubePodInfo, regexIPv4)
if err != nil {
e2e.Logf("Error checking logs: %v", err)
return true
}
return found
}).
WithPolling(20 * time.Second).
WithTimeout(2 * time.Minute).
Should(o.BeFalse())
case IPv6:
// This is an expected failure when pod network in IPv6 address family
// whereas external gateway is set with IPv4 address
expectError(err)
g.By("verifying ovnkube-node logs report APB sync failure for mismatched IPv4 gateway on IPv6 cluster")
o.Eventually(func() bool {
found, err := checkLogs(ovnkubePodInfo, regexIPv4)
if err != nil {
e2e.Logf("Error checking logs: %v", err)
return false
}
return found
}).
WithPolling(20 * time.Second).
WithTimeout(2 * time.Minute).
Should(o.BeTrue())
}

g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameIPv4))
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameIPv4, "--ignore-not-found").Execute()
expectNoError(err)

g.By(fmt.Sprintf("deleting pod %s", podNameIPv4))
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute()
expectNoError(err)

// Set external gateway address supporting Dual Stack and make sure
// pod ip address(es) match with desired address family.
setNamespaceExternalGateway(f, "10.10.10.1,fd00:10:244:2::6")
podIPs, err = createPod(f.ClientSet, f.Namespace.Name, "test-dual-stack-pod")
o.Expect(err).NotTo(o.HaveOccurred())
apbPolicyNameDualStack := "static-egress-route-dual-stack"
g.By(fmt.Sprintf("applying dual-stack AdminPolicyBasedExternalRoute %s with gateways 10.10.10.1 and fd00:10:244:2::6", apbPolicyNameDualStack))
setNamespaceExternalGateway(apbPolicyNameDualStack, []string{"10.10.10.1", "fd00:10:244:2::6"}, labelKey, labelValue)

podNameDualStack := "test-dual-stack-pod"
g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameDualStack, f.Namespace.Name))
pod, err = createPod(f.ClientSet, f.Namespace.Name, podNameDualStack)
expectNoError(err)
podIPs = pod.Status.PodIPs
e2e.Logf("pod IPs are %v after setting external gw with Dual Stack address", podIPs)

g.By("verifying pod IP address family matches cluster")
o.Expect(getIPFamily(podIPs)).To(o.Equal(podIPFamily))

g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName))
ovnkubePodInfo, err = ovnkubePod(oc, pod.Spec.NodeName)
expectNoError(err)

regexDualStack, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameDualStack, f.Namespace.Name, podNameDualStack))
expectNoError(err)
g.By("verifying ovnkube-node logs do not report APB sync failure for dual-stack gateway")
o.Consistently(func() bool {
found, err := checkLogs(ovnkubePodInfo, regexDualStack)
if err != nil {
e2e.Logf("Error checking logs: %v", err)
return true
}
return found
}).
WithPolling(20 * time.Second).
WithTimeout(2 * time.Minute).
Should(o.BeFalse())

g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameDualStack))
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameDualStack, "--ignore-not-found").Execute()
expectNoError(err)

g.By(fmt.Sprintf("deleting pod %s", podNameDualStack))
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute()
expectNoError(err)
})
})
})
76 changes: 59 additions & 17 deletions test/extended/networking/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package networking

import (
"bytes"
"context"
"encoding/json"
"errors"
Expand All @@ -11,6 +12,7 @@ import (
"path/filepath"
"strconv"
"strings"
"text/template"
"time"

netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
Expand Down Expand Up @@ -317,18 +319,54 @@ func modifyNetworkConfig(configClient configv1client.Interface, autoAssignCIDRs,
expectNoError(kubeAPIServerRollout.Err())
}

func setNamespaceExternalGateway(f *e2e.Framework, gatewayIP string) {
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
ns, err := f.ClientSet.CoreV1().Namespaces().Get(context.Background(), f.Namespace.Name, metav1.GetOptions{})
expectNoError(err)
if ns.Annotations == nil {
ns.Annotations = make(map[string]string)
}
ns.Annotations["k8s.ovn.org/routing-external-gws"] = gatewayIP
_, err = f.ClientSet.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
return err
const adminPolicyBasedExternalRouteTemplate = `
apiVersion: k8s.ovn.org/v1
kind: AdminPolicyBasedExternalRoute
metadata:
name: {{ .Name }}
spec:
from:
namespaceSelector:
matchLabels:
{{ .LabelKey | quote }}: {{ .LabelValue | quote }}
nextHops:
static:
{{- range .IPs }}
- ip: {{ . | quote }}
{{- end }}`

type adminPolicyBasedExternalRouteParams struct {
Name string
LabelKey string
LabelValue string
IPs []string
}

func renderAdminPolicyBasedExternalRoute(params adminPolicyBasedExternalRouteParams) (string, error) {
tmpl, err := template.New("adminPolicyBasedExternalRoute").Funcs(template.FuncMap{
"quote": func(s string) string {
return fmt.Sprintf("%q", s)
},
}).Parse(adminPolicyBasedExternalRouteTemplate)
if err != nil {
return "", err
}
var manifest bytes.Buffer
if err := tmpl.Execute(&manifest, params); err != nil {
return "", err
}
return manifest.String(), nil
}

func setNamespaceExternalGateway(apbPolicyName string, gatewayIPs []string, labelKey string, labelValue string) {
manifest, err := renderAdminPolicyBasedExternalRoute(adminPolicyBasedExternalRouteParams{
Name: apbPolicyName,
LabelKey: labelKey,
LabelValue: labelValue,
IPs: gatewayIPs,
})
expectNoError(err)
expectNoError(applyManifest("", manifest))
}

// findAppropriateNodes tries to find a source and destination for a type of node connectivity
Expand Down Expand Up @@ -464,9 +502,14 @@ func networkAttachmentDefinitionClient(config *rest.Config) (dynamic.Namespaceab
}

func GetIPFamilyForCluster(f *e2e.Framework) IPFamily {
podIPs, err := createPod(f.ClientSet, f.Namespace.Name, "test-ip-family-pod")
pod, err := createPod(f.ClientSet, f.Namespace.Name, "test-ip-family-pod")
expectNoError(err)
podIPFamily := getIPFamily(pod.Status.PodIPs)
// Delete the temporary pod
err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
expectNoError(err)
return getIPFamily(podIPs)
// Return the IP family of the pod
return podIPFamily
}

func getIPFamily(podIPs []corev1.PodIP) IPFamily {
Expand All @@ -493,21 +536,20 @@ func getIPFamily(podIPs []corev1.PodIP) IPFamily {
}
}

func createPod(client k8sclient.Interface, ns, generateName string) ([]corev1.PodIP, error) {
func createPod(client k8sclient.Interface, ns, generateName string) (*corev1.Pod, error) {
pod := e2epod.NewAgnhostPod(ns, "", nil, nil, nil)
pod.ObjectMeta.GenerateName = generateName
execPod, err := client.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
expectNoError(err, "failed to create new pod in namespace: %s", ns)
var podIPs []corev1.PodIP
var retrievedPod *corev1.Pod
err = wait.PollImmediate(poll, 2*time.Minute, func() (bool, error) {
retrievedPod, err := client.CoreV1().Pods(execPod.Namespace).Get(context.TODO(), execPod.Name, metav1.GetOptions{})
retrievedPod, err = client.CoreV1().Pods(execPod.Namespace).Get(context.TODO(), execPod.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
podIPs = retrievedPod.Status.PodIPs
return retrievedPod.Status.Phase == corev1.PodRunning, nil
})
return podIPs, err
return retrievedPod, err
}

// SubnetIPs enumerates all IP addresses in an IP subnet (starting with the provided IP address and including the broadcast address).
Expand Down