diff --git a/test/e2e/framework/network/BUILD b/test/e2e/framework/network/BUILD index 0be68270156..a391b9ccaf1 100644 --- a/test/e2e/framework/network/BUILD +++ b/test/e2e/framework/network/BUILD @@ -24,6 +24,7 @@ go_library( "//test/e2e/framework/ssh:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/test/e2e/framework/network/utils.go b/test/e2e/framework/network/utils.go index 37d74d737bc..e89d5c79ae0 100644 --- a/test/e2e/framework/network/utils.go +++ b/test/e2e/framework/network/utils.go @@ -46,6 +46,7 @@ import ( e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" imageutils "k8s.io/kubernetes/test/utils/image" + netutils "k8s.io/utils/net" ) const ( @@ -99,6 +100,11 @@ func EnableSCTP(config *NetworkingTestConfig) { config.SCTPEnabled = true } +// EnableDualStack create Dual Stack services +func EnableDualStack(config *NetworkingTestConfig) { + config.DualStackEnabled = true +} + // UseHostNetwork run the test container with HostNetwork=true. func UseHostNetwork(config *NetworkingTestConfig) { config.HostNetwork = true @@ -161,6 +167,8 @@ type NetworkingTestConfig struct { // if the test pods are listening on sctp port. We need this as sctp tests // are marked as disruptive as they may load the sctp module. SCTPEnabled bool + // DualStackEnabled enables dual stack on services + DualStackEnabled bool // EndpointPods are the pods belonging to the Service created by this // test config. Each invocation of `setup` creates a service with // 1 pod per node running the netexecImage. @@ -173,17 +181,23 @@ type NetworkingTestConfig struct { // SessionAffinityService is a Service with SessionAffinity=ClientIP // spanning over all endpointPods. SessionAffinityService *v1.Service - // ExternalAddrs is a list of external IPs of nodes in the cluster. + // ExternalAddr is a external IP of a node in the cluster. ExternalAddr string + // SecondaryExternalAddr is a external IP of the secondary IP family of a node in the cluster. + SecondaryExternalAddr string // Nodes is a list of nodes in the cluster. Nodes []v1.Node // MaxTries is the number of retries tolerated for tests run against // endpoints and services created by this config. MaxTries int - // The ClusterIP of the Service reated by this test config. + // The ClusterIP of the Service created by this test config. ClusterIP string + // The SecondaryClusterIP of the Service created by this test config. + SecondaryClusterIP string // External ip of first node for use in nodePort testing. NodeIP string + // External ip of other IP family of first node for use in nodePort testing. + SecondaryNodeIP string // The http/udp/sctp nodePorts of the Service. NodeHTTPPort int NodeUDPPort int @@ -649,6 +663,10 @@ func (config *NetworkingTestConfig) createNodePortServiceSpec(svcName string, se if config.SCTPEnabled { res.Spec.Ports = append(res.Spec.Ports, v1.ServicePort{Port: ClusterSCTPPort, Name: "sctp", Protocol: v1.ProtocolSCTP, TargetPort: intstr.FromInt(EndpointSCTPPort)}) } + if config.DualStackEnabled { + requireDual := v1.IPFamilyPolicyRequireDualStack + res.Spec.IPFamilyPolicy = &requireDual + } return res } @@ -733,7 +751,6 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) { framework.ExpectNoError(framework.WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute)) nodeList, err := e2enode.GetReadySchedulableNodes(config.f.ClientSet) framework.ExpectNoError(err) - config.ExternalAddr = e2enode.FirstAddress(nodeList, v1.NodeExternalIP) e2eskipper.SkipUnlessNodeCountIsAtLeast(2) config.Nodes = nodeList.Items @@ -754,11 +771,34 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) { continue } } + + // obtain the ClusterIP config.ClusterIP = config.NodePortService.Spec.ClusterIP + if config.DualStackEnabled { + config.SecondaryClusterIP = config.NodePortService.Spec.ClusterIPs[1] + } + + // Obtain the primary IP family of the Cluster based on the first ClusterIP + family := v1.IPv4Protocol + secondaryFamily := v1.IPv6Protocol + if netutils.IsIPv6String(config.ClusterIP) { + family = v1.IPv6Protocol + secondaryFamily = v1.IPv4Protocol + } + // Get Node IPs from the cluster, ExternalIPs take precedence + config.ExternalAddr = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, family) if config.ExternalAddr != "" { config.NodeIP = config.ExternalAddr } else { - config.NodeIP = e2enode.FirstAddress(nodeList, v1.NodeInternalIP) + config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, family) + } + if config.DualStackEnabled { + config.SecondaryExternalAddr = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, secondaryFamily) + if config.SecondaryExternalAddr != "" { + config.SecondaryNodeIP = config.SecondaryExternalAddr + } else { + config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, secondaryFamily) + } } ginkgo.By("Waiting for NodePort service to expose endpoint") diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index d5a827e7188..c44c5bcd097 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -20,11 +20,12 @@ import ( "context" "encoding/json" "fmt" - netutil "k8s.io/utils/net" "net" "strings" "time" + netutil "k8s.io/utils/net" + "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -249,6 +250,17 @@ func GetInternalIP(node *v1.Node) (string, error) { return host, nil } +// FirstAddressByTypeAndFamily returns the first address of the given type and family of each node. +func FirstAddressByTypeAndFamily(nodelist *v1.NodeList, addrType v1.NodeAddressType, family v1.IPFamily) string { + for _, n := range nodelist.Items { + addresses := GetAddressesByTypeAndFamily(&n, addrType, family) + if len(addresses) > 0 { + return addresses[0] + } + } + return "" +} + // GetAddressesByTypeAndFamily returns a list of addresses of the given addressType for the given node // and filtered by IPFamily func GetAddressesByTypeAndFamily(node *v1.Node, addressType v1.NodeAddressType, family v1.IPFamily) (ips []string) { diff --git a/test/e2e/network/dual_stack.go b/test/e2e/network/dual_stack.go index 9a5fbe88b1a..0f7209ee7a8 100644 --- a/test/e2e/network/dual_stack.go +++ b/test/e2e/network/dual_stack.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net" + "strings" "time" "github.com/onsi/ginkgo" @@ -32,6 +33,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" + e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" imageutils "k8s.io/kubernetes/test/utils/image" @@ -435,6 +437,148 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { }) // TODO (khenidak add slice validation logic, since endpoint controller only operates // on primary ClusterIP + + // Service Granular Checks as in k8s.io/kubernetes/test/e2e/network/networking.go + // but using the secondary IP, so we run the same tests for each ClusterIP family + ginkgo.Describe("Granular Checks: Services Secondary IP Family", func() { + + ginkgo.It("should function for pod-Service: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for pod-Service: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + // Once basic tests checking for the sctp module not to be loaded are implemented, this + // needs to be marked as [Disruptive] + ginkgo.It("should function for pod-Service: sctp [Feature:SCTPConnectivity][Disruptive]", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.EnableSCTP) + ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort)) + config.DialFromTestContainer("sctp", config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeSCTPPort)) + config.DialFromTestContainer("sctp", config.SecondaryNodeIP, config.NodeSCTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for node-Service: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork) + ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromNode("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeHTTPPort)) + config.DialFromNode("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for node-Service: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork) + ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromNode("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeUDPPort)) + config.DialFromNode("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for endpoint-Service: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromEndpointContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeHTTPPort)) + config.DialFromEndpointContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for endpoint-Service: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromEndpointContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeUDPPort)) + config.DialFromEndpointContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should update endpoints: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + config.DeleteNetProxyPod() + + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) + }) + + ginkgo.It("should update endpoints: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + config.DeleteNetProxyPod() + + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) + }) + + // [LinuxOnly]: Windows does not support session affinity. + ginkgo.It("should function for client IP based session affinity: http [LinuxOnly]", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort)) + + // Check if number of endpoints returned are exactly one. + eps, err := config.GetEndpointsFromTestContainer("http", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort, e2enetwork.SessionAffinityChecks) + if err != nil { + framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err) + } + if len(eps) == 0 { + framework.Failf("Unexpected no endpoints return") + } + if len(eps) > 1 { + framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps) + } + }) + + // [LinuxOnly]: Windows does not support session affinity. + ginkgo.It("should function for client IP based session affinity: udp [LinuxOnly]", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort)) + + // Check if number of endpoints returned are exactly one. + eps, err := config.GetEndpointsFromTestContainer("udp", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort, e2enetwork.SessionAffinityChecks) + if err != nil { + framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err) + } + if len(eps) == 0 { + framework.Failf("Unexpected no endpoints return") + } + if len(eps) > 1 { + framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps) + } + }) + + ginkgo.It("should be able to handle large requests: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + message := strings.Repeat("42", 1000) + config.DialEchoFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, message) + }) + + ginkgo.It("should be able to handle large requests: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + message := "n" + strings.Repeat("o", 1999) + config.DialEchoFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, message) + }) + }) }) func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) {