diff --git a/test/e2e/framework/network/BUILD b/test/e2e/framework/network/BUILD index 0be68270156..a391b9ccaf1 100644 --- a/test/e2e/framework/network/BUILD +++ b/test/e2e/framework/network/BUILD @@ -24,6 +24,7 @@ go_library( "//test/e2e/framework/ssh:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/test/e2e/framework/network/utils.go b/test/e2e/framework/network/utils.go index 338604c271e..0d47cfeac7a 100644 --- a/test/e2e/framework/network/utils.go +++ b/test/e2e/framework/network/utils.go @@ -46,6 +46,7 @@ import ( e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" imageutils "k8s.io/kubernetes/test/utils/image" + netutils "k8s.io/utils/net" ) const ( @@ -99,6 +100,11 @@ func EnableSCTP(config *NetworkingTestConfig) { config.SCTPEnabled = true } +// EnableDualStack create Dual Stack services +func EnableDualStack(config *NetworkingTestConfig) { + config.DualStackEnabled = true +} + // UseHostNetwork run the test container with HostNetwork=true. func UseHostNetwork(config *NetworkingTestConfig) { config.HostNetwork = true @@ -161,6 +167,8 @@ type NetworkingTestConfig struct { // if the test pods are listening on sctp port. We need this as sctp tests // are marked as disruptive as they may load the sctp module. SCTPEnabled bool + // DualStackEnabled enables dual stack on services + DualStackEnabled bool // EndpointPods are the pods belonging to the Service created by this // test config. Each invocation of `setup` creates a service with // 1 pod per node running the netexecImage. @@ -173,17 +181,21 @@ type NetworkingTestConfig struct { // SessionAffinityService is a Service with SessionAffinity=ClientIP // spanning over all endpointPods. SessionAffinityService *v1.Service - // ExternalAddrs is a list of external IPs of nodes in the cluster. - ExternalAddr string // Nodes is a list of nodes in the cluster. Nodes []v1.Node // MaxTries is the number of retries tolerated for tests run against // endpoints and services created by this config. MaxTries int - // The ClusterIP of the Service reated by this test config. + // The ClusterIP of the Service created by this test config. ClusterIP string - // External ip of first node for use in nodePort testing. + // The SecondaryClusterIP of the Service created by this test config. + SecondaryClusterIP string + // NodeIP it's an ExternalIP if the node has one, + // or an InternalIP if not, for use in nodePort testing. NodeIP string + // SecondaryNodeIP it's an ExternalIP of the secondary IP family if the node has one, + // or an InternalIP if not, for usein nodePort testing. + SecondaryNodeIP string // The http/udp/sctp nodePorts of the Service. NodeHTTPPort int NodeUDPPort int @@ -649,6 +661,10 @@ func (config *NetworkingTestConfig) createNodePortServiceSpec(svcName string, se if config.SCTPEnabled { res.Spec.Ports = append(res.Spec.Ports, v1.ServicePort{Port: ClusterSCTPPort, Name: "sctp", Protocol: v1.ProtocolSCTP, TargetPort: intstr.FromInt(EndpointSCTPPort)}) } + if config.DualStackEnabled { + requireDual := v1.IPFamilyPolicyRequireDualStack + res.Spec.IPFamilyPolicy = &requireDual + } return res } @@ -740,7 +756,6 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) { framework.ExpectNoError(framework.WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute)) nodeList, err := e2enode.GetReadySchedulableNodes(config.f.ClientSet) framework.ExpectNoError(err) - config.ExternalAddr = e2enode.FirstAddress(nodeList, v1.NodeExternalIP) e2eskipper.SkipUnlessNodeCountIsAtLeast(2) config.Nodes = nodeList.Items @@ -761,11 +776,32 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) { continue } } + + // obtain the ClusterIP config.ClusterIP = config.NodePortService.Spec.ClusterIP - if config.ExternalAddr != "" { - config.NodeIP = config.ExternalAddr - } else { - config.NodeIP = e2enode.FirstAddress(nodeList, v1.NodeInternalIP) + if config.DualStackEnabled { + config.SecondaryClusterIP = config.NodePortService.Spec.ClusterIPs[1] + } + + // Obtain the primary IP family of the Cluster based on the first ClusterIP + // TODO: Eventually we should just be getting these from Spec.IPFamilies + // but for now that would only if the feature gate is enabled. + family := v1.IPv4Protocol + secondaryFamily := v1.IPv6Protocol + if netutils.IsIPv6String(config.ClusterIP) { + family = v1.IPv6Protocol + secondaryFamily = v1.IPv4Protocol + } + // Get Node IPs from the cluster, ExternalIPs take precedence + config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, family) + if config.NodeIP == "" { + config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, family) + } + if config.DualStackEnabled { + config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, secondaryFamily) + if config.SecondaryNodeIP == "" { + config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, secondaryFamily) + } } ginkgo.By("Waiting for NodePort service to expose endpoint") diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index d5a827e7188..ce33c29cb5d 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - netutil "k8s.io/utils/net" "net" "strings" "time" @@ -42,6 +41,7 @@ import ( clientset "k8s.io/client-go/kubernetes" clientretry "k8s.io/client-go/util/retry" e2elog "k8s.io/kubernetes/test/e2e/framework/log" + netutil "k8s.io/utils/net" ) const ( @@ -249,6 +249,17 @@ func GetInternalIP(node *v1.Node) (string, error) { return host, nil } +// FirstAddressByTypeAndFamily returns the first address that matches the given type and family of the list of nodes +func FirstAddressByTypeAndFamily(nodelist *v1.NodeList, addrType v1.NodeAddressType, family v1.IPFamily) string { + for _, n := range nodelist.Items { + addresses := GetAddressesByTypeAndFamily(&n, addrType, family) + if len(addresses) > 0 { + return addresses[0] + } + } + return "" +} + // GetAddressesByTypeAndFamily returns a list of addresses of the given addressType for the given node // and filtered by IPFamily func GetAddressesByTypeAndFamily(node *v1.Node, addressType v1.NodeAddressType, family v1.IPFamily) (ips []string) { diff --git a/test/e2e/network/dual_stack.go b/test/e2e/network/dual_stack.go index 9a5fbe88b1a..19f7727554e 100644 --- a/test/e2e/network/dual_stack.go +++ b/test/e2e/network/dual_stack.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net" + "strings" "time" "github.com/onsi/ginkgo" @@ -32,6 +33,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" + e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" imageutils "k8s.io/kubernetes/test/utils/image" @@ -435,6 +437,147 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { }) // TODO (khenidak add slice validation logic, since endpoint controller only operates // on primary ClusterIP + + // Service Granular Checks as in k8s.io/kubernetes/test/e2e/network/networking.go + // but using the secondary IP, so we run the same tests for each ClusterIP family + ginkgo.Describe("Granular Checks: Services Secondary IP Family", func() { + + ginkgo.It("should function for pod-Service: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for pod-Service: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + // [Disruptive] because it conflicts with tests that call CheckSCTPModuleLoadedOnNodes + ginkgo.It("should function for pod-Service: sctp [Feature:SCTPConnectivity][Disruptive]", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.EnableSCTP) + ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort)) + config.DialFromTestContainer("sctp", config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeSCTPPort)) + config.DialFromTestContainer("sctp", config.SecondaryNodeIP, config.NodeSCTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for node-Service: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork) + ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromNode("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeHTTPPort)) + config.DialFromNode("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for node-Service: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork) + ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromNode("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeUDPPort)) + config.DialFromNode("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for endpoint-Service: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromEndpointContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeHTTPPort)) + config.DialFromEndpointContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should function for endpoint-Service: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromEndpointContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeUDPPort)) + config.DialFromEndpointContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + }) + + ginkgo.It("should update endpoints: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + + config.DeleteNetProxyPod() + + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) + }) + + ginkgo.It("should update endpoints: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + + config.DeleteNetProxyPod() + + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) + }) + + // [LinuxOnly]: Windows does not support session affinity. + ginkgo.It("should function for client IP based session affinity: http [LinuxOnly]", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort)) + + // Check if number of endpoints returned are exactly one. + eps, err := config.GetEndpointsFromTestContainer("http", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort, e2enetwork.SessionAffinityChecks) + if err != nil { + framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err) + } + if len(eps) == 0 { + framework.Failf("Unexpected no endpoints return") + } + if len(eps) > 1 { + framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps) + } + }) + + // [LinuxOnly]: Windows does not support session affinity. + ginkgo.It("should function for client IP based session affinity: udp [LinuxOnly]", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort)) + + // Check if number of endpoints returned are exactly one. + eps, err := config.GetEndpointsFromTestContainer("udp", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort, e2enetwork.SessionAffinityChecks) + if err != nil { + framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err) + } + if len(eps) == 0 { + framework.Failf("Unexpected no endpoints return") + } + if len(eps) > 1 { + framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps) + } + }) + + ginkgo.It("should be able to handle large requests: http", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort)) + message := strings.Repeat("42", 1000) + config.DialEchoFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, message) + }) + + ginkgo.It("should be able to handle large requests: udp", func() { + config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort)) + message := "n" + strings.Repeat("o", 1999) + config.DialEchoFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, message) + }) + }) }) func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) {