mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 10:51:29 +00:00
Merge pull request #96484 from aojea/e2etest
add e2e test for dual-stack secondary service IPs
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
379ed6644d
@ -24,6 +24,7 @@ go_library(
|
||||
"//test/e2e/framework/ssh:go_default_library",
|
||||
"//test/utils/image:go_default_library",
|
||||
"//vendor/github.com/onsi/ginkgo:go_default_library",
|
||||
"//vendor/k8s.io/utils/net:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -46,6 +46,7 @@ import (
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
netutils "k8s.io/utils/net"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -99,6 +100,11 @@ func EnableSCTP(config *NetworkingTestConfig) {
|
||||
config.SCTPEnabled = true
|
||||
}
|
||||
|
||||
// EnableDualStack create Dual Stack services
|
||||
func EnableDualStack(config *NetworkingTestConfig) {
|
||||
config.DualStackEnabled = true
|
||||
}
|
||||
|
||||
// UseHostNetwork run the test container with HostNetwork=true.
|
||||
func UseHostNetwork(config *NetworkingTestConfig) {
|
||||
config.HostNetwork = true
|
||||
@ -161,6 +167,8 @@ type NetworkingTestConfig struct {
|
||||
// if the test pods are listening on sctp port. We need this as sctp tests
|
||||
// are marked as disruptive as they may load the sctp module.
|
||||
SCTPEnabled bool
|
||||
// DualStackEnabled enables dual stack on services
|
||||
DualStackEnabled bool
|
||||
// EndpointPods are the pods belonging to the Service created by this
|
||||
// test config. Each invocation of `setup` creates a service with
|
||||
// 1 pod per node running the netexecImage.
|
||||
@ -173,17 +181,21 @@ type NetworkingTestConfig struct {
|
||||
// SessionAffinityService is a Service with SessionAffinity=ClientIP
|
||||
// spanning over all endpointPods.
|
||||
SessionAffinityService *v1.Service
|
||||
// ExternalAddrs is a list of external IPs of nodes in the cluster.
|
||||
ExternalAddr string
|
||||
// Nodes is a list of nodes in the cluster.
|
||||
Nodes []v1.Node
|
||||
// MaxTries is the number of retries tolerated for tests run against
|
||||
// endpoints and services created by this config.
|
||||
MaxTries int
|
||||
// The ClusterIP of the Service reated by this test config.
|
||||
// The ClusterIP of the Service created by this test config.
|
||||
ClusterIP string
|
||||
// External ip of first node for use in nodePort testing.
|
||||
// The SecondaryClusterIP of the Service created by this test config.
|
||||
SecondaryClusterIP string
|
||||
// NodeIP it's an ExternalIP if the node has one,
|
||||
// or an InternalIP if not, for use in nodePort testing.
|
||||
NodeIP string
|
||||
// SecondaryNodeIP it's an ExternalIP of the secondary IP family if the node has one,
|
||||
// or an InternalIP if not, for usein nodePort testing.
|
||||
SecondaryNodeIP string
|
||||
// The http/udp/sctp nodePorts of the Service.
|
||||
NodeHTTPPort int
|
||||
NodeUDPPort int
|
||||
@ -649,6 +661,10 @@ func (config *NetworkingTestConfig) createNodePortServiceSpec(svcName string, se
|
||||
if config.SCTPEnabled {
|
||||
res.Spec.Ports = append(res.Spec.Ports, v1.ServicePort{Port: ClusterSCTPPort, Name: "sctp", Protocol: v1.ProtocolSCTP, TargetPort: intstr.FromInt(EndpointSCTPPort)})
|
||||
}
|
||||
if config.DualStackEnabled {
|
||||
requireDual := v1.IPFamilyPolicyRequireDualStack
|
||||
res.Spec.IPFamilyPolicy = &requireDual
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
@ -740,7 +756,6 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
|
||||
framework.ExpectNoError(framework.WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute))
|
||||
nodeList, err := e2enode.GetReadySchedulableNodes(config.f.ClientSet)
|
||||
framework.ExpectNoError(err)
|
||||
config.ExternalAddr = e2enode.FirstAddress(nodeList, v1.NodeExternalIP)
|
||||
|
||||
e2eskipper.SkipUnlessNodeCountIsAtLeast(2)
|
||||
config.Nodes = nodeList.Items
|
||||
@ -761,11 +776,32 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// obtain the ClusterIP
|
||||
config.ClusterIP = config.NodePortService.Spec.ClusterIP
|
||||
if config.ExternalAddr != "" {
|
||||
config.NodeIP = config.ExternalAddr
|
||||
} else {
|
||||
config.NodeIP = e2enode.FirstAddress(nodeList, v1.NodeInternalIP)
|
||||
if config.DualStackEnabled {
|
||||
config.SecondaryClusterIP = config.NodePortService.Spec.ClusterIPs[1]
|
||||
}
|
||||
|
||||
// Obtain the primary IP family of the Cluster based on the first ClusterIP
|
||||
// TODO: Eventually we should just be getting these from Spec.IPFamilies
|
||||
// but for now that would only if the feature gate is enabled.
|
||||
family := v1.IPv4Protocol
|
||||
secondaryFamily := v1.IPv6Protocol
|
||||
if netutils.IsIPv6String(config.ClusterIP) {
|
||||
family = v1.IPv6Protocol
|
||||
secondaryFamily = v1.IPv4Protocol
|
||||
}
|
||||
// Get Node IPs from the cluster, ExternalIPs take precedence
|
||||
config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, family)
|
||||
if config.NodeIP == "" {
|
||||
config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, family)
|
||||
}
|
||||
if config.DualStackEnabled {
|
||||
config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, secondaryFamily)
|
||||
if config.SecondaryNodeIP == "" {
|
||||
config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, secondaryFamily)
|
||||
}
|
||||
}
|
||||
|
||||
ginkgo.By("Waiting for NodePort service to expose endpoint")
|
||||
|
||||
@ -20,7 +20,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
netutil "k8s.io/utils/net"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
@ -42,6 +41,7 @@ import (
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
clientretry "k8s.io/client-go/util/retry"
|
||||
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
|
||||
netutil "k8s.io/utils/net"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -249,6 +249,17 @@ func GetInternalIP(node *v1.Node) (string, error) {
|
||||
return host, nil
|
||||
}
|
||||
|
||||
// FirstAddressByTypeAndFamily returns the first address that matches the given type and family of the list of nodes
|
||||
func FirstAddressByTypeAndFamily(nodelist *v1.NodeList, addrType v1.NodeAddressType, family v1.IPFamily) string {
|
||||
for _, n := range nodelist.Items {
|
||||
addresses := GetAddressesByTypeAndFamily(&n, addrType, family)
|
||||
if len(addresses) > 0 {
|
||||
return addresses[0]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetAddressesByTypeAndFamily returns a list of addresses of the given addressType for the given node
|
||||
// and filtered by IPFamily
|
||||
func GetAddressesByTypeAndFamily(node *v1.Node, addressType v1.NodeAddressType, family v1.IPFamily) (ips []string) {
|
||||
|
||||
@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
@ -32,6 +33,7 @@ import (
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
|
||||
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
@ -435,6 +437,147 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() {
|
||||
})
|
||||
// TODO (khenidak add slice validation logic, since endpoint controller only operates
|
||||
// on primary ClusterIP
|
||||
|
||||
// Service Granular Checks as in k8s.io/kubernetes/test/e2e/network/networking.go
|
||||
// but using the secondary IP, so we run the same tests for each ClusterIP family
|
||||
ginkgo.Describe("Granular Checks: Services Secondary IP Family", func() {
|
||||
|
||||
ginkgo.It("should function for pod-Service: http", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
|
||||
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeHTTPPort))
|
||||
config.DialFromTestContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should function for pod-Service: udp", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
|
||||
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeUDPPort))
|
||||
config.DialFromTestContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
// [Disruptive] because it conflicts with tests that call CheckSCTPModuleLoadedOnNodes
|
||||
ginkgo.It("should function for pod-Service: sctp [Feature:SCTPConnectivity][Disruptive]", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.EnableSCTP)
|
||||
ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort))
|
||||
config.DialFromTestContainer("sctp", config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeSCTPPort))
|
||||
config.DialFromTestContainer("sctp", config.SecondaryNodeIP, config.NodeSCTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should function for node-Service: http", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork)
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
|
||||
config.DialFromNode("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeHTTPPort))
|
||||
config.DialFromNode("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should function for node-Service: udp", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork)
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
|
||||
config.DialFromNode("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeUDPPort))
|
||||
config.DialFromNode("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should function for endpoint-Service: http", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
|
||||
config.DialFromEndpointContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeHTTPPort))
|
||||
config.DialFromEndpointContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should function for endpoint-Service: udp", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
|
||||
config.DialFromEndpointContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeUDPPort))
|
||||
config.DialFromEndpointContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should update endpoints: http", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
|
||||
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
config.DeleteNetProxyPod()
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
|
||||
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
ginkgo.It("should update endpoints: udp", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
|
||||
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
|
||||
|
||||
config.DeleteNetProxyPod()
|
||||
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
|
||||
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames())
|
||||
})
|
||||
|
||||
// [LinuxOnly]: Windows does not support session affinity.
|
||||
ginkgo.It("should function for client IP based session affinity: http [LinuxOnly]", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort))
|
||||
|
||||
// Check if number of endpoints returned are exactly one.
|
||||
eps, err := config.GetEndpointsFromTestContainer("http", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort, e2enetwork.SessionAffinityChecks)
|
||||
if err != nil {
|
||||
framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err)
|
||||
}
|
||||
if len(eps) == 0 {
|
||||
framework.Failf("Unexpected no endpoints return")
|
||||
}
|
||||
if len(eps) > 1 {
|
||||
framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps)
|
||||
}
|
||||
})
|
||||
|
||||
// [LinuxOnly]: Windows does not support session affinity.
|
||||
ginkgo.It("should function for client IP based session affinity: udp [LinuxOnly]", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort))
|
||||
|
||||
// Check if number of endpoints returned are exactly one.
|
||||
eps, err := config.GetEndpointsFromTestContainer("udp", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort, e2enetwork.SessionAffinityChecks)
|
||||
if err != nil {
|
||||
framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err)
|
||||
}
|
||||
if len(eps) == 0 {
|
||||
framework.Failf("Unexpected no endpoints return")
|
||||
}
|
||||
if len(eps) > 1 {
|
||||
framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps)
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should be able to handle large requests: http", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
|
||||
message := strings.Repeat("42", 1000)
|
||||
config.DialEchoFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, message)
|
||||
})
|
||||
|
||||
ginkgo.It("should be able to handle large requests: udp", func() {
|
||||
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
|
||||
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
|
||||
message := "n" + strings.Repeat("o", 1999)
|
||||
config.DialEchoFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, message)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user