Merge pull request #114600 from aojea/e2e_external_nodeport

e2e: don't assume NodeIP are reachable
This commit is contained in:
Kubernetes Prow Robot 2022-12-20 04:53:43 -08:00 committed by GitHub
commit 05a0ce83a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -156,9 +156,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
ns1 := f.Namespace.Name // LB1 in ns1 on TCP ns1 := f.Namespace.Name // LB1 in ns1 on TCP
framework.Logf("namespace for TCP test: %s", ns1) framework.Logf("namespace for TCP test: %s", ns1)
nodeIP, err := getRandomNodeIP(ctx, cs)
framework.ExpectNoError(err, "Could not obtain a valid Node IP")
ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1) ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1)
tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName) tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName)
tcpService, err := tcpJig.CreateTCPService(ctx, nil) tcpService, err := tcpJig.CreateTCPService(ctx, nil)
@ -188,9 +185,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = tcpJig.CheckServiceReachability(ctx, tcpService, execPod) err = tcpJig.CheckServiceReachability(ctx, tcpService, execPod)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("hitting the TCP service's NodePort")
e2eservice.TestReachableHTTP(ctx, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
// Change the services to LoadBalancer. // Change the services to LoadBalancer.
// Here we test that LoadBalancers can receive static IP addresses. This isn't // Here we test that LoadBalancers can receive static IP addresses. This isn't
@ -261,9 +255,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = tcpJig.CheckServiceReachability(ctx, tcpService, execPod) err = tcpJig.CheckServiceReachability(ctx, tcpService, execPod)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("hitting the TCP service's NodePort")
e2eservice.TestReachableHTTP(ctx, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the TCP service's LoadBalancer") ginkgo.By("hitting the TCP service's LoadBalancer")
e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerLagTimeout) e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerLagTimeout)
@ -282,12 +273,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
} }
framework.Logf("TCP node port: %d", tcpNodePort) framework.Logf("TCP node port: %d", tcpNodePort)
ginkgo.By("hitting the TCP service's new NodePort")
e2eservice.TestReachableHTTP(ctx, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("checking the old TCP NodePort is closed")
testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the TCP service's LoadBalancer") ginkgo.By("hitting the TCP service's LoadBalancer")
e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerLagTimeout) e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerLagTimeout)
@ -312,9 +297,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
framework.Logf("service port TCP: %d", svcPort) framework.Logf("service port TCP: %d", svcPort)
ginkgo.By("hitting the TCP service's NodePort")
e2eservice.TestReachableHTTP(ctx, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the TCP service's LoadBalancer") ginkgo.By("hitting the TCP service's LoadBalancer")
e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerCreateTimeout) e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerCreateTimeout)
@ -322,9 +304,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = tcpJig.Scale(ctx, 0) err = tcpJig.Scale(ctx, 0)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort")
testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer") ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer")
testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
@ -332,9 +311,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = tcpJig.Scale(ctx, 1) err = tcpJig.Scale(ctx, 1)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("hitting the TCP service's NodePort")
e2eservice.TestReachableHTTP(ctx, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the TCP service's LoadBalancer") ginkgo.By("hitting the TCP service's LoadBalancer")
e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerCreateTimeout) e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerCreateTimeout)
@ -352,9 +328,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
_, err = tcpJig.WaitForLoadBalancerDestroy(ctx, tcpIngressIP, svcPort, loadBalancerCreateTimeout) _, err = tcpJig.WaitForLoadBalancerDestroy(ctx, tcpIngressIP, svcPort, loadBalancerCreateTimeout)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("checking the TCP NodePort is closed")
testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("checking the TCP LoadBalancer is closed") ginkgo.By("checking the TCP LoadBalancer is closed")
testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
}) })
@ -373,9 +346,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
ns2 := f.Namespace.Name // LB1 in ns2 on TCP ns2 := f.Namespace.Name // LB1 in ns2 on TCP
framework.Logf("namespace for TCP test: %s", ns2) framework.Logf("namespace for TCP test: %s", ns2)
nodeIP, err := getRandomNodeIP(ctx, cs)
framework.ExpectNoError(err, "Could not obtain a valid Node IP")
ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2) ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2)
udpJig := e2eservice.NewTestJig(cs, ns2, serviceName) udpJig := e2eservice.NewTestJig(cs, ns2, serviceName)
udpService, err := udpJig.CreateUDPService(ctx, nil) udpService, err := udpJig.CreateUDPService(ctx, nil)
@ -405,9 +375,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = udpJig.CheckServiceReachability(ctx, udpService, execPod) err = udpJig.CheckServiceReachability(ctx, udpService, execPod)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("hitting the UDP service's NodePort")
testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
// Change the services to LoadBalancer. // Change the services to LoadBalancer.
// Here we test that LoadBalancers can receive static IP addresses. This isn't // Here we test that LoadBalancers can receive static IP addresses. This isn't
@ -471,9 +438,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = udpJig.CheckServiceReachability(ctx, udpService, execPod) err = udpJig.CheckServiceReachability(ctx, udpService, execPod)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("hitting the UDP service's NodePort")
testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the UDP service's LoadBalancer") ginkgo.By("hitting the UDP service's LoadBalancer")
testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
@ -492,11 +456,8 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
} }
framework.Logf("UDP node port: %d", udpNodePort) framework.Logf("UDP node port: %d", udpNodePort)
ginkgo.By("hitting the UDP service's new NodePort") err = udpJig.CheckServiceReachability(ctx, udpService, execPod)
testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) framework.ExpectNoError(err)
ginkgo.By("checking the old UDP NodePort is closed")
testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the UDP service's LoadBalancer") ginkgo.By("hitting the UDP service's LoadBalancer")
testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
@ -523,7 +484,8 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
framework.Logf("service port UDP: %d", svcPort) framework.Logf("service port UDP: %d", svcPort)
ginkgo.By("hitting the UDP service's NodePort") ginkgo.By("hitting the UDP service's NodePort")
testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) err = udpJig.CheckServiceReachability(ctx, udpService, execPod)
framework.ExpectNoError(err)
ginkgo.By("hitting the UDP service's LoadBalancer") ginkgo.By("hitting the UDP service's LoadBalancer")
testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
@ -532,9 +494,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
err = udpJig.Scale(ctx, 0) err = udpJig.Scale(ctx, 0)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort")
testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer") ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer")
testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
@ -543,7 +502,8 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("hitting the UDP service's NodePort") ginkgo.By("hitting the UDP service's NodePort")
testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) err = udpJig.CheckServiceReachability(ctx, udpService, execPod)
framework.ExpectNoError(err)
ginkgo.By("hitting the UDP service's LoadBalancer") ginkgo.By("hitting the UDP service's LoadBalancer")
testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
@ -562,9 +522,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
_, err = udpJig.WaitForLoadBalancerDestroy(ctx, udpIngressIP, svcPort, loadBalancerCreateTimeout) _, err = udpJig.WaitForLoadBalancerDestroy(ctx, udpIngressIP, svcPort, loadBalancerCreateTimeout)
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("checking the UDP NodePort is closed")
testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("checking the UDP LoadBalancer is closed") ginkgo.By("checking the UDP LoadBalancer is closed")
testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
}) })
@ -956,9 +913,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
ns1 := f.Namespace.Name // LB1 in ns1 on TCP ns1 := f.Namespace.Name // LB1 in ns1 on TCP
framework.Logf("namespace for TCP test: %s", ns1) framework.Logf("namespace for TCP test: %s", ns1)
nodeIP, err := e2enode.PickIP(ctx, cs) // for later
framework.ExpectNoError(err)
ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1) ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1)
tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName) tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName)
tcpService, err := tcpJig.CreateTCPService(ctx, nil) tcpService, err := tcpJig.CreateTCPService(ctx, nil)
@ -1058,9 +1012,6 @@ var _ = common.SIGDescribe("LoadBalancers", func() {
} }
framework.Logf("TCP node port: %d", tcpNodePort) framework.Logf("TCP node port: %d", tcpNodePort)
ginkgo.By("hitting the TCP service's new NodePort")
e2eservice.TestReachableHTTP(ctx, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
ginkgo.By("hitting the TCP service's LoadBalancer") ginkgo.By("hitting the TCP service's LoadBalancer")
e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerLagTimeout) e2eservice.TestReachableHTTP(ctx, tcpIngressIP, svcPort, loadBalancerLagTimeout)
}) })
@ -1939,28 +1890,3 @@ func testRollingUpdateLBConnectivityDisruption(ctx context.Context, f *framework
// assert that the load balancer address is still reachable after the rolling updates are finished // assert that the load balancer address is still reachable after the rolling updates are finished
e2eservice.TestReachableHTTP(ctx, lbNameOrAddress, svcPort, timeout) e2eservice.TestReachableHTTP(ctx, lbNameOrAddress, svcPort, timeout)
} }
// getRandomNodeIP gets an IP address from a random worker node.
// These tests exercise traffic coming from outside the traffic,
// so it prefers ExternalIPs over InternalIPs.
func getRandomNodeIP(ctx context.Context, cs clientset.Interface) (string, error) {
family := v1.IPv4Protocol
if framework.TestContext.ClusterIsIPv6() {
family = v1.IPv6Protocol
}
node, err := e2enode.GetRandomReadySchedulableNode(ctx, cs)
if err != nil {
return "", err
}
ips := e2enode.GetAddressesByTypeAndFamily(node, v1.NodeExternalIP, family)
if len(ips) > 0 {
return ips[0], nil
}
ips = e2enode.GetAddressesByTypeAndFamily(node, v1.NodeInternalIP, family)
if len(ips) > 0 {
return ips[0], nil
}
return "", fmt.Errorf("node %v does not contain any valid IP", node)
}