From 56def96d9d50fe22e0115f8c396ad80f33047370 Mon Sep 17 00:00:00 2001 From: jay vyas Date: Sun, 4 Oct 2020 15:20:03 -0400 Subject: [PATCH 1/2] Update DialFromNode to return values as is done w/ other tests. Update comments to clarify missing probability check function (since they need to be updated anyways b/c of the return value introduced) --- test/e2e/common/networking.go | 10 ++++- test/e2e/framework/network/utils.go | 29 +++++++------ test/e2e/network/networking.go | 66 ++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/test/e2e/common/networking.go b/test/e2e/common/networking.go index f08520f9e9d..4774526015a 100644 --- a/test/e2e/common/networking.go +++ b/test/e2e/common/networking.go @@ -103,7 +103,10 @@ var _ = ginkgo.Describe("[sig-network] Networking", func() { framework.ConformanceIt("should function for node-pod communication: http [LinuxOnly] [NodeConformance]", func() { config := e2enetwork.NewCoreNetworkingTestConfig(f, true) for _, endpointPod := range config.EndpointPods { - config.DialFromNode("http", endpointPod.Status.PodIP, e2enetwork.EndpointHTTPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + err := config.DialFromNode("http", endpointPod.Status.PodIP, e2enetwork.EndpointHTTPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + if err != nil { + framework.Failf("Error dialing HTTP node to pod %v", err) + } } }) @@ -117,7 +120,10 @@ var _ = ginkgo.Describe("[sig-network] Networking", func() { framework.ConformanceIt("should function for node-pod communication: udp [LinuxOnly] [NodeConformance]", func() { config := e2enetwork.NewCoreNetworkingTestConfig(f, true) for _, endpointPod := range config.EndpointPods { - config.DialFromNode("udp", endpointPod.Status.PodIP, e2enetwork.EndpointUDPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + err := config.DialFromNode("udp", endpointPod.Status.PodIP, e2enetwork.EndpointUDPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + if err != nil { + framework.Failf("Error dialing UDP from node to pod: %v", err) + } } }) }) diff --git a/test/e2e/framework/network/utils.go b/test/e2e/framework/network/utils.go index aff1704b04f..49166d317c5 100644 --- a/test/e2e/framework/network/utils.go +++ b/test/e2e/framework/network/utils.go @@ -380,17 +380,20 @@ func (config *NetworkingTestConfig) GetHTTPCodeFromTestContainer(path, targetIP return code, nil } -// DialFromNode executes a tcp or udp request based on protocol via kubectl exec +// DialFromNode executes a tcp or udp curl/echo request based on protocol via kubectl exec // in a test container running with host networking. -// - minTries is the minimum number of curl attempts required before declaring -// success. Set to 0 if you'd like to return as soon as all endpoints respond -// at least once. -// - maxTries is the maximum number of curl attempts. If this many attempts pass -// and we don't see all expected endpoints, the test fails. -// maxTries == minTries will confirm that we see the expected endpoints and no -// more for maxTries. Use this if you want to eg: fail a readiness check on a -// pod and confirm it doesn't show up as an endpoint. -func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { +// - minTries is the minimum number of curl/echo attempts required before declaring +// success. If 0, then we return as soon as all endpoints succeed. +// - There is no logical change to test results if faillures happen AFTER endpoints have succeeded, +// hence over-padding minTries will NOT reverse a successfull result and is thus not very useful yet +// (See the TODO about checking probability, which isnt implemented yet). +// - maxTries is the maximum number of curl/echo attempts before an error is returned. The +// smaller this number is, the less 'slack' there is for declaring success. +// - if maxTries < expectedEps, this test is guaranteed to return an error, because all endpoints wont be hit. +// - maxTries == minTries will return as soon as all endpoints succeed (or fail once maxTries is reached without +// success on all endpoints). +// In general its prudent to have a high enough level of minTries to guarantee that all pods get a fair chance at receiving traffic. +func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) error { var cmd string if protocol == "udp" { cmd = fmt.Sprintf("echo hostName | nc -w 1 -u %s %d", targetIP, targetPort) @@ -424,8 +427,8 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ // Check against i+1 so we exit if minTries == maxTries. if eps.Equal(expectedEps) && i+1 >= minTries { - framework.Logf("Found all expected endpoints: %+v", eps.List()) - return + framework.Logf("Found all %+v expected endpoints: %+v", len(eps.List()), eps.List()) + return nil } framework.Logf("Waiting for %+v endpoints (expected=%+v, actual=%+v)", expectedEps.Difference(eps).List(), expectedEps.List(), eps.List()) @@ -435,7 +438,7 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ } config.diagnoseMissingEndpoints(eps) - framework.Failf("Failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", maxTries, cmd, eps, expectedEps) + return fmt.Errorf("failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", maxTries, cmd, eps, expectedEps) } // GetSelfURL executes a curl against the given path via kubectl exec into a diff --git a/test/e2e/network/networking.go b/test/e2e/network/networking.go index 0df3eefa4fd..69257b04c59 100644 --- a/test/e2e/network/networking.go +++ b/test/e2e/network/networking.go @@ -210,18 +210,29 @@ var _ = SIGDescribe("Networking", func() { ginkgo.It("should function for node-Service: http", func() { config := e2enetwork.NewNetworkingTestConfig(f, true, false) ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.NodeIP, config.ClusterIP, e2enetwork.ClusterHTTPPort)) - config.DialFromNode("http", config.ClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + err := config.DialFromNode("http", config.ClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + if err != nil { + framework.Failf("failed dialing endpoint, %v", err) + } ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeHTTPPort)) - config.DialFromNode("http", config.NodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + err = config.DialFromNode("http", config.NodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + if err != nil { + framework.Failf("failed dialing endpoint, %v", err) + } }) ginkgo.It("should function for node-Service: udp", func() { config := e2enetwork.NewNetworkingTestConfig(f, true, false) ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.NodeIP, config.ClusterIP, e2enetwork.ClusterUDPPort)) - config.DialFromNode("udp", config.ClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) - + err := config.DialFromNode("udp", config.ClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + if err != nil { + framework.Failf("failed dialing endpoint, %v", err) + } ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUDPPort)) - config.DialFromNode("udp", config.NodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + err = config.DialFromNode("udp", config.NodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + if err != nil { + framework.Failf("failed dialing endpoint, %v", err) + } }) ginkgo.It("should function for endpoint-Service: http", func() { @@ -305,14 +316,15 @@ var _ = SIGDescribe("Networking", func() { ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, e2enetwork.ClusterHTTPPort)) err := config.DialFromTestContainer("http", config.ClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) if err != nil { - framework.Failf("failed dialing endpoint, %v", err) + framework.Failf("failed dialing endpoint (initial), %v", err) } + ginkgo.By("Deleting a pod which, will be replaced with a new endpoint") config.DeleteNetProxyPod() - ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, e2enetwork.ClusterHTTPPort)) + ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP) (endpoint recovery)", config.TestContainerPod.Name, config.ClusterIP, e2enetwork.ClusterHTTPPort)) err = config.DialFromTestContainer("http", config.ClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) if err != nil { - framework.Failf("failed dialing endpoint, %v", err) + framework.Failf("failed dialing endpoint (recovery), %v", err) } }) @@ -321,39 +333,53 @@ var _ = SIGDescribe("Networking", func() { ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, e2enetwork.ClusterUDPPort)) err := config.DialFromTestContainer("udp", config.ClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames()) if err != nil { - framework.Failf("failed dialing endpoint, %v", err) + framework.Failf("failed dialing endpoint (initial), %v", err) } + ginkgo.By("Deleting a pod which, will be replaced with a new endpoint") config.DeleteNetProxyPod() - ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, e2enetwork.ClusterUDPPort)) + ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP) (endpoint recovery)", config.TestContainerPod.Name, config.ClusterIP, e2enetwork.ClusterUDPPort)) err = config.DialFromTestContainer("udp", config.ClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) if err != nil { - framework.Failf("failed dialing endpoint, %v", err) + framework.Failf("failed dialing endpoint (recovery), %v", err) } }) // Slow because we confirm that the nodePort doesn't serve traffic, which requires a period of polling. ginkgo.It("should update nodePort: http [Slow]", func() { config := e2enetwork.NewNetworkingTestConfig(f, true, false) - ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeHTTPPort)) - config.DialFromNode("http", config.NodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) - + ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP) and getting ALL host endpoints", config.NodeIP, config.NodeIP, config.NodeHTTPPort)) + err := config.DialFromNode("http", config.NodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames()) + if err != nil { + framework.Failf("Error dialing http from node: %v", err) + } + ginkgo.By("Deleting the node port access point") config.DeleteNodePortService() - ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeHTTPPort)) - config.DialFromNode("http", config.NodeIP, config.NodeHTTPPort, config.MaxTries, config.MaxTries, sets.NewString()) + ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP) and getting ZERO host endpoints", config.NodeIP, config.NodeIP, config.NodeHTTPPort)) + err = config.DialFromNode("http", config.NodeIP, config.NodeHTTPPort, config.MaxTries, config.MaxTries, sets.NewString()) + if err != nil { + framework.Failf("Error dialing http from node: %v", err) + } }) // Slow because we confirm that the nodePort doesn't serve traffic, which requires a period of polling. ginkgo.It("should update nodePort: udp [Slow]", func() { config := e2enetwork.NewNetworkingTestConfig(f, true, false) - ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUDPPort)) - config.DialFromNode("udp", config.NodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP) and getting ALL host endpoints", config.NodeIP, config.NodeIP, config.NodeUDPPort)) + err := config.DialFromNode("udp", config.NodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames()) + if err != nil { + framework.Failf("Failure validating that nodePort service WAS forwarding properly: %v", err) + } + ginkgo.By("Deleting the node port access point") config.DeleteNodePortService() - ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUDPPort)) - config.DialFromNode("udp", config.NodeIP, config.NodeUDPPort, config.MaxTries, config.MaxTries, sets.NewString()) + ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP) and getting ZERO host endpoints", config.NodeIP, config.NodeIP, config.NodeUDPPort)) + err = config.DialFromNode("udp", config.NodeIP, config.NodeUDPPort, config.MaxTries, config.MaxTries, sets.NewString()) + if err != nil { + framework.Failf("Failure validating that node port service STOPPED removed properly: %v", err) + } }) // [LinuxOnly]: Windows does not support session affinity. From 8630dc924ad67a110baec00147f3e66d7e9ee8cb Mon Sep 17 00:00:00 2001 From: jay vyas Date: Mon, 5 Oct 2020 05:15:30 -0400 Subject: [PATCH 2/2] Update test/e2e/framework/network/utils.go add logging about min/max interval Co-authored-by: Antonio Ojea --- test/e2e/framework/network/utils.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/e2e/framework/network/utils.go b/test/e2e/framework/network/utils.go index 49166d317c5..051456b6a0f 100644 --- a/test/e2e/framework/network/utils.go +++ b/test/e2e/framework/network/utils.go @@ -380,12 +380,12 @@ func (config *NetworkingTestConfig) GetHTTPCodeFromTestContainer(path, targetIP return code, nil } -// DialFromNode executes a tcp or udp curl/echo request based on protocol via kubectl exec +// DialFromNode executes a tcp/udp curl/nc request based on protocol via kubectl exec // in a test container running with host networking. -// - minTries is the minimum number of curl/echo attempts required before declaring +// - minTries is the minimum number of curl/nc attempts required before declaring // success. If 0, then we return as soon as all endpoints succeed. // - There is no logical change to test results if faillures happen AFTER endpoints have succeeded, -// hence over-padding minTries will NOT reverse a successfull result and is thus not very useful yet +// hence over-padding minTries will NOT reverse a successful result and is thus not very useful yet // (See the TODO about checking probability, which isnt implemented yet). // - maxTries is the maximum number of curl/echo attempts before an error is returned. The // smaller this number is, the less 'slack' there is for declaring success. @@ -411,6 +411,7 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ eps := sets.NewString() filterCmd := fmt.Sprintf("%s | grep -v '^\\s*$'", cmd) + framework.Logf("Going to poll %v on port %v at least %v times, with a maximum of %v tries before failing", targetIP, targetPort, minTries, maxTries) for i := 0; i < maxTries; i++ { stdout, stderr, err := config.f.ExecShellInPodWithFullOutput(config.HostTestContainerPod.Name, filterCmd) if err != nil || len(stderr) > 0 { @@ -427,7 +428,7 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ // Check against i+1 so we exit if minTries == maxTries. if eps.Equal(expectedEps) && i+1 >= minTries { - framework.Logf("Found all %+v expected endpoints: %+v", len(eps.List()), eps.List()) + framework.Logf("Found all %d expected endpoints: %+v", eps.Len(), eps.List()) return nil } @@ -438,7 +439,7 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ } config.diagnoseMissingEndpoints(eps) - return fmt.Errorf("failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", maxTries, cmd, eps, expectedEps) + return fmt.Errorf("failed to find expected endpoints, \ntries %d\nCommand %v\nretrieved %v\nexpected %v", maxTries, cmd, eps, expectedEps) } // GetSelfURL executes a curl against the given path via kubectl exec into a