github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-04 09:49:50 +00:00
Code Issues Projects Releases Wiki Activity

implement breadth first try

Browse Source
This commit is contained in:
jay vyas 2020-08-11 11:32:46 -04:00
parent ccbfa84173
commit 56e2adbd79
2 changed files with 38 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
test/e2e/common/networking.go
View File

@ -18,6 +18,7 @@ package common
import ( import (
"github.com/onsi/ginkgo" "github.com/onsi/ginkgo"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
@ -29,21 +30,42 @@ var _ = ginkgo.Describe("[sig-network] Networking", func() {
ginkgo.Describe("Granular Checks: Pods", func() { ginkgo.Describe("Granular Checks: Pods", func() {
checkNodeConnectivity := func(config *e2enetwork.NetworkingTestConfig, protocol string, port int) { checkNodeConnectivity := func(config *e2enetwork.NetworkingTestConfig, protocol string, port int) {
errors := []error{} // breadth first poll to quickly estimate failure.
failedPodsByHost := map[string][]*v1.Pod{}
// First time, we'll quickly try all pods, breadth first.
for _, endpointPod := range config.EndpointPods { for _, endpointPod := range config.EndpointPods {
if err := config.DialFromTestContainer(protocol, endpointPod.Status.PodIP, port, config.MaxTries, 0, sets.NewString(endpointPod.Name)); err != nil { framework.Logf("Breadth first check of %v on host %v...", endpointPod.Status.PodIP, endpointPod.Status.HostIP)
errors = append(errors, err) if err := config.DialFromTestContainer(protocol, endpointPod.Status.PodIP, port, 1, 0, sets.NewString(endpointPod.Name)); err != nil {
framework.Logf("Warning: Test failure (%v) will occur due to %v", len(errors)+1, err) // convenient error message for diagnosis... how many pods failed, and on what hosts? if _, ok := failedPodsByHost[endpointPod.Status.HostIP]; !ok {
} else { failedPodsByHost[endpointPod.Status.HostIP] = []*v1.Pod{}
framework.Logf("Was able to reach %v on %v ", endpointPod.Status.PodIP, endpointPod.Status.HostIP) }
failedPodsByHost[endpointPod.Status.HostIP] = append(failedPodsByHost[endpointPod.Status.HostIP], endpointPod)
framework.Logf("...failed...will try again in next pass")
} }
} }
if len(errors) > 0 { errors := []error{}
framework.Logf("Pod polling failure summary:") // Second time, we pass through pods more carefully...
for _, e := range errors { framework.Logf("Going to retry %v out of %v pods....", len(failedPodsByHost), len(config.EndpointPods))
framework.Logf("%v", e) for host, failedPods := range failedPodsByHost {
framework.Logf("Doublechecking %v pods in host %v which werent seen the first time.", len(failedPods), host)
for _, endpointPod := range failedPods {
framework.Logf("Now attempting to probe pod [[[ %v ]]]", endpointPod.Status.PodIP)
if err := config.DialFromTestContainer(protocol, endpointPod.Status.PodIP, port, config.MaxTries, 0, sets.NewString(endpointPod.Name)); err != nil {
errors = append(errors, err)
framework.Logf("Warning: Test failure (%v) will occur due to %v", len(errors), err) // convenient error message for diagnosis... how many pods failed, and on what hosts?
} else {
framework.Logf("Was able to reach %v on %v ", endpointPod.Status.PodIP, endpointPod.Status.HostIP)
}
framework.Logf("... Done probing pod [[[ %v ]]]", endpointPod.Status.PodIP)
} }
framework.Failf("Failed due to %v errors polling %v pods", len(errors), len(config.EndpointPods)) framework.Logf("succeeded at polling %v out of %v connections", len(config.EndpointPods)-len(errors), len(config.EndpointPods))
}
if len(errors) > 0 {
framework.Logf("pod polling failure summary:")
for _, e := range errors {
framework.Logf("Collected error: %v", e)
}
framework.Failf("failed, %v out of %v connections failed", len(errors), len(config.EndpointPods))
} }
} }

6
test/e2e/framework/network/utils.go
View File

@ -275,6 +275,7 @@ func (config *NetworkingTestConfig) DialFromContainer(protocol, dialCommand, con
// Check against i+1 so we exit if minTries == maxTries. // Check against i+1 so we exit if minTries == maxTries.
if (responses.Equal(expectedResponses) || responses.Len() == 0 && expectedResponses.Len() == 0) && i+1 >= minTries { if (responses.Equal(expectedResponses) || responses.Len() == 0 && expectedResponses.Len() == 0) && i+1 >= minTries {
framework.Logf("reached %v after %v/%v tries", targetIP, i, maxTries)
return nil return nil
} }
// TODO: get rid of this delay #36281 // TODO: get rid of this delay #36281
@ -283,7 +284,10 @@ func (config *NetworkingTestConfig) DialFromContainer(protocol, dialCommand, con
if dialCommand == echoHostname { if dialCommand == echoHostname {
config.diagnoseMissingEndpoints(responses) config.diagnoseMissingEndpoints(responses)
} }
return fmt.Errorf("did not find expected responses... \nTries %d\nCommand %v\nretrieved %v\nexpected %v", maxTries, cmd, responses, expectedResponses) returnMsg := fmt.Errorf("did not find expected responses... \nTries %d\nCommand %v\nretrieved %v\nexpected %v", maxTries, cmd, responses, expectedResponses)
framework.Logf("encountered error during dial (%v)", returnMsg)
return returnMsg
} }
// GetEndpointsFromTestContainer executes a curl via kubectl exec in a test container. // GetEndpointsFromTestContainer executes a curl via kubectl exec in a test container.