From 024299780e21e3107abd2c56024b9fa4cd6247ca Mon Sep 17 00:00:00 2001 From: bprashanth Date: Thu, 3 Nov 2016 22:10:47 -0700 Subject: [PATCH] Stricter timeouts for nodePort curling --- test/e2e/density.go | 2 +- test/e2e/empty.go | 2 +- test/e2e/framework/networking_utils.go | 6 +++--- test/e2e/framework/util.go | 10 +++++++--- test/e2e/load.go | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/e2e/density.go b/test/e2e/density.go index 5d0e8ece1e7..5789b41b09d 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -313,7 +313,7 @@ var _ = framework.KubeDescribe("Density", func() { // In large clusters we may get to this point but still have a bunch // of nodes without Routes created. Since this would make a node // unschedulable, we need to wait until all of them are schedulable. - framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c)) + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.NodeSchedulableTimeout)) masters, nodes = framework.GetMasterAndWorkerNodesOrDie(c) nodeCount = len(nodes.Items) Expect(nodeCount).NotTo(BeZero()) diff --git a/test/e2e/empty.go b/test/e2e/empty.go index 8f4a364e284..10b15175b41 100644 --- a/test/e2e/empty.go +++ b/test/e2e/empty.go @@ -32,7 +32,7 @@ var _ = framework.KubeDescribe("Empty [Feature:Empty]", func() { ns := f.Namespace.Name // TODO: respect --allow-notready-nodes flag in those functions. - framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c)) + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.NodeSchedulableTimeout)) framework.WaitForAllNodesHealthy(c, time.Minute) err := framework.CheckTestingNSDeletedExcept(c, ns) diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go index 19fd6c9e0a7..0c3f536ba6c 100644 --- a/test/e2e/framework/networking_utils.go +++ b/test/e2e/framework/networking_utils.go @@ -223,7 +223,7 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ // busybox timeout doesn't support non-integer values. cmd = fmt.Sprintf("echo 'hostName' | timeout -t 2 nc -w 1 -u %s %d", targetIP, targetPort) } else { - cmd = fmt.Sprintf("curl -q -s --connect-timeout 1 http://%s:%d/hostName", targetIP, targetPort) + cmd = fmt.Sprintf("timeout -t 15 curl -q -s --connect-timeout 1 --max-time 10 http://%s:%d/hostName", targetIP, targetPort) } // TODO: This simply tells us that we can reach the endpoints. Check that @@ -435,7 +435,7 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) { config.setupCore(selector) By("Getting node addresses") - ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet)) + ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute)) nodeList := GetReadySchedulableNodesOrDie(config.f.ClientSet) config.ExternalAddrs = NodeAddresses(nodeList, api.NodeExternalIP) if len(config.ExternalAddrs) < 2 { @@ -486,7 +486,7 @@ func shuffleNodes(nodes []api.Node) []api.Node { } func (config *NetworkingTestConfig) createNetProxyPods(podName string, selector map[string]string) []*api.Pod { - ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet)) + ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute)) nodeList := GetReadySchedulableNodesOrDie(config.f.ClientSet) // To make this test work reasonably fast in large clusters, diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index d6a12f7ed5a..294743d134f 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -170,6 +170,10 @@ const ( // TODO(justinsb): Avoid hardcoding this. awsMasterIP = "172.20.0.9" + + // Default time to wait for nodes to become schedulable. + // Set so high for scale tests. + NodeSchedulableTimeout = 4 * time.Hour ) var ( @@ -2437,11 +2441,11 @@ func GetReadySchedulableNodesOrDie(c clientset.Interface) (nodes *api.NodeList) return nodes } -func WaitForAllNodesSchedulable(c clientset.Interface) error { - Logf("Waiting up to %v for all (but %d) nodes to be schedulable", 4*time.Hour, TestContext.AllowedNotReadyNodes) +func WaitForAllNodesSchedulable(c clientset.Interface, timeout time.Duration) error { + Logf("Waiting up to %v for all (but %d) nodes to be schedulable", timeout, TestContext.AllowedNotReadyNodes) var notSchedulable []*api.Node - return wait.PollImmediate(30*time.Second, 4*time.Hour, func() (bool, error) { + return wait.PollImmediate(30*time.Second, timeout, func() (bool, error) { notSchedulable = nil opts := api.ListOptions{ ResourceVersion: "0", diff --git a/test/e2e/load.go b/test/e2e/load.go index 543b6e75bf9..bfd8f69ded9 100644 --- a/test/e2e/load.go +++ b/test/e2e/load.go @@ -101,7 +101,7 @@ var _ = framework.KubeDescribe("Load capacity", func() { // In large clusters we may get to this point but still have a bunch // of nodes without Routes created. Since this would make a node // unschedulable, we need to wait until all of them are schedulable. - framework.ExpectNoError(framework.WaitForAllNodesSchedulable(clientset)) + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(clientset, framework.NodeSchedulableTimeout)) ns = f.Namespace.Name nodes := framework.GetReadySchedulableNodesOrDie(clientset)