diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 0f7bc590e5b..092eeffc82b 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -18,6 +18,7 @@ package e2e import ( "fmt" + "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" @@ -86,7 +87,7 @@ func (f *Framework) afterEach() { } // Check whether all nodes are ready after the test. - if err := allNodesReady(f.Client); err != nil { + if err := allNodesReady(f.Client, time.Minute); err != nil { Failf("All nodes should be ready after test, %v", err) } diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index bd3ee2bc26c..3f4b6f5fd9a 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -413,7 +413,7 @@ var _ = Describe("Nodes", func() { AfterEach(func() { By("checking whether all nodes are healthy") - if err := allNodesReady(c); err != nil { + if err := allNodesReady(c, time.Minute); err != nil { Failf("Not all nodes are ready: %v", err) } By(fmt.Sprintf("destroying namespace for this suite %s", ns)) diff --git a/test/e2e/util.go b/test/e2e/util.go index 31aab8eefcb..6cdef39b23f 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1432,8 +1432,9 @@ func waitForNodeToBeNotReady(c *client.Client, name string, timeout time.Duratio func isNodeReadySetAsExpected(node *api.Node, wantReady bool) bool { // Check the node readiness condition (logging all). for i, cond := range node.Status.Conditions { - Logf("Node %s condition %d/%d: type: %v, status: %v", - node.Name, i+1, len(node.Status.Conditions), cond.Type, cond.Status) + Logf("Node %s condition %d/%d: type: %v, status: %v, reason: %q, message: %q, last transistion time: %v", + node.Name, i+1, len(node.Status.Conditions), cond.Type, cond.Status, + cond.Reason, cond.Message, cond.LastTransitionTime) // Ensure that the condition type is readiness and the status // matches as desired. if cond.Type == api.NodeReady && (cond.Status == api.ConditionTrue) == wantReady { @@ -1466,15 +1467,28 @@ func waitForNodeToBe(c *client.Client, name string, wantReady bool, timeout time } // checks whether all registered nodes are ready -func allNodesReady(c *client.Client) error { - nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) - Expect(err).NotTo(HaveOccurred()) +func allNodesReady(c *client.Client, timeout time.Duration) error { + Logf("Waiting up to %v for all nodes to be ready", timeout) + var notReady []api.Node - for _, node := range nodes.Items { - if isNodeReadySetAsExpected(&node, false) { - notReady = append(notReady, node) + err := wait.Poll(poll, timeout, func() (bool, error) { + notReady = nil + nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) + if err != nil { + return false, err } + for _, node := range nodes.Items { + if !isNodeReadySetAsExpected(&node, true) { + notReady = append(notReady, node) + } + } + return len(notReady) == 0, nil + }) + + if err != nil && err != wait.ErrWaitTimeout { + return err } + if len(notReady) > 0 { return fmt.Errorf("Not ready nodes: %v", notReady) }