Adding polling to all node readiness

This commit is contained in:
Kris Rousey 2015-07-09 11:38:10 -07:00
parent 127fe8d4a5
commit 3058ddad19
3 changed files with 25 additions and 10 deletions

View File

@ -18,6 +18,7 @@ package e2e
import (
"fmt"
"time"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
@ -86,7 +87,7 @@ func (f *Framework) afterEach() {
}
// Check whether all nodes are ready after the test.
if err := allNodesReady(f.Client); err != nil {
if err := allNodesReady(f.Client, time.Minute); err != nil {
Failf("All nodes should be ready after test, %v", err)
}

View File

@ -413,7 +413,7 @@ var _ = Describe("Nodes", func() {
AfterEach(func() {
By("checking whether all nodes are healthy")
if err := allNodesReady(c); err != nil {
if err := allNodesReady(c, time.Minute); err != nil {
Failf("Not all nodes are ready: %v", err)
}
By(fmt.Sprintf("destroying namespace for this suite %s", ns))

View File

@ -1432,8 +1432,9 @@ func waitForNodeToBeNotReady(c *client.Client, name string, timeout time.Duratio
func isNodeReadySetAsExpected(node *api.Node, wantReady bool) bool {
// Check the node readiness condition (logging all).
for i, cond := range node.Status.Conditions {
Logf("Node %s condition %d/%d: type: %v, status: %v",
node.Name, i+1, len(node.Status.Conditions), cond.Type, cond.Status)
Logf("Node %s condition %d/%d: type: %v, status: %v, reason: %q, message: %q, last transistion time: %v",
node.Name, i+1, len(node.Status.Conditions), cond.Type, cond.Status,
cond.Reason, cond.Message, cond.LastTransitionTime)
// Ensure that the condition type is readiness and the status
// matches as desired.
if cond.Type == api.NodeReady && (cond.Status == api.ConditionTrue) == wantReady {
@ -1466,15 +1467,28 @@ func waitForNodeToBe(c *client.Client, name string, wantReady bool, timeout time
}
// checks whether all registered nodes are ready
func allNodesReady(c *client.Client) error {
nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
func allNodesReady(c *client.Client, timeout time.Duration) error {
Logf("Waiting up to %v for all nodes to be ready", timeout)
var notReady []api.Node
for _, node := range nodes.Items {
if isNodeReadySetAsExpected(&node, false) {
notReady = append(notReady, node)
err := wait.Poll(poll, timeout, func() (bool, error) {
notReady = nil
nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
if err != nil {
return false, err
}
for _, node := range nodes.Items {
if !isNodeReadySetAsExpected(&node, true) {
notReady = append(notReady, node)
}
}
return len(notReady) == 0, nil
})
if err != nil && err != wait.ErrWaitTimeout {
return err
}
if len(notReady) > 0 {
return fmt.Errorf("Not ready nodes: %v", notReady)
}