Merge pull request #4243 from zmerlynn/wait_for_pod_timeout

Add a timeout to waitForPodRunning

test/e2e/basic.go

@@ -123,7 +123,11 @@ func TestBasicImage(c *client.Client, test string, image string) bool {
 	// Wait for the pods to enter the running state. Waiting loops until the pods
 	// are running so non-running pods cause a timeout for this test.
 	for _, pod := range pods.Items {
-		waitForPodRunning(c, pod.Name)
+		err = waitForPodRunning(c, pod.Name, 300*time.Second)
+		if err != nil {
+			glog.Errorf("waitForPodRunningFailed: %v", err.Error())
+			return false
+		}
 	}
 
 	// Try to make sure we get a hostIP for each pod.

test/e2e/events.go

@@ -80,7 +80,8 @@ var _ = Describe("Events", func() {
 		}()
 
 		By("waiting for the pod to start running")
-		waitForPodRunning(c, pod.Name)
+		err = waitForPodRunning(c, pod.Name, 300*time.Second)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("verifying the pod is in kubernetes")
 		pods, err := podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))

test/e2e/pods.go

@@ -145,7 +145,8 @@ var _ = Describe("Pods", func() {
 		}()
 
 		By("waiting for the pod to start running")
-		waitForPodRunning(c, pod.Name)
+		err = waitForPodRunning(c, pod.Name, 300*time.Second)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("verifying the pod is in kubernetes")
 		pods, err := podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
@@ -168,7 +169,8 @@ var _ = Describe("Pods", func() {
 		}
 
 		By("waiting for the updated pod to start running")
-		waitForPodRunning(c, pod.Name)
+		err = waitForPodRunning(c, pod.Name, 300*time.Second)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("verifying the updated pod is in kubernetes")
 		pods, err = podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
@@ -203,7 +205,8 @@ var _ = Describe("Pods", func() {
 			defer GinkgoRecover()
 			c.Pods(api.NamespaceDefault).Delete(serverPod.Name)
 		}()
-		waitForPodRunning(c, serverPod.Name)
+		err = waitForPodRunning(c, serverPod.Name, 300*time.Second)
+		Expect(err).NotTo(HaveOccurred())
 
 		// This service exposes port 8080 of the test pod as a service on port 8765
 		// TODO(filbranden): We would like to use a unique service name such as:

test/e2e/service.go

@@ -116,7 +116,8 @@ var _ = Describe("Services", func() {
 		}()
 
 		By("waiting for the pod to start running")
-		waitForPodRunning(c, pod.Name)
+		err = waitForPodRunning(c, pod.Name, 300*time.Second)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("retrieving the pod")
 		pod, err = podClient.Get(pod.Name)

test/e2e/util.go

@@ -17,6 +17,7 @@ limitations under the License.
 package e2e
 
 import (
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"path/filepath"
@@ -29,6 +30,8 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
 	"github.com/golang/glog"
+
+	. "github.com/onsi/ginkgo"
 )
 
 type testContextType struct {
@@ -41,19 +44,20 @@ type testContextType struct {
 
 var testContext testContextType
 
-func waitForPodRunning(c *client.Client, id string) {
+func waitForPodRunning(c *client.Client, id string, tryFor time.Duration) error {
-	for {
+	trySecs := int(tryFor.Seconds())
+	for i := 0; i <= trySecs; i += 5 {
 		time.Sleep(5 * time.Second)
 		pod, err := c.Pods(api.NamespaceDefault).Get(id)
 		if err != nil {
-			glog.Warningf("Get pod %s failed: %v", id, err)
+			return fmt.Errorf("Get pod %s failed: %v", id, err.Error())
-			continue
 		}
 		if pod.Status.Phase == api.PodRunning {
-			break
+			return nil
 		}
-		glog.Infof("Waiting for pod %s status to be %q (found %q)", id, api.PodRunning, pod.Status.Phase)
+		By(fmt.Sprintf("Waiting for pod %s status to be %q (found %q) (%d secs)", id, api.PodRunning, pod.Status.Phase, i))
 	}
+	return fmt.Errorf("Gave up waiting for pod %s to be running after %d seconds", id, trySecs)
 }
 
 // waitForPodNotPending returns false if it took too long for the pod to go out of pending state.