Make basic end to end test robustly wait for hostIP

contrib/for-demos/serve_hostname/Dockerfile

@@ -1,8 +1,6 @@
-# https://github.com/thockin/serve_hostname
+FROM ubuntu:14.04
-
-FROM scratch
 MAINTAINER Tim Hockin <thockin@google.com>
-ADD serve_hostname serve_hostname
+ADD serve_hostname /serve_hostname
-ADD serve_hostname.go serve_hostname.go
+ADD serve_hostname.go /serve_hostname.go
 EXPOSE 9376
 ENTRYPOINT ["/serve_hostname"]

hack/e2e-suite/private.sh

@@ -32,5 +32,5 @@ if [[ "${KUBERNETES_PROVIDER}" != "gce" ]] && [[ "${KUBERNETES_PROVIDER}" != "gk
 fi
 
 # Run the basic.sh test, but using this image.
-export POD_IMG_SRV="container.cloud.google.com/_b_k8s_test/serve_hostname"
+export POD_IMG_SRV="gcr.io/_b_k8s_test/serve_hostname:1.0"
 source "${KUBE_ROOT}/hack/e2e-suite/basic.sh"

test/e2e/basic.go

@@ -20,48 +20,25 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"strings"
+	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/golang/glog"
 )
 
 // A basic test to check the deployment of an image using
 // a replication controller. The image serves its hostname
 // which is checked for each replica.
-func TestBasicImage(c *client.Client, image string) bool {
+func TestBasicImage(c *client.Client, test string, image string) bool {
+	testStart := time.Now()
 	ns := api.NamespaceDefault
-	// TODO(satnam6502): Generate a unique name to allow
+	name := "my-hostname-" + test + "-" + string(util.NewUUID())
-	//                   parallel executions of this test.
-	name := "my-hostname"
 	replicas := 2
 
-	// Attmept to delete a controller that might have been
+	// Create a replication controller for a service
-	// left lying around from a previously aborted test.
-	controllers, err := c.ReplicationControllers(ns).List(labels.Everything())
-	if err != nil {
-		glog.Infof("Failed to list replication controllers: %v", err)
-		return false
-	}
-	for _, cnt := range controllers.Items {
-		if cnt.Name == name {
-			glog.Infof("Found a straggler %s controller", name)
-			// Delete any pods controlled by this replicaiton controller.
-			cnt.Spec.Replicas = 0
-			if _, err := c.ReplicationControllers(ns).Update(&cnt); err != nil {
-				glog.Warningf("Failed to resize straggler controller to zero: %v", err)
-			}
-			// Delete the controller
-			if err = c.ReplicationControllers(ns).Delete(name); err != nil {
-				glog.Warningf("Failed to delete straggler replication controller: %v", err)
-			}
-			break
-		}
-	}
-
-	// Create a replication controller  for a service
 	// that serves its hostname on port 8080.
 	// The source for the Docker containter kubernetes/serve_hostname is
 	// in contrib/for-demos/serve_hostname
@@ -92,76 +69,116 @@ func TestBasicImage(c *client.Client, image string) bool {
 		},
 	})
 	if err != nil {
-		glog.Infof("Failed to create replication controller for %s: %v", name, err)
+		glog.Infof("Failed to create replication controller %s: %v", name, err)
 		return false
 	}
+	// Cleanup the replication controller when we are done.
+	defer func() {
+		// Resize the replication controller to zero to get rid of pods.
+		controller.Spec.Replicas = 0
+		if _, err = c.ReplicationControllers(ns).Update(controller); err != nil {
+			glog.Errorf("Failed to resize replication controller %s to zero: %v", name, err)
+		}
 
-	// List the pods.
+		// Delete the replication controller.
-	// pods, err := c.Pods(ns).List(labels.Set{"name": name}.AsSelector())
+		if err = c.ReplicationControllers(ns).Delete(name); err != nil {
-	pods, err := c.Pods(ns).List(labels.SelectorFromSet(labels.Set(map[string]string{"name": name})))
+			glog.Errorf("Failed to delete replication controller %s: %v", name, err)
+		}
+
+		// Report running time for test.
+		glog.Infof("Controller %s test took %v seconds", name, time.Since(testStart).Seconds())
+	}()
+
+	// List the pods, making sure we observe all the replicas.
+	listTimeout := time.Minute
+	label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
+	pods, err := c.Pods(ns).List(label)
 	if err != nil {
-		glog.Errorf("Failed to list pods before wait for running check: %v", err)
+		glog.Errorf("Controller %s: Failed to list pods: %v", name, err)
-		return false
 	}
-	for i, pod := range pods.Items {
+	t := time.Now()
-		glog.Infof("Replica %d: %s\n", i+1, pod.Name)
+	for {
+		glog.Infof("Controller %s: Found %d pods out of %d", name, len(pods.Items), replicas)
+		if len(pods.Items) == replicas {
+			break
+		}
+		if time.Since(t) > listTimeout {
+			glog.Errorf("Controller %s: Gave up waiting for %d pods to come up after seeing only %d pods after %v seconds",
+				name, replicas, len(pods.Items), time.Since(t).Seconds())
+			return false
+		}
+		time.Sleep(10 * time.Second)
+		pods, err = c.Pods(ns).List(label)
+		if err != nil {
+			glog.Errorf("Controller %s: Failed to list pods: %v", name, err)
+		}
 	}
 
+	for i, pod := range pods.Items {
+		glog.Infof("Controller %s: Replica %d: pod: %s hostPort: %s\n", name, i+1, pod.Name, pod.Status.HostIP)
+	}
 	// Wait for the pods to enter the running state. Waiting loops until the pods
 	// are running so non-running pods cause a timeout for this test.
 	for _, pod := range pods.Items {
 		waitForPodRunning(c, pod.Name)
 	}
 
-	// List the pods again to get the host IP information.
+	// Try to make sure we get a hostIP for each pod.
-	pods, err = c.Pods(ns).List(labels.SelectorFromSet(labels.Set(map[string]string{"name": name})))
+	hostIPTimeout := 2 * time.Minute
+	t = time.Now()
+	for i, pod := range pods.Items {
+		for {
+			p, err := c.Pods(ns).Get(pod.Name)
+			if err != nil {
+				glog.Errorf("Controller %s: Failed to get status for pod %s: %v", name, pod.Name, err)
+				return false
+			}
+			if p.Status.HostIP != "" {
+				glog.Infof("Controller %s: Replica %d has hostIP: %s", name, i+1, p.Status.HostIP)
+				break
+			}
+			if time.Since(t) >= hostIPTimeout {
+				glog.Errorf("Controller %s: Gave up waiting for hostIP of replica %d after %v seconds",
+					name, i, time.Since(t).Seconds())
+				return false
+			}
+			glog.Infof("Controller %s: Retrying to get the hostIP of replica %d", name, i+1)
+			time.Sleep(5 * time.Second)
+		}
+	}
+
+	// Re-fetch the pod information to update the host port information.
+	pods, err = c.Pods(ns).List(label)
 	if err != nil {
-		glog.Errorf("Failed to list pods after wait for running check: %v", err)
+		glog.Errorf("Controller %s: Failed to list pods: %v", name, err)
-		return false
 	}
 
 	// Verify that something is listening.
 	for i, pod := range pods.Items {
 		glog.Infof("Pod: %s hostIP: %s", pod.Name, pod.Status.HostIP)
-		if pod.Status.HostIP == "" {
-			glog.Warningf("Skipping test of GET response since hostIP is not available")
-			continue
-		}
 		resp, err := http.Get(fmt.Sprintf("http://%s:8080", pod.Status.HostIP))
 		if err != nil {
-			glog.Errorf("Failed to GET from replica %d: %v", i+1, err)
+			glog.Errorf("Controller %s: Failed to GET from replica %d: %v", name, i+1, err)
 			return false
 		}
 		defer resp.Body.Close()
 		if resp.StatusCode != http.StatusOK {
-			glog.Errorf("Expected OK status code for replica %d but got %d", i+1, resp.StatusCode)
+			glog.Errorf("Controller %s: Expected OK status code for replica %d but got %d", name, i+1, resp.StatusCode)
 			return false
 		}
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			glog.Errorf("Failed to read the body of the GET response from replica %d: %v", i+1, err)
+			glog.Errorf("Controller %s: Failed to read the body of the GET response from replica %d: %v",
+				name, i+1, err)
 			return false
 		}
-		// The body should be the pod name although we may need to skip a newline
+		// The body should be the pod name.
-		// character at the end of the response.
+		if string(body) != pod.Name {
-		if !strings.HasPrefix(string(body), pod.Name) {
+			glog.Errorf("Controller %s: Replica %d expected response %s but got %s",
-			glog.Errorf("From replica %d expected response %s but got %s", i+1, pod.Name, string(body))
+				name, i+1, pod.Name, string(body))
 			return false
 		}
-		glog.Infof("Got expected result from replica %d: %s", i+1, string(body))
+		glog.Infof("Controller %s: Got expected result from replica %d: %s", name, i+1, string(body))
-	}
-
-	// Resize the replication controller to zero to get rid of pods.
-	controller.Spec.Replicas = 0
-	if _, err = c.ReplicationControllers(ns).Update(controller); err != nil {
-		glog.Errorf("Failed to resize replication controller to zero: %v", err)
-		return false
-	}
-
-	// Delete the replication controller.
-	if err = c.ReplicationControllers(ns).Delete(name); err != nil {
-		glog.Errorf("Failed to delete replication controller %s: %v", name, err)
-		return false
 	}
 
 	return true
@@ -170,5 +187,5 @@ func TestBasicImage(c *client.Client, image string) bool {
 // TestBasic performs the TestBasicImage check with the
 // image kubernetes/serve_hostname
 func TestBasic(c *client.Client) bool {
-	return TestBasicImage(c, "kubernetes/serve_hostname")
+	return TestBasicImage(c, "basic", "kubernetes/serve_hostname:1.0")
 }

test/e2e/private.go

@@ -22,7 +22,7 @@ import (
 )
 
 // A basic test to check the deployment of the
-// container.cloud.google.com/_b_k8s_test/serve_hostname image
+// contaier gcr.io/_b_k8s_test/serve_hostname image
 // with the TestBasicImage test. This test is only supported
 // for the providers GCE and GKE.
 func TestPrivate(c *client.Client) bool {
@@ -31,5 +31,5 @@ func TestPrivate(c *client.Client) bool {
 		return true
 	}
 	glog.Info("Calling out to TestBasic")
-	return TestBasicImage(c, "container.cloud.google.com/_b_k8s_test/serve_hostname")
+	return TestBasicImage(c, "private", "gcr.io/_b_k8s_test/serve_hostname:1.0")
 }