From f1d6439c2ee71dc8feee3045b160663d948d5f6f Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Wed, 1 Jul 2015 14:58:37 -0700
Subject: [PATCH] Add timeout to service endpoint resolution e2e test

---
 test/e2e/service.go | 41 ++++++++++++++++++++---------------------
 test/e2e/util.go    |  3 +++
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/test/e2e/service.go b/test/e2e/service.go
index cfe061796bf..4d08eb523eb 100644
--- a/test/e2e/service.go
+++ b/test/e2e/service.go
@@ -894,29 +894,28 @@ func validatePortsOrFail(endpoints map[string][]int, expectedEndpoints map[strin
 	}
 }
 
-func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedEndpoints map[string][]int) {
-	By(fmt.Sprintf("Validating endpoints %v with on service %s/%s", expectedEndpoints, ns, serviceName))
-	for {
-		endpoints, err := c.Endpoints(ns).Get(serviceName)
-		if err == nil {
-			By(fmt.Sprintf("Found endpoints %v", endpoints))
-
-			portsByIp := getPortsByIp(endpoints.Subsets)
-
-			By(fmt.Sprintf("Found ports by ip %v", portsByIp))
-			if len(portsByIp) == len(expectedEndpoints) {
-				expectedPortsByIp := translatePodNameToIpOrFail(c, ns, expectedEndpoints)
-				validatePortsOrFail(portsByIp, expectedPortsByIp)
-				break
-			} else {
-				By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", portsByIp, expectedEndpoints))
-			}
-		} else {
-			By(fmt.Sprintf("Failed to get endpoints: %v (ignoring for 1 second)", err))
+func validateEndpointsOrFail(c *client.Client, namespace, serviceName string, expectedEndpoints map[string][]int) {
+	By(fmt.Sprintf("Waiting up to %v for service %s in namespace %s to expose endpoints %v", serviceStartTimeout, serviceName, namespace, expectedEndpoints))
+	for start := time.Now(); time.Since(start) < serviceStartTimeout; time.Sleep(5 * time.Second) {
+		endpoints, err := c.Endpoints(namespace).Get(serviceName)
+		if err != nil {
+			Logf("Get endpoints failed (%v elapsed, ignoring for 5s): %v", time.Since(start), err)
+			continue
 		}
-		time.Sleep(time.Second)
+		Logf("Found endpoints %v", endpoints)
+
+		portsByIp := getPortsByIp(endpoints.Subsets)
+		Logf("Found ports by ip %v", portsByIp)
+
+		if len(portsByIp) == len(expectedEndpoints) {
+			expectedPortsByIp := translatePodNameToIpOrFail(c, namespace, expectedEndpoints)
+			validatePortsOrFail(portsByIp, expectedPortsByIp)
+			By(fmt.Sprintf("Successfully validated that service %s in namespace %s exposes endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, time.Since(start)))
+			return
+		}
+		Logf("Unexpected number of endpoints: found %v, expected %v (%v elapsed, ignoring for 5s)", portsByIp, expectedEndpoints, time.Since(start))
 	}
-	By(fmt.Sprintf("successfully validated endpoints %v with on service %s/%s", expectedEndpoints, ns, serviceName))
+	Failf("Timed out waiting for service %s in namespace %s to expose endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, serviceStartTimeout)
 }
 
 func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]string, containerPorts []api.ContainerPort) {
diff --git a/test/e2e/util.go b/test/e2e/util.go
index 940a3a2e60f..001454f130b 100644
--- a/test/e2e/util.go
+++ b/test/e2e/util.go
@@ -59,6 +59,9 @@ const (
 	// TODO: Make this 30 seconds once #4566 is resolved.
 	podStartTimeout = 5 * time.Minute
 
+	// How long to wait for a service endpoint to be resolvable.
+	serviceStartTimeout = 1 * time.Minute
+
 	// String used to mark pod deletion
 	nonExist = "NonExist"