Merge pull request #4681 from jlowdermilk/cadvisor-e2e

Deflake cadvisor e2e by adding retry loop

test/e2e/cadvisor.go

@@ -34,21 +34,31 @@ var _ = Describe("Cadvisor", func() {
 		expectNoError(err)
 	})
 
-	It("cadvisor should be healthy on every node.", func() {
+	It("should be healthy on every node.", func() {
-		CheckCadvisorHealthOnAllNodes(c)
+		CheckCadvisorHealthOnAllNodes(c, 5*time.Minute)
 	})
 })
 
-func CheckCadvisorHealthOnAllNodes(c *client.Client) {
+func CheckCadvisorHealthOnAllNodes(c *client.Client, timeout time.Duration) {
 	By("getting list of nodes")
 	nodeList, err := c.Nodes().List()
 	expectNoError(err)
-	for _, node := range nodeList.Items {
+	var errors []error
-		// cadvisor is not accessible directly unless its port (4194 by default) is exposed.
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
-		// Here, we access '/stats/' REST endpoint on the kubelet which polls cadvisor internally.
+		errors = []error{}
-		statsResource := fmt.Sprintf("api/v1beta1/proxy/minions/%s/stats/", node.Name)
+		for _, node := range nodeList.Items {
-		By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource))
+			// cadvisor is not accessible directly unless its port (4194 by default) is exposed.
-		_, err = c.Get().AbsPath(statsResource).Timeout(1 * time.Second).Do().Raw()
+			// Here, we access '/stats/' REST endpoint on the kubelet which polls cadvisor internally.
-		expectNoError(err)
+			statsResource := fmt.Sprintf("api/v1beta1/proxy/minions/%s/stats/", node.Name)
+			By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource))
+			_, err = c.Get().AbsPath(statsResource).Timeout(1 * time.Second).Do().Raw()
+			if err != nil {
+				errors = append(errors, err)
+			}
+		}
+		if len(errors) == 0 {
+			return
+		}
 	}
+	Failf("Timed out after %v waiting for cadvisor to be healthy on all nodes. Errors:\n%v", timeout, errors)
 }