From 4361f8f05217f86296dd4d87d7f27d56bc657371 Mon Sep 17 00:00:00 2001
From: Antonio Ojea <antonio.ojea.garcia@gmail.com>
Date: Wed, 11 Mar 2020 14:32:03 +0100
Subject: [PATCH] wait until /metrics are ready on e2e test

Some e2e tests depend on the controller-manager to expose metrics
on the path /metrics.

It may happen that when the test runs, the pod is not available or the
URL not ready, causing it to fail.

Previously, the test were waiting until the pod was running, but we
need to wait until the /metrics URL is ready.
---
 test/e2e/framework/metrics/BUILD              |  1 +
 test/e2e/framework/metrics/metrics_grabber.go | 21 ++++++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/test/e2e/framework/metrics/BUILD b/test/e2e/framework/metrics/BUILD
index 21cd2a08f10..415fd3a0172 100644
--- a/test/e2e/framework/metrics/BUILD
+++ b/test/e2e/framework/metrics/BUILD
@@ -28,6 +28,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
         "//test/e2e/framework/log:go_default_library",
diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go
index 84ee88ce191..85f3ba3e7c1 100644
--- a/test/e2e/framework/metrics/metrics_grabber.go
+++ b/test/e2e/framework/metrics/metrics_grabber.go
@@ -20,9 +20,11 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/master/ports"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -164,10 +166,27 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error)
 		return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
 	}
 
+	var err error
 	podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
 	g.waitForControllerManagerReadyOnce.Do(func() {
-		e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem)
+		if runningErr := e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem); runningErr != nil {
+			err = fmt.Errorf("error waiting for controller manager pod to be running: %w", err)
+			return
+		}
+
+		var lastMetricsFetchErr error
+		if metricsWaitErr := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
+			_, lastMetricsFetchErr = g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
+			return lastMetricsFetchErr == nil, nil
+		}); metricsWaitErr != nil {
+			err = fmt.Errorf("error waiting for controller manager pod to expose metrics: %v; %v", err, lastMetricsFetchErr)
+			return
+		}
 	})
+	if err != nil {
+		return ControllerManagerMetrics{}, err
+	}
+
 	output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
 	if err != nil {
 		return ControllerManagerMetrics{}, err