diff --git a/test/e2e/framework/metrics/BUILD b/test/e2e/framework/metrics/BUILD index 081ec489116..21cd2a08f10 100644 --- a/test/e2e/framework/metrics/BUILD +++ b/test/e2e/framework/metrics/BUILD @@ -31,6 +31,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library", "//test/e2e/framework/log:go_default_library", + "//test/e2e/framework/pod:go_default_library", "//test/e2e/perftype:go_default_library", "//test/e2e/system:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index fdfd51f9997..84ee88ce191 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -19,11 +19,13 @@ package metrics import ( "context" "fmt" + "sync" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/master/ports" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/system" "k8s.io/klog" @@ -40,15 +42,16 @@ type Collection struct { // Grabber provides functions which grab metrics from components type Grabber struct { - client clientset.Interface - externalClient clientset.Interface - grabFromAPIServer bool - grabFromControllerManager bool - grabFromKubelets bool - grabFromScheduler bool - grabFromClusterAutoscaler bool - masterName string - registeredMaster bool + client clientset.Interface + externalClient clientset.Interface + grabFromAPIServer bool + grabFromControllerManager bool + grabFromKubelets bool + grabFromScheduler bool + grabFromClusterAutoscaler bool + masterName string + registeredMaster bool + waitForControllerManagerReadyOnce sync.Once } // NewMetricsGrabber returns new metrics which are initialized. @@ -160,7 +163,12 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) if !g.registeredMaster { return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering") } - output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort) + + podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName) + g.waitForControllerManagerReadyOnce.Do(func() { + e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem) + }) + output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort) if err != nil { return ControllerManagerMetrics{}, err }