e2e: wait for controller manager pod to be ready

The MetricsGrabber may use the controller-manager pod
to gather metrics, however, it doesn't wait until
it is ready to serve, failing the test if this is the
case.

We wait until the controller-manager pod is running
before trying to get metrics from it.
This commit is contained in:
Kubernetes Prow Robot 2020-02-29 07:54:40 -08:00 committed by Antonio Ojea
parent 56987164f8
commit b1c1187cca
2 changed files with 19 additions and 10 deletions

View File

@ -32,6 +32,7 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library", "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
"//test/e2e/framework/log:go_default_library", "//test/e2e/framework/log:go_default_library",
"//test/e2e/framework/pod:go_default_library",
"//test/e2e/perftype:go_default_library", "//test/e2e/perftype:go_default_library",
"//test/e2e/system:go_default_library", "//test/e2e/system:go_default_library",
"//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/klog:go_default_library",

View File

@ -19,12 +19,14 @@ package metrics
import ( import (
"context" "context"
"fmt" "fmt"
"sync"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
api "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/master/ports"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
"k8s.io/kubernetes/test/e2e/system" "k8s.io/kubernetes/test/e2e/system"
"k8s.io/klog" "k8s.io/klog"
@ -50,6 +52,7 @@ type Grabber struct {
grabFromClusterAutoscaler bool grabFromClusterAutoscaler bool
masterName string masterName string
registeredMaster bool registeredMaster bool
waitForControllerManagerReadyOnce sync.Once
} }
// NewMetricsGrabber returns new metrics which are initialized. // NewMetricsGrabber returns new metrics which are initialized.
@ -161,7 +164,12 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error)
if !g.registeredMaster { if !g.registeredMaster {
return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering") return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
} }
output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
g.waitForControllerManagerReadyOnce.Do(func() {
e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem)
})
output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
if err != nil { if err != nil {
return ControllerManagerMetrics{}, err return ControllerManagerMetrics{}, err
} }