Merge pull request #92485 from oomichi/metrics-grabber

Avoid DeprecatedMightBeMasterNode() in e2e metrics
This commit is contained in:
Kubernetes Prow Robot 2020-06-26 00:07:54 -07:00 committed by GitHub
commit 0eaa85483b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 46 additions and 37 deletions

View File

@ -31,7 +31,6 @@ go_library(
"//test/e2e/framework/log:go_default_library", "//test/e2e/framework/log:go_default_library",
"//test/e2e/framework/pod:go_default_library", "//test/e2e/framework/pod:go_default_library",
"//test/e2e/perftype:go_default_library", "//test/e2e/perftype:go_default_library",
"//test/e2e/system:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library",
], ],
) )

View File

@ -19,6 +19,7 @@ package metrics
import ( import (
"context" "context"
"fmt" "fmt"
"regexp"
"sync" "sync"
"time" "time"
@ -27,7 +28,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
"k8s.io/kubernetes/test/e2e/system"
"k8s.io/klog/v2" "k8s.io/klog/v2"
) )
@ -61,38 +61,48 @@ type Grabber struct {
grabFromKubelets bool grabFromKubelets bool
grabFromScheduler bool grabFromScheduler bool
grabFromClusterAutoscaler bool grabFromClusterAutoscaler bool
masterName string kubeScheduler string
registeredMaster bool kubeControllerManager string
waitForControllerManagerReadyOnce sync.Once waitForControllerManagerReadyOnce sync.Once
} }
// NewMetricsGrabber returns new metrics which are initialized. // NewMetricsGrabber returns new metrics which are initialized.
func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) { func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) {
registeredMaster := false
masterName := "" kubeScheduler := ""
nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) kubeControllerManager := ""
regKubeScheduler := regexp.MustCompile("kube-scheduler-.*")
regKubeControllerManager := regexp.MustCompile("kube-controller-manager-.*")
podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{})
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(nodeList.Items) < 1 { if len(podList.Items) < 1 {
klog.Warning("Can't find any Nodes in the API server to grab metrics from") klog.Warningf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem)
} }
for _, node := range nodeList.Items { for _, pod := range podList.Items {
if system.DeprecatedMightBeMasterNode(node.Name) { if regKubeScheduler.MatchString(pod.Name) {
registeredMaster = true kubeScheduler = pod.Name
masterName = node.Name }
if regKubeControllerManager.MatchString(pod.Name) {
kubeControllerManager = pod.Name
}
if kubeScheduler != "" && kubeControllerManager != "" {
break break
} }
} }
if !registeredMaster { if kubeScheduler == "" {
scheduler = false scheduler = false
klog.Warningf("Can't find kube-scheduler pod. Grabbing metrics from kube-scheduler is disabled.")
}
if kubeControllerManager == "" {
controllers = false controllers = false
clusterAutoscaler = ec != nil klog.Warningf("Can't find kube-controller-manager pod. Grabbing metrics from kube-controller-manager is disabled.")
if clusterAutoscaler { }
klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.") if ec == nil {
} else { klog.Warningf("Did not receive an external client interface. Grabbing metrics from ClusterAutoscaler is disabled.")
klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.")
}
} }
return &Grabber{ return &Grabber{
@ -103,14 +113,14 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b
grabFromKubelets: kubelets, grabFromKubelets: kubelets,
grabFromScheduler: scheduler, grabFromScheduler: scheduler,
grabFromClusterAutoscaler: clusterAutoscaler, grabFromClusterAutoscaler: clusterAutoscaler,
masterName: masterName, kubeScheduler: kubeScheduler,
registeredMaster: registeredMaster, kubeControllerManager: kubeControllerManager,
}, nil }, nil
} }
// HasRegisteredMaster returns if metrics grabber was able to find a master node // HasControlPlanePods returns true if metrics grabber was able to find control-plane pods
func (g *Grabber) HasRegisteredMaster() bool { func (g *Grabber) HasControlPlanePods() bool {
return g.registeredMaster return g.kubeScheduler != "" && g.kubeControllerManager != ""
} }
// GrabFromKubelet returns metrics from kubelet // GrabFromKubelet returns metrics from kubelet
@ -139,10 +149,10 @@ func (g *Grabber) grabFromKubeletInternal(nodeName string, kubeletPort int) (Kub
// GrabFromScheduler returns metrics from scheduler // GrabFromScheduler returns metrics from scheduler
func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) { func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
if !g.registeredMaster { if g.kubeScheduler == "" {
return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering") return SchedulerMetrics{}, fmt.Errorf("kube-scheduler pod is not registered. Skipping Scheduler's metrics gathering")
} }
output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, insecureSchedulerPort) output, err := g.getMetricsFromPod(g.client, g.kubeScheduler, metav1.NamespaceSystem, insecureSchedulerPort)
if err != nil { if err != nil {
return SchedulerMetrics{}, err return SchedulerMetrics{}, err
} }
@ -151,8 +161,8 @@ func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
// GrabFromClusterAutoscaler returns metrics from cluster autoscaler // GrabFromClusterAutoscaler returns metrics from cluster autoscaler
func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) { func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) {
if !g.registeredMaster && g.externalClient == nil { if !g.HasControlPlanePods() && g.externalClient == nil {
return ClusterAutoscalerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ClusterAutoscaler's metrics gathering") return ClusterAutoscalerMetrics{}, fmt.Errorf("Did not find control-plane pods. Skipping ClusterAutoscaler's metrics gathering")
} }
var client clientset.Interface var client clientset.Interface
var namespace string var namespace string
@ -172,12 +182,12 @@ func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error)
// GrabFromControllerManager returns metrics from controller manager // GrabFromControllerManager returns metrics from controller manager
func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) { func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) {
if !g.registeredMaster { if g.kubeControllerManager == "" {
return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering") return ControllerManagerMetrics{}, fmt.Errorf("kube-controller-manager pod is not registered. Skipping ControllerManager's metrics gathering")
} }
var err error var err error
podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName) podName := g.kubeControllerManager
g.waitForControllerManagerReadyOnce.Do(func() { g.waitForControllerManagerReadyOnce.Do(func() {
if readyErr := e2epod.WaitForPodsReady(g.client, metav1.NamespaceSystem, podName, 0); readyErr != nil { if readyErr := e2epod.WaitForPodsReady(g.client, metav1.NamespaceSystem, podName, 0); readyErr != nil {
err = fmt.Errorf("error waiting for controller manager pod to be ready: %w", readyErr) err = fmt.Errorf("error waiting for controller manager pod to be ready: %w", readyErr)

View File

@ -625,7 +625,7 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
framework.ExpectNoError(err, "Error creating metrics grabber: %v", err) framework.ExpectNoError(err, "Error creating metrics grabber: %v", err)
} }
if !metricsGrabber.HasRegisteredMaster() { if !metricsGrabber.HasControlPlanePods() {
framework.Logf("Warning: Environment does not support getting controller-manager metrics") framework.Logf("Warning: Environment does not support getting controller-manager metrics")
return opCounts{} return opCounts{}
} }

View File

@ -100,7 +100,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() { ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() {
var err error var err error
if !metricsGrabber.HasRegisteredMaster() { if !metricsGrabber.HasControlPlanePods() {
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping") e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
} }
@ -146,7 +146,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() { ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() {
var err error var err error
if !metricsGrabber.HasRegisteredMaster() { if !metricsGrabber.HasControlPlanePods() {
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping") e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
} }
@ -446,7 +446,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
} }
ginkgo.BeforeEach(func() { ginkgo.BeforeEach(func() {
if !metricsGrabber.HasRegisteredMaster() { if !metricsGrabber.HasControlPlanePods() {
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping") e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
} }