From 176c8e219f4c7b4c15d34b92c50bfa5ba02b3aba Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Wed, 24 Jun 2020 19:58:43 +0000 Subject: [PATCH] Avoid DeprecatedMightBeMasterNode() in e2e metrics As its name, DeprecatedMightBeMasterNode is deprecated. In e2e metrics, the function was used for knowing master node name to get metrics from kube-scheduler and kube-controller-manager pods. This make e2e metrics get these metrics directly by getting those pod names without calling DeprecatedMightBeMasterNode(). --- test/e2e/framework/metrics/BUILD | 1 - test/e2e/framework/metrics/metrics_grabber.go | 74 +++++++++++-------- test/e2e/storage/testsuites/base.go | 2 +- test/e2e/storage/volume_metrics.go | 6 +- 4 files changed, 46 insertions(+), 37 deletions(-) diff --git a/test/e2e/framework/metrics/BUILD b/test/e2e/framework/metrics/BUILD index 806f8d21a81..d9d53d28af5 100644 --- a/test/e2e/framework/metrics/BUILD +++ b/test/e2e/framework/metrics/BUILD @@ -31,7 +31,6 @@ go_library( "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/e2e/perftype:go_default_library", - "//test/e2e/system:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", ], ) diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index 0fc7303d9ad..1d341cc9360 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -19,6 +19,7 @@ package metrics import ( "context" "fmt" + "regexp" "sync" "time" @@ -27,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - "k8s.io/kubernetes/test/e2e/system" "k8s.io/klog/v2" ) @@ -61,38 +61,48 @@ type Grabber struct { grabFromKubelets bool grabFromScheduler bool grabFromClusterAutoscaler bool - masterName string - registeredMaster bool + kubeScheduler string + kubeControllerManager string waitForControllerManagerReadyOnce sync.Once } // NewMetricsGrabber returns new metrics which are initialized. func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) { - registeredMaster := false - masterName := "" - nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + + kubeScheduler := "" + kubeControllerManager := "" + + regKubeScheduler := regexp.MustCompile("kube-scheduler-.*") + regKubeControllerManager := regexp.MustCompile("kube-controller-manager-.*") + + podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, err } - if len(nodeList.Items) < 1 { - klog.Warning("Can't find any Nodes in the API server to grab metrics from") + if len(podList.Items) < 1 { + klog.Warningf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem) } - for _, node := range nodeList.Items { - if system.DeprecatedMightBeMasterNode(node.Name) { - registeredMaster = true - masterName = node.Name + for _, pod := range podList.Items { + if regKubeScheduler.MatchString(pod.Name) { + kubeScheduler = pod.Name + } + if regKubeControllerManager.MatchString(pod.Name) { + kubeControllerManager = pod.Name + } + if kubeScheduler != "" && kubeControllerManager != "" { break } } - if !registeredMaster { + if kubeScheduler == "" { scheduler = false + klog.Warningf("Can't find kube-scheduler pod. Grabbing metrics from kube-scheduler is disabled.") + } + if kubeControllerManager == "" { controllers = false - clusterAutoscaler = ec != nil - if clusterAutoscaler { - klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.") - } else { - klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.") - } + klog.Warningf("Can't find kube-controller-manager pod. Grabbing metrics from kube-controller-manager is disabled.") + } + if ec == nil { + klog.Warningf("Did not receive an external client interface. Grabbing metrics from ClusterAutoscaler is disabled.") } return &Grabber{ @@ -103,14 +113,14 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b grabFromKubelets: kubelets, grabFromScheduler: scheduler, grabFromClusterAutoscaler: clusterAutoscaler, - masterName: masterName, - registeredMaster: registeredMaster, + kubeScheduler: kubeScheduler, + kubeControllerManager: kubeControllerManager, }, nil } -// HasRegisteredMaster returns if metrics grabber was able to find a master node -func (g *Grabber) HasRegisteredMaster() bool { - return g.registeredMaster +// HasControlPlanePods returns true if metrics grabber was able to find control-plane pods +func (g *Grabber) HasControlPlanePods() bool { + return g.kubeScheduler != "" && g.kubeControllerManager != "" } // GrabFromKubelet returns metrics from kubelet @@ -139,10 +149,10 @@ func (g *Grabber) grabFromKubeletInternal(nodeName string, kubeletPort int) (Kub // GrabFromScheduler returns metrics from scheduler func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) { - if !g.registeredMaster { - return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering") + if g.kubeScheduler == "" { + return SchedulerMetrics{}, fmt.Errorf("kube-scheduler pod is not registered. Skipping Scheduler's metrics gathering") } - output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, insecureSchedulerPort) + output, err := g.getMetricsFromPod(g.client, g.kubeScheduler, metav1.NamespaceSystem, insecureSchedulerPort) if err != nil { return SchedulerMetrics{}, err } @@ -151,8 +161,8 @@ func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) { // GrabFromClusterAutoscaler returns metrics from cluster autoscaler func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) { - if !g.registeredMaster && g.externalClient == nil { - return ClusterAutoscalerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ClusterAutoscaler's metrics gathering") + if !g.HasControlPlanePods() && g.externalClient == nil { + return ClusterAutoscalerMetrics{}, fmt.Errorf("Did not find control-plane pods. Skipping ClusterAutoscaler's metrics gathering") } var client clientset.Interface var namespace string @@ -172,12 +182,12 @@ func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) // GrabFromControllerManager returns metrics from controller manager func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) { - if !g.registeredMaster { - return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering") + if g.kubeControllerManager == "" { + return ControllerManagerMetrics{}, fmt.Errorf("kube-controller-manager pod is not registered. Skipping ControllerManager's metrics gathering") } var err error - podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName) + podName := g.kubeControllerManager g.waitForControllerManagerReadyOnce.Do(func() { if readyErr := e2epod.WaitForPodsReady(g.client, metav1.NamespaceSystem, podName, 0); readyErr != nil { err = fmt.Errorf("error waiting for controller manager pod to be ready: %w", readyErr) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 73c051300ba..626bd9c4240 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -625,7 +625,7 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts { framework.ExpectNoError(err, "Error creating metrics grabber: %v", err) } - if !metricsGrabber.HasRegisteredMaster() { + if !metricsGrabber.HasControlPlanePods() { framework.Logf("Warning: Environment does not support getting controller-manager metrics") return opCounts{} } diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index 614dd7fb4bb..eae8360e216 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -100,7 +100,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() { ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() { var err error - if !metricsGrabber.HasRegisteredMaster() { + if !metricsGrabber.HasControlPlanePods() { e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping") } @@ -146,7 +146,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() { ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() { var err error - if !metricsGrabber.HasRegisteredMaster() { + if !metricsGrabber.HasControlPlanePods() { e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping") } @@ -446,7 +446,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() { } ginkgo.BeforeEach(func() { - if !metricsGrabber.HasRegisteredMaster() { + if !metricsGrabber.HasControlPlanePods() { e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping") }