mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Merge pull request #92485 from oomichi/metrics-grabber
Avoid DeprecatedMightBeMasterNode() in e2e metrics
This commit is contained in:
commit
0eaa85483b
@ -31,7 +31,6 @@ go_library(
|
|||||||
"//test/e2e/framework/log:go_default_library",
|
"//test/e2e/framework/log:go_default_library",
|
||||||
"//test/e2e/framework/pod:go_default_library",
|
"//test/e2e/framework/pod:go_default_library",
|
||||||
"//test/e2e/perftype:go_default_library",
|
"//test/e2e/perftype:go_default_library",
|
||||||
"//test/e2e/system:go_default_library",
|
|
||||||
"//vendor/k8s.io/klog/v2:go_default_library",
|
"//vendor/k8s.io/klog/v2:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -19,6 +19,7 @@ package metrics
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"regexp"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -27,7 +28,6 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/util/wait"
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||||
"k8s.io/kubernetes/test/e2e/system"
|
|
||||||
|
|
||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
)
|
)
|
||||||
@ -61,38 +61,48 @@ type Grabber struct {
|
|||||||
grabFromKubelets bool
|
grabFromKubelets bool
|
||||||
grabFromScheduler bool
|
grabFromScheduler bool
|
||||||
grabFromClusterAutoscaler bool
|
grabFromClusterAutoscaler bool
|
||||||
masterName string
|
kubeScheduler string
|
||||||
registeredMaster bool
|
kubeControllerManager string
|
||||||
waitForControllerManagerReadyOnce sync.Once
|
waitForControllerManagerReadyOnce sync.Once
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewMetricsGrabber returns new metrics which are initialized.
|
// NewMetricsGrabber returns new metrics which are initialized.
|
||||||
func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) {
|
func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) {
|
||||||
registeredMaster := false
|
|
||||||
masterName := ""
|
kubeScheduler := ""
|
||||||
nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
|
kubeControllerManager := ""
|
||||||
|
|
||||||
|
regKubeScheduler := regexp.MustCompile("kube-scheduler-.*")
|
||||||
|
regKubeControllerManager := regexp.MustCompile("kube-controller-manager-.*")
|
||||||
|
|
||||||
|
podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if len(nodeList.Items) < 1 {
|
if len(podList.Items) < 1 {
|
||||||
klog.Warning("Can't find any Nodes in the API server to grab metrics from")
|
klog.Warningf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem)
|
||||||
}
|
}
|
||||||
for _, node := range nodeList.Items {
|
for _, pod := range podList.Items {
|
||||||
if system.DeprecatedMightBeMasterNode(node.Name) {
|
if regKubeScheduler.MatchString(pod.Name) {
|
||||||
registeredMaster = true
|
kubeScheduler = pod.Name
|
||||||
masterName = node.Name
|
}
|
||||||
|
if regKubeControllerManager.MatchString(pod.Name) {
|
||||||
|
kubeControllerManager = pod.Name
|
||||||
|
}
|
||||||
|
if kubeScheduler != "" && kubeControllerManager != "" {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !registeredMaster {
|
if kubeScheduler == "" {
|
||||||
scheduler = false
|
scheduler = false
|
||||||
|
klog.Warningf("Can't find kube-scheduler pod. Grabbing metrics from kube-scheduler is disabled.")
|
||||||
|
}
|
||||||
|
if kubeControllerManager == "" {
|
||||||
controllers = false
|
controllers = false
|
||||||
clusterAutoscaler = ec != nil
|
klog.Warningf("Can't find kube-controller-manager pod. Grabbing metrics from kube-controller-manager is disabled.")
|
||||||
if clusterAutoscaler {
|
}
|
||||||
klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.")
|
if ec == nil {
|
||||||
} else {
|
klog.Warningf("Did not receive an external client interface. Grabbing metrics from ClusterAutoscaler is disabled.")
|
||||||
klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Grabber{
|
return &Grabber{
|
||||||
@ -103,14 +113,14 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b
|
|||||||
grabFromKubelets: kubelets,
|
grabFromKubelets: kubelets,
|
||||||
grabFromScheduler: scheduler,
|
grabFromScheduler: scheduler,
|
||||||
grabFromClusterAutoscaler: clusterAutoscaler,
|
grabFromClusterAutoscaler: clusterAutoscaler,
|
||||||
masterName: masterName,
|
kubeScheduler: kubeScheduler,
|
||||||
registeredMaster: registeredMaster,
|
kubeControllerManager: kubeControllerManager,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// HasRegisteredMaster returns if metrics grabber was able to find a master node
|
// HasControlPlanePods returns true if metrics grabber was able to find control-plane pods
|
||||||
func (g *Grabber) HasRegisteredMaster() bool {
|
func (g *Grabber) HasControlPlanePods() bool {
|
||||||
return g.registeredMaster
|
return g.kubeScheduler != "" && g.kubeControllerManager != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// GrabFromKubelet returns metrics from kubelet
|
// GrabFromKubelet returns metrics from kubelet
|
||||||
@ -139,10 +149,10 @@ func (g *Grabber) grabFromKubeletInternal(nodeName string, kubeletPort int) (Kub
|
|||||||
|
|
||||||
// GrabFromScheduler returns metrics from scheduler
|
// GrabFromScheduler returns metrics from scheduler
|
||||||
func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
|
func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
|
||||||
if !g.registeredMaster {
|
if g.kubeScheduler == "" {
|
||||||
return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering")
|
return SchedulerMetrics{}, fmt.Errorf("kube-scheduler pod is not registered. Skipping Scheduler's metrics gathering")
|
||||||
}
|
}
|
||||||
output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, insecureSchedulerPort)
|
output, err := g.getMetricsFromPod(g.client, g.kubeScheduler, metav1.NamespaceSystem, insecureSchedulerPort)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return SchedulerMetrics{}, err
|
return SchedulerMetrics{}, err
|
||||||
}
|
}
|
||||||
@ -151,8 +161,8 @@ func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
|
|||||||
|
|
||||||
// GrabFromClusterAutoscaler returns metrics from cluster autoscaler
|
// GrabFromClusterAutoscaler returns metrics from cluster autoscaler
|
||||||
func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) {
|
func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) {
|
||||||
if !g.registeredMaster && g.externalClient == nil {
|
if !g.HasControlPlanePods() && g.externalClient == nil {
|
||||||
return ClusterAutoscalerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ClusterAutoscaler's metrics gathering")
|
return ClusterAutoscalerMetrics{}, fmt.Errorf("Did not find control-plane pods. Skipping ClusterAutoscaler's metrics gathering")
|
||||||
}
|
}
|
||||||
var client clientset.Interface
|
var client clientset.Interface
|
||||||
var namespace string
|
var namespace string
|
||||||
@ -172,12 +182,12 @@ func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error)
|
|||||||
|
|
||||||
// GrabFromControllerManager returns metrics from controller manager
|
// GrabFromControllerManager returns metrics from controller manager
|
||||||
func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) {
|
func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) {
|
||||||
if !g.registeredMaster {
|
if g.kubeControllerManager == "" {
|
||||||
return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
|
return ControllerManagerMetrics{}, fmt.Errorf("kube-controller-manager pod is not registered. Skipping ControllerManager's metrics gathering")
|
||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
|
podName := g.kubeControllerManager
|
||||||
g.waitForControllerManagerReadyOnce.Do(func() {
|
g.waitForControllerManagerReadyOnce.Do(func() {
|
||||||
if readyErr := e2epod.WaitForPodsReady(g.client, metav1.NamespaceSystem, podName, 0); readyErr != nil {
|
if readyErr := e2epod.WaitForPodsReady(g.client, metav1.NamespaceSystem, podName, 0); readyErr != nil {
|
||||||
err = fmt.Errorf("error waiting for controller manager pod to be ready: %w", readyErr)
|
err = fmt.Errorf("error waiting for controller manager pod to be ready: %w", readyErr)
|
||||||
|
@ -625,7 +625,7 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
|
|||||||
framework.ExpectNoError(err, "Error creating metrics grabber: %v", err)
|
framework.ExpectNoError(err, "Error creating metrics grabber: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !metricsGrabber.HasRegisteredMaster() {
|
if !metricsGrabber.HasControlPlanePods() {
|
||||||
framework.Logf("Warning: Environment does not support getting controller-manager metrics")
|
framework.Logf("Warning: Environment does not support getting controller-manager metrics")
|
||||||
return opCounts{}
|
return opCounts{}
|
||||||
}
|
}
|
||||||
|
@ -100,7 +100,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
|
|||||||
ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() {
|
ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
if !metricsGrabber.HasRegisteredMaster() {
|
if !metricsGrabber.HasControlPlanePods() {
|
||||||
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
|
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -146,7 +146,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
|
|||||||
ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() {
|
ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
if !metricsGrabber.HasRegisteredMaster() {
|
if !metricsGrabber.HasControlPlanePods() {
|
||||||
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
|
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -446,7 +446,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ginkgo.BeforeEach(func() {
|
ginkgo.BeforeEach(func() {
|
||||||
if !metricsGrabber.HasRegisteredMaster() {
|
if !metricsGrabber.HasControlPlanePods() {
|
||||||
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
|
e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user