Merge pull request #83535 from wgliang/cleanup/scheduler_goroutines

Add a metric to track number of scheduler binding and prioritizing goroutines

pkg/scheduler/core/generic_scheduler.go

@@ -732,7 +732,11 @@ func PrioritizeNodes(
 		if priorityConfigs[i].Function != nil {
 			wg.Add(1)
 			go func(index int) {
-				defer wg.Done()
+				metrics.SchedulerGoroutines.WithLabelValues("prioritizing_legacy").Inc()
+				defer func() {
+					metrics.SchedulerGoroutines.WithLabelValues("prioritizing_legacy").Dec()
+					wg.Done()
+				}()
 				var err error
 				results[index], err = priorityConfigs[index].Function(pod, nodeNameToInfo, nodes)
 				if err != nil {
@@ -766,7 +770,11 @@ func PrioritizeNodes(
 		}
 		wg.Add(1)
 		go func(index int) {
-			defer wg.Done()
+			metrics.SchedulerGoroutines.WithLabelValues("prioritizing_mapreduce").Inc()
+			defer func() {
+				metrics.SchedulerGoroutines.WithLabelValues("prioritizing_mapreduce").Dec()
+				wg.Done()
+			}()
 			if err := priorityConfigs[index].Reduce(pod, meta, nodeNameToInfo, results[index]); err != nil {
 				appendError(err)
 			}
@@ -812,7 +820,11 @@ func PrioritizeNodes(
 			}
 			wg.Add(1)
 			go func(extIndex int) {
-				defer wg.Done()
+				metrics.SchedulerGoroutines.WithLabelValues("prioritizing_extender").Inc()
+				defer func() {
+					metrics.SchedulerGoroutines.WithLabelValues("prioritizing_extender").Dec()
+					wg.Done()
+				}()
 				prioritizedList, weight, err := extenders[extIndex].Prioritize(pod, nodes)
 				if err != nil {
 					// Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities

pkg/scheduler/metrics/metrics.go

@@ -211,7 +211,6 @@ var (
 			Help:           "Total preemption attempts in the cluster till now",
 			StabilityLevel: metrics.ALPHA,
 		})
-
 	pendingPods = metrics.NewGaugeVec(
 		&metrics.GaugeOpts{
 			Subsystem:      SchedulerSubsystem,
@@ -219,6 +218,13 @@ var (
 			Help:           "Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulableQ.",
 			StabilityLevel: metrics.ALPHA,
 		}, []string{"queue"})
+	SchedulerGoroutines = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Subsystem:      SchedulerSubsystem,
+			Name:           "scheduler_goroutines",
+			Help:           "Number of running goroutines split by the work they do such as binding.",
+			StabilityLevel: metrics.ALPHA,
+		}, []string{"work"})
 
 	PodSchedulingDuration = metrics.NewHistogram(
 		&metrics.HistogramOpts{
@@ -279,6 +285,7 @@ var (
 		PodSchedulingAttempts,
 		FrameworkExtensionPointDuration,
 		SchedulerQueueIncomingPods,
+		SchedulerGoroutines,
 	}
 )
 

pkg/scheduler/scheduler.go

@@ -675,6 +675,9 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) {
 	}
 	// bind the pod to its host asynchronously (we can do this b/c of the assumption step above).
 	go func() {
+		metrics.SchedulerGoroutines.WithLabelValues("binding").Inc()
+		defer metrics.SchedulerGoroutines.WithLabelValues("binding").Dec()
+
 		// Bind volumes first before Pod
 		if !allBound {
 			err := sched.bindVolumes(assumedPod)