Merge pull request #46245 from ravisantoshgudimetla/metrics_additions

Automatic merge from submit-queue (batch tested with PRs 57266, 58187, 58186, 46245, 56509). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Added metrics for predicate and priority evaluation 

**What this PR does / why we need it**:

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #45972

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

pkg/scheduler/core/BUILD

@@ -50,6 +50,7 @@ go_library(
         "//pkg/scheduler/algorithm/predicates:go_default_library",
         "//pkg/scheduler/algorithm/priorities/util:go_default_library",
         "//pkg/scheduler/api:go_default_library",
+        "//pkg/scheduler/metrics:go_default_library",
         "//pkg/scheduler/schedulercache:go_default_library",
         "//pkg/scheduler/util:go_default_library",
         "//pkg/scheduler/volumebinder:go_default_library",

pkg/scheduler/core/generic_scheduler.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/algorithm"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	"k8s.io/kubernetes/pkg/scheduler/schedulercache"
 	"k8s.io/kubernetes/pkg/scheduler/util"
 
@@ -131,6 +132,7 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister
 	}
 
 	trace.Step("Computing predicates")
+	startPredicateEvalTime := time.Now()
 	filteredNodes, failedPredicateMap, err := findNodesThatFit(pod, g.cachedNodeInfoMap, nodes, g.predicates, g.extenders, g.predicateMetaProducer, g.equivalenceCache, g.schedulingQueue)
 	if err != nil {
 		return "", err
@@ -143,11 +145,13 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister
 			FailedPredicates: failedPredicateMap,
 		}
 	}
+	metrics.SchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPredicateEvalTime))
 
 	trace.Step("Prioritizing")
-
+	startPriorityEvalTime := time.Now()
 	// When only one node after predicate, just use it.
 	if len(filteredNodes) == 1 {
+		metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime))
 		return filteredNodes[0].Name, nil
 	}
 
@@ -156,6 +160,7 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister
 	if err != nil {
 		return "", err
 	}
+	metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime))
 
 	trace.Step("Selecting host")
 	return g.selectHost(priorityList)

pkg/scheduler/metrics/metrics.go

@@ -25,8 +25,7 @@ import (
 
 const schedulerSubsystem = "scheduler"
 
-var BindingSaturationReportInterval = 1 * time.Second
-
+// All the histogram based metrics have 1ms as size for the smallest bucket.
 var (
 	E2eSchedulingLatency = prometheus.NewHistogram(
 		prometheus.HistogramOpts{
@@ -44,6 +43,22 @@ var (
 			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
+	SchedulingAlgorithmPredicateEvaluationDuration = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Subsystem: schedulerSubsystem,
+			Name:      "scheduling_algorithm_predicate_evaluation",
+			Help:      "Scheduling algorithm predicate evaluation duration",
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
+		},
+	)
+	SchedulingAlgorithmPriorityEvaluationDuration = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Subsystem: schedulerSubsystem,
+			Name:      "scheduling_algorithm_priority_evaluation",
+			Help:      "Scheduling algorithm priority evaluation duration",
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
+		},
+	)
 	BindingLatency = prometheus.NewHistogram(
 		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
@@ -63,6 +78,8 @@ func Register() {
 		prometheus.MustRegister(E2eSchedulingLatency)
 		prometheus.MustRegister(SchedulingAlgorithmLatency)
 		prometheus.MustRegister(BindingLatency)
+		prometheus.MustRegister(SchedulingAlgorithmPredicateEvaluationDuration)
+		prometheus.MustRegister(SchedulingAlgorithmPriorityEvaluationDuration)
 	})
 }