diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 2f49b0733f6..15babf64aee 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -192,14 +192,16 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister FailedPredicates: failedPredicateMap, } } - metrics.SchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPredicateEvalTime)) + metrics.SchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInSeconds(startPredicateEvalTime)) + metrics.DeprecatedSchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPredicateEvalTime)) metrics.SchedulingLatency.WithLabelValues(metrics.PredicateEvaluation).Observe(metrics.SinceInSeconds(startPredicateEvalTime)) trace.Step("Prioritizing") startPriorityEvalTime := time.Now() // When only one node after predicate, just use it. if len(filteredNodes) == 1 { - metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) + metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInSeconds(startPriorityEvalTime)) + metrics.DeprecatedSchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) return ScheduleResult{ SuggestedHost: filteredNodes[0].Name, EvaluatedNodes: 1 + len(failedPredicateMap), @@ -212,7 +214,8 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister if err != nil { return result, err } - metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) + metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInSeconds(startPriorityEvalTime)) + metrics.DeprecatedSchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) metrics.SchedulingLatency.WithLabelValues(metrics.PriorityEvaluation).Observe(metrics.SinceInSeconds(startPriorityEvalTime)) trace.Step("Selecting host") diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go index b05ff33eb2e..1d31f5cdfe9 100644 --- a/pkg/scheduler/metrics/metrics.go +++ b/pkg/scheduler/metrics/metrics.go @@ -71,50 +71,98 @@ var ( []string{OperationLabel}, ) E2eSchedulingLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "e2e_scheduling_latency_seconds", + Help: "E2e scheduling latency in seconds (scheduling algorithm + binding)", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedE2eSchedulingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "e2e_scheduling_latency_microseconds", - Help: "E2e scheduling latency (scheduling algorithm + binding)", + Help: "E2e scheduling latency in microseconds (scheduling algorithm + binding)", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_latency_seconds", + Help: "Scheduling algorithm latency in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_latency_microseconds", - Help: "Scheduling algorithm latency", + Help: "Scheduling algorithm latency in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmPredicateEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_predicate_evaluation_seconds", + Help: "Scheduling algorithm predicate evaluation duration in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmPredicateEvaluationDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_predicate_evaluation", - Help: "Scheduling algorithm predicate evaluation duration", + Help: "Scheduling algorithm predicate evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmPriorityEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_priority_evaluation_seconds", + Help: "Scheduling algorithm priority evaluation duration in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmPriorityEvaluationDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_priority_evaluation", - Help: "Scheduling algorithm priority evaluation duration", + Help: "Scheduling algorithm priority evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmPremptionEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_preemption_evaluation_seconds", + Help: "Scheduling algorithm preemption evaluation duration in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmPremptionEvaluationDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_preemption_evaluation", - Help: "Scheduling algorithm preemption evaluation duration", + Help: "Scheduling algorithm preemption evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) BindingLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "binding_latency_seconds", + Help: "Binding latency in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedBindingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "binding_latency_microseconds", - Help: "Binding latency", + Help: "Binding latency in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -135,11 +183,17 @@ var ( scheduleAttempts, SchedulingLatency, E2eSchedulingLatency, + DeprecatedE2eSchedulingLatency, SchedulingAlgorithmLatency, + DeprecatedSchedulingAlgorithmLatency, BindingLatency, + DeprecatedBindingLatency, SchedulingAlgorithmPredicateEvaluationDuration, + DeprecatedSchedulingAlgorithmPredicateEvaluationDuration, SchedulingAlgorithmPriorityEvaluationDuration, + DeprecatedSchedulingAlgorithmPriorityEvaluationDuration, SchedulingAlgorithmPremptionEvaluationDuration, + DeprecatedSchedulingAlgorithmPremptionEvaluationDuration, PreemptionVictims, PreemptionAttempts, } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 999f25b2bfe..b1266e2e363 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -444,7 +444,8 @@ func (sched *Scheduler) bind(assumed *v1.Pod, b *v1.Binding) error { return err } - metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) + metrics.BindingLatency.Observe(metrics.SinceInSeconds(bindingStart)) + metrics.DeprecatedBindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) metrics.SchedulingLatency.WithLabelValues(metrics.Binding).Observe(metrics.SinceInSeconds(bindingStart)) sched.config.Recorder.Eventf(assumed, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v/%v to %v", assumed.Namespace, assumed.Name, b.Target.Name) return nil @@ -487,7 +488,8 @@ func (sched *Scheduler) scheduleOne() { preemptionStartTime := time.Now() sched.preempt(pod, fitError) metrics.PreemptionAttempts.Inc() - metrics.SchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime)) + metrics.SchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInSeconds(preemptionStartTime)) + metrics.DeprecatedSchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime)) metrics.SchedulingLatency.WithLabelValues(metrics.PreemptionEvaluation).Observe(metrics.SinceInSeconds(preemptionStartTime)) } // Pod did not fit anywhere, so it is counted as a failure. If preemption @@ -500,7 +502,8 @@ func (sched *Scheduler) scheduleOne() { } return } - metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) + metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInSeconds(start)) + metrics.DeprecatedSchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) // Tell the cache to assume that a pod now is running on a given node, even though it hasn't been bound yet. // This allows us to keep scheduling without waiting on binding to occur. assumedPod := pod.DeepCopy() @@ -579,7 +582,8 @@ func (sched *Scheduler) scheduleOne() { Name: scheduleResult.SuggestedHost, }, }) - metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) + metrics.E2eSchedulingLatency.Observe(metrics.SinceInSeconds(start)) + metrics.DeprecatedE2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) if err != nil { klog.Errorf("error binding pod: %v", err) metrics.PodScheduleErrors.Inc()