From 5b42a84cdeb9ec464d186d0dc62de5bb85eaae3c Mon Sep 17 00:00:00 2001 From: danielqsj Date: Wed, 26 Dec 2018 16:31:29 +0800 Subject: [PATCH 1/2] Change scheduler metrics to conform guideline --- pkg/scheduler/core/generic_scheduler.go | 9 ++-- pkg/scheduler/metrics/metrics.go | 66 ++++++++++++++++++++++--- pkg/scheduler/scheduler.go | 12 +++-- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 2f49b0733f6..15babf64aee 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -192,14 +192,16 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister FailedPredicates: failedPredicateMap, } } - metrics.SchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPredicateEvalTime)) + metrics.SchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInSeconds(startPredicateEvalTime)) + metrics.DeprecatedSchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPredicateEvalTime)) metrics.SchedulingLatency.WithLabelValues(metrics.PredicateEvaluation).Observe(metrics.SinceInSeconds(startPredicateEvalTime)) trace.Step("Prioritizing") startPriorityEvalTime := time.Now() // When only one node after predicate, just use it. if len(filteredNodes) == 1 { - metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) + metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInSeconds(startPriorityEvalTime)) + metrics.DeprecatedSchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) return ScheduleResult{ SuggestedHost: filteredNodes[0].Name, EvaluatedNodes: 1 + len(failedPredicateMap), @@ -212,7 +214,8 @@ func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister if err != nil { return result, err } - metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) + metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInSeconds(startPriorityEvalTime)) + metrics.DeprecatedSchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime)) metrics.SchedulingLatency.WithLabelValues(metrics.PriorityEvaluation).Observe(metrics.SinceInSeconds(startPriorityEvalTime)) trace.Step("Selecting host") diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go index b05ff33eb2e..1d31f5cdfe9 100644 --- a/pkg/scheduler/metrics/metrics.go +++ b/pkg/scheduler/metrics/metrics.go @@ -71,50 +71,98 @@ var ( []string{OperationLabel}, ) E2eSchedulingLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "e2e_scheduling_latency_seconds", + Help: "E2e scheduling latency in seconds (scheduling algorithm + binding)", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedE2eSchedulingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "e2e_scheduling_latency_microseconds", - Help: "E2e scheduling latency (scheduling algorithm + binding)", + Help: "E2e scheduling latency in microseconds (scheduling algorithm + binding)", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_latency_seconds", + Help: "Scheduling algorithm latency in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_latency_microseconds", - Help: "Scheduling algorithm latency", + Help: "Scheduling algorithm latency in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmPredicateEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_predicate_evaluation_seconds", + Help: "Scheduling algorithm predicate evaluation duration in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmPredicateEvaluationDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_predicate_evaluation", - Help: "Scheduling algorithm predicate evaluation duration", + Help: "Scheduling algorithm predicate evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmPriorityEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_priority_evaluation_seconds", + Help: "Scheduling algorithm priority evaluation duration in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmPriorityEvaluationDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_priority_evaluation", - Help: "Scheduling algorithm priority evaluation duration", + Help: "Scheduling algorithm priority evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) SchedulingAlgorithmPremptionEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "scheduling_algorithm_preemption_evaluation_seconds", + Help: "Scheduling algorithm preemption evaluation duration in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedSchedulingAlgorithmPremptionEvaluationDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_preemption_evaluation", - Help: "Scheduling algorithm preemption evaluation duration", + Help: "Scheduling algorithm preemption evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) BindingLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: SchedulerSubsystem, + Name: "binding_latency_seconds", + Help: "Binding latency in seconds", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + ) + DeprecatedBindingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "binding_latency_microseconds", - Help: "Binding latency", + Help: "Binding latency in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -135,11 +183,17 @@ var ( scheduleAttempts, SchedulingLatency, E2eSchedulingLatency, + DeprecatedE2eSchedulingLatency, SchedulingAlgorithmLatency, + DeprecatedSchedulingAlgorithmLatency, BindingLatency, + DeprecatedBindingLatency, SchedulingAlgorithmPredicateEvaluationDuration, + DeprecatedSchedulingAlgorithmPredicateEvaluationDuration, SchedulingAlgorithmPriorityEvaluationDuration, + DeprecatedSchedulingAlgorithmPriorityEvaluationDuration, SchedulingAlgorithmPremptionEvaluationDuration, + DeprecatedSchedulingAlgorithmPremptionEvaluationDuration, PreemptionVictims, PreemptionAttempts, } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 999f25b2bfe..b1266e2e363 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -444,7 +444,8 @@ func (sched *Scheduler) bind(assumed *v1.Pod, b *v1.Binding) error { return err } - metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) + metrics.BindingLatency.Observe(metrics.SinceInSeconds(bindingStart)) + metrics.DeprecatedBindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) metrics.SchedulingLatency.WithLabelValues(metrics.Binding).Observe(metrics.SinceInSeconds(bindingStart)) sched.config.Recorder.Eventf(assumed, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v/%v to %v", assumed.Namespace, assumed.Name, b.Target.Name) return nil @@ -487,7 +488,8 @@ func (sched *Scheduler) scheduleOne() { preemptionStartTime := time.Now() sched.preempt(pod, fitError) metrics.PreemptionAttempts.Inc() - metrics.SchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime)) + metrics.SchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInSeconds(preemptionStartTime)) + metrics.DeprecatedSchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime)) metrics.SchedulingLatency.WithLabelValues(metrics.PreemptionEvaluation).Observe(metrics.SinceInSeconds(preemptionStartTime)) } // Pod did not fit anywhere, so it is counted as a failure. If preemption @@ -500,7 +502,8 @@ func (sched *Scheduler) scheduleOne() { } return } - metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) + metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInSeconds(start)) + metrics.DeprecatedSchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) // Tell the cache to assume that a pod now is running on a given node, even though it hasn't been bound yet. // This allows us to keep scheduling without waiting on binding to occur. assumedPod := pod.DeepCopy() @@ -579,7 +582,8 @@ func (sched *Scheduler) scheduleOne() { Name: scheduleResult.SuggestedHost, }, }) - metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) + metrics.E2eSchedulingLatency.Observe(metrics.SinceInSeconds(start)) + metrics.DeprecatedE2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) if err != nil { klog.Errorf("error binding pod: %v", err) metrics.PodScheduleErrors.Inc() From 0f516f751d04e6cc6d0fca371b23fa0f8fc47e0e Mon Sep 17 00:00:00 2001 From: danielqsj Date: Tue, 8 Jan 2019 13:07:16 +0800 Subject: [PATCH 2/2] Mark deprecated in related scheduler metrics --- pkg/scheduler/metrics/metrics.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go index 1d31f5cdfe9..edc9b9becb6 100644 --- a/pkg/scheduler/metrics/metrics.go +++ b/pkg/scheduler/metrics/metrics.go @@ -82,7 +82,7 @@ var ( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "e2e_scheduling_latency_microseconds", - Help: "E2e scheduling latency in microseconds (scheduling algorithm + binding)", + Help: "(Deprecated) E2e scheduling latency in microseconds (scheduling algorithm + binding)", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -98,7 +98,7 @@ var ( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_latency_microseconds", - Help: "Scheduling algorithm latency in microseconds", + Help: "(Deprecated) Scheduling algorithm latency in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -114,7 +114,7 @@ var ( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_predicate_evaluation", - Help: "Scheduling algorithm predicate evaluation duration in microseconds", + Help: "(Deprecated) Scheduling algorithm predicate evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -130,7 +130,7 @@ var ( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_priority_evaluation", - Help: "Scheduling algorithm priority evaluation duration in microseconds", + Help: "(Deprecated) Scheduling algorithm priority evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -146,7 +146,7 @@ var ( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "scheduling_algorithm_preemption_evaluation", - Help: "Scheduling algorithm preemption evaluation duration in microseconds", + Help: "(Deprecated) Scheduling algorithm preemption evaluation duration in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) @@ -162,7 +162,7 @@ var ( prometheus.HistogramOpts{ Subsystem: SchedulerSubsystem, Name: "binding_latency_microseconds", - Help: "Binding latency in microseconds", + Help: "(Deprecated) Binding latency in microseconds", Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, )