From c9c4be66d3ed697f36d4eb6c20c069a1fdcc6495 Mon Sep 17 00:00:00 2001 From: Shingo Omura Date: Thu, 30 Jan 2020 20:02:54 +0900 Subject: [PATCH] Fix pending_pods, schedule_attempts_total was not recorded because metric initializations are too early. This causes actual metric instance become no-op. modification made in thie commit to make sure actual metric instance won't be no-op metrics: - re-initialize scheduler/metrics.PodSchedule{Successes, Failure, Errors} after metric creation - scheduler/metrics.Register() should be called before initializing SchedulingQueue, --- pkg/scheduler/metrics/metrics.go | 6 ++++++ pkg/scheduler/scheduler.go | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go index 76968ef78ee..095b991ce94 100644 --- a/pkg/scheduler/metrics/metrics.go +++ b/pkg/scheduler/metrics/metrics.go @@ -56,10 +56,13 @@ var ( StabilityLevel: metrics.ALPHA, }, []string{"result"}) // PodScheduleSuccesses counts how many pods were scheduled. + // This metric will be initialized again in Register() to assure the metric is not no-op metric. PodScheduleSuccesses = scheduleAttempts.With(metrics.Labels{"result": "scheduled"}) // PodScheduleFailures counts how many pods could not be scheduled. + // This metric will be initialized again in Register() to assure the metric is not no-op metric. PodScheduleFailures = scheduleAttempts.With(metrics.Labels{"result": "unschedulable"}) // PodScheduleErrors counts how many pods could not be scheduled due to a scheduler error. + // This metric will be initialized again in Register() to assure the metric is not no-op metric. PodScheduleErrors = scheduleAttempts.With(metrics.Labels{"result": "error"}) DeprecatedSchedulingDuration = metrics.NewSummaryVec( &metrics.SummaryOpts{ @@ -262,6 +265,9 @@ func Register() { legacyregistry.MustRegister(metric) } volumeschedulingmetrics.RegisterVolumeSchedulingMetrics() + PodScheduleSuccesses = scheduleAttempts.With(metrics.Labels{"result": "scheduled"}) + PodScheduleFailures = scheduleAttempts.With(metrics.Labels{"result": "unschedulable"}) + PodScheduleErrors = scheduleAttempts.With(metrics.Labels{"result": "error"}) }) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index d1045c92dda..a4f9a057133 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -291,6 +291,8 @@ func New(client clientset.Interface, nodeInfoSnapshot: snapshot, } + metrics.Register() + var sched *Scheduler source := options.schedulerAlgorithmSource switch { @@ -322,7 +324,6 @@ func New(client clientset.Interface, default: return nil, fmt.Errorf("unsupported algorithm source: %v", source) } - metrics.Register() // Additional tweaks to the config produced by the configurator. sched.Recorder = recorder sched.DisablePreemption = options.disablePreemption