From 47a21aa285c4399a8adb58f5f88ee43d475b1966 Mon Sep 17 00:00:00 2001
From: Hongchao Deng <hongchaodeng1@gmail.com>
Date: Mon, 4 Jan 2016 14:50:37 -0800
Subject: [PATCH 1/2] scheduler metrics should only observe successful ops

---
 plugin/pkg/scheduler/scheduler.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/plugin/pkg/scheduler/scheduler.go b/plugin/pkg/scheduler/scheduler.go
index cbfcbc8fd62..16f5afc103d 100644
--- a/plugin/pkg/scheduler/scheduler.go
+++ b/plugin/pkg/scheduler/scheduler.go
@@ -125,17 +125,15 @@ func (s *Scheduler) scheduleOne() {
 
 	glog.V(3).Infof("Attempting to schedule: %+v", pod)
 	start := time.Now()
-	defer func() {
-		metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
-	}()
 	dest, err := s.config.Algorithm.Schedule(pod, s.config.NodeLister)
-	metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
 	if err != nil {
 		glog.V(1).Infof("Failed to schedule: %+v", pod)
 		s.config.Recorder.Eventf(pod, api.EventTypeWarning, "FailedScheduling", "%v", err)
 		s.config.Error(pod, err)
 		return
 	}
+	metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
+
 	b := &api.Binding{
 		ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name},
 		Target: api.ObjectReference{
@@ -149,17 +147,19 @@ func (s *Scheduler) scheduleOne() {
 	s.config.Modeler.LockedAction(func() {
 		bindingStart := time.Now()
 		err := s.config.Binder.Bind(b)
-		metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart))
 		if err != nil {
 			glog.V(1).Infof("Failed to bind pod: %+v", err)
 			s.config.Recorder.Eventf(pod, api.EventTypeNormal, "FailedScheduling", "Binding rejected: %v", err)
 			s.config.Error(pod, err)
 			return
 		}
+		metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart))
 		s.config.Recorder.Eventf(pod, api.EventTypeNormal, "Scheduled", "Successfully assigned %v to %v", pod.Name, dest)
 		// tell the model to assume that this binding took effect.
 		assumed := *pod
 		assumed.Spec.NodeName = dest
 		s.config.Modeler.AssumePod(&assumed)
 	})
+
+	metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
 }

From 2744447dbacf8c9462079c22e002a87422f815fd Mon Sep 17 00:00:00 2001
From: Hongchao Deng <hongchaodeng1@gmail.com>
Date: Mon, 4 Jan 2016 14:50:49 -0800
Subject: [PATCH 2/2] scheduler metrics: change Summary to Histogram

---
 plugin/pkg/scheduler/metrics/metrics.go | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/plugin/pkg/scheduler/metrics/metrics.go b/plugin/pkg/scheduler/metrics/metrics.go
index db6258a31d9..f3c07ca9af3 100644
--- a/plugin/pkg/scheduler/metrics/metrics.go
+++ b/plugin/pkg/scheduler/metrics/metrics.go
@@ -28,28 +28,28 @@ const schedulerSubsystem = "scheduler"
 var BindingSaturationReportInterval = 1 * time.Second
 
 var (
-	E2eSchedulingLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
+	E2eSchedulingLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
 			Name:      "e2e_scheduling_latency_microseconds",
 			Help:      "E2e scheduling latency (scheduling algorithm + binding)",
-			MaxAge:    time.Hour,
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
-	SchedulingAlgorithmLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
+	SchedulingAlgorithmLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
 			Name:      "scheduling_algorithm_latency_microseconds",
 			Help:      "Scheduling algorithm latency",
-			MaxAge:    time.Hour,
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
-	BindingLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
+	BindingLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
 			Name:      "binding_latency_microseconds",
 			Help:      "Binding latency",
-			MaxAge:    time.Hour,
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
 	BindingRateLimiterSaturation = prometheus.NewGauge(