leaderelection: Instrument for when slowpath is exercised

Signed-off-by: Eric Lin <exlin@google.com>

staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go

@@ -315,6 +315,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) {
 				observedRawRecord: observedRawRecord,
 				observedTime:      test.observedTime,
 				clock:             clock,
+				metrics:           globalMetricsFactory.newLeaderMetrics(),
 			}
 			if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) {
 				if test.retryAfter != 0 {
@@ -491,6 +492,7 @@ func testReleaseLease(t *testing.T, objectType string) {
 				observedRawRecord: observedRawRecord,
 				observedTime:      test.observedTime,
 				clock:             clock.RealClock{},
+				metrics:           globalMetricsFactory.newLeaderMetrics(),
 			}
 			if !le.tryAcquireOrRenew(context.Background()) {
 				t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", true)

staging/src/k8s.io/client-go/tools/leaderelection/metrics.go

@@ -26,24 +26,26 @@ import (
 type leaderMetricsAdapter interface {
 	leaderOn(name string)
 	leaderOff(name string)
+	slowpathExercised(name string)
 }
 
-// GaugeMetric represents a single numerical value that can arbitrarily go up
+// LeaderMetric instruments metrics used in leader election.
-// and down.
+type LeaderMetric interface {
-type SwitchMetric interface {
 	On(name string)
 	Off(name string)
+	SlowpathExercised(name string)
 }
 
 type noopMetric struct{}
 
-func (noopMetric) On(name string)  {}
+func (noopMetric) On(name string)                {}
-func (noopMetric) Off(name string) {}
+func (noopMetric) Off(name string)               {}
+func (noopMetric) SlowpathExercised(name string) {}
 
 // defaultLeaderMetrics expects the caller to lock before setting any metrics.
 type defaultLeaderMetrics struct {
 	// leader's value indicates if the current process is the owner of name lease
-	leader SwitchMetric
+	leader LeaderMetric
 }
 
 func (m *defaultLeaderMetrics) leaderOn(name string) {
@@ -60,19 +62,27 @@ func (m *defaultLeaderMetrics) leaderOff(name string) {
 	m.leader.Off(name)
 }
 
+func (m *defaultLeaderMetrics) slowpathExercised(name string) {
+	if m == nil {
+		return
+	}
+	m.leader.SlowpathExercised(name)
+}
+
 type noMetrics struct{}
 
-func (noMetrics) leaderOn(name string)  {}
+func (noMetrics) leaderOn(name string)          {}
-func (noMetrics) leaderOff(name string) {}
+func (noMetrics) leaderOff(name string)         {}
+func (noMetrics) slowpathExercised(name string) {}
 
 // MetricsProvider generates various metrics used by the leader election.
 type MetricsProvider interface {
-	NewLeaderMetric() SwitchMetric
+	NewLeaderMetric() LeaderMetric
 }
 
 type noopMetricsProvider struct{}
 
-func (_ noopMetricsProvider) NewLeaderMetric() SwitchMetric {
+func (noopMetricsProvider) NewLeaderMetric() LeaderMetric {
 	return noopMetric{}
 }
 

staging/src/k8s.io/component-base/metrics/prometheus/clientgo/leaderelection/metrics.go

@@ -28,27 +28,41 @@ var (
 		StabilityLevel: k8smetrics.ALPHA,
 		Help:           "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.",
 	}, []string{"name"})
+	// A cumulative counter should be sufficient to get a rough ratio of slow path
+	// exercised given the leader election frequency is specified explicitly. So that
+	// to avoid the overhead to report a counter exercising fastpath.
+	leaderSlowpathCounter = k8smetrics.NewCounterVec(&k8smetrics.CounterOpts{
+		Name:           "leader_election_slowpath_total",
+		StabilityLevel: k8smetrics.ALPHA,
+		Help:           "Total number of slow path exercised in renewing leader leases. 'name' is the string used to identify the lease. Please make sure to group by name.",
+	}, []string{"name"})
 )
 
 func init() {
 	legacyregistry.MustRegister(leaderGauge)
+	legacyregistry.MustRegister(leaderSlowpathCounter)
 	leaderelection.SetProvider(prometheusMetricsProvider{})
 }
 
 type prometheusMetricsProvider struct{}
 
-func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric {
+func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric {
-	return &switchAdapter{gauge: leaderGauge}
+	return &leaderAdapter{gauge: leaderGauge, counter: leaderSlowpathCounter}
 }
 
-type switchAdapter struct {
+type leaderAdapter struct {
-	gauge *k8smetrics.GaugeVec
+	gauge   *k8smetrics.GaugeVec
+	counter *k8smetrics.CounterVec
 }
 
-func (s *switchAdapter) On(name string) {
+func (s *leaderAdapter) On(name string) {
 	s.gauge.WithLabelValues(name).Set(1.0)
 }
 
-func (s *switchAdapter) Off(name string) {
+func (s *leaderAdapter) Off(name string) {
 	s.gauge.WithLabelValues(name).Set(0.0)
 }
+
+func (s *leaderAdapter) SlowpathExercised(name string) {
+	s.counter.WithLabelValues(name).Inc()
+}