diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go index 8c94b35a217..de481e0adfc 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go @@ -315,6 +315,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { observedRawRecord: observedRawRecord, observedTime: test.observedTime, clock: clock, + metrics: globalMetricsFactory.newLeaderMetrics(), } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { if test.retryAfter != 0 { @@ -491,6 +492,7 @@ func testReleaseLease(t *testing.T, objectType string) { observedRawRecord: observedRawRecord, observedTime: test.observedTime, clock: clock.RealClock{}, + metrics: globalMetricsFactory.newLeaderMetrics(), } if !le.tryAcquireOrRenew(context.Background()) { t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", true) diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go b/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go index 65917bf88e1..7438345fb15 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go @@ -26,24 +26,26 @@ import ( type leaderMetricsAdapter interface { leaderOn(name string) leaderOff(name string) + slowpathExercised(name string) } -// GaugeMetric represents a single numerical value that can arbitrarily go up -// and down. -type SwitchMetric interface { +// LeaderMetric instruments metrics used in leader election. +type LeaderMetric interface { On(name string) Off(name string) + SlowpathExercised(name string) } type noopMetric struct{} -func (noopMetric) On(name string) {} -func (noopMetric) Off(name string) {} +func (noopMetric) On(name string) {} +func (noopMetric) Off(name string) {} +func (noopMetric) SlowpathExercised(name string) {} // defaultLeaderMetrics expects the caller to lock before setting any metrics. type defaultLeaderMetrics struct { // leader's value indicates if the current process is the owner of name lease - leader SwitchMetric + leader LeaderMetric } func (m *defaultLeaderMetrics) leaderOn(name string) { @@ -60,19 +62,27 @@ func (m *defaultLeaderMetrics) leaderOff(name string) { m.leader.Off(name) } +func (m *defaultLeaderMetrics) slowpathExercised(name string) { + if m == nil { + return + } + m.leader.SlowpathExercised(name) +} + type noMetrics struct{} -func (noMetrics) leaderOn(name string) {} -func (noMetrics) leaderOff(name string) {} +func (noMetrics) leaderOn(name string) {} +func (noMetrics) leaderOff(name string) {} +func (noMetrics) slowpathExercised(name string) {} // MetricsProvider generates various metrics used by the leader election. type MetricsProvider interface { - NewLeaderMetric() SwitchMetric + NewLeaderMetric() LeaderMetric } type noopMetricsProvider struct{} -func (_ noopMetricsProvider) NewLeaderMetric() SwitchMetric { +func (noopMetricsProvider) NewLeaderMetric() LeaderMetric { return noopMetric{} } diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/clientgo/leaderelection/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/clientgo/leaderelection/metrics.go index fd85d136cb0..004767bd766 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/clientgo/leaderelection/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/clientgo/leaderelection/metrics.go @@ -28,27 +28,41 @@ var ( StabilityLevel: k8smetrics.ALPHA, Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.", }, []string{"name"}) + // A cumulative counter should be sufficient to get a rough ratio of slow path + // exercised given the leader election frequency is specified explicitly. So that + // to avoid the overhead to report a counter exercising fastpath. + leaderSlowpathCounter = k8smetrics.NewCounterVec(&k8smetrics.CounterOpts{ + Name: "leader_election_slowpath_total", + StabilityLevel: k8smetrics.ALPHA, + Help: "Total number of slow path exercised in renewing leader leases. 'name' is the string used to identify the lease. Please make sure to group by name.", + }, []string{"name"}) ) func init() { legacyregistry.MustRegister(leaderGauge) + legacyregistry.MustRegister(leaderSlowpathCounter) leaderelection.SetProvider(prometheusMetricsProvider{}) } type prometheusMetricsProvider struct{} -func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric { - return &switchAdapter{gauge: leaderGauge} +func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric { + return &leaderAdapter{gauge: leaderGauge, counter: leaderSlowpathCounter} } -type switchAdapter struct { - gauge *k8smetrics.GaugeVec +type leaderAdapter struct { + gauge *k8smetrics.GaugeVec + counter *k8smetrics.CounterVec } -func (s *switchAdapter) On(name string) { +func (s *leaderAdapter) On(name string) { s.gauge.WithLabelValues(name).Set(1.0) } -func (s *switchAdapter) Off(name string) { +func (s *leaderAdapter) Off(name string) { s.gauge.WithLabelValues(name).Set(0.0) } + +func (s *leaderAdapter) SlowpathExercised(name string) { + s.counter.WithLabelValues(name).Inc() +}