Merge pull request #109626 from logicalhan/watch-cache-metrics

refactor watch cache metrics so that they are in a single place

staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -408,7 +408,7 @@ func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
 		successfulList = true
 		c.ready.set(true)
 		klog.V(1).Infof("cacher (%v): initialized", c.objectType.String())
-		watchCacheInitializations.WithLabelValues(c.objectType.String()).Inc()
+		metrics.WatchCacheInitializations.WithLabelValues(c.objectType.String()).Inc()
 	})
 	defer func() {
 		if successfulList {
@@ -773,7 +773,7 @@ func (c *Cacher) dispatchEvents() {
 				c.dispatchEvent(&event)
 			}
 			lastProcessedResourceVersion = event.ResourceVersion
-			eventsCounter.WithLabelValues(c.objectType.String()).Inc()
+			metrics.EventsCounter.WithLabelValues(c.objectType.String()).Inc()
 		case <-bookmarkTimer.C():
 			bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25))
 			// Never send a bookmark event if we did not see an event here, this is fine
@@ -1212,7 +1212,7 @@ func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer) bool {
 		// Since we don't want to block on it infinitely,
 		// we simply terminate it.
 		klog.V(1).Infof("Forcing %v watcher close due to unresponsiveness: %v. len(c.input) = %v, len(c.result) = %v", c.objectType.String(), c.identifier, len(c.input), len(c.result))
-		terminatedWatchersCounter.WithLabelValues(c.objectType.String()).Inc()
+		metrics.TerminatedWatchersCounter.WithLabelValues(c.objectType.String()).Inc()
 		c.forget()
 	}
 
@@ -1392,7 +1392,7 @@ func (c *cacheWatcher) processInterval(ctx context.Context, cacheInterval *watch
 
 	objType := c.objectType.String()
 	if initEventCount > 0 {
-		initCounter.WithLabelValues(objType).Add(float64(initEventCount))
+		metrics.InitCounter.WithLabelValues(objType).Add(float64(initEventCount))
 	}
 	processingTime := time.Since(startTime)
 	if processingTime > initProcessThreshold {

staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics.go

@@ -1,124 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cacher
-
-import (
-	"k8s.io/component-base/metrics"
-	"k8s.io/component-base/metrics/legacyregistry"
-)
-
-const (
-	namespace = "apiserver"
-	subsystem = "watch_cache"
-)
-
-/*
- * By default, all the following metrics are defined as falling under
- * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
- *
- * Promoting the stability level of the metric is a responsibility of the component owner, since it
- * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
- * the metric stability policy.
- */
-var (
-	initCounter = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Name:           "apiserver_init_events_total",
-			Help:           "Counter of init events processed in watch cache broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-
-	eventsCounter = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Namespace:      namespace,
-			Subsystem:      subsystem,
-			Name:           "events_dispatched_total",
-			Help:           "Counter of events dispatched in watch cache broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-
-	terminatedWatchersCounter = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Name:           "apiserver_terminated_watchers_total",
-			Help:           "Counter of watchers closed due to unresponsiveness broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-
-	watchCacheCapacityIncreaseTotal = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Name:           "watch_cache_capacity_increase_total",
-			Help:           "Total number of watch cache capacity increase events broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-
-	watchCacheCapacityDecreaseTotal = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Name:           "watch_cache_capacity_decrease_total",
-			Help:           "Total number of watch cache capacity decrease events broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-
-	watchCacheCapacity = metrics.NewGaugeVec(
-		&metrics.GaugeOpts{
-			Name:           "watch_cache_capacity",
-			Help:           "Total capacity of watch cache broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-
-	watchCacheInitializations = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Namespace:      namespace,
-			Subsystem:      subsystem,
-			Name:           "initializations_total",
-			Help:           "Counter of watch cache initializations broken by resource type.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
-)
-
-func init() {
-	legacyregistry.MustRegister(initCounter)
-	legacyregistry.MustRegister(eventsCounter)
-	legacyregistry.MustRegister(terminatedWatchersCounter)
-	legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal)
-	legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal)
-	legacyregistry.MustRegister(watchCacheCapacity)
-	legacyregistry.MustRegister(watchCacheInitializations)
-}
-
-// recordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations.
-func recordsWatchCacheCapacityChange(objType string, old, new int) {
-	watchCacheCapacity.WithLabelValues(objType).Set(float64(new))
-	if old < new {
-		watchCacheCapacityIncreaseTotal.WithLabelValues(objType).Inc()
-		return
-	}
-	watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc()
-}

staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics/metrics.go

@@ -23,6 +23,11 @@ import (
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
+const (
+	namespace = "apiserver"
+	subsystem = "watch_cache"
+)
+
 /*
  * By default, all the following metrics are defined as falling under
  * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
@@ -34,7 +39,8 @@ import (
 var (
 	listCacheCount = compbasemetrics.NewCounterVec(
 		&compbasemetrics.CounterOpts{
-			Name:           "apiserver_cache_list_total",
+			Namespace:      namespace,
+			Name:           "cache_list_total",
 			Help:           "Number of LIST requests served from watch cache",
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
@@ -42,7 +48,8 @@ var (
 	)
 	listCacheNumFetched = compbasemetrics.NewCounterVec(
 		&compbasemetrics.CounterOpts{
-			Name:           "apiserver_cache_list_fetched_objects_total",
+			Namespace:      namespace,
+			Name:           "cache_list_fetched_objects_total",
 			Help:           "Number of objects read from watch cache in the course of serving a LIST request",
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
@@ -50,12 +57,84 @@ var (
 	)
 	listCacheNumReturned = compbasemetrics.NewCounterVec(
 		&compbasemetrics.CounterOpts{
-			Name:           "apiserver_cache_list_returned_objects_total",
+			Namespace:      namespace,
+			Name:           "cache_list_returned_objects_total",
 			Help:           "Number of objects returned for a LIST request from watch cache",
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{"resource_prefix"},
 	)
+	InitCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Namespace:      namespace,
+			Name:           "init_events_total",
+			Help:           "Counter of init events processed in watch cache broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	EventsCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "events_dispatched_total",
+			Help:           "Counter of events dispatched in watch cache broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	TerminatedWatchersCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Namespace:      namespace,
+			Name:           "terminated_watchers_total",
+			Help:           "Counter of watchers closed due to unresponsiveness broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	watchCacheCapacityIncreaseTotal = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Subsystem:      subsystem,
+			Name:           "capacity_increase_total",
+			Help:           "Total number of watch cache capacity increase events broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	watchCacheCapacityDecreaseTotal = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Subsystem:      subsystem,
+			Name:           "capacity_decrease_total",
+			Help:           "Total number of watch cache capacity decrease events broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	WatchCacheCapacity = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Subsystem:      subsystem,
+			Name:           "capacity",
+			Help:           "Total capacity of watch cache broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	WatchCacheInitializations = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "initializations_total",
+			Help:           "Counter of watch cache initializations broken by resource type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
 )
 
 var registerMetrics sync.Once
@@ -67,6 +146,13 @@ func Register() {
 		legacyregistry.MustRegister(listCacheCount)
 		legacyregistry.MustRegister(listCacheNumFetched)
 		legacyregistry.MustRegister(listCacheNumReturned)
+		legacyregistry.MustRegister(InitCounter)
+		legacyregistry.MustRegister(EventsCounter)
+		legacyregistry.MustRegister(TerminatedWatchersCounter)
+		legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal)
+		legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal)
+		legacyregistry.MustRegister(WatchCacheCapacity)
+		legacyregistry.MustRegister(WatchCacheInitializations)
 	})
 }
 
@@ -76,3 +162,13 @@ func RecordListCacheMetrics(resourcePrefix, indexName string, numFetched, numRet
 	listCacheNumFetched.WithLabelValues(resourcePrefix, indexName).Add(float64(numFetched))
 	listCacheNumReturned.WithLabelValues(resourcePrefix).Add(float64(numReturned))
 }
+
+// RecordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations.
+func RecordsWatchCacheCapacityChange(objType string, old, new int) {
+	WatchCacheCapacity.WithLabelValues(objType).Set(float64(new))
+	if old < new {
+		WatchCacheCapacity.WithLabelValues(objType).Inc()
+		return
+	}
+	watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc()
+}

staging/src/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/cacher/metrics"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/clock"
@@ -220,7 +221,7 @@ func newWatchCache(
 		objectType:          objectType,
 	}
 	objType := objectType.String()
-	watchCacheCapacity.WithLabelValues(objType).Set(float64(wc.capacity))
+	metrics.WatchCacheCapacity.WithLabelValues(objType).Set(float64(wc.capacity))
 	wc.cond = sync.NewCond(wc.RLocker())
 	wc.indexValidator = wc.isIndexValidLocked
 
@@ -385,7 +386,7 @@ func (w *watchCache) doCacheResizeLocked(capacity int) {
 		newCache[i%capacity] = w.cache[i%w.capacity]
 	}
 	w.cache = newCache
-	recordsWatchCacheCapacityChange(w.objectType.String(), w.capacity, capacity)
+	metrics.RecordsWatchCacheCapacityChange(w.objectType.String(), w.capacity, capacity)
 	w.capacity = capacity
 }