diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go index e6eab856afa..90c5fbdafb4 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go @@ -408,7 +408,7 @@ func (c *Cacher) startCaching(stopChannel <-chan struct{}) { successfulList = true c.ready.set(true) klog.V(1).Infof("cacher (%v): initialized", c.objectType.String()) - watchCacheInitializations.WithLabelValues(c.objectType.String()).Inc() + metrics.WatchCacheInitializations.WithLabelValues(c.objectType.String()).Inc() }) defer func() { if successfulList { @@ -773,7 +773,7 @@ func (c *Cacher) dispatchEvents() { c.dispatchEvent(&event) } lastProcessedResourceVersion = event.ResourceVersion - eventsCounter.WithLabelValues(c.objectType.String()).Inc() + metrics.EventsCounter.WithLabelValues(c.objectType.String()).Inc() case <-bookmarkTimer.C(): bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25)) // Never send a bookmark event if we did not see an event here, this is fine @@ -1212,7 +1212,7 @@ func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer) bool { // Since we don't want to block on it infinitely, // we simply terminate it. klog.V(1).Infof("Forcing %v watcher close due to unresponsiveness: %v. len(c.input) = %v, len(c.result) = %v", c.objectType.String(), c.identifier, len(c.input), len(c.result)) - terminatedWatchersCounter.WithLabelValues(c.objectType.String()).Inc() + metrics.TerminatedWatchersCounter.WithLabelValues(c.objectType.String()).Inc() c.forget() } @@ -1392,7 +1392,7 @@ func (c *cacheWatcher) processInterval(ctx context.Context, cacheInterval *watch objType := c.objectType.String() if initEventCount > 0 { - initCounter.WithLabelValues(objType).Add(float64(initEventCount)) + metrics.InitCounter.WithLabelValues(objType).Add(float64(initEventCount)) } processingTime := time.Since(startTime) if processingTime > initProcessThreshold { diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics.go deleted file mode 100644 index 8e97980f5ae..00000000000 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cacher - -import ( - "k8s.io/component-base/metrics" - "k8s.io/component-base/metrics/legacyregistry" -) - -const ( - namespace = "apiserver" - subsystem = "watch_cache" -) - -/* - * By default, all the following metrics are defined as falling under - * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes) - * - * Promoting the stability level of the metric is a responsibility of the component owner, since it - * involves explicitly acknowledging support for the metric across multiple releases, in accordance with - * the metric stability policy. - */ -var ( - initCounter = metrics.NewCounterVec( - &metrics.CounterOpts{ - Name: "apiserver_init_events_total", - Help: "Counter of init events processed in watch cache broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) - - eventsCounter = metrics.NewCounterVec( - &metrics.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "events_dispatched_total", - Help: "Counter of events dispatched in watch cache broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) - - terminatedWatchersCounter = metrics.NewCounterVec( - &metrics.CounterOpts{ - Name: "apiserver_terminated_watchers_total", - Help: "Counter of watchers closed due to unresponsiveness broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) - - watchCacheCapacityIncreaseTotal = metrics.NewCounterVec( - &metrics.CounterOpts{ - Name: "watch_cache_capacity_increase_total", - Help: "Total number of watch cache capacity increase events broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) - - watchCacheCapacityDecreaseTotal = metrics.NewCounterVec( - &metrics.CounterOpts{ - Name: "watch_cache_capacity_decrease_total", - Help: "Total number of watch cache capacity decrease events broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) - - watchCacheCapacity = metrics.NewGaugeVec( - &metrics.GaugeOpts{ - Name: "watch_cache_capacity", - Help: "Total capacity of watch cache broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) - - watchCacheInitializations = metrics.NewCounterVec( - &metrics.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "initializations_total", - Help: "Counter of watch cache initializations broken by resource type.", - StabilityLevel: metrics.ALPHA, - }, - []string{"resource"}, - ) -) - -func init() { - legacyregistry.MustRegister(initCounter) - legacyregistry.MustRegister(eventsCounter) - legacyregistry.MustRegister(terminatedWatchersCounter) - legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal) - legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal) - legacyregistry.MustRegister(watchCacheCapacity) - legacyregistry.MustRegister(watchCacheInitializations) -} - -// recordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations. -func recordsWatchCacheCapacityChange(objType string, old, new int) { - watchCacheCapacity.WithLabelValues(objType).Set(float64(new)) - if old < new { - watchCacheCapacityIncreaseTotal.WithLabelValues(objType).Inc() - return - } - watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc() -} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics/metrics.go index 652730332e1..ffebf5e5b48 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/metrics/metrics.go @@ -23,6 +23,11 @@ import ( "k8s.io/component-base/metrics/legacyregistry" ) +const ( + namespace = "apiserver" + subsystem = "watch_cache" +) + /* * By default, all the following metrics are defined as falling under * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes) @@ -34,7 +39,8 @@ import ( var ( listCacheCount = compbasemetrics.NewCounterVec( &compbasemetrics.CounterOpts{ - Name: "apiserver_cache_list_total", + Namespace: namespace, + Name: "cache_list_total", Help: "Number of LIST requests served from watch cache", StabilityLevel: compbasemetrics.ALPHA, }, @@ -42,7 +48,8 @@ var ( ) listCacheNumFetched = compbasemetrics.NewCounterVec( &compbasemetrics.CounterOpts{ - Name: "apiserver_cache_list_fetched_objects_total", + Namespace: namespace, + Name: "cache_list_fetched_objects_total", Help: "Number of objects read from watch cache in the course of serving a LIST request", StabilityLevel: compbasemetrics.ALPHA, }, @@ -50,12 +57,84 @@ var ( ) listCacheNumReturned = compbasemetrics.NewCounterVec( &compbasemetrics.CounterOpts{ - Name: "apiserver_cache_list_returned_objects_total", + Namespace: namespace, + Name: "cache_list_returned_objects_total", Help: "Number of objects returned for a LIST request from watch cache", StabilityLevel: compbasemetrics.ALPHA, }, []string{"resource_prefix"}, ) + InitCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Namespace: namespace, + Name: "init_events_total", + Help: "Counter of init events processed in watch cache broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) + + EventsCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "events_dispatched_total", + Help: "Counter of events dispatched in watch cache broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) + + TerminatedWatchersCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Namespace: namespace, + Name: "terminated_watchers_total", + Help: "Counter of watchers closed due to unresponsiveness broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) + + watchCacheCapacityIncreaseTotal = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Subsystem: subsystem, + Name: "capacity_increase_total", + Help: "Total number of watch cache capacity increase events broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) + + watchCacheCapacityDecreaseTotal = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Subsystem: subsystem, + Name: "capacity_decrease_total", + Help: "Total number of watch cache capacity decrease events broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) + + WatchCacheCapacity = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ + Subsystem: subsystem, + Name: "capacity", + Help: "Total capacity of watch cache broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) + + WatchCacheInitializations = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "initializations_total", + Help: "Counter of watch cache initializations broken by resource type.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"resource"}, + ) ) var registerMetrics sync.Once @@ -67,6 +146,13 @@ func Register() { legacyregistry.MustRegister(listCacheCount) legacyregistry.MustRegister(listCacheNumFetched) legacyregistry.MustRegister(listCacheNumReturned) + legacyregistry.MustRegister(InitCounter) + legacyregistry.MustRegister(EventsCounter) + legacyregistry.MustRegister(TerminatedWatchersCounter) + legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal) + legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal) + legacyregistry.MustRegister(WatchCacheCapacity) + legacyregistry.MustRegister(WatchCacheInitializations) }) } @@ -76,3 +162,13 @@ func RecordListCacheMetrics(resourcePrefix, indexName string, numFetched, numRet listCacheNumFetched.WithLabelValues(resourcePrefix, indexName).Add(float64(numFetched)) listCacheNumReturned.WithLabelValues(resourcePrefix).Add(float64(numReturned)) } + +// RecordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations. +func RecordsWatchCacheCapacityChange(objType string, old, new int) { + WatchCacheCapacity.WithLabelValues(objType).Set(float64(new)) + if old < new { + WatchCacheCapacity.WithLabelValues(objType).Inc() + return + } + watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc() +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go index 3c9bd926893..65e9fde6591 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/cacher/metrics" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" "k8s.io/utils/clock" @@ -220,7 +221,7 @@ func newWatchCache( objectType: objectType, } objType := objectType.String() - watchCacheCapacity.WithLabelValues(objType).Set(float64(wc.capacity)) + metrics.WatchCacheCapacity.WithLabelValues(objType).Set(float64(wc.capacity)) wc.cond = sync.NewCond(wc.RLocker()) wc.indexValidator = wc.isIndexValidLocked @@ -385,7 +386,7 @@ func (w *watchCache) doCacheResizeLocked(capacity int) { newCache[i%capacity] = w.cache[i%w.capacity] } w.cache = newCache - recordsWatchCacheCapacityChange(w.objectType.String(), w.capacity, capacity) + metrics.RecordsWatchCacheCapacityChange(w.objectType.String(), w.capacity, capacity) w.capacity = capacity }