From 1b74ea0f29c3ddbb302bf8fdb5d8087186278bf4 Mon Sep 17 00:00:00 2001 From: Han Kang Date: Thu, 6 Oct 2022 17:10:23 -0700 Subject: [PATCH 1/4] add meta-metrics for metrics framework Change-Id: I15c2947e910419e6cb07db22a40fc0667fcd128b --- .../k8s.io/component-base/metrics/counter.go | 4 +-- .../k8s.io/component-base/metrics/gauge.go | 4 +-- .../component-base/metrics/histogram.go | 4 +-- .../k8s.io/component-base/metrics/metric.go | 17 ++++++++++ .../k8s.io/component-base/metrics/registry.go | 33 +++++++++++++++++-- .../k8s.io/component-base/metrics/summary.go | 4 +-- .../metrics/timing_histogram.go | 4 +-- 7 files changed, 58 insertions(+), 12 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/counter.go b/staging/src/k8s.io/component-base/metrics/counter.go index 78c211a0ede..5664a68a900 100644 --- a/staging/src/k8s.io/component-base/metrics/counter.go +++ b/staging/src/k8s.io/component-base/metrics/counter.go @@ -44,7 +44,7 @@ func NewCounter(opts *CounterOpts) *Counter { kc := &Counter{ CounterOpts: opts, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } kc.setPrometheusCounter(noop) kc.lazyInit(kc, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)) @@ -129,7 +129,7 @@ func NewCounterVec(opts *CounterOpts, labels []string) *CounterVec { CounterVec: noopCounterVec, CounterOpts: opts, originalLabels: labels, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } cv.lazyInit(cv, fqName) return cv diff --git a/staging/src/k8s.io/component-base/metrics/gauge.go b/staging/src/k8s.io/component-base/metrics/gauge.go index 04041bab652..89631115acb 100644 --- a/staging/src/k8s.io/component-base/metrics/gauge.go +++ b/staging/src/k8s.io/component-base/metrics/gauge.go @@ -46,7 +46,7 @@ func NewGauge(opts *GaugeOpts) *Gauge { kc := &Gauge{ GaugeOpts: opts, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } kc.setPrometheusGauge(noop) kc.lazyInit(kc, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)) @@ -115,7 +115,7 @@ func NewGaugeVec(opts *GaugeOpts, labels []string) *GaugeVec { GaugeVec: noopGaugeVec, GaugeOpts: opts, originalLabels: labels, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } cv.lazyInit(cv, fqName) return cv diff --git a/staging/src/k8s.io/component-base/metrics/histogram.go b/staging/src/k8s.io/component-base/metrics/histogram.go index 838f09e17fd..1bf8ba887e0 100644 --- a/staging/src/k8s.io/component-base/metrics/histogram.go +++ b/staging/src/k8s.io/component-base/metrics/histogram.go @@ -52,7 +52,7 @@ func NewHistogram(opts *HistogramOpts) *Histogram { h := &Histogram{ HistogramOpts: opts, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } h.setPrometheusHistogram(noopMetric{}) h.lazyInit(h, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)) @@ -119,7 +119,7 @@ func NewHistogramVec(opts *HistogramOpts, labels []string) *HistogramVec { HistogramVec: noopHistogramVec, HistogramOpts: opts, originalLabels: labels, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } v.lazyInit(v, fqName) return v diff --git a/staging/src/k8s.io/component-base/metrics/metric.go b/staging/src/k8s.io/component-base/metrics/metric.go index 2980a972382..d5ccb277f30 100644 --- a/staging/src/k8s.io/component-base/metrics/metric.go +++ b/staging/src/k8s.io/component-base/metrics/metric.go @@ -72,6 +72,7 @@ type lazyMetric struct { markDeprecationOnce sync.Once createOnce sync.Once self kubeCollector + stabilityLevel StabilityLevel } func (r *lazyMetric) IsCreated() bool { @@ -88,6 +89,14 @@ func (r *lazyMetric) lazyInit(self kubeCollector, fqName string) { r.self = self } +func stabilityLevelOrDefault(sl string) string { + if sl == "" { + return string(INTERNAL) + } else { + return sl + } +} + // preprocessMetric figures out whether the lazy metric should be hidden or not. // This method takes a Version argument which should be the version of the binary in which // this code is currently being executed. A metric can be hidden under two conditions: @@ -149,6 +158,7 @@ func (r *lazyMetric) Create(version *semver.Version) bool { if r.IsHidden() { return false } + r.createOnce.Do(func() { r.createLock.Lock() defer r.createLock.Unlock() @@ -159,6 +169,13 @@ func (r *lazyMetric) Create(version *semver.Version) bool { r.self.initializeMetric() } }) + sl := r.stabilityLevel + deprecatedV := r.self.DeprecatedVersion() + dv := "" + if deprecatedV != nil { + dv = deprecatedV.String() + } + defer registeredMetrics.WithLabelValues(string(sl), dv).Inc() return r.IsCreated() } diff --git a/staging/src/k8s.io/component-base/metrics/registry.go b/staging/src/k8s.io/component-base/metrics/registry.go index 5d744397eee..c0eafeaefe2 100644 --- a/staging/src/k8s.io/component-base/metrics/registry.go +++ b/staging/src/k8s.io/component-base/metrics/registry.go @@ -36,6 +36,31 @@ var ( registries []*kubeRegistry // stores all registries created by NewKubeRegistry() registriesLock sync.RWMutex disabledMetrics = map[string]struct{}{} + + registeredMetrics = NewCounterVec( + &CounterOpts{ + Name: "registered_metric_total", + Help: "The count of registered metrics broken by stability level and deprecation version.", + StabilityLevel: ALPHA, + }, + []string{"stability_level", "deprecated_version"}, + ) + + disabledMetricsTotal = NewCounter( + &CounterOpts{ + Name: "disabled_metric_total", + Help: "The count of disabled metrics.", + StabilityLevel: ALPHA, + }, + ) + + hiddenMetricsTotal = NewCounter( + &CounterOpts{ + Name: "hidden_metric_total", + Help: "The count of hidden metrics.", + StabilityLevel: ALPHA, + }, + ) ) // shouldHide be used to check if a specific metric with deprecated version should be hidden @@ -67,6 +92,7 @@ func SetDisabledMetric(name string) { disabledMetricsLock.Lock() defer disabledMetricsLock.Unlock() disabledMetrics[name] = struct{}{} + disabledMetricsTotal.Inc() } // SetShowHidden will enable showing hidden metrics. This will no-opt @@ -250,6 +276,7 @@ func (kr *kubeRegistry) trackHiddenCollector(c Registerable) { defer kr.hiddenCollectorsLock.Unlock() kr.hiddenCollectors[c.FQName()] = c + hiddenMetricsTotal.Inc() } // trackStableCollectors stores all custom collectors. @@ -329,9 +356,11 @@ func newKubeRegistry(v apimachineryversion.Info) *kubeRegistry { return r } -// NewKubeRegistry creates a new vanilla Registry without any Collectors -// pre-registered. +// NewKubeRegistry creates a new vanilla Registry func NewKubeRegistry() KubeRegistry { r := newKubeRegistry(BuildVersion()) + r.MustRegister(registeredMetrics) + r.MustRegister(disabledMetricsTotal) + r.MustRegister(hiddenMetricsTotal) return r } diff --git a/staging/src/k8s.io/component-base/metrics/summary.go b/staging/src/k8s.io/component-base/metrics/summary.go index 0198f724834..d40421645af 100644 --- a/staging/src/k8s.io/component-base/metrics/summary.go +++ b/staging/src/k8s.io/component-base/metrics/summary.go @@ -49,7 +49,7 @@ func NewSummary(opts *SummaryOpts) *Summary { s := &Summary{ SummaryOpts: opts, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } s.setPrometheusSummary(noopMetric{}) s.lazyInit(s, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)) @@ -118,7 +118,7 @@ func NewSummaryVec(opts *SummaryOpts, labels []string) *SummaryVec { v := &SummaryVec{ SummaryOpts: opts, originalLabels: labels, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } v.lazyInit(v, fqName) return v diff --git a/staging/src/k8s.io/component-base/metrics/timing_histogram.go b/staging/src/k8s.io/component-base/metrics/timing_histogram.go index c015a04ea9e..793a049fd46 100644 --- a/staging/src/k8s.io/component-base/metrics/timing_histogram.go +++ b/staging/src/k8s.io/component-base/metrics/timing_histogram.go @@ -57,7 +57,7 @@ func NewTestableTimingHistogram(nowFunc func() time.Time, opts *TimingHistogramO h := &TimingHistogram{ TimingHistogramOpts: opts, nowFunc: nowFunc, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } h.setPrometheusHistogram(noopMetric{}) h.lazyInit(h, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)) @@ -136,7 +136,7 @@ func NewTestableTimingHistogramVec(nowFunc func() time.Time, opts *TimingHistogr TimingHistogramOpts: opts, nowFunc: nowFunc, originalLabels: labels, - lazyMetric: lazyMetric{}, + lazyMetric: lazyMetric{stabilityLevel: opts.StabilityLevel}, } v.lazyInit(v, fqName) return v From 3cad84b347ba087896813cfb7bb54ed95141e80b Mon Sep 17 00:00:00 2001 From: Han Kang Date: Thu, 6 Oct 2022 17:13:41 -0700 Subject: [PATCH 2/4] remove unused code Change-Id: Id46edb8bbcd043288a4c38692dcf99dd6d34fa1c --- staging/src/k8s.io/component-base/metrics/metric.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/metric.go b/staging/src/k8s.io/component-base/metrics/metric.go index d5ccb277f30..10389a6e15b 100644 --- a/staging/src/k8s.io/component-base/metrics/metric.go +++ b/staging/src/k8s.io/component-base/metrics/metric.go @@ -89,14 +89,6 @@ func (r *lazyMetric) lazyInit(self kubeCollector, fqName string) { r.self = self } -func stabilityLevelOrDefault(sl string) string { - if sl == "" { - return string(INTERNAL) - } else { - return sl - } -} - // preprocessMetric figures out whether the lazy metric should be hidden or not. // This method takes a Version argument which should be the version of the binary in which // this code is currently being executed. A metric can be hidden under two conditions: From e3a1d6c914607ec134b00d36a5493862cf278bba Mon Sep 17 00:00:00 2001 From: Han Kang Date: Fri, 7 Oct 2022 08:40:14 -0700 Subject: [PATCH 3/4] move automatic registration of meta-metrics to legacy registry Change-Id: I06608184db62b3e893efc0c0a42963f9ffaaf677 --- .../component-base/metrics/legacyregistry/registry.go | 1 + staging/src/k8s.io/component-base/metrics/registry.go | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/metrics/legacyregistry/registry.go b/staging/src/k8s.io/component-base/metrics/legacyregistry/registry.go index 386df2be858..ed0f1c348b4 100644 --- a/staging/src/k8s.io/component-base/metrics/legacyregistry/registry.go +++ b/staging/src/k8s.io/component-base/metrics/legacyregistry/registry.go @@ -47,6 +47,7 @@ var ( func init() { RawMustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) RawMustRegister(collectors.NewGoCollector(collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll))) + defaultRegistry.RegisterMetaMetrics() } // Handler returns an HTTP handler for the DefaultGatherer. It is diff --git a/staging/src/k8s.io/component-base/metrics/registry.go b/staging/src/k8s.io/component-base/metrics/registry.go index c0eafeaefe2..af7d1b8bffd 100644 --- a/staging/src/k8s.io/component-base/metrics/registry.go +++ b/staging/src/k8s.io/component-base/metrics/registry.go @@ -155,6 +155,8 @@ type KubeRegistry interface { // Reset invokes the Reset() function on all items in the registry // which are added as resettables. Reset() + // RegisterMetaMetrics registers metrics about the number of registered metrics. + RegisterMetaMetrics() } // kubeRegistry is a wrapper around a prometheus registry-type object. Upon initialization @@ -359,8 +361,11 @@ func newKubeRegistry(v apimachineryversion.Info) *kubeRegistry { // NewKubeRegistry creates a new vanilla Registry func NewKubeRegistry() KubeRegistry { r := newKubeRegistry(BuildVersion()) + return r +} + +func (r *kubeRegistry) RegisterMetaMetrics() { r.MustRegister(registeredMetrics) r.MustRegister(disabledMetricsTotal) r.MustRegister(hiddenMetricsTotal) - return r } From 1b3c3465318543ce94e038f73cd9aa56407cd510 Mon Sep 17 00:00:00 2001 From: Han Kang Date: Fri, 7 Oct 2022 09:31:15 -0700 Subject: [PATCH 4/4] remove defer Change-Id: Idbe0ee3349f91ea58a15be4eb9d3892c3e6a3693 --- staging/src/k8s.io/component-base/metrics/metric.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/metrics/metric.go b/staging/src/k8s.io/component-base/metrics/metric.go index 10389a6e15b..cf5bccfa7b9 100644 --- a/staging/src/k8s.io/component-base/metrics/metric.go +++ b/staging/src/k8s.io/component-base/metrics/metric.go @@ -167,7 +167,7 @@ func (r *lazyMetric) Create(version *semver.Version) bool { if deprecatedV != nil { dv = deprecatedV.String() } - defer registeredMetrics.WithLabelValues(string(sl), dv).Inc() + registeredMetrics.WithLabelValues(string(sl), dv).Inc() return r.IsCreated() }