From e17a3b5d214da2b5d35851b3aba068cc5f3c96c6 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Thu, 2 Dec 2021 18:49:20 +0000 Subject: [PATCH 01/10] first pass metrics --- .../component-base/metrics/prometheus/controllers/metrics.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go new file mode 100644 index 00000000000..e69de29bb2d From a520d5e74b468f0854fc58e514d4585bdd6c4370 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Tue, 7 Dec 2021 00:47:03 +0000 Subject: [PATCH 02/10] added controller metrics function to component base --- .../metrics/prometheus/controllers/metrics.go | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index e69de29bb2d..2ff6471ba7c 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -0,0 +1,56 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + // Metrics provides access to all controllerreadiness metrics. + Metrics = newControllerMetrics() +) + +// ControllerMetrics includes all the metrics of the proxy server. +type ControllerMetrics struct { + controllerInstanceCount *prometheus.GaugeVec +} + +// newControllerMetrics create a new ControllerMetrics, configured with default metric names. +func newControllerMetrics() *ControllerMetrics { + controllerInstanceCount := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "managed_controller_instance_count", + Help: "Instances of individual controllers currently running", + }, + []string{"controller_name", "controller_manager"}, + ) + prometheus.MustRegister(controllerInstanceCount) + return &ControllerMetrics{ + controllerInstanceCount: controllerInstanceCount, + } +} + +// ControllerStarted sets the controllerInstanceCount to 1. +func (a *ControllerMetrics) ControllerStarted(controllerName string, controllerManager string) { + a.controllerInstanceCount.With(prometheus.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(1)) +} + +// ControllerStopped sets the controllerInstanceCount to 0. +func (a *ControllerMetrics) ControllerStopped(controllerName string, controllerManager string) { + a.controllerInstanceCount.With(prometheus.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(0)) +} From bdedfb7653774737e9ff3da45b87d7bc438b3f50 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Thu, 9 Dec 2021 00:37:08 +0000 Subject: [PATCH 03/10] removed dependency on external prometheus --- .../metrics/prometheus/controllers/metrics.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index 2ff6471ba7c..e3930bf7497 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -17,7 +17,8 @@ limitations under the License. package controllers import ( - "github.com/prometheus/client_golang/prometheus" + k8smetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) var ( @@ -27,19 +28,19 @@ var ( // ControllerMetrics includes all the metrics of the proxy server. type ControllerMetrics struct { - controllerInstanceCount *prometheus.GaugeVec + controllerInstanceCount *k8smetrics.GaugeVec } // newControllerMetrics create a new ControllerMetrics, configured with default metric names. func newControllerMetrics() *ControllerMetrics { - controllerInstanceCount := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + controllerInstanceCount := k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ Name: "managed_controller_instance_count", Help: "Instances of individual controllers currently running", }, []string{"controller_name", "controller_manager"}, ) - prometheus.MustRegister(controllerInstanceCount) + legacyregistry.MustRegister(controllerInstanceCount) return &ControllerMetrics{ controllerInstanceCount: controllerInstanceCount, } @@ -47,10 +48,10 @@ func newControllerMetrics() *ControllerMetrics { // ControllerStarted sets the controllerInstanceCount to 1. func (a *ControllerMetrics) ControllerStarted(controllerName string, controllerManager string) { - a.controllerInstanceCount.With(prometheus.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(1)) + a.controllerInstanceCount.With(k8smetrics.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(1)) } // ControllerStopped sets the controllerInstanceCount to 0. func (a *ControllerMetrics) ControllerStopped(controllerName string, controllerManager string) { - a.controllerInstanceCount.With(prometheus.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(0)) + a.controllerInstanceCount.With(k8smetrics.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(0)) } From 2987e65a572c4e928191168dd64e41d055edb4c4 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Fri, 10 Dec 2021 21:07:14 +0000 Subject: [PATCH 04/10] updated to add StabilityLevel --- .../metrics/prometheus/controllers/metrics.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index e3930bf7497..015482e4218 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -35,8 +35,9 @@ type ControllerMetrics struct { func newControllerMetrics() *ControllerMetrics { controllerInstanceCount := k8smetrics.NewGaugeVec( &k8smetrics.GaugeOpts{ - Name: "managed_controller_instance_count", - Help: "Instances of individual controllers currently running", + Name: "managed_controller_instance_count", + Help: "Instances of individual controllers currently running", + StabilityLevel: k8smetrics.ALPHA, }, []string{"controller_name", "controller_manager"}, ) @@ -46,6 +47,11 @@ func newControllerMetrics() *ControllerMetrics { } } +// controller_name and controller_manager should be updated to validate against bounded lists +// this will allow us to confirm they are populated with "good" values. + +// These values use set instead of inc/dec to avoid accidentally double counting +// a controller that starts but fails to properly signal when it crashes. // ControllerStarted sets the controllerInstanceCount to 1. func (a *ControllerMetrics) ControllerStarted(controllerName string, controllerManager string) { a.controllerInstanceCount.With(k8smetrics.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(1)) From 5947ee3653c9ee280850a62aeeea933d2143c4c1 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Tue, 14 Dec 2021 21:16:48 +0000 Subject: [PATCH 05/10] updated comments --- .../component-base/metrics/prometheus/controllers/metrics.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index 015482e4218..dcf13e1add8 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -47,12 +47,9 @@ func newControllerMetrics() *ControllerMetrics { } } -// controller_name and controller_manager should be updated to validate against bounded lists -// this will allow us to confirm they are populated with "good" values. - +// ControllerStarted sets the controllerInstanceCount to 1. // These values use set instead of inc/dec to avoid accidentally double counting // a controller that starts but fails to properly signal when it crashes. -// ControllerStarted sets the controllerInstanceCount to 1. func (a *ControllerMetrics) ControllerStarted(controllerName string, controllerManager string) { a.controllerInstanceCount.With(k8smetrics.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(1)) } From 89c375bc745fd3101aedd6ee5241b63fea9d85f2 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Tue, 14 Dec 2021 21:22:32 +0000 Subject: [PATCH 06/10] updated with fixes based on comments by dgrisonnet --- .../metrics/prometheus/controllers/metrics.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index dcf13e1add8..afa692bb196 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -35,11 +35,11 @@ type ControllerMetrics struct { func newControllerMetrics() *ControllerMetrics { controllerInstanceCount := k8smetrics.NewGaugeVec( &k8smetrics.GaugeOpts{ - Name: "managed_controller_instance_count", + Name: "managed_controller_instances", Help: "Instances of individual controllers currently running", StabilityLevel: k8smetrics.ALPHA, }, - []string{"controller_name", "controller_manager"}, + []string{"name", "manager"}, ) legacyregistry.MustRegister(controllerInstanceCount) return &ControllerMetrics{ @@ -50,11 +50,11 @@ func newControllerMetrics() *ControllerMetrics { // ControllerStarted sets the controllerInstanceCount to 1. // These values use set instead of inc/dec to avoid accidentally double counting // a controller that starts but fails to properly signal when it crashes. -func (a *ControllerMetrics) ControllerStarted(controllerName string, controllerManager string) { - a.controllerInstanceCount.With(k8smetrics.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(1)) +func (a *ControllerMetrics) ControllerStarted(name string, manager string) { + a.controllerInstanceCount.With(k8smetrics.Labels{"name": name, "manager": manager}).Set(float64(1)) } // ControllerStopped sets the controllerInstanceCount to 0. -func (a *ControllerMetrics) ControllerStopped(controllerName string, controllerManager string) { - a.controllerInstanceCount.With(k8smetrics.Labels{"controller_name": controllerName, "controller_manager": controllerManager}).Set(float64(0)) +func (a *ControllerMetrics) ControllerStopped(name string, manager string) { + a.controllerInstanceCount.With(k8smetrics.Labels{"name": name, "manager": manager}).Set(float64(0)) } From daf7edb8f461e98eb5a0147c85c8bed05f9665ad Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Wed, 15 Dec 2021 23:47:34 +0000 Subject: [PATCH 07/10] added sync and register func --- .../metrics/prometheus/controllers/metrics.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index afa692bb196..92ed735b854 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -17,13 +17,14 @@ limitations under the License. package controllers import ( + "sync" + k8smetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) var ( - // Metrics provides access to all controllerreadiness metrics. - Metrics = newControllerMetrics() + once sync.Once ) // ControllerMetrics includes all the metrics of the proxy server. @@ -32,7 +33,7 @@ type ControllerMetrics struct { } // newControllerMetrics create a new ControllerMetrics, configured with default metric names. -func newControllerMetrics() *ControllerMetrics { +func NewControllerMetrics() *ControllerMetrics { controllerInstanceCount := k8smetrics.NewGaugeVec( &k8smetrics.GaugeOpts{ Name: "managed_controller_instances", @@ -41,10 +42,18 @@ func newControllerMetrics() *ControllerMetrics { }, []string{"name", "manager"}, ) - legacyregistry.MustRegister(controllerInstanceCount) - return &ControllerMetrics{ + //legacyregistry.MustRegister(controllerInstanceCount) + controllerMetrics := &ControllerMetrics{ controllerInstanceCount: controllerInstanceCount, } + once.Do(func() { + controllerMetrics.Register() + }) + return controllerMetrics +} + +func (a *ControllerMetrics) Register() { + legacyregistry.MustRegister(a.controllerInstanceCount) } // ControllerStarted sets the controllerInstanceCount to 1. From 9529c8ff8e4c6ee6ba49e04e47de582a2e22a8f5 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Thu, 16 Dec 2021 17:54:34 +0000 Subject: [PATCH 08/10] updated help field to better indicate metric intent --- .../component-base/metrics/prometheus/controllers/metrics.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index 92ed735b854..70492c87085 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -32,17 +32,16 @@ type ControllerMetrics struct { controllerInstanceCount *k8smetrics.GaugeVec } -// newControllerMetrics create a new ControllerMetrics, configured with default metric names. +// NewControllerMetrics create a new ControllerMetrics, configured with default metric names. func NewControllerMetrics() *ControllerMetrics { controllerInstanceCount := k8smetrics.NewGaugeVec( &k8smetrics.GaugeOpts{ Name: "managed_controller_instances", - Help: "Instances of individual controllers currently running", + Help: "Indicates where instances of a controller are currently running", StabilityLevel: k8smetrics.ALPHA, }, []string{"name", "manager"}, ) - //legacyregistry.MustRegister(controllerInstanceCount) controllerMetrics := &ControllerMetrics{ controllerInstanceCount: controllerInstanceCount, } From b0d52b491e1c5b47f4d7400a6d20205ffd471535 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Wed, 5 Jan 2022 18:19:14 +0000 Subject: [PATCH 09/10] updated name of metric --- .../component-base/metrics/prometheus/controllers/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index 70492c87085..01891cc09f9 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -36,7 +36,7 @@ type ControllerMetrics struct { func NewControllerMetrics() *ControllerMetrics { controllerInstanceCount := k8smetrics.NewGaugeVec( &k8smetrics.GaugeOpts{ - Name: "managed_controller_instances", + Name: "managed_controller_started", Help: "Indicates where instances of a controller are currently running", StabilityLevel: k8smetrics.ALPHA, }, From 6d2dd9952bd99c0c7ada5bedf4d2c8fe7390b548 Mon Sep 17 00:00:00 2001 From: JD Nurme Date: Tue, 18 Jan 2022 19:33:41 +0000 Subject: [PATCH 10/10] updated name to address han's comments --- .../component-base/metrics/prometheus/controllers/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go index 01891cc09f9..c1a117e586c 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/controllers/metrics.go @@ -36,7 +36,7 @@ type ControllerMetrics struct { func NewControllerMetrics() *ControllerMetrics { controllerInstanceCount := k8smetrics.NewGaugeVec( &k8smetrics.GaugeOpts{ - Name: "managed_controller_started", + Name: "running_managed_controllers", Help: "Indicates where instances of a controller are currently running", StabilityLevel: k8smetrics.ALPHA, },