From c449c2735f0cb9fcc1809cc77a72a06cc2f44f55 Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Wed, 5 Dec 2018 19:05:30 -0800 Subject: [PATCH 1/3] Log metric registration error --- pkg/util/workqueue/prometheus/prometheus.go | 29 ++++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pkg/util/workqueue/prometheus/prometheus.go b/pkg/util/workqueue/prometheus/prometheus.go index 6f5cf38f09d..21840cf49ce 100644 --- a/pkg/util/workqueue/prometheus/prometheus.go +++ b/pkg/util/workqueue/prometheus/prometheus.go @@ -18,6 +18,7 @@ package prometheus import ( "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "github.com/prometheus/client_golang/prometheus" ) @@ -37,7 +38,9 @@ func (prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetr Name: "depth", Help: "Current depth of workqueue: " + name, }) - prometheus.Register(depth) + if err := prometheus.Register(depth); err != nil { + klog.Errorf("failed to register depth metric %v: %v", name, err) + } return depth } @@ -47,7 +50,9 @@ func (prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMet Name: "adds", Help: "Total number of adds handled by workqueue: " + name, }) - prometheus.Register(adds) + if err := prometheus.Register(adds); err != nil { + klog.Errorf("failed to register adds metric %v: %v", name, err) + } return adds } @@ -57,7 +62,9 @@ func (prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.Summary Name: "queue_latency", Help: "How long an item stays in workqueue" + name + " before being requested.", }) - prometheus.Register(latency) + if err := prometheus.Register(latency); err != nil { + klog.Errorf("failed to register latency metric %v: %v", name, err) + } return latency } @@ -67,7 +74,9 @@ func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.Su Name: "work_duration", Help: "How long processing an item from workqueue" + name + " takes.", }) - prometheus.Register(workDuration) + if err := prometheus.Register(workDuration); err != nil { + klog.Errorf("failed to register work_duration metric %v: %v", name, err) + } return workDuration } @@ -80,7 +89,9 @@ func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) wor "values indicate stuck threads. One can deduce the number of stuck " + "threads by observing the rate at which this increases.", }) - prometheus.Register(unfinished) + if err := prometheus.Register(unfinished); err != nil { + klog.Errorf("failed to register unfinished_work_seconds metric %v: %v", name, err) + } return unfinished } @@ -91,7 +102,9 @@ func (prometheusMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(na Help: "How many microseconds has the longest running " + "processor for " + name + " been running.", }) - prometheus.Register(unfinished) + if err := prometheus.Register(unfinished); err != nil { + klog.Errorf("failed to register longest_running_processor_microseconds metric %v: %v", name, err) + } return unfinished } @@ -101,6 +114,8 @@ func (prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.Counter Name: "retries", Help: "Total number of retries handled by workqueue: " + name, }) - prometheus.Register(retries) + if err := prometheus.Register(retries); err != nil { + klog.Errorf("failed to register retries metric %v: %v", name, err) + } return retries } From 7c6d9d634b5d08540629ab893b90df6b6806f5eb Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Thu, 6 Dec 2018 16:36:08 -0800 Subject: [PATCH 2/3] Fix crd workqueue metric names --- .../controller/crdregistration/crdregistration_controller.go | 2 +- .../pkg/controller/finalizer/crd_finalizer.go | 2 +- .../pkg/controller/status/naming_controller.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/master/controller/crdregistration/crdregistration_controller.go b/pkg/master/controller/crdregistration/crdregistration_controller.go index 1670a5b0780..50b9f6ea0d4 100644 --- a/pkg/master/controller/crdregistration/crdregistration_controller.go +++ b/pkg/master/controller/crdregistration/crdregistration_controller.go @@ -68,7 +68,7 @@ func NewAutoRegistrationController(crdinformer crdinformers.CustomResourceDefini crdSynced: crdinformer.Informer().HasSynced, apiServiceRegistration: apiServiceRegistration, syncedInitialSet: make(chan struct{}), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd-autoregister"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_autoregistration_controller"), } c.syncHandler = c.handleVersionUpdate diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go index c2ebdcf1709..e3e3d27c0b0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go @@ -79,7 +79,7 @@ func NewCRDFinalizer( crdLister: crdInformer.Lister(), crdSynced: crdInformer.Informer().HasSynced, crClientGetter: crClientGetter, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CustomResourceDefinition-CRDFinalizer"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_finalizer"), } crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go index fe5c9479847..1bea5f440b9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go @@ -66,7 +66,7 @@ func NewNamingConditionController( crdClient: crdClient, crdLister: crdInformer.Lister(), crdSynced: crdInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CustomResourceDefinition-NamingConditionController"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_naming_condition_controller"), } informerIndexer := crdInformer.Informer().GetIndexer() From b109fda56df4a049c7eaebccb6f47b85bf09cbf6 Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Thu, 6 Dec 2018 16:36:14 -0800 Subject: [PATCH 3/3] generated --- pkg/util/workqueue/prometheus/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/util/workqueue/prometheus/BUILD b/pkg/util/workqueue/prometheus/BUILD index 62a98289a50..d9708f7eb4f 100644 --- a/pkg/util/workqueue/prometheus/BUILD +++ b/pkg/util/workqueue/prometheus/BUILD @@ -12,6 +12,7 @@ go_library( deps = [ "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], )