diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 8cebb69e066..f210e5f46d0 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -43,6 +43,8 @@ go_library( "//pkg/quota/install:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/rbac/rest:go_default_library", + "//pkg/util/reflector/prometheus:go_default_library", + "//pkg/util/workqueue/prometheus:go_default_library", "//pkg/version:go_default_library", "//plugin/pkg/admission/admit:go_default_library", "//plugin/pkg/admission/alwayspullimages:go_default_library", diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index a8e8356b4c9..d35cd7d8c01 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -83,6 +83,9 @@ import ( rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" + + _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration + _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration ) const etcdRetryLimit = 60 diff --git a/cmd/kube-controller-manager/BUILD b/cmd/kube-controller-manager/BUILD index 96bd4e5d630..6d7de6abe5a 100644 --- a/cmd/kube-controller-manager/BUILD +++ b/cmd/kube-controller-manager/BUILD @@ -30,6 +30,7 @@ go_library( "//cmd/kube-controller-manager/app:go_default_library", "//cmd/kube-controller-manager/app/options:go_default_library", "//pkg/client/metrics/prometheus:go_default_library", + "//pkg/util/reflector/prometheus:go_default_library", "//pkg/util/workqueue/prometheus:go_default_library", "//pkg/version/prometheus:go_default_library", "//pkg/version/verflag:go_default_library", diff --git a/cmd/kube-controller-manager/controller-manager.go b/cmd/kube-controller-manager/controller-manager.go index 739ba2acc4f..d4a827cdd17 100644 --- a/cmd/kube-controller-manager/controller-manager.go +++ b/cmd/kube-controller-manager/controller-manager.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/cmd/kube-controller-manager/app" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration + _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration "k8s.io/kubernetes/pkg/version/verflag" diff --git a/federation/cmd/federation-controller-manager/BUILD b/federation/cmd/federation-controller-manager/BUILD index cd8a215893b..0a610f149c2 100644 --- a/federation/cmd/federation-controller-manager/BUILD +++ b/federation/cmd/federation-controller-manager/BUILD @@ -21,6 +21,7 @@ go_library( deps = [ "//federation/cmd/federation-controller-manager/app:go_default_library", "//federation/cmd/federation-controller-manager/app/options:go_default_library", + "//pkg/util/reflector/prometheus:go_default_library", "//pkg/util/workqueue/prometheus:go_default_library", "//pkg/version/verflag:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", diff --git a/federation/cmd/federation-controller-manager/controller-manager.go b/federation/cmd/federation-controller-manager/controller-manager.go index 2becafa875f..69ba6f21ef5 100644 --- a/federation/cmd/federation-controller-manager/controller-manager.go +++ b/federation/cmd/federation-controller-manager/controller-manager.go @@ -26,6 +26,7 @@ import ( "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/federation/cmd/federation-controller-manager/app" "k8s.io/kubernetes/federation/cmd/federation-controller-manager/app/options" + _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration "k8s.io/kubernetes/pkg/version/verflag" ) diff --git a/pkg/controller/garbagecollector/BUILD b/pkg/controller/garbagecollector/BUILD index 9fcd9602c17..51568904605 100644 --- a/pkg/controller/garbagecollector/BUILD +++ b/pkg/controller/garbagecollector/BUILD @@ -26,6 +26,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/controller/garbagecollector/metaonly:go_default_library", "//pkg/util/metrics:go_default_library", + "//pkg/util/reflector/prometheus:go_default_library", "//pkg/util/workqueue/prometheus:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", diff --git a/pkg/controller/garbagecollector/garbagecollector.go b/pkg/controller/garbagecollector/garbagecollector.go index ddf7c7ffaa3..c722da346da 100644 --- a/pkg/controller/garbagecollector/garbagecollector.go +++ b/pkg/controller/garbagecollector/garbagecollector.go @@ -36,6 +36,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/garbagecollector/metaonly" + _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration // install the prometheus plugin _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // import known versions diff --git a/pkg/util/BUILD b/pkg/util/BUILD index e81de7cd0f1..a2916a9a137 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -44,6 +44,7 @@ filegroup( "//pkg/util/parsers:all-srcs", "//pkg/util/pointer:all-srcs", "//pkg/util/procfs:all-srcs", + "//pkg/util/reflector/prometheus:all-srcs", "//pkg/util/removeall:all-srcs", "//pkg/util/resourcecontainer:all-srcs", "//pkg/util/rlimit:all-srcs", diff --git a/pkg/util/reflector/prometheus/BUILD b/pkg/util/reflector/prometheus/BUILD new file mode 100644 index 00000000000..f3d62fa0875 --- /dev/null +++ b/pkg/util/reflector/prometheus/BUILD @@ -0,0 +1,31 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["prometheus.go"], + tags = ["automanaged"], + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/util/reflector/prometheus/prometheus.go b/pkg/util/reflector/prometheus/prometheus.go new file mode 100644 index 00000000000..ab16edf4911 --- /dev/null +++ b/pkg/util/reflector/prometheus/prometheus.go @@ -0,0 +1,127 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package prometheus sets the cache DefaultMetricsFactory to produce +// prometheus metrics. To use this package, you just have to import it. +package prometheus + +import ( + "k8s.io/client-go/tools/cache" + + "github.com/prometheus/client_golang/prometheus" +) + +const reflectorSubsystem = "reflector" + +var ( + listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: reflectorSubsystem, + Name: "lists_total", + Help: "Total number of API lists done by the reflectors", + }, []string{"name"}) + + listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Subsystem: reflectorSubsystem, + Name: "list_duration_seconds", + Help: "How long an API list takes to return and decode for the reflectors", + }, []string{"name"}) + + itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Subsystem: reflectorSubsystem, + Name: "items_per_list", + Help: "How many items an API list returns to the reflectors", + }, []string{"name"}) + + watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: reflectorSubsystem, + Name: "watches_total", + Help: "Total number of API watches done by the reflectors", + }, []string{"name"}) + + shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: reflectorSubsystem, + Name: "short_watches_total", + Help: "Total number of short API watches done by the reflectors", + }, []string{"name"}) + + watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Subsystem: reflectorSubsystem, + Name: "watch_duration_seconds", + Help: "How long an API watch takes to return and decode for the reflectors", + }, []string{"name"}) + + itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Subsystem: reflectorSubsystem, + Name: "items_per_watch", + Help: "How many items an API watch returns to the reflectors", + }, []string{"name"}) +) + +func init() { + prometheus.MustRegister(listsTotal) + prometheus.MustRegister(listsDuration) + prometheus.MustRegister(itemsPerList) + prometheus.MustRegister(watchesTotal) + prometheus.MustRegister(shortWatchesTotal) + prometheus.MustRegister(watchDuration) + prometheus.MustRegister(itemsPerWatch) + + cache.SetReflectorMetricsProvider(prometheusMetricsProvider{}) +} + +type prometheusMetricsProvider struct{} + +func (prometheusMetricsProvider) NewListsMetric(name string) cache.CounterMetric { + return listsTotal.WithLabelValues(name) +} + +// use summary to get averages and percentiles +func (prometheusMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric { + return listsDuration.WithLabelValues(name) +} + +// use summary to get averages and percentiles +func (prometheusMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric { + return itemsPerList.WithLabelValues(name) +} + +func (prometheusMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric { + return watchesTotal.WithLabelValues(name) +} + +func (prometheusMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric { + return shortWatchesTotal.WithLabelValues(name) +} + +// use summary to get averages and percentiles +func (prometheusMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric { + return watchDuration.WithLabelValues(name) +} + +// use summary to get averages and percentiles +func (prometheusMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric { + return itemsPerWatch.WithLabelValues(name) +} + +func (prometheusMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric { + rv := prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: name, + Name: "last_resource_version", + Help: "last resource version seen for the reflectors", + }) + prometheus.MustRegister(rv) + return rv +} diff --git a/plugin/pkg/admission/resourcequota/BUILD b/plugin/pkg/admission/resourcequota/BUILD index f4f18bd77a1..7f7a577e2e2 100644 --- a/plugin/pkg/admission/resourcequota/BUILD +++ b/plugin/pkg/admission/resourcequota/BUILD @@ -25,6 +25,7 @@ go_library( "//pkg/client/listers/core/internalversion:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", "//pkg/quota:go_default_library", + "//pkg/util/reflector/prometheus:go_default_library", "//pkg/util/workqueue/prometheus:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota/install:go_default_library", diff --git a/plugin/pkg/admission/resourcequota/controller.go b/plugin/pkg/admission/resourcequota/controller.go index 0dcd9b1521b..ab32bbf31ba 100644 --- a/plugin/pkg/admission/resourcequota/controller.go +++ b/plugin/pkg/admission/resourcequota/controller.go @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/quota" + _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota" ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher.go index 13f7cf29438..9c7839e57ad 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher.go @@ -196,6 +196,7 @@ type Cacher struct { func NewCacherFromConfig(config CacherConfig) *Cacher { watchCache := newWatchCache(config.CacheCapacity, config.KeyFunc, config.GetAttrsFunc) listerWatcher := newCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc) + reflectorName := "storage/cacher.go:" + config.ResourcePrefix // Give this error when it is constructed rather than when you get the // first watch item, because it's much easier to track down that way. @@ -212,7 +213,7 @@ func NewCacherFromConfig(config CacherConfig) *Cacher { copier: config.Copier, objectType: reflect.TypeOf(config.Type), watchCache: watchCache, - reflector: cache.NewReflector(listerWatcher, config.Type, watchCache, 0), + reflector: cache.NewNamedReflector(reflectorName, listerWatcher, config.Type, watchCache, 0), versioner: config.Versioner, triggerFunc: config.TriggerPublisherFunc, watcherIdx: 0, diff --git a/staging/src/k8s.io/client-go/tools/cache/BUILD b/staging/src/k8s.io/client-go/tools/cache/BUILD index 9b542466aa3..6c5fd44edd5 100644 --- a/staging/src/k8s.io/client-go/tools/cache/BUILD +++ b/staging/src/k8s.io/client-go/tools/cache/BUILD @@ -56,6 +56,7 @@ go_library( "mutation_cache.go", "mutation_detector.go", "reflector.go", + "reflector_metrics.go", "shared_informer.go", "store.go", "thread_safe_store.go", diff --git a/staging/src/k8s.io/client-go/tools/cache/reflector.go b/staging/src/k8s.io/client-go/tools/cache/reflector.go index c09a7865337..ca73ba9233c 100644 --- a/staging/src/k8s.io/client-go/tools/cache/reflector.go +++ b/staging/src/k8s.io/client-go/tools/cache/reflector.go @@ -48,6 +48,8 @@ import ( type Reflector struct { // name identifies this reflector. By default it will be a file:line if possible. name string + // metrics tracks basic metric information about the reflector + metrics *reflectorMetrics // The type of object we expect to place in the store. expectedType reflect.Type @@ -99,7 +101,9 @@ func NewReflector(lw ListerWatcher, expectedType interface{}, store Store, resyn // NewNamedReflector same as NewReflector, but with a specified name for logging func NewNamedReflector(name string, lw ListerWatcher, expectedType interface{}, store Store, resyncPeriod time.Duration) *Reflector { r := &Reflector{ - name: name, + name: name, + // we need this to be unique per process (some names are still the same)but obvious who it belongs to + metrics: newReflectorMetrics(makeValidPromethusMetricName(fmt.Sprintf("reflector_"+name+"_%07d", rand.Intn(1000000)))), listerWatcher: lw, store: store, expectedType: reflect.TypeOf(expectedType), @@ -110,6 +114,11 @@ func NewNamedReflector(name string, lw ListerWatcher, expectedType interface{}, return r } +func makeValidPromethusMetricName(in string) string { + // this isn't perfect, but it removes our common characters + return strings.NewReplacer("/", "_", ".", "_", "-", "_").Replace(in) +} + // internalPackages are packages that ignored when creating a default reflector name. These packages are in the common // call chains to NewReflector, so they'd be low entropy names for reflectors var internalPackages = []string{"client-go/tools/cache/", "/runtime/asm_"} @@ -231,10 +240,13 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { // to be served from cache and potentially be delayed relative to // etcd contents. Reflector framework will catch up via Watch() eventually. options := metav1.ListOptions{ResourceVersion: "0"} + r.metrics.numberOfLists.Inc() + start := r.clock.Now() list, err := r.listerWatcher.List(options) if err != nil { return fmt.Errorf("%s: Failed to list %v: %v", r.name, r.expectedType, err) } + r.metrics.listDuration.Observe(time.Since(start).Seconds()) listMetaInterface, err := meta.ListAccessor(list) if err != nil { return fmt.Errorf("%s: Unable to understand list result %#v: %v", r.name, list, err) @@ -244,6 +256,7 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { if err != nil { return fmt.Errorf("%s: Unable to understand list result %#v (%v)", r.name, list, err) } + r.metrics.numberOfItemsInList.Observe(float64(len(items))) if err := r.syncWith(items, resourceVersion); err != nil { return fmt.Errorf("%s: Unable to sync list result: %v", r.name, err) } @@ -282,6 +295,7 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { TimeoutSeconds: &timemoutseconds, } + r.metrics.numberOfWatches.Inc() w, err := r.listerWatcher.Watch(options) if err != nil { switch err { @@ -333,6 +347,11 @@ func (r *Reflector) watchHandler(w watch.Interface, resourceVersion *string, err // Stopping the watcher should be idempotent and if we return from this function there's no way // we're coming back in with the same watch interface. defer w.Stop() + // update metrics + defer func() { + r.metrics.numberOfItemsInWatch.Observe(float64(eventCount)) + r.metrics.watchDuration.Observe(time.Since(start).Seconds()) + }() loop: for { @@ -388,8 +407,8 @@ loop: watchDuration := r.clock.Now().Sub(start) if watchDuration < 1*time.Second && eventCount == 0 { - glog.V(4).Infof("%s: Unexpected watch close - watch lasted less than a second and no items received", r.name) - return errors.New("very short watch") + r.metrics.numberOfShortWatches.Inc() + return fmt.Errorf("very short watch: %s: Unexpected watch close - watch lasted less than a second and no items received", r.name) } glog.V(4).Infof("%s: Watch close - %v total %v items received", r.name, r.expectedType, eventCount) return nil @@ -407,4 +426,9 @@ func (r *Reflector) setLastSyncResourceVersion(v string) { r.lastSyncResourceVersionMutex.Lock() defer r.lastSyncResourceVersionMutex.Unlock() r.lastSyncResourceVersion = v + + rv, err := strconv.Atoi(v) + if err == nil { + r.metrics.lastResourceVersion.Set(float64(rv)) + } } diff --git a/staging/src/k8s.io/client-go/tools/cache/reflector_metrics.go b/staging/src/k8s.io/client-go/tools/cache/reflector_metrics.go new file mode 100644 index 00000000000..0945e5c3a2a --- /dev/null +++ b/staging/src/k8s.io/client-go/tools/cache/reflector_metrics.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file provides abstractions for setting the provider (e.g., prometheus) +// of metrics. + +package cache + +import ( + "sync" +) + +// GaugeMetric represents a single numerical value that can arbitrarily go up +// and down. +type GaugeMetric interface { + Set(float64) +} + +// CounterMetric represents a single numerical value that only ever +// goes up. +type CounterMetric interface { + Inc() +} + +// SummaryMetric captures individual observations. +type SummaryMetric interface { + Observe(float64) +} + +type noopMetric struct{} + +func (noopMetric) Inc() {} +func (noopMetric) Dec() {} +func (noopMetric) Observe(float64) {} +func (noopMetric) Set(float64) {} + +type reflectorMetrics struct { + numberOfLists CounterMetric + listDuration SummaryMetric + numberOfItemsInList SummaryMetric + + numberOfWatches CounterMetric + numberOfShortWatches CounterMetric + watchDuration SummaryMetric + numberOfItemsInWatch SummaryMetric + + lastResourceVersion GaugeMetric +} + +// MetricsProvider generates various metrics used by the reflector. +type MetricsProvider interface { + NewListsMetric(name string) CounterMetric + NewListDurationMetric(name string) SummaryMetric + NewItemsInListMetric(name string) SummaryMetric + + NewWatchesMetric(name string) CounterMetric + NewShortWatchesMetric(name string) CounterMetric + NewWatchDurationMetric(name string) SummaryMetric + NewItemsInWatchMetric(name string) SummaryMetric + + NewLastResourceVersionMetric(name string) GaugeMetric +} + +type noopMetricsProvider struct{} + +func (noopMetricsProvider) NewListsMetric(name string) CounterMetric { return noopMetric{} } +func (noopMetricsProvider) NewListDurationMetric(name string) SummaryMetric { return noopMetric{} } +func (noopMetricsProvider) NewItemsInListMetric(name string) SummaryMetric { return noopMetric{} } +func (noopMetricsProvider) NewWatchesMetric(name string) CounterMetric { return noopMetric{} } +func (noopMetricsProvider) NewShortWatchesMetric(name string) CounterMetric { return noopMetric{} } +func (noopMetricsProvider) NewWatchDurationMetric(name string) SummaryMetric { return noopMetric{} } +func (noopMetricsProvider) NewItemsInWatchMetric(name string) SummaryMetric { return noopMetric{} } +func (noopMetricsProvider) NewLastResourceVersionMetric(name string) GaugeMetric { + return noopMetric{} +} + +var metricsFactory = struct { + metricsProvider MetricsProvider + setProviders sync.Once +}{ + metricsProvider: noopMetricsProvider{}, +} + +func newReflectorMetrics(name string) *reflectorMetrics { + var ret *reflectorMetrics + if len(name) == 0 { + return ret + } + return &reflectorMetrics{ + numberOfLists: metricsFactory.metricsProvider.NewListsMetric(name), + listDuration: metricsFactory.metricsProvider.NewListDurationMetric(name), + numberOfItemsInList: metricsFactory.metricsProvider.NewItemsInListMetric(name), + numberOfWatches: metricsFactory.metricsProvider.NewWatchesMetric(name), + numberOfShortWatches: metricsFactory.metricsProvider.NewShortWatchesMetric(name), + watchDuration: metricsFactory.metricsProvider.NewWatchDurationMetric(name), + numberOfItemsInWatch: metricsFactory.metricsProvider.NewItemsInWatchMetric(name), + lastResourceVersion: metricsFactory.metricsProvider.NewLastResourceVersionMetric(name), + } +} + +// SetReflectorMetricsProvider sets the metrics provider +func SetReflectorMetricsProvider(metricsProvider MetricsProvider) { + metricsFactory.setProviders.Do(func() { + metricsFactory.metricsProvider = metricsProvider + }) +}