Merge pull request #48224 from deads2k/controller-11-reflectormetrics

Automatic merge from submit-queue (batch tested with PRs 48224, 45431, 45946, 48775, 49396)

add reflector metrics

This adds metrics (optionally prometheus) to reflectors so that you can see when one reflector is behaving poorly and just how poorly its doing.

@eparis 

```release-note
Adds metrics for checking reflector health.
```

pkg/controller/garbagecollector/BUILD

@@ -26,6 +26,7 @@ go_library(
         "//pkg/controller:go_default_library",
         "//pkg/controller/garbagecollector/metaonly:go_default_library",
         "//pkg/util/metrics:go_default_library",
+        "//pkg/util/reflector/prometheus:go_default_library",
         "//pkg/util/workqueue/prometheus:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",

pkg/controller/garbagecollector/garbagecollector.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector/metaonly"
+	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
 	// install the prometheus plugin
 	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	// import known versions

pkg/util/BUILD

@@ -44,6 +44,7 @@ filegroup(
         "//pkg/util/parsers:all-srcs",
         "//pkg/util/pointer:all-srcs",
         "//pkg/util/procfs:all-srcs",
+        "//pkg/util/reflector/prometheus:all-srcs",
         "//pkg/util/removeall:all-srcs",
         "//pkg/util/resourcecontainer:all-srcs",
         "//pkg/util/rlimit:all-srcs",

pkg/util/reflector/prometheus/BUILD

@@ -0,0 +1,31 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["prometheus.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/util/reflector/prometheus/prometheus.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package prometheus sets the cache DefaultMetricsFactory to produce
+// prometheus metrics. To use this package, you just have to import it.
+package prometheus
+
+import (
+	"k8s.io/client-go/tools/cache"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const reflectorSubsystem = "reflector"
+
+var (
+	listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "lists_total",
+		Help:      "Total number of API lists done by the reflectors",
+	}, []string{"name"})
+
+	listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "list_duration_seconds",
+		Help:      "How long an API list takes to return and decode for the reflectors",
+	}, []string{"name"})
+
+	itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "items_per_list",
+		Help:      "How many items an API list returns to the reflectors",
+	}, []string{"name"})
+
+	watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "watches_total",
+		Help:      "Total number of API watches done by the reflectors",
+	}, []string{"name"})
+
+	shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "short_watches_total",
+		Help:      "Total number of short API watches done by the reflectors",
+	}, []string{"name"})
+
+	watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "watch_duration_seconds",
+		Help:      "How long an API watch takes to return and decode for the reflectors",
+	}, []string{"name"})
+
+	itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Subsystem: reflectorSubsystem,
+		Name:      "items_per_watch",
+		Help:      "How many items an API watch returns to the reflectors",
+	}, []string{"name"})
+)
+
+func init() {
+	prometheus.MustRegister(listsTotal)
+	prometheus.MustRegister(listsDuration)
+	prometheus.MustRegister(itemsPerList)
+	prometheus.MustRegister(watchesTotal)
+	prometheus.MustRegister(shortWatchesTotal)
+	prometheus.MustRegister(watchDuration)
+	prometheus.MustRegister(itemsPerWatch)
+
+	cache.SetReflectorMetricsProvider(prometheusMetricsProvider{})
+}
+
+type prometheusMetricsProvider struct{}
+
+func (prometheusMetricsProvider) NewListsMetric(name string) cache.CounterMetric {
+	return listsTotal.WithLabelValues(name)
+}
+
+// use summary to get averages and percentiles
+func (prometheusMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric {
+	return listsDuration.WithLabelValues(name)
+}
+
+// use summary to get averages and percentiles
+func (prometheusMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric {
+	return itemsPerList.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric {
+	return watchesTotal.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric {
+	return shortWatchesTotal.WithLabelValues(name)
+}
+
+// use summary to get averages and percentiles
+func (prometheusMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric {
+	return watchDuration.WithLabelValues(name)
+}
+
+// use summary to get averages and percentiles
+func (prometheusMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric {
+	return itemsPerWatch.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric {
+	rv := prometheus.NewGauge(prometheus.GaugeOpts{
+		Subsystem: name,
+		Name:      "last_resource_version",
+		Help:      "last resource version seen for the reflectors",
+	})
+	prometheus.MustRegister(rv)
+	return rv
+}