From 466980dd747e06e55451301c624eecccfa505123 Mon Sep 17 00:00:00 2001 From: Han Kang Date: Thu, 22 Aug 2019 15:38:42 -0700 Subject: [PATCH] migrate kube-apiserver metrics to stability framework --- cmd/kube-apiserver/BUILD | 3 +- cmd/kube-apiserver/apiserver.go | 4 +- cmd/kube-apiserver/app/BUILD | 4 +- cmd/kube-apiserver/app/server.go | 4 +- pkg/ssh/BUILD | 3 +- pkg/ssh/ssh.go | 25 ++-- staging/publishing/import-restrictions.yaml | 2 + .../pkg/apiserver/conversion/BUILD | 2 + .../pkg/apiserver/conversion/metrics.go | 21 +-- .../apiserver/pkg/admission/metrics/BUILD | 3 +- .../pkg/admission/metrics/metrics.go | 44 ++++--- staging/src/k8s.io/apiserver/pkg/audit/BUILD | 3 +- .../src/k8s.io/apiserver/pkg/audit/metrics.go | 43 +++--- .../pkg/authentication/request/x509/BUILD | 3 +- .../pkg/authentication/request/x509/x509.go | 11 +- .../apiserver/pkg/endpoints/filters/BUILD | 3 +- .../pkg/endpoints/filters/authentication.go | 17 +-- .../apiserver/pkg/endpoints/metrics/BUILD | 2 + .../pkg/endpoints/metrics/metrics.go | 123 ++++++++++-------- .../k8s.io/apiserver/pkg/server/routes/BUILD | 2 +- .../apiserver/pkg/server/routes/metrics.go | 7 +- .../k8s.io/apiserver/pkg/storage/cacher/BUILD | 3 +- .../apiserver/pkg/storage/cacher/cacher.go | 18 +-- .../apiserver/pkg/storage/etcd3/metrics/BUILD | 5 +- .../pkg/storage/etcd3/metrics/metrics.go | 36 ++--- .../pkg/storage/storagebackend/factory/BUILD | 1 + .../storage/storagebackend/factory/etcd3.go | 5 + .../k8s.io/apiserver/pkg/storage/value/BUILD | 2 + .../apiserver/pkg/storage/value/metrics.go | 116 +++++++++-------- .../pkg/storage/value/metrics_test.go | 10 +- .../pkg/storage/value/transformer_test.go | 12 +- staging/src/k8s.io/kube-aggregator/go.mod | 1 - .../pkg/controllers/status/BUILD | 3 +- .../pkg/controllers/status/metrics.go | 15 ++- .../k8s.io/legacy-cloud-providers/gce/BUILD | 2 + .../gce/token_source.go | 29 +++-- .../src/k8s.io/legacy-cloud-providers/go.sum | 1 + vendor/modules.txt | 4 + 38 files changed, 334 insertions(+), 258 deletions(-) diff --git a/cmd/kube-apiserver/BUILD b/cmd/kube-apiserver/BUILD index 1e9207a7801..a2c7666db2d 100644 --- a/cmd/kube-apiserver/BUILD +++ b/cmd/kube-apiserver/BUILD @@ -20,9 +20,8 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kube-apiserver", deps = [ "//cmd/kube-apiserver/app:go_default_library", - "//pkg/util/prometheusclientgo:go_default_library", - "//pkg/version/prometheus:go_default_library", "//staging/src/k8s.io/component-base/logs:go_default_library", + "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library", ], ) diff --git a/cmd/kube-apiserver/apiserver.go b/cmd/kube-apiserver/apiserver.go index ca524ab62c4..577de1bbc25 100644 --- a/cmd/kube-apiserver/apiserver.go +++ b/cmd/kube-apiserver/apiserver.go @@ -24,12 +24,12 @@ import ( "time" "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins "k8s.io/kubernetes/cmd/kube-apiserver/app" - _ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins - _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration ) func main() { + rand.Seed(time.Now().UnixNano()) command := app.NewAPIServerCommand() diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 70be68a87b9..660766736c9 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -30,8 +30,6 @@ go_library( "//pkg/registry/rbac/rest:go_default_library", "//pkg/serviceaccount:go_default_library", "//pkg/util/flag:go_default_library", - "//pkg/util/workqueue/prometheus:go_default_library", - "//pkg/version:go_default_library", "//pkg/version/verflag:go_default_library", "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", @@ -67,6 +65,8 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", + "//staging/src/k8s.io/component-base/metrics/prometheus/workqueue:go_default_library", + "//staging/src/k8s.io/component-base/version:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library", diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 3cfccf0fa04..b4b73cb863a 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -58,6 +58,8 @@ import ( cloudprovider "k8s.io/cloud-provider" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" + _ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration + "k8s.io/component-base/version" "k8s.io/klog" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -80,8 +82,6 @@ import ( rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" "k8s.io/kubernetes/pkg/serviceaccount" utilflag "k8s.io/kubernetes/pkg/util/flag" - _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration - "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/version/verflag" "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" ) diff --git a/pkg/ssh/BUILD b/pkg/ssh/BUILD index 2cc03ea717c..ac08e456edf 100644 --- a/pkg/ssh/BUILD +++ b/pkg/ssh/BUILD @@ -25,7 +25,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/golang.org/x/crypto/ssh:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/pkg/ssh/ssh.go b/pkg/ssh/ssh.go index 4263456b231..f3d68b4225e 100644 --- a/pkg/ssh/ssh.go +++ b/pkg/ssh/ssh.go @@ -36,33 +36,36 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" "golang.org/x/crypto/ssh" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog" ) var ( - tunnelOpenCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "ssh_tunnel_open_count", - Help: "Counter of ssh tunnel total open attempts", + tunnelOpenCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Name: "ssh_tunnel_open_count", + Help: "Counter of ssh tunnel total open attempts", + StabilityLevel: metrics.ALPHA, }, ) - tunnelOpenFailCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "ssh_tunnel_open_fail_count", - Help: "Counter of ssh tunnel failed open attempts", + tunnelOpenFailCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Name: "ssh_tunnel_open_fail_count", + Help: "Counter of ssh tunnel failed open attempts", + StabilityLevel: metrics.ALPHA, }, ) ) func init() { - prometheus.MustRegister(tunnelOpenCounter) - prometheus.MustRegister(tunnelOpenFailCounter) + legacyregistry.MustRegister(tunnelOpenCounter) + legacyregistry.MustRegister(tunnelOpenFailCounter) } // TODO: Unit tests for this code, we can spin up a test SSH server with instructions here: diff --git a/staging/publishing/import-restrictions.yaml b/staging/publishing/import-restrictions.yaml index 26cab4d3558..f66b33f292c 100644 --- a/staging/publishing/import-restrictions.yaml +++ b/staging/publishing/import-restrictions.yaml @@ -233,6 +233,8 @@ - k8s.io/utils - k8s.io/apiserver/pkg/util/feature - k8s.io/component-base/featuregate + - k8s.io/component-base/metrics + - k8s.io/component-base/metrics/legacyregistry - baseImportPath: "./vendor/k8s.io/node-api/" allowedImports: diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD index e06289dd6fb..244f0064950 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD @@ -30,6 +30,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/utils/trace:go_default_library", ], diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/metrics.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/metrics.go index 89090d7f999..dcd96232a5c 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/metrics.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/metrics.go @@ -26,6 +26,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) var ( @@ -36,19 +38,19 @@ var ( type converterMetricFactory struct { // A map from a converter name to it's metric. Allows the converterMetric to be created // again with the same metric for a specific converter (e.g. 'webhook'). - durations map[string]*prometheus.HistogramVec + durations map[string]*metrics.HistogramVec factoryLock sync.Mutex } func newConverterMertricFactory() *converterMetricFactory { - return &converterMetricFactory{durations: map[string]*prometheus.HistogramVec{}, factoryLock: sync.Mutex{}} + return &converterMetricFactory{durations: map[string]*metrics.HistogramVec{}, factoryLock: sync.Mutex{}} } var _ crConverterInterface = &converterMetric{} type converterMetric struct { delegate crConverterInterface - latencies *prometheus.HistogramVec + latencies *metrics.HistogramVec crdName string } @@ -57,14 +59,15 @@ func (c *converterMetricFactory) addMetrics(converterName string, crdName string defer c.factoryLock.Unlock() metric, exists := c.durations[converterName] if !exists { - metric = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: fmt.Sprintf("apiserver_crd_%s_conversion_duration_seconds", converterName), - Help: fmt.Sprintf("CRD %s conversion duration in seconds", converterName), - Buckets: latencyBuckets, + metric = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Name: fmt.Sprintf("apiserver_crd_%s_conversion_duration_seconds", converterName), + Help: fmt.Sprintf("CRD %s conversion duration in seconds", converterName), + Buckets: latencyBuckets, + StabilityLevel: metrics.ALPHA, }, []string{"crd_name", "from_version", "to_version", "succeeded"}) - err := prometheus.Register(metric) + err := legacyregistry.Register(metric) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/metrics/BUILD index e0f92b550d1..e15ed9a25b2 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/metrics/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/metrics/BUILD @@ -8,7 +8,8 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go index dbdc5db783c..a1ada6258f6 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go @@ -22,9 +22,9 @@ import ( "strconv" "time" - "github.com/prometheus/client_golang/prometheus" - "k8s.io/apiserver/pkg/admission" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) const ( @@ -154,33 +154,35 @@ func (m *AdmissionMetrics) ObserveWebhook(elapsed time.Duration, rejected bool, } type metricSet struct { - latencies *prometheus.HistogramVec - latenciesSummary *prometheus.SummaryVec + latencies *metrics.HistogramVec + latenciesSummary *metrics.SummaryVec } func newMetricSet(name string, labels []string, helpTemplate string, hasSummary bool) *metricSet { - var summary *prometheus.SummaryVec + var summary *metrics.SummaryVec if hasSummary { - summary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: fmt.Sprintf("%s_admission_duration_seconds_summary", name), - Help: fmt.Sprintf(helpTemplate, "latency summary in seconds"), - MaxAge: latencySummaryMaxAge, + summary = metrics.NewSummaryVec( + &metrics.SummaryOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: fmt.Sprintf("%s_admission_duration_seconds_summary", name), + Help: fmt.Sprintf(helpTemplate, "latency summary in seconds"), + MaxAge: latencySummaryMaxAge, + StabilityLevel: metrics.ALPHA, }, labels, ) } return &metricSet{ - latencies: prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: fmt.Sprintf("%s_admission_duration_seconds", name), - Help: fmt.Sprintf(helpTemplate, "latency histogram in seconds"), - Buckets: latencyBuckets, + latencies: metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: fmt.Sprintf("%s_admission_duration_seconds", name), + Help: fmt.Sprintf(helpTemplate, "latency histogram in seconds"), + Buckets: latencyBuckets, + StabilityLevel: metrics.ALPHA, }, labels, ), @@ -191,9 +193,9 @@ func newMetricSet(name string, labels []string, helpTemplate string, hasSummary // MustRegister registers all the prometheus metrics in the metricSet. func (m *metricSet) mustRegister() { - prometheus.MustRegister(m.latencies) + legacyregistry.MustRegister(m.latencies) if m.latenciesSummary != nil { - prometheus.MustRegister(m.latenciesSummary) + legacyregistry.MustRegister(m.latenciesSummary) } } diff --git a/staging/src/k8s.io/apiserver/pkg/audit/BUILD b/staging/src/k8s.io/apiserver/pkg/audit/BUILD index f6923a2828a..92726d869cf 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/audit/BUILD @@ -34,8 +34,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/audit/metrics.go b/staging/src/k8s.io/apiserver/pkg/audit/metrics.go index 9b81b30cc26..6c51d1385dd 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/audit/metrics.go @@ -19,8 +19,9 @@ package audit import ( "fmt" - "github.com/prometheus/client_golang/prometheus" auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog" ) @@ -29,45 +30,49 @@ const ( ) var ( - eventCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: "event_total", - Help: "Counter of audit events generated and sent to the audit backend.", + eventCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Subsystem: subsystem, + Name: "event_total", + Help: "Counter of audit events generated and sent to the audit backend.", + StabilityLevel: metrics.ALPHA, }) - errorCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ + errorCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ Subsystem: subsystem, Name: "error_total", Help: "Counter of audit events that failed to be audited properly. " + "Plugin identifies the plugin affected by the error.", + StabilityLevel: metrics.ALPHA, }, []string{"plugin"}, ) - levelCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: "level_total", - Help: "Counter of policy levels for audit events (1 per request).", + levelCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: subsystem, + Name: "level_total", + Help: "Counter of policy levels for audit events (1 per request).", + StabilityLevel: metrics.ALPHA, }, []string{"level"}, ) - ApiserverAuditDroppedCounter = prometheus.NewCounter( - prometheus.CounterOpts{ + ApiserverAuditDroppedCounter = metrics.NewCounter( + &metrics.CounterOpts{ Subsystem: subsystem, Name: "requests_rejected_total", Help: "Counter of apiserver requests rejected due to an error " + "in audit logging backend.", + StabilityLevel: metrics.ALPHA, }, ) ) func init() { - prometheus.MustRegister(eventCounter) - prometheus.MustRegister(errorCounter) - prometheus.MustRegister(levelCounter) - prometheus.MustRegister(ApiserverAuditDroppedCounter) + legacyregistry.MustRegister(eventCounter) + legacyregistry.MustRegister(errorCounter) + legacyregistry.MustRegister(levelCounter) + legacyregistry.MustRegister(ApiserverAuditDroppedCounter) } // ObserveEvent updates the relevant prometheus metrics for the generated audit event. diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD index 399197ea136..f75340b599d 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD @@ -36,7 +36,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go index 5e91034aa24..364ac2d94ec 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go @@ -23,16 +23,16 @@ import ( "net/http" "time" - "github.com/prometheus/client_golang/prometheus" - utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) -var clientCertificateExpirationHistogram = prometheus.NewHistogram( - prometheus.HistogramOpts{ +var clientCertificateExpirationHistogram = metrics.NewHistogram( + &metrics.HistogramOpts{ Namespace: "apiserver", Subsystem: "client", Name: "certificate_expiration_seconds", @@ -53,11 +53,12 @@ var clientCertificateExpirationHistogram = prometheus.NewHistogram( (6 * 30 * 24 * time.Hour).Seconds(), (12 * 30 * 24 * time.Hour).Seconds(), }, + StabilityLevel: metrics.ALPHA, }, ) func init() { - prometheus.MustRegister(clientCertificateExpirationHistogram) + legacyregistry.MustRegister(clientCertificateExpirationHistogram) } // UserConversion defines an interface for extracting user info from a client certificate chain diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD index 0fc8e02381b..d57d8292745 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD @@ -68,7 +68,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go index d9f70efac26..17b456a031e 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go @@ -21,29 +21,30 @@ import ( "net/http" "strings" - "github.com/prometheus/client_golang/prometheus" - "k8s.io/klog" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/klog" ) var ( - authenticatedUserCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "authenticated_user_requests", - Help: "Counter of authenticated requests broken out by username.", + authenticatedUserCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "authenticated_user_requests", + Help: "Counter of authenticated requests broken out by username.", + StabilityLevel: metrics.ALPHA, }, []string{"username"}, ) ) func init() { - prometheus.MustRegister(authenticatedUserCounter) + legacyregistry.MustRegister(authenticatedUserCounter) } // WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD index 4df68923804..95e8093e922 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD @@ -25,6 +25,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go index ea0146d5c81..42949645685 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go @@ -27,6 +27,9 @@ import ( "sync" "time" + restful "github.com/emicklei/go-restful" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" @@ -34,15 +37,14 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" - - restful "github.com/emicklei/go-restful" - "github.com/prometheus/client_golang/prometheus" + compbasemetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) // resettableCollector is the interface implemented by prometheus.MetricVec // that can be used by Prometheus to collect metrics and reset their values. type resettableCollector interface { - prometheus.Collector + compbasemetrics.Registerable Reset() } @@ -53,10 +55,11 @@ const ( var ( // TODO(a-robinson): Add unit tests for the handling of these metrics once // the upstream library supports it. - requestCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_request_total", - Help: "Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, client, and HTTP response contentType and code.", + requestCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "apiserver_request_total", + Help: "Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, client, and HTTP response contentType and code.", + StabilityLevel: compbasemetrics.ALPHA, }, // The label_name contentType doesn't follow the label_name convention defined here: // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/instrumentation.md @@ -64,22 +67,24 @@ var ( // should be all lowercase and separated by underscores. []string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component", "client", "contentType", "code"}, ) - deprecatedRequestCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_request_count", - Help: "(Deprecated) Counter of apiserver requests broken out for each verb, group, version, resource, scope, component, client, and HTTP response contentType and code.", + deprecatedRequestCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "apiserver_request_count", + Help: "(Deprecated) Counter of apiserver requests broken out for each verb, group, version, resource, scope, component, client, and HTTP response contentType and code.", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "group", "version", "resource", "subresource", "scope", "component", "client", "contentType", "code"}, ) - longRunningRequestGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "apiserver_longrunning_gauge", - Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.", + longRunningRequestGauge = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ + Name: "apiserver_longrunning_gauge", + Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, ) - requestLatencies = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + requestLatencies = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ Name: "apiserver_request_duration_seconds", Help: "Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component.", // This metric is used for verifying api call latencies SLO, @@ -87,81 +92,91 @@ var ( // Thus we customize buckets significantly, to empower both usecases. Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}, + StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component"}, ) - deprecatedRequestLatencies = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + deprecatedRequestLatencies = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ Name: "apiserver_request_latencies", Help: "(Deprecated) Response latency distribution in microseconds for each verb, group, version, resource, subresource, scope and component.", // Use buckets ranging from 125 ms to 8 seconds. - Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), + Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), + StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, ) - deprecatedRequestLatenciesSummary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ + deprecatedRequestLatenciesSummary = compbasemetrics.NewSummaryVec( + &compbasemetrics.SummaryOpts{ Name: "apiserver_request_latencies_summary", Help: "(Deprecated) Response latency summary in microseconds for each verb, group, version, resource, subresource, scope and component.", // Make the sliding window of 5h. // TODO: The value for this should be based on our SLI definition (medium term). - MaxAge: 5 * time.Hour, + MaxAge: 5 * time.Hour, + StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, ) - responseSizes = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + responseSizes = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ Name: "apiserver_response_sizes", Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.", // Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB). - Buckets: prometheus.ExponentialBuckets(1000, 10.0, 7), + Buckets: prometheus.ExponentialBuckets(1000, 10.0, 7), + StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, ) // DroppedRequests is a number of requests dropped with 'Try again later' response" - DroppedRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_dropped_requests_total", - Help: "Number of requests dropped with 'Try again later' response", + DroppedRequests = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "apiserver_dropped_requests_total", + Help: "Number of requests dropped with 'Try again later' response", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"requestKind"}, ) - DeprecatedDroppedRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_dropped_requests", - Help: "(Deprecated) Number of requests dropped with 'Try again later' response", + DeprecatedDroppedRequests = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "apiserver_dropped_requests", + Help: "(Deprecated) Number of requests dropped with 'Try again later' response", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"requestKind"}, ) // RegisteredWatchers is a number of currently registered watchers splitted by resource. - RegisteredWatchers = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "apiserver_registered_watchers", - Help: "Number of currently registered watchers for a given resources", + RegisteredWatchers = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ + Name: "apiserver_registered_watchers", + Help: "Number of currently registered watchers for a given resources", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"group", "version", "kind"}, ) - WatchEvents = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_watch_events_total", - Help: "Number of events sent in watch clients", + WatchEvents = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "apiserver_watch_events_total", + Help: "Number of events sent in watch clients", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"group", "version", "kind"}, ) - WatchEventsSizes = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "apiserver_watch_events_sizes", - Help: "Watch event size distribution in bytes", - Buckets: prometheus.ExponentialBuckets(1024, 2.0, 8), // 1K, 2K, 4K, 8K, ..., 128K. + WatchEventsSizes = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ + Name: "apiserver_watch_events_sizes", + Help: "Watch event size distribution in bytes", + Buckets: prometheus.ExponentialBuckets(1024, 2.0, 8), // 1K, 2K, 4K, 8K, ..., 128K. + StabilityLevel: compbasemetrics.ALPHA, }, []string{"group", "version", "kind"}, ) // Because of volatality of the base metric this is pre-aggregated one. Instead of reporing current usage all the time // it reports maximal usage during the last second. - currentInflightRequests = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "apiserver_current_inflight_requests", - Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.", + currentInflightRequests = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ + Name: "apiserver_current_inflight_requests", + Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"requestKind"}, ) @@ -197,7 +212,7 @@ var registerMetrics sync.Once func Register() { registerMetrics.Do(func() { for _, metric := range metrics { - prometheus.MustRegister(metric) + legacyregistry.MustRegister(metric) } }) } @@ -241,7 +256,7 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp if requestInfo == nil { requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path} } - var g prometheus.Gauge + var g compbasemetrics.GaugeMetric scope := CleanScope(requestInfo) // We don't use verb from , as for the healthy path // MonitorRequest is called from InstrumentRouteFunc which is registered diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD b/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD index e6e09a1ab46..57ac2c70f7e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD @@ -26,9 +26,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go index f9ae46f64e7..7b4591c8736 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go @@ -23,8 +23,7 @@ import ( apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/apiserver/pkg/server/mux" etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics" - - "github.com/prometheus/client_golang/prometheus" + "k8s.io/component-base/metrics/legacyregistry" ) // DefaultMetrics installs the default prometheus metrics handler @@ -33,7 +32,7 @@ type DefaultMetrics struct{} // Install adds the DefaultMetrics handler func (m DefaultMetrics) Install(c *mux.PathRecorderMux) { register() - c.Handle("/metrics", prometheus.Handler()) + c.Handle("/metrics", legacyregistry.Handler()) } // MetricsWithReset install the prometheus metrics handler extended with support for the DELETE method @@ -43,7 +42,7 @@ type MetricsWithReset struct{} // Install adds the MetricsWithReset handler func (m MetricsWithReset) Install(c *mux.PathRecorderMux) { register() - defaultMetricsHandler := prometheus.Handler().ServeHTTP + defaultMetricsHandler := legacyregistry.Handler().ServeHTTP c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { if req.Method == "DELETE" { apimetrics.Reset() diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD index 7978c79fa6d..aadbd8037d8 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD @@ -27,7 +27,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/trace:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go index 65ef690f484..f27db4e9d17 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go @@ -24,8 +24,6 @@ import ( "sync" "time" - "k8s.io/klog" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,16 +39,18 @@ import ( "k8s.io/apiserver/pkg/storage" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/cache" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/klog" utiltrace "k8s.io/utils/trace" - - "github.com/prometheus/client_golang/prometheus" ) var ( - initCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_init_events_total", - Help: "Counter of init events processed in watchcache broken by resource type", + initCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "apiserver_init_events_total", + Help: "Counter of init events processed in watchcache broken by resource type", + StabilityLevel: metrics.ALPHA, }, []string{"resource"}, ) @@ -64,7 +64,7 @@ const ( ) func init() { - prometheus.MustRegister(initCounter) + legacyregistry.MustRegister(initCounter) } // Config contains the configuration for a given Cache. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/BUILD index 0a42ce60971..13324471c83 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/BUILD @@ -10,7 +10,10 @@ go_library( srcs = ["metrics.go"], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/etcd3/metrics", importpath = "k8s.io/apiserver/pkg/storage/etcd3/metrics", - deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"], + deps = [ + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", + ], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go index 68f8b3158a0..c32326c0464 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go @@ -20,29 +20,33 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" + compbasemetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) var ( - etcdRequestLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "etcd_request_duration_seconds", - Help: "Etcd request latency in seconds for each operation and object type.", + etcdRequestLatency = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ + Name: "etcd_request_duration_seconds", + Help: "Etcd request latency in seconds for each operation and object type.", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"operation", "type"}, ) - objectCounts = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "etcd_object_counts", - Help: "Number of stored objects at the time of last check split by kind.", + objectCounts = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ + Name: "etcd_object_counts", + Help: "Number of stored objects at the time of last check split by kind.", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"resource"}, ) - deprecatedEtcdRequestLatenciesSummary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Name: "etcd_request_latencies_summary", - Help: "(Deprecated) Etcd request latency summary in microseconds for each operation and object type.", + deprecatedEtcdRequestLatenciesSummary = compbasemetrics.NewSummaryVec( + &compbasemetrics.SummaryOpts{ + Name: "etcd_request_latencies_summary", + Help: "(Deprecated) Etcd request latency summary in microseconds for each operation and object type.", + StabilityLevel: compbasemetrics.ALPHA, }, []string{"operation", "type"}, ) @@ -54,11 +58,11 @@ var registerMetrics sync.Once func Register() { // Register the metrics. registerMetrics.Do(func() { - prometheus.MustRegister(etcdRequestLatency) - prometheus.MustRegister(objectCounts) + legacyregistry.MustRegister(etcdRequestLatency) + legacyregistry.MustRegister(objectCounts) // TODO(danielqsj): Remove the following metrics, they are deprecated - prometheus.MustRegister(deprecatedEtcdRequestLatenciesSummary) + legacyregistry.MustRegister(deprecatedEtcdRequestLatenciesSummary) }) } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD index 108ddbc269e..5783f0ef776 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD @@ -39,6 +39,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", "//vendor/github.com/grpc-ecosystem/go-grpc-prometheus:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 801f8d6c920..eadb684f56c 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -34,6 +34,7 @@ import ( "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/value" + "k8s.io/component-base/metrics/legacyregistry" ) // The short keepalive timeout and interval have been chosen to aggressively @@ -46,6 +47,10 @@ const keepaliveTimeout = 10 * time.Second // on heavily loaded arm64 CPUs (issue #64649) const dialTimeout = 20 * time.Second +func init() { + legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics) +} + func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) { // constructing the etcd v3 client blocks and times out if etcd is not available. // retry in a loop in the background until we successfully create the client, storing the client or error encountered diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD index 9c4716b41ff..8ddbb531961 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD @@ -30,6 +30,8 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value", importpath = "k8s.io/apiserver/pkg/storage/value", deps = [ + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/google.golang.org/grpc/status:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go index 9da584bcdc9..179d2b98ee2 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go @@ -20,9 +20,11 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc/status" - "github.com/prometheus/client_golang/prometheus" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) const ( @@ -31,84 +33,92 @@ const ( ) var ( - transformerLatencies = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + transformerLatencies = metrics.NewHistogramVec( + &metrics.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "transformation_duration_seconds", Help: "Latencies in seconds of value transformation operations.", // In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when // external KMS is involved latencies may climb into milliseconds. - Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14), + Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14), + StabilityLevel: metrics.ALPHA, }, []string{"transformation_type"}, ) - deprecatedTransformerLatencies = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + deprecatedTransformerLatencies = metrics.NewHistogramVec( + &metrics.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "transformation_latencies_microseconds", Help: "(Deprecated) Latencies in microseconds of value transformation operations.", // In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when // external KMS is involved latencies may climb into milliseconds. - Buckets: prometheus.ExponentialBuckets(5, 2, 14), + Buckets: prometheus.ExponentialBuckets(5, 2, 14), + StabilityLevel: metrics.ALPHA, }, []string{"transformation_type"}, ) - transformerOperationsTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "transformation_operations_total", - Help: "Total number of transformations.", + transformerOperationsTotal = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "transformation_operations_total", + Help: "Total number of transformations.", + StabilityLevel: metrics.ALPHA, }, []string{"transformation_type", "transformer_prefix", "status"}, ) - deprecatedTransformerFailuresTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "transformation_failures_total", - Help: "(Deprecated) Total number of failed transformation operations.", + deprecatedTransformerFailuresTotal = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "transformation_failures_total", + Help: "(Deprecated) Total number of failed transformation operations.", + StabilityLevel: metrics.ALPHA, }, []string{"transformation_type"}, ) - envelopeTransformationCacheMissTotal = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "envelope_transformation_cache_misses_total", - Help: "Total number of cache misses while accessing key decryption key(KEK).", + envelopeTransformationCacheMissTotal = metrics.NewCounter( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "envelope_transformation_cache_misses_total", + Help: "Total number of cache misses while accessing key decryption key(KEK).", + StabilityLevel: metrics.ALPHA, }, ) - dataKeyGenerationLatencies = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "data_key_generation_duration_seconds", - Help: "Latencies in seconds of data encryption key(DEK) generation operations.", - Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14), + dataKeyGenerationLatencies = metrics.NewHistogram( + &metrics.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "data_key_generation_duration_seconds", + Help: "Latencies in seconds of data encryption key(DEK) generation operations.", + Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14), + StabilityLevel: metrics.ALPHA, }, ) - deprecatedDataKeyGenerationLatencies = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "data_key_generation_latencies_microseconds", - Help: "(Deprecated) Latencies in microseconds of data encryption key(DEK) generation operations.", - Buckets: prometheus.ExponentialBuckets(5, 2, 14), + deprecatedDataKeyGenerationLatencies = metrics.NewHistogram( + &metrics.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "data_key_generation_latencies_microseconds", + Help: "(Deprecated) Latencies in microseconds of data encryption key(DEK) generation operations.", + Buckets: prometheus.ExponentialBuckets(5, 2, 14), + StabilityLevel: metrics.ALPHA, }, ) - dataKeyGenerationFailuresTotal = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "data_key_generation_failures_total", - Help: "Total number of failed data encryption key(DEK) generation operations.", + dataKeyGenerationFailuresTotal = metrics.NewCounter( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "data_key_generation_failures_total", + Help: "Total number of failed data encryption key(DEK) generation operations.", + StabilityLevel: metrics.ALPHA, }, ) ) @@ -117,14 +127,14 @@ var registerMetrics sync.Once func RegisterMetrics() { registerMetrics.Do(func() { - prometheus.MustRegister(transformerLatencies) - prometheus.MustRegister(deprecatedTransformerLatencies) - prometheus.MustRegister(transformerOperationsTotal) - prometheus.MustRegister(deprecatedTransformerFailuresTotal) - prometheus.MustRegister(envelopeTransformationCacheMissTotal) - prometheus.MustRegister(dataKeyGenerationLatencies) - prometheus.MustRegister(deprecatedDataKeyGenerationLatencies) - prometheus.MustRegister(dataKeyGenerationFailuresTotal) + legacyregistry.MustRegister(transformerLatencies) + legacyregistry.MustRegister(deprecatedTransformerLatencies) + legacyregistry.MustRegister(transformerOperationsTotal) + legacyregistry.MustRegister(deprecatedTransformerFailuresTotal) + legacyregistry.MustRegister(envelopeTransformationCacheMissTotal) + legacyregistry.MustRegister(dataKeyGenerationLatencies) + legacyregistry.MustRegister(deprecatedDataKeyGenerationLatencies) + legacyregistry.MustRegister(dataKeyGenerationFailuresTotal) }) } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go index bfdbb404962..d67edc3d7e0 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go @@ -44,10 +44,10 @@ func TestTotals(t *testing.T) { }, error: errors.New("foo"), want: ` - # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. + # HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations. # TYPE apiserver_storage_transformation_failures_total counter apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 `, @@ -59,7 +59,7 @@ func TestTotals(t *testing.T) { "apiserver_storage_transformation_failures_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 `, @@ -72,10 +72,10 @@ func TestTotals(t *testing.T) { }, error: status.Error(codes.FailedPrecondition, "foo"), want: ` - # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. + # HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations. # TYPE apiserver_storage_transformation_failures_total counter apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 `, diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go index 3ae9e3e35b3..95427586754 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go @@ -128,7 +128,7 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1 `, @@ -142,7 +142,7 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1 `, @@ -156,7 +156,7 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="other:"} 1 `, @@ -170,7 +170,7 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="unknown"} 1 `, @@ -214,7 +214,7 @@ func TestPrefixToMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 `, @@ -228,7 +228,7 @@ func TestPrefixToMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # TYPE apiserver_storage_transformation_operations_total counter apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="to_storage",transformer_prefix="other:"} 1 `, diff --git a/staging/src/k8s.io/kube-aggregator/go.mod b/staging/src/k8s.io/kube-aggregator/go.mod index b08c87d504d..58bb883560c 100644 --- a/staging/src/k8s.io/kube-aggregator/go.mod +++ b/staging/src/k8s.io/kube-aggregator/go.mod @@ -10,7 +10,6 @@ require ( github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 // indirect github.com/go-openapi/spec v0.19.2 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d - github.com/prometheus/client_golang v0.9.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.3.0 diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD index 543eb3eae39..cb7c2119f3c 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD @@ -25,13 +25,14 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/controllers:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/metrics.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/metrics.go index 539641ea93b..47a08ad4238 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/metrics.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/metrics.go @@ -17,19 +17,20 @@ limitations under the License. package apiserver import ( - "github.com/prometheus/client_golang/prometheus" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) var ( - unavailableCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ + unavailableCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ Name: "aggregator_unavailable_apiservice_count", Help: "Counter of APIServices which are marked as unavailable broken down by APIService name and reason.", }, []string{"name", "reason"}, ) - unavailableGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + unavailableGauge = metrics.NewGaugeVec( + &metrics.GaugeOpts{ Name: "aggregator_unavailable_apiservice", Help: "Gauge of APIServices which are marked as unavailable broken down by APIService name.", }, @@ -38,6 +39,6 @@ var ( ) func init() { - prometheus.MustRegister(unavailableCounter) - prometheus.MustRegister(unavailableGauge) + legacyregistry.MustRegister(unavailableCounter) + legacyregistry.MustRegister(unavailableGauge) } diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/BUILD b/staging/src/k8s.io/legacy-cloud-providers/gce/BUILD index 1354a0ab1b7..05ab6e8420c 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/gce/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/BUILD @@ -66,6 +66,8 @@ go_library( "//staging/src/k8s.io/cloud-provider/volume:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/errors:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/cloud.google.com/go/compute/metadata:go_default_library", "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud:go_default_library", "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/filter:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/token_source.go b/staging/src/k8s.io/legacy-cloud-providers/gce/token_source.go index f4670eca9b7..154652d3e76 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/gce/token_source.go +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/token_source.go @@ -25,12 +25,13 @@ import ( "strings" "time" - "k8s.io/client-go/util/flowcontrol" - - "github.com/prometheus/client_golang/prometheus" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" + + "k8s.io/client-go/util/flowcontrol" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) const ( @@ -41,23 +42,25 @@ const ( ) var ( - getTokenCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "get_token_count", - Help: "Counter of total Token() requests to the alternate token source", + getTokenCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Name: "get_token_count", + Help: "Counter of total Token() requests to the alternate token source", + StabilityLevel: metrics.ALPHA, }, ) - getTokenFailCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "get_token_fail_count", - Help: "Counter of failed Token() requests to the alternate token source", + getTokenFailCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Name: "get_token_fail_count", + Help: "Counter of failed Token() requests to the alternate token source", + StabilityLevel: metrics.ALPHA, }, ) ) func init() { - prometheus.MustRegister(getTokenCounter) - prometheus.MustRegister(getTokenFailCounter) + legacyregistry.MustRegister(getTokenCounter) + legacyregistry.MustRegister(getTokenFailCounter) } // AltTokenSource is the structure holding the data for the functionality needed to generates tokens diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.sum b/staging/src/k8s.io/legacy-cloud-providers/go.sum index b957b8a3475..e115e8c35ad 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.sum +++ b/staging/src/k8s.io/legacy-cloud-providers/go.sum @@ -33,6 +33,7 @@ github.com/aws/aws-sdk-go v1.16.26 h1:GWkl3rkRO/JGRTWoLLIqwf7AWC4/W/1hMOUZqmX0js github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/vendor/modules.txt b/vendor/modules.txt index 5baa6d4cc3d..fce383986db 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1638,6 +1638,10 @@ k8s.io/component-base/featuregate/testing k8s.io/component-base/logs k8s.io/component-base/metrics k8s.io/component-base/metrics/legacyregistry +k8s.io/component-base/metrics/prometheus/clientgo +k8s.io/component-base/metrics/prometheus/clientgo/leaderelection +k8s.io/component-base/metrics/prometheus/restclient +k8s.io/component-base/metrics/prometheus/workqueue k8s.io/component-base/version # k8s.io/cri-api v0.0.0 => ./staging/src/k8s.io/cri-api k8s.io/cri-api/pkg/apis