From 3940e4f0533a7ee8e50ec939cdcb44c33d4a0ae9 Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Mon, 6 Nov 2017 14:14:33 -0800 Subject: [PATCH] Add admission metrics --- .../src/k8s.io/apiserver/pkg/admission/BUILD | 3 + .../k8s.io/apiserver/pkg/admission/chain.go | 34 +++- .../k8s.io/apiserver/pkg/admission/metrics.go | 161 ++++++++++++++++++ .../pkg/admission/plugin/webhook/admission.go | 3 + .../k8s.io/apiserver/pkg/admission/plugins.go | 17 +- 5 files changed, 210 insertions(+), 8 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/admission/metrics.go diff --git a/staging/src/k8s.io/apiserver/pkg/admission/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/BUILD index 3d8927eb3d9..fc40a140a2a 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/BUILD @@ -32,12 +32,15 @@ go_library( "errors.go", "handler.go", "interfaces.go", + "metrics.go", "plugins.go", ], importpath = "k8s.io/apiserver/pkg/admission", deps = [ "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/admission/chain.go b/staging/src/k8s.io/apiserver/pkg/admission/chain.go index 65c7a526187..ab7e2179762 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/chain.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/chain.go @@ -16,22 +16,38 @@ limitations under the License. package admission -// chainAdmissionHandler is an instance of admission.Interface that performs admission control using a chain of admission handlers -type chainAdmissionHandler []Interface +import "time" + +// chainAdmissionHandler is an instance of admission.Interface that performs admission control using +// a chain of admission handlers +type chainAdmissionHandler []NamedHandler // NewChainHandler creates a new chain handler from an array of handlers. Used for testing. -func NewChainHandler(handlers ...Interface) chainAdmissionHandler { +func NewChainHandler(handlers ...NamedHandler) chainAdmissionHandler { return chainAdmissionHandler(handlers) } +const ( + stepValidating = "validating" + stepMutating = "mutating" +) + // Admit performs an admission control check using a chain of handlers, and returns immediately on first error func (admissionHandler chainAdmissionHandler) Admit(a Attributes) error { + var err error + start := time.Now() + defer func() { + ObserveAdmissionStep(time.Since(start), err != nil, a, stepMutating) + }() + for _, handler := range admissionHandler { if !handler.Handles(a.GetOperation()) { continue } if mutator, ok := handler.(MutationInterface); ok { - err := mutator.Admit(a) + t := time.Now() + err = mutator.Admit(a) + ObserveAdmissionController(time.Since(t), err != nil, handler, a) if err != nil { return err } @@ -42,12 +58,20 @@ func (admissionHandler chainAdmissionHandler) Admit(a Attributes) error { // Validate performs an admission control check using a chain of handlers, and returns immediately on first error func (admissionHandler chainAdmissionHandler) Validate(a Attributes) error { + var err error + start := time.Now() + defer func() { + ObserveAdmissionStep(time.Since(start), err != nil, a, stepValidating) + }() + for _, handler := range admissionHandler { if !handler.Handles(a.GetOperation()) { continue } if validator, ok := handler.(ValidationInterface); ok { - err := validator.Validate(a) + t := time.Now() + err = validator.Validate(a) + ObserveAdmissionController(time.Since(t), err != nil, handler, a) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics.go b/staging/src/k8s.io/apiserver/pkg/admission/metrics.go new file mode 100644 index 00000000000..0d0135b64c1 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/admission/metrics.go @@ -0,0 +1,161 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "fmt" + "time" + + "k8s.io/api/admissionregistration/v1alpha1" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace = "apiserver" + subsystem = "admission" +) + +var ( + latencyBuckets = prometheus.ExponentialBuckets(10000, 2.0, 8) + latencySummaryMaxAge = 5 * time.Hour + + // Admission step metrics. Each step is identified by a distinct type label value. + stepMetrics = newAdmissionMetrics("step_", + []string{"operation", "group", "version", "resource", "subresource", "type"}, + "Admission sub-step %s, broken out for each operation and API resource and step type (validating or mutating).") + + // Build-in admission controller metrics. Each admission controller is identified by name. + controllerMetrics = newAdmissionMetrics("controller_", + []string{"name", "type", "operation", "group", "version", "resource", "subresource"}, + "Admission controller %s, identified by name and broken out for each operation and API resource and type (validating or mutating).") + + // External admission webhook metrics. Each webhook is identified by name. + externalWebhookMetrics = newAdmissionMetrics("external_webhook_", + []string{"name", "type", "operation", "group", "version", "resource", "subresource"}, + "External admission webhook %s, identified by name and broken out for each operation and API resource and type (validating or mutating).") +) + +func init() { + stepMetrics.mustRegister() + controllerMetrics.mustRegister() + externalWebhookMetrics.mustRegister() +} + +// namedHandler requires each admission.Interface be named, primarly for metrics tracking purposes. +type NamedHandler interface { + Interface + GetName() string +} + +// ObserveAdmissionStep records admission related metrics for a admission step, identified by step type. +func ObserveAdmissionStep(elapsed time.Duration, rejected bool, attr Attributes, stepType string) { + gvr := attr.GetResource() + stepMetrics.observe(elapsed, rejected, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), stepType) +} + +// ObserveAdmissionController records admission related metrics for a build-in admission controller, identified by it's plugin handler name. +func ObserveAdmissionController(elapsed time.Duration, rejected bool, handler NamedHandler, attr Attributes) { + t := typeToLabel(handler) + gvr := attr.GetResource() + controllerMetrics.observe(elapsed, rejected, handler.GetName(), t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource()) +} + +// ObserveExternalWebhook records admission related metrics for a external admission webhook. +func ObserveExternalWebhook(elapsed time.Duration, rejected bool, hook *v1alpha1.ExternalAdmissionHook, attr Attributes) { + t := "validating" // TODO: pass in type (validating|mutating) once mutating webhook functionality has been implemented + gvr := attr.GetResource() + externalWebhookMetrics.observe(elapsed, rejected, hook.Name, t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource()) +} + +func typeToLabel(i Interface) string { + switch i.(type) { + case ValidationInterface: + return "validating" + case MutationInterface: + return "mutating" + default: + return "UNRECOGNIZED_ADMISSION_TYPE" + } +} + +type admissionMetrics struct { + total *prometheus.CounterVec + rejectedTotal *prometheus.CounterVec + latencies *prometheus.HistogramVec + latenciesSummary *prometheus.SummaryVec +} + +func (m *admissionMetrics) mustRegister() { + prometheus.MustRegister(m.total) + prometheus.MustRegister(m.rejectedTotal) + prometheus.MustRegister(m.latencies) + prometheus.MustRegister(m.latenciesSummary) +} + +func (m *admissionMetrics) observe(elapsed time.Duration, rejected bool, labels ...string) { + elapsedMicroseconds := float64(elapsed / time.Microsecond) + m.total.WithLabelValues(labels...).Inc() + if rejected { + m.rejectedTotal.WithLabelValues(labels...).Inc() + } + m.latencies.WithLabelValues(labels...).Observe(elapsedMicroseconds) + m.latenciesSummary.WithLabelValues(labels...).Observe(elapsedMicroseconds) +} + +func newAdmissionMetrics(name string, labels []string, helpTemplate string) *admissionMetrics { + return &admissionMetrics{ + total: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: fmt.Sprintf("%stotal", name), + Help: fmt.Sprintf(helpTemplate, "count"), + }, + labels, + ), + rejectedTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: fmt.Sprintf("%srejected_total", name), + Help: fmt.Sprintf(helpTemplate, "rejected count"), + }, + labels, + ), + latencies: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: fmt.Sprintf("%slatencies", name), + Help: fmt.Sprintf(helpTemplate, "latency histogram"), + Buckets: latencyBuckets, + }, + labels, + ), + latenciesSummary: prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: fmt.Sprintf("%slatencies_summary", name), + Help: fmt.Sprintf(helpTemplate, "latency summary"), + MaxAge: latencySummaryMaxAge, + }, + labels, + ), + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go index 737009068c1..f48bfb44dce 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go @@ -26,6 +26,7 @@ import ( "net" "net/url" "sync" + "time" "github.com/golang/glog" lru "github.com/hashicorp/golang-lru" @@ -306,7 +307,9 @@ func (a *GenericAdmissionWebhook) Admit(attr admission.Attributes) error { go func(hook *v1alpha1.Webhook) { defer wg.Done() + t := time.Now() err := a.callHook(ctx, hook, versionedAttr) + admission.ObserveExternalWebhook(time.Since(t), err != nil, hook, attr) if err == nil { return } diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugins.go b/staging/src/k8s.io/apiserver/pkg/admission/plugins.go index 172ac337b34..f1ee98e83a3 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugins.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugins.go @@ -39,6 +39,16 @@ type Plugins struct { registry map[string]Factory } +// pluginHandler associates name with a admission.Interface handler. +type pluginHandler struct { + Interface + name string +} + +func (h *pluginHandler) GetName() string { + return h.name +} + // All registered admission options. var ( // PluginEnabledFn checks whether a plugin is enabled. By default, if you ask about it, it's enabled. @@ -121,7 +131,7 @@ func splitStream(config io.Reader) (io.Reader, io.Reader, error) { // NewFromPlugins returns an admission.Interface that will enforce admission control decisions of all // the given plugins. func (ps *Plugins) NewFromPlugins(pluginNames []string, configProvider ConfigProvider, pluginInitializer PluginInitializer) (Interface, error) { - plugins := []Interface{} + handlers := []NamedHandler{} for _, pluginName := range pluginNames { pluginConfig, err := configProvider.ConfigFor(pluginName) if err != nil { @@ -133,10 +143,11 @@ func (ps *Plugins) NewFromPlugins(pluginNames []string, configProvider ConfigPro return nil, err } if plugin != nil { - plugins = append(plugins, plugin) + handler := &pluginHandler{Interface: plugin, name: pluginName} + handlers = append(handlers, handler) } } - return chainAdmissionHandler(plugins), nil + return chainAdmissionHandler(handlers), nil } // InitPlugin creates an instance of the named interface.