Add admission metrics

staging/src/k8s.io/apiserver/pkg/admission/BUILD

@@ -32,12 +32,15 @@ go_library(
         "errors.go",
         "handler.go",
         "interfaces.go",
+        "metrics.go",
         "plugins.go",
     ],
     importpath = "k8s.io/apiserver/pkg/admission",
     deps = [
         "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library",

staging/src/k8s.io/apiserver/pkg/admission/chain.go

@@ -16,22 +16,38 @@ limitations under the License.
 
 package admission
 
-// chainAdmissionHandler is an instance of admission.Interface that performs admission control using a chain of admission handlers
-type chainAdmissionHandler []Interface
+import "time"
+
+// chainAdmissionHandler is an instance of admission.Interface that performs admission control using
+// a chain of admission handlers
+type chainAdmissionHandler []NamedHandler
 
 // NewChainHandler creates a new chain handler from an array of handlers. Used for testing.
-func NewChainHandler(handlers ...Interface) chainAdmissionHandler {
+func NewChainHandler(handlers ...NamedHandler) chainAdmissionHandler {
 	return chainAdmissionHandler(handlers)
 }
 
+const (
+	stepValidating = "validating"
+	stepMutating   = "mutating"
+)
+
 // Admit performs an admission control check using a chain of handlers, and returns immediately on first error
 func (admissionHandler chainAdmissionHandler) Admit(a Attributes) error {
+	var err error
+	start := time.Now()
+	defer func() {
+		ObserveAdmissionStep(time.Since(start), err != nil, a, stepMutating)
+	}()
+
 	for _, handler := range admissionHandler {
 		if !handler.Handles(a.GetOperation()) {
 			continue
 		}
 		if mutator, ok := handler.(MutationInterface); ok {
-			err := mutator.Admit(a)
+			t := time.Now()
+			err = mutator.Admit(a)
+			ObserveAdmissionController(time.Since(t), err != nil, handler, a)
 			if err != nil {
 				return err
 			}
@@ -42,12 +58,20 @@ func (admissionHandler chainAdmissionHandler) Admit(a Attributes) error {
 
 // Validate performs an admission control check using a chain of handlers, and returns immediately on first error
 func (admissionHandler chainAdmissionHandler) Validate(a Attributes) error {
+	var err error
+	start := time.Now()
+	defer func() {
+		ObserveAdmissionStep(time.Since(start), err != nil, a, stepValidating)
+	}()
+
 	for _, handler := range admissionHandler {
 		if !handler.Handles(a.GetOperation()) {
 			continue
 		}
 		if validator, ok := handler.(ValidationInterface); ok {
-			err := validator.Validate(a)
+			t := time.Now()
+			err = validator.Validate(a)
+			ObserveAdmissionController(time.Since(t), err != nil, handler, a)
 			if err != nil {
 				return err
 			}

staging/src/k8s.io/apiserver/pkg/admission/metrics.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package admission
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/api/admissionregistration/v1alpha1"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "admission"
+)
+
+var (
+	latencyBuckets       = prometheus.ExponentialBuckets(10000, 2.0, 8)
+	latencySummaryMaxAge = 5 * time.Hour
+
+	// Admission step metrics. Each step is identified by a distinct type label value.
+	stepMetrics = newAdmissionMetrics("step_",
+		[]string{"operation", "group", "version", "resource", "subresource", "type"},
+		"Admission sub-step %s, broken out for each operation and API resource and step type (validating or mutating).")
+
+	// Build-in admission controller metrics. Each admission controller is identified by name.
+	controllerMetrics = newAdmissionMetrics("controller_",
+		[]string{"name", "type", "operation", "group", "version", "resource", "subresource"},
+		"Admission controller %s, identified by name and broken out for each operation and API resource and type (validating or mutating).")
+
+	// External admission webhook metrics. Each webhook is identified by name.
+	externalWebhookMetrics = newAdmissionMetrics("external_webhook_",
+		[]string{"name", "type", "operation", "group", "version", "resource", "subresource"},
+		"External admission webhook %s, identified by name and broken out for each operation and API resource and type (validating or mutating).")
+)
+
+func init() {
+	stepMetrics.mustRegister()
+	controllerMetrics.mustRegister()
+	externalWebhookMetrics.mustRegister()
+}
+
+// namedHandler requires each admission.Interface be named, primarly for metrics tracking purposes.
+type NamedHandler interface {
+	Interface
+	GetName() string
+}
+
+// ObserveAdmissionStep records admission related metrics for a admission step, identified by step type.
+func ObserveAdmissionStep(elapsed time.Duration, rejected bool, attr Attributes, stepType string) {
+	gvr := attr.GetResource()
+	stepMetrics.observe(elapsed, rejected, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), stepType)
+}
+
+// ObserveAdmissionController records admission related metrics for a build-in admission controller, identified by it's plugin handler name.
+func ObserveAdmissionController(elapsed time.Duration, rejected bool, handler NamedHandler, attr Attributes) {
+	t := typeToLabel(handler)
+	gvr := attr.GetResource()
+	controllerMetrics.observe(elapsed, rejected, handler.GetName(), t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource())
+}
+
+// ObserveExternalWebhook records admission related metrics for a external admission webhook.
+func ObserveExternalWebhook(elapsed time.Duration, rejected bool, hook *v1alpha1.ExternalAdmissionHook, attr Attributes) {
+	t := "validating" // TODO: pass in type (validating|mutating) once mutating webhook functionality has been implemented
+	gvr := attr.GetResource()
+	externalWebhookMetrics.observe(elapsed, rejected, hook.Name, t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource())
+}
+
+func typeToLabel(i Interface) string {
+	switch i.(type) {
+	case ValidationInterface:
+		return "validating"
+	case MutationInterface:
+		return "mutating"
+	default:
+		return "UNRECOGNIZED_ADMISSION_TYPE"
+	}
+}
+
+type admissionMetrics struct {
+	total            *prometheus.CounterVec
+	rejectedTotal    *prometheus.CounterVec
+	latencies        *prometheus.HistogramVec
+	latenciesSummary *prometheus.SummaryVec
+}
+
+func (m *admissionMetrics) mustRegister() {
+	prometheus.MustRegister(m.total)
+	prometheus.MustRegister(m.rejectedTotal)
+	prometheus.MustRegister(m.latencies)
+	prometheus.MustRegister(m.latenciesSummary)
+}
+
+func (m *admissionMetrics) observe(elapsed time.Duration, rejected bool, labels ...string) {
+	elapsedMicroseconds := float64(elapsed / time.Microsecond)
+	m.total.WithLabelValues(labels...).Inc()
+	if rejected {
+		m.rejectedTotal.WithLabelValues(labels...).Inc()
+	}
+	m.latencies.WithLabelValues(labels...).Observe(elapsedMicroseconds)
+	m.latenciesSummary.WithLabelValues(labels...).Observe(elapsedMicroseconds)
+}
+
+func newAdmissionMetrics(name string, labels []string, helpTemplate string) *admissionMetrics {
+	return &admissionMetrics{
+		total: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: subsystem,
+				Name:      fmt.Sprintf("%stotal", name),
+				Help:      fmt.Sprintf(helpTemplate, "count"),
+			},
+			labels,
+		),
+		rejectedTotal: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: subsystem,
+				Name:      fmt.Sprintf("%srejected_total", name),
+				Help:      fmt.Sprintf(helpTemplate, "rejected count"),
+			},
+			labels,
+		),
+		latencies: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Namespace: namespace,
+				Subsystem: subsystem,
+				Name:      fmt.Sprintf("%slatencies", name),
+				Help:      fmt.Sprintf(helpTemplate, "latency histogram"),
+				Buckets:   latencyBuckets,
+			},
+			labels,
+		),
+		latenciesSummary: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Namespace: namespace,
+				Subsystem: subsystem,
+				Name:      fmt.Sprintf("%slatencies_summary", name),
+				Help:      fmt.Sprintf(helpTemplate, "latency summary"),
+				MaxAge:    latencySummaryMaxAge,
+			},
+			labels,
+		),
+	}
+}

staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/url"
 	"sync"
+	"time"
 
 	"github.com/golang/glog"
 	lru "github.com/hashicorp/golang-lru"
@@ -306,7 +307,9 @@ func (a *GenericAdmissionWebhook) Admit(attr admission.Attributes) error {
 		go func(hook *v1alpha1.Webhook) {
 			defer wg.Done()
 
+			t := time.Now()
 			err := a.callHook(ctx, hook, versionedAttr)
+			admission.ObserveExternalWebhook(time.Since(t), err != nil, hook, attr)
 			if err == nil {
 				return
 			}

staging/src/k8s.io/apiserver/pkg/admission/plugins.go

@@ -39,6 +39,16 @@ type Plugins struct {
 	registry map[string]Factory
 }
 
+// pluginHandler associates name with a admission.Interface handler.
+type pluginHandler struct {
+	Interface
+	name string
+}
+
+func (h *pluginHandler) GetName() string {
+	return h.name
+}
+
 // All registered admission options.
 var (
 	// PluginEnabledFn checks whether a plugin is enabled.  By default, if you ask about it, it's enabled.
@@ -121,7 +131,7 @@ func splitStream(config io.Reader) (io.Reader, io.Reader, error) {
 // NewFromPlugins returns an admission.Interface that will enforce admission control decisions of all
 // the given plugins.
 func (ps *Plugins) NewFromPlugins(pluginNames []string, configProvider ConfigProvider, pluginInitializer PluginInitializer) (Interface, error) {
-	plugins := []Interface{}
+	handlers := []NamedHandler{}
 	for _, pluginName := range pluginNames {
 		pluginConfig, err := configProvider.ConfigFor(pluginName)
 		if err != nil {
@@ -133,10 +143,11 @@ func (ps *Plugins) NewFromPlugins(pluginNames []string, configProvider ConfigPro
 			return nil, err
 		}
 		if plugin != nil {
-			plugins = append(plugins, plugin)
+			handler := &pluginHandler{Interface: plugin, name: pluginName}
+			handlers = append(handlers, handler)
 		}
 	}
-	return chainAdmissionHandler(plugins), nil
+	return chainAdmissionHandler(handlers), nil
 }
 
 // InitPlugin creates an instance of the named interface.