Merge pull request #86265 from mikedanese/cachestats

auth: add metrics to token cache

staging/src/k8s.io/apiserver/pkg/authentication/token/cache/BUILD

@@ -29,6 +29,7 @@ go_library(
         "cache_simple.go",
         "cache_striped.go",
         "cached_token_authenticator.go",
+        "stats.go",
     ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/authentication/token/cache",
     importpath = "k8s.io/apiserver/pkg/authentication/token/cache",
@@ -37,6 +38,8 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/cache:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
         "//vendor/golang.org/x/sync/singleflight:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go

@@ -109,19 +109,29 @@ func newWithClock(authenticator authenticator.Token, cacheErrs bool, successTTL,
 
 // AuthenticateToken implements authenticator.Token
 func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) {
+	doneAuthenticating := stats.authenticating()
+
 	auds, audsOk := authenticator.AudiencesFrom(ctx)
 
 	key := keyFunc(a.hashPool, auds, token)
 	if record, ok := a.cache.get(key); ok {
+		// Record cache hit
+		doneAuthenticating(true)
 		return record.resp, record.ok, record.err
 	}
 
+	// Record cache miss
+	doneBlocking := stats.blocking()
+	defer doneBlocking()
+	defer doneAuthenticating(false)
+
 	type lookup struct {
 		resp *authenticator.Response
 		ok   bool
 	}
 
 	c := a.group.DoChan(key, func() (val interface{}, err error) {
+		doneFetching := stats.fetching()
 		// We're leaving the request handling stack so we need to handle crashes
 		// ourselves. Log a stack trace and return a 500 if something panics.
 		defer func() {
@@ -134,6 +144,7 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token
 				buf = buf[:runtime.Stack(buf, false)]
 				klog.Errorf("%v\n%s", r, buf)
 			}
+			doneFetching(err == nil)
 		}()
 
 		// Check again for a cached record. We may have raced with a fetch.

staging/src/k8s.io/apiserver/pkg/authentication/token/cache/stats.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+var (
+	requestLatency = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "request_duration_seconds",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+	requestCount = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "request_count",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+	fetchCount = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "fetch_count",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+	blockCount = metrics.NewGauge(
+		&metrics.GaugeOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "block_count",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+)
+
+func init() {
+	legacyregistry.MustRegister(
+		requestLatency,
+		requestCount,
+		fetchCount,
+		blockCount,
+	)
+}
+
+const (
+	hitTag  = "hit"
+	missTag = "miss"
+
+	fetchActiveTag = "active"
+	fetchFailedTag = "error"
+	fetchOkTag     = "ok"
+)
+
+type statsCollector struct{}
+
+var stats = statsCollector{}
+
+func (statsCollector) authenticating() func(hit bool) {
+	start := time.Now()
+	return func(hit bool) {
+		var tag string
+		if hit {
+			tag = hitTag
+		} else {
+			tag = missTag
+		}
+
+		latency := time.Since(start)
+
+		requestCount.WithLabelValues(tag).Inc()
+		requestLatency.WithLabelValues(tag).Observe(float64(latency.Milliseconds()) / 1000)
+	}
+}
+
+func (statsCollector) blocking() func() {
+	blockCount.Inc()
+	return blockCount.Dec
+}
+
+func (statsCollector) fetching() func(ok bool) {
+	fetchCount.WithLabelValues(fetchActiveTag).Inc()
+	return func(ok bool) {
+		var tag string
+		if ok {
+			tag = fetchOkTag
+		} else {
+			tag = fetchFailedTag
+		}
+
+		fetchCount.WithLabelValues(tag).Dec()
+	}
+}