From 3f3bc37e05879ec97a64b4833a315a3c7b1186a4 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Fri, 13 Dec 2019 14:03:05 -0800 Subject: [PATCH] auth: add metrics to token cache --- .../pkg/authentication/token/cache/BUILD | 3 + .../token/cache/cached_token_authenticator.go | 11 ++ .../pkg/authentication/token/cache/stats.go | 120 ++++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 staging/src/k8s.io/apiserver/pkg/authentication/token/cache/stats.go diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/BUILD b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/BUILD index 771634d747a..e646a57eb8b 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/BUILD @@ -29,6 +29,7 @@ go_library( "cache_simple.go", "cache_striped.go", "cached_token_authenticator.go", + "stats.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/authentication/token/cache", importpath = "k8s.io/apiserver/pkg/authentication/token/cache", @@ -37,6 +38,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/cache:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/golang.org/x/sync/singleflight:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go index 66a9fc577f5..8edd6b646cf 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go @@ -109,19 +109,29 @@ func newWithClock(authenticator authenticator.Token, cacheErrs bool, successTTL, // AuthenticateToken implements authenticator.Token func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) { + doneAuthenticating := stats.authenticating() + auds, audsOk := authenticator.AudiencesFrom(ctx) key := keyFunc(a.hashPool, auds, token) if record, ok := a.cache.get(key); ok { + // Record cache hit + doneAuthenticating(true) return record.resp, record.ok, record.err } + // Record cache miss + doneBlocking := stats.blocking() + defer doneBlocking() + defer doneAuthenticating(false) + type lookup struct { resp *authenticator.Response ok bool } c := a.group.DoChan(key, func() (val interface{}, err error) { + doneFetching := stats.fetching() // We're leaving the request handling stack so we need to handle crashes // ourselves. Log a stack trace and return a 500 if something panics. defer func() { @@ -134,6 +144,7 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token buf = buf[:runtime.Stack(buf, false)] klog.Errorf("%v\n%s", r, buf) } + doneFetching(err == nil) }() // Check again for a cached record. We may have raced with a fetch. diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/stats.go b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/stats.go new file mode 100644 index 00000000000..8435f6834d9 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/stats.go @@ -0,0 +1,120 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "time" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +var ( + requestLatency = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Namespace: "authentication", + Subsystem: "token_cache", + Name: "request_duration_seconds", + StabilityLevel: metrics.ALPHA, + }, + []string{"status"}, + ) + requestCount = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: "authentication", + Subsystem: "token_cache", + Name: "request_count", + StabilityLevel: metrics.ALPHA, + }, + []string{"status"}, + ) + fetchCount = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Namespace: "authentication", + Subsystem: "token_cache", + Name: "fetch_count", + StabilityLevel: metrics.ALPHA, + }, + []string{"status"}, + ) + blockCount = metrics.NewGauge( + &metrics.GaugeOpts{ + Namespace: "authentication", + Subsystem: "token_cache", + Name: "block_count", + StabilityLevel: metrics.ALPHA, + }, + ) +) + +func init() { + legacyregistry.MustRegister( + requestLatency, + requestCount, + fetchCount, + blockCount, + ) +} + +const ( + hitTag = "hit" + missTag = "miss" + + fetchActiveTag = "active" + fetchFailedTag = "error" + fetchOkTag = "ok" +) + +type statsCollector struct{} + +var stats = statsCollector{} + +func (statsCollector) authenticating() func(hit bool) { + start := time.Now() + return func(hit bool) { + var tag string + if hit { + tag = hitTag + } else { + tag = missTag + } + + latency := time.Since(start) + + requestCount.WithLabelValues(tag).Inc() + requestLatency.WithLabelValues(tag).Observe(float64(latency.Milliseconds()) / 1000) + } +} + +func (statsCollector) blocking() func() { + blockCount.Inc() + return blockCount.Dec +} + +func (statsCollector) fetching() func(ok bool) { + fetchCount.WithLabelValues(fetchActiveTag).Inc() + return func(ok bool) { + var tag string + if ok { + tag = fetchOkTag + } else { + tag = fetchFailedTag + } + + fetchCount.WithLabelValues(tag).Dec() + } +}