adds metrics for authentication webhook

This commit is contained in:
Lukasz Szaszkiewicz 2021-02-23 08:39:25 +01:00
parent 6010cbe593
commit 322c18c147
9 changed files with 326 additions and 32 deletions

View File

@ -41,11 +41,11 @@ import (
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) {
// Get clients, if provided
var (
tokenClient authenticationclient.TokenReviewInterface
tokenClient authenticationclient.AuthenticationV1Interface
sarClient authorizationclient.SubjectAccessReviewInterface
)
if client != nil && !reflect.ValueOf(client).IsNil() {
tokenClient = client.AuthenticationV1().TokenReviews()
tokenClient = client.AuthenticationV1()
sarClient = client.AuthorizationV1().SubjectAccessReviews()
}
@ -65,7 +65,7 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
}
// BuildAuthn creates an authenticator compatible with the kubelet's needs
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) {
func BuildAuthn(client authenticationclient.AuthenticationV1Interface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) {
var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent
var err error
if len(authn.X509.ClientCAFile) > 0 {

View File

@ -42,7 +42,7 @@ type DelegatingAuthenticatorConfig struct {
Anonymous bool
// TokenAccessReviewClient is a client to do token review. It can be nil. Then every token is ignored.
TokenAccessReviewClient authenticationclient.TokenReviewInterface
TokenAccessReviewClient authenticationclient.AuthenticationV1Interface
// TokenAccessReviewTimeout specifies a time limit for requests made by the authorization webhook client.
TokenAccessReviewTimeout time.Duration
@ -91,7 +91,10 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
if c.WebhookRetryBackoff == nil {
return nil, nil, errors.New("retry backoff parameters for delegating authentication webhook has not been specified")
}
tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences, *c.WebhookRetryBackoff, c.TokenAccessReviewTimeout)
tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences, *c.WebhookRetryBackoff, c.TokenAccessReviewTimeout, webhooktoken.AuthenticatorMetrics{
RecordRequestTotal: RecordRequestTotal,
RecordRequestLatency: RecordRequestLatency,
})
if err != nil {
return nil, nil, err
}

View File

@ -0,0 +1,69 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package authenticatorfactory
import (
"context"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
type registerables []compbasemetrics.Registerable
// init registers all metrics.
func init() {
for _, metric := range metrics {
legacyregistry.MustRegister(metric)
}
}
var (
requestTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_delegated_authn_request_total",
Help: "Number of HTTP requests partitioned by status code.",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"code"},
)
requestLatency = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_delegated_authn_request_duration_seconds",
Help: "Request latency in seconds. Broken down by status code.",
Buckets: []float64{0.25, 0.5, 0.7, 1, 1.5, 3, 5, 10},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"code"},
)
metrics = registerables{
requestTotal,
requestLatency,
}
)
// RecordRequestTotal increments the total number of requests for the delegated authentication.
func RecordRequestTotal(ctx context.Context, code string) {
requestTotal.WithContext(ctx).WithLabelValues(code).Inc()
}
// RecordRequestLatency measures request latency in seconds for the delegated authentication. Broken down by status code.
func RecordRequestLatency(ctx context.Context, code string, latency float64) {
requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency)
}

View File

@ -295,7 +295,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
// configure token review
if client != nil {
cfg.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews()
cfg.TokenAccessReviewClient = client.AuthenticationV1()
}
// get the clientCA information

View File

@ -0,0 +1,35 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"context"
)
// AuthenticatorMetrics specifies a set of methods that are used to register various metrics
type AuthenticatorMetrics struct {
// RecordRequestTotal increments the total number of requests for webhooks
RecordRequestTotal func(ctx context.Context, code string)
// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
RecordRequestLatency func(ctx context.Context, code string, latency float64)
}
type noopMetrics struct{}
func (noopMetrics) RequestTotal(context.Context, string) {}
func (noopMetrics) RequestLatency(context.Context, string, float64) {}

View File

@ -0,0 +1,117 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"context"
"testing"
)
func TestAuthenticatorMetrics(t *testing.T) {
scenarios := []struct {
name string
clientCert, clientKey, clientCA []byte
serverCert, serverKey, serverCA []byte
authnFakeServiceStatusCode int
authFakeServiceDeny bool
expectedRegisteredStatusCode string
wantErr bool
}{
{
name: "happy path",
clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
expectedRegisteredStatusCode: "200",
},
{
name: "an internal error returned from the webhook",
clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
authnFakeServiceStatusCode: 500,
expectedRegisteredStatusCode: "500",
},
{
name: "incorrect client certificate used, the webhook not called, an error is recorded",
clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
serverCert: serverCert, serverKey: serverKey, serverCA: badCACert,
expectedRegisteredStatusCode: "<error>",
wantErr: true,
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
service := new(mockV1Service)
service.statusCode = scenario.authnFakeServiceStatusCode
if service.statusCode == 0 {
service.statusCode = 200
}
service.allow = !scenario.authFakeServiceDeny
server, err := NewV1TestServer(service, scenario.serverCert, scenario.serverKey, scenario.serverCA)
if err != nil {
t.Errorf("%s: failed to create server: %v", scenario.name, err)
return
}
defer server.Close()
fakeAuthnMetrics := &fakeAuthenticatorMetrics{}
authnMetrics := AuthenticatorMetrics{
RecordRequestTotal: fakeAuthnMetrics.RequestTotal,
RecordRequestLatency: fakeAuthnMetrics.RequestLatency,
}
wh, err := newV1TokenAuthenticator(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, nil, authnMetrics)
if err != nil {
t.Error("failed to create client")
return
}
_, _, err = wh.AuthenticateToken(context.Background(), "t0k3n")
if scenario.wantErr {
if err == nil {
t.Errorf("expected error making authorization request: %v", err)
}
}
if fakeAuthnMetrics.totalCode != scenario.expectedRegisteredStatusCode {
t.Errorf("incorrect status code recorded for RecordRequestTotal method, expected = %v, got %v", scenario.expectedRegisteredStatusCode, fakeAuthnMetrics.totalCode)
}
if fakeAuthnMetrics.latencyCode != scenario.expectedRegisteredStatusCode {
t.Errorf("incorrect status code recorded for RecordRequestLatency method, expected = %v, got %v", scenario.expectedRegisteredStatusCode, fakeAuthnMetrics.latencyCode)
}
})
}
}
type fakeAuthenticatorMetrics struct {
totalCode string
latency float64
latencyCode string
}
func (f *fakeAuthenticatorMetrics) RequestTotal(_ context.Context, code string) {
f.totalCode = code
}
func (f *fakeAuthenticatorMetrics) RequestLatency(_ context.Context, code string, latency float64) {
f.latency = latency
f.latencyCode = code
}

View File

@ -21,6 +21,7 @@ import (
"context"
"errors"
"fmt"
"strconv"
"time"
authenticationv1 "k8s.io/api/authentication/v1"
@ -35,6 +36,7 @@ import (
"k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/kubernetes/scheme"
authenticationv1client "k8s.io/client-go/kubernetes/typed/authentication/v1"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
)
@ -48,7 +50,7 @@ func DefaultRetryBackoff() *wait.Backoff {
var _ authenticator.Token = (*WebhookTokenAuthenticator)(nil)
type tokenReviewer interface {
Create(ctx context.Context, review *authenticationv1.TokenReview, _ metav1.CreateOptions) (*authenticationv1.TokenReview, error)
Create(ctx context.Context, review *authenticationv1.TokenReview, _ metav1.CreateOptions) (*authenticationv1.TokenReview, int, error)
}
type WebhookTokenAuthenticator struct {
@ -56,14 +58,16 @@ type WebhookTokenAuthenticator struct {
retryBackoff wait.Backoff
implicitAuds authenticator.Audiences
requestTimeout time.Duration
metrics AuthenticatorMetrics
}
// NewFromInterface creates a webhook authenticator using the given tokenReview
// client. It is recommend to wrap this authenticator with the token cache
// authenticator implemented in
// k8s.io/apiserver/pkg/authentication/token/cache.
func NewFromInterface(tokenReview authenticationv1client.TokenReviewInterface, implicitAuds authenticator.Audiences, retryBackoff wait.Backoff, requestTimeout time.Duration) (*WebhookTokenAuthenticator, error) {
return newWithBackoff(tokenReview, retryBackoff, implicitAuds, requestTimeout)
func NewFromInterface(tokenReview authenticationv1client.AuthenticationV1Interface, implicitAuds authenticator.Audiences, retryBackoff wait.Backoff, requestTimeout time.Duration, metrics AuthenticatorMetrics) (*WebhookTokenAuthenticator, error) {
tokenReviewClient := &tokenReviewV1Client{tokenReview.RESTClient()}
return newWithBackoff(tokenReviewClient, retryBackoff, implicitAuds, requestTimeout, metrics)
}
// New creates a new WebhookTokenAuthenticator from the provided kubeconfig
@ -75,12 +79,21 @@ func New(kubeConfigFile string, version string, implicitAuds authenticator.Audie
if err != nil {
return nil, err
}
return newWithBackoff(tokenReview, retryBackoff, implicitAuds, time.Duration(0))
return newWithBackoff(tokenReview, retryBackoff, implicitAuds, time.Duration(0), AuthenticatorMetrics{
RecordRequestTotal: noopMetrics{}.RequestTotal,
RecordRequestLatency: noopMetrics{}.RequestLatency,
})
}
// newWithBackoff allows tests to skip the sleep.
func newWithBackoff(tokenReview tokenReviewer, retryBackoff wait.Backoff, implicitAuds authenticator.Audiences, requestTimeout time.Duration) (*WebhookTokenAuthenticator, error) {
return &WebhookTokenAuthenticator{tokenReview, retryBackoff, implicitAuds, requestTimeout}, nil
func newWithBackoff(tokenReview tokenReviewer, retryBackoff wait.Backoff, implicitAuds authenticator.Audiences, requestTimeout time.Duration, metrics AuthenticatorMetrics) (*WebhookTokenAuthenticator, error) {
return &WebhookTokenAuthenticator{
tokenReview,
retryBackoff,
implicitAuds,
requestTimeout,
metrics,
}, nil
}
// AuthenticateToken implements the authenticator.Token interface.
@ -120,7 +133,22 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token
// WithExponentialBackoff will return tokenreview create error (tokenReviewErr) if any.
if err := webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error {
var tokenReviewErr error
result, tokenReviewErr = w.tokenReview.Create(ctx, r, metav1.CreateOptions{})
var statusCode int
start := time.Now()
result, statusCode, tokenReviewErr = w.tokenReview.Create(ctx, r, metav1.CreateOptions{})
latency := time.Now().Sub(start)
if statusCode != 0 {
w.metrics.RecordRequestTotal(ctx, strconv.Itoa(statusCode))
w.metrics.RecordRequestLatency(ctx, strconv.Itoa(statusCode), latency.Seconds())
return tokenReviewErr
}
if tokenReviewErr != nil {
w.metrics.RecordRequestTotal(ctx, "<error>")
w.metrics.RecordRequestLatency(ctx, "<error>", latency.Seconds())
}
return tokenReviewErr
}, webhook.DefaultShouldRetry); err != nil {
// An error here indicates bad configuration or an outage. Log for debugging.
@ -186,7 +214,7 @@ func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, r
if err != nil {
return nil, err
}
return &tokenReviewV1Client{gw}, nil
return &tokenReviewV1ClientGW{gw.RestClient}, nil
case authenticationv1beta1.SchemeGroupVersion.Version:
groupVersions := []schema.GroupVersion{authenticationv1beta1.SchemeGroupVersion}
@ -197,7 +225,7 @@ func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, r
if err != nil {
return nil, err
}
return &tokenReviewV1beta1Client{gw}, nil
return &tokenReviewV1beta1ClientGW{gw.RestClient}, nil
default:
return nil, fmt.Errorf(
@ -211,28 +239,60 @@ func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, r
}
type tokenReviewV1Client struct {
w *webhook.GenericWebhook
client rest.Interface
}
func (t *tokenReviewV1Client) Create(ctx context.Context, review *authenticationv1.TokenReview, _ metav1.CreateOptions) (*authenticationv1.TokenReview, error) {
result := &authenticationv1.TokenReview{}
err := t.w.RestClient.Post().Body(review).Do(ctx).Into(result)
return result, err
// Create takes the representation of a tokenReview and creates it. Returns the server's representation of the tokenReview, HTTP status code and an error, if there is any.
func (c *tokenReviewV1Client) Create(ctx context.Context, tokenReview *authenticationv1.TokenReview, opts metav1.CreateOptions) (result *authenticationv1.TokenReview, statusCode int, err error) {
result = &authenticationv1.TokenReview{}
restResult := c.client.Post().
Resource("tokenreviews").
VersionedParams(&opts, scheme.ParameterCodec).
Body(tokenReview).
Do(ctx)
restResult.StatusCode(&statusCode)
err = restResult.Into(result)
return
}
type tokenReviewV1beta1Client struct {
w *webhook.GenericWebhook
// tokenReviewV1ClientGW used by the generic webhook, doesn't specify GVR.
type tokenReviewV1ClientGW struct {
client rest.Interface
}
func (t *tokenReviewV1beta1Client) Create(ctx context.Context, review *authenticationv1.TokenReview, _ metav1.CreateOptions) (*authenticationv1.TokenReview, error) {
// Create takes the representation of a tokenReview and creates it. Returns the server's representation of the tokenReview, HTTP status code and an error, if there is any.
func (c *tokenReviewV1ClientGW) Create(ctx context.Context, tokenReview *authenticationv1.TokenReview, opts metav1.CreateOptions) (result *authenticationv1.TokenReview, statusCode int, err error) {
result = &authenticationv1.TokenReview{}
restResult := c.client.Post().
Body(tokenReview).
Do(ctx)
restResult.StatusCode(&statusCode)
err = restResult.Into(result)
return
}
// tokenReviewV1beta1ClientGW used by the generic webhook, doesn't specify GVR.
type tokenReviewV1beta1ClientGW struct {
client rest.Interface
}
func (t *tokenReviewV1beta1ClientGW) Create(ctx context.Context, review *authenticationv1.TokenReview, _ metav1.CreateOptions) (*authenticationv1.TokenReview, int, error) {
var statusCode int
v1beta1Review := &authenticationv1beta1.TokenReview{Spec: v1SpecToV1beta1Spec(&review.Spec)}
v1beta1Result := &authenticationv1beta1.TokenReview{}
err := t.w.RestClient.Post().Body(v1beta1Review).Do(ctx).Into(v1beta1Result)
restResult := t.client.Post().Body(v1beta1Review).Do(ctx)
restResult.StatusCode(&statusCode)
err := restResult.Into(v1beta1Result)
if err != nil {
return nil, err
return nil, statusCode, err
}
review.Status = v1beta1StatusToV1Status(&v1beta1Result.Status)
return review, nil
return review, statusCode, nil
}
func v1SpecToV1beta1Spec(in *authenticationv1.TokenReviewSpec) authenticationv1beta1.TokenReviewSpec {

View File

@ -178,7 +178,7 @@ func (m *mockV1Service) HTTPStatusCode() int { return m.statusCode }
// newV1TokenAuthenticator creates a temporary kubeconfig file from the provided
// arguments and attempts to load a new WebhookTokenAuthenticator from it.
func newV1TokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) {
func newV1TokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, implicitAuds authenticator.Audiences, metrics AuthenticatorMetrics) (authenticator.Token, error) {
tempfile, err := ioutil.TempFile("", "")
if err != nil {
return nil, err
@ -206,7 +206,7 @@ func newV1TokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte,
return nil, err
}
authn, err := newWithBackoff(c, testRetryBackoff, implicitAuds, 10*time.Second)
authn, err := newWithBackoff(c, testRetryBackoff, implicitAuds, 10*time.Second, metrics)
if err != nil {
return nil, err
}
@ -267,7 +267,7 @@ func TestV1TLSConfig(t *testing.T) {
}
defer server.Close()
wh, err := newV1TokenAuthenticator(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, nil)
wh, err := newV1TokenAuthenticator(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, nil, noopAuthenticatorMetrics())
if err != nil {
t.Errorf("%s: failed to create client: %v", tt.test, err)
return
@ -490,7 +490,7 @@ func TestV1WebhookTokenAuthenticator(t *testing.T) {
token := "my-s3cr3t-t0ken" // Fake token for testing.
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
wh, err := newV1TokenAuthenticator(s.URL, clientCert, clientKey, caCert, 0, tt.implicitAuds)
wh, err := newV1TokenAuthenticator(s.URL, clientCert, clientKey, caCert, 0, tt.implicitAuds, noopAuthenticatorMetrics())
if err != nil {
t.Fatal(err)
}
@ -563,7 +563,7 @@ func TestV1WebhookCacheAndRetry(t *testing.T) {
defer s.Close()
// Create an authenticator that caches successful responses "forever" (100 days).
wh, err := newV1TokenAuthenticator(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, nil)
wh, err := newV1TokenAuthenticator(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, nil, noopAuthenticatorMetrics())
if err != nil {
t.Fatal(err)
}
@ -690,3 +690,10 @@ func TestV1WebhookCacheAndRetry(t *testing.T) {
})
}
}
func noopAuthenticatorMetrics() AuthenticatorMetrics {
return AuthenticatorMetrics{
RecordRequestTotal: noopMetrics{}.RequestTotal,
RecordRequestLatency: noopMetrics{}.RequestLatency,
}
}

View File

@ -200,7 +200,10 @@ func newV1beta1TokenAuthenticator(serverURL string, clientCert, clientKey, ca []
return nil, err
}
authn, err := newWithBackoff(c, testRetryBackoff, implicitAuds, 10*time.Second)
authn, err := newWithBackoff(c, testRetryBackoff, implicitAuds, 10*time.Second, AuthenticatorMetrics{
RecordRequestTotal: noopMetrics{}.RequestTotal,
RecordRequestLatency: noopMetrics{}.RequestLatency,
})
if err != nil {
return nil, err
}