adds metrics for authorization webhook

This commit is contained in:
Lukasz Szaszkiewicz 2021-03-17 16:30:40 +01:00
parent 696d0f5772
commit 4a2aef00d6
9 changed files with 321 additions and 32 deletions

View File

@ -42,11 +42,11 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
// Get clients, if provided
var (
tokenClient authenticationclient.AuthenticationV1Interface
sarClient authorizationclient.SubjectAccessReviewInterface
sarClient authorizationclient.AuthorizationV1Interface
)
if client != nil && !reflect.ValueOf(client).IsNil() {
tokenClient = client.AuthenticationV1()
sarClient = client.AuthorizationV1().SubjectAccessReviews()
sarClient = client.AuthorizationV1()
}
authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication)
@ -102,7 +102,7 @@ func BuildAuthn(client authenticationclient.AuthenticationV1Interface, authn kub
}
// BuildAuthz creates an authorizer compatible with the kubelet's needs
func BuildAuthz(client authorizationclient.SubjectAccessReviewInterface, authz kubeletconfig.KubeletAuthorization) (authorizer.Authorizer, error) {
func BuildAuthz(client authorizationclient.AuthorizationV1Interface, authz kubeletconfig.KubeletAuthorization) (authorizer.Authorizer, error) {
switch authz.Mode {
case kubeletconfig.KubeletAuthorizationModeAlwaysAllow:
return authorizerfactory.NewAlwaysAllowAuthorizer(), nil

View File

@ -29,7 +29,7 @@ import (
// DelegatingAuthorizerConfig is the minimal configuration needed to create an authenticator
// built to delegate authorization to a kube API server
type DelegatingAuthorizerConfig struct {
SubjectAccessReviewClient authorizationclient.SubjectAccessReviewInterface
SubjectAccessReviewClient authorizationclient.AuthorizationV1Interface
// AllowCacheTTL is the length of time that a successful authorization response will be cached
AllowCacheTTL time.Duration
@ -54,5 +54,9 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
c.AllowCacheTTL,
c.DenyCacheTTL,
*c.WebhookRetryBackoff,
webhook.AuthorizerMetrics{
RecordRequestTotal: RecordRequestTotal,
RecordRequestLatency: RecordRequestLatency,
},
)
}

View File

@ -0,0 +1,69 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package authorizerfactory
import (
"context"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
type registerables []compbasemetrics.Registerable
// init registers all metrics
func init() {
for _, metric := range metrics {
legacyregistry.MustRegister(metric)
}
}
var (
requestTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_delegated_authz_request_total",
Help: "Number of HTTP requests partitioned by status code.",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"code"},
)
requestLatency = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_delegated_authz_request_duration_seconds",
Help: "Request latency in seconds. Broken down by status code.",
Buckets: []float64{0.25, 0.5, 0.7, 1, 1.5, 3, 5, 10},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"code"},
)
metrics = registerables{
requestTotal,
requestLatency,
}
)
// RecordRequestTotal increments the total number of requests for the delegated authorization.
func RecordRequestTotal(ctx context.Context, code string) {
requestTotal.WithContext(ctx).WithLabelValues(code).Add(1)
}
// RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code.
func RecordRequestLatency(ctx context.Context, code string, latency float64) {
requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency)
}

View File

@ -193,7 +193,7 @@ func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interfac
klog.Warning("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.")
} else {
cfg := authorizerfactory.DelegatingAuthorizerConfig{
SubjectAccessReviewClient: client.AuthorizationV1().SubjectAccessReviews(),
SubjectAccessReviewClient: client.AuthorizationV1(),
AllowCacheTTL: s.AllowCacheTTL,
DenyCacheTTL: s.DenyCacheTTL,
WebhookRetryBackoff: s.WebhookRetryBackoff,

View File

@ -0,0 +1,35 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"context"
)
// AuthorizerMetrics specifies a set of methods that are used to register various metrics for the webhook authorizer
type AuthorizerMetrics struct {
// RecordRequestTotal increments the total number of requests for the webhook authorizer
RecordRequestTotal func(ctx context.Context, code string)
// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
RecordRequestLatency func(ctx context.Context, code string, latency float64)
}
type noopMetrics struct{}
func (noopMetrics) RecordRequestTotal(context.Context, string) {}
func (noopMetrics) RecordRequestLatency(context.Context, string, float64) {}

View File

@ -0,0 +1,121 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"context"
"testing"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
func TestAuthorizerMetrics(t *testing.T) {
scenarios := []struct {
name string
clientCert, clientKey, clientCA []byte
serverCert, serverKey, serverCA []byte
authzFakeServiceStatusCode int
authFakeServiceDeny bool
expectedRegisteredStatusCode string
wantErr bool
}{
{
name: "happy path",
clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
expectedRegisteredStatusCode: "200",
},
{
name: "an internal error returned from the webhook",
clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
authzFakeServiceStatusCode: 500,
expectedRegisteredStatusCode: "500",
},
{
name: "incorrect client certificate used, the webhook not called, an error is recorded",
clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
serverCert: serverCert, serverKey: serverKey, serverCA: badCACert,
expectedRegisteredStatusCode: "<error>",
wantErr: true,
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
service := new(mockV1Service)
service.statusCode = scenario.authzFakeServiceStatusCode
if service.statusCode == 0 {
service.statusCode = 200
}
service.allow = !scenario.authFakeServiceDeny
server, err := NewV1TestServer(service, scenario.serverCert, scenario.serverKey, scenario.serverCA)
if err != nil {
t.Errorf("%s: failed to create server: %v", scenario.name, err)
return
}
defer server.Close()
fakeAuthzMetrics := &fakeAuthorizerMetrics{}
authzMetrics := AuthorizerMetrics{
RecordRequestTotal: fakeAuthzMetrics.RequestTotal,
RecordRequestLatency: fakeAuthzMetrics.RequestLatency,
}
wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, authzMetrics)
if err != nil {
t.Error("failed to create client")
return
}
attr := authorizer.AttributesRecord{User: &user.DefaultInfo{}}
_, _, err = wh.Authorize(context.Background(), attr)
if scenario.wantErr {
if err == nil {
t.Errorf("expected error making authorization request: %v", err)
}
}
if fakeAuthzMetrics.totalCode != scenario.expectedRegisteredStatusCode {
t.Errorf("incorrect status code recorded for RecordRequestTotal method, expected = %v, got %v", scenario.expectedRegisteredStatusCode, fakeAuthzMetrics.totalCode)
}
if fakeAuthzMetrics.latencyCode != scenario.expectedRegisteredStatusCode {
t.Errorf("incorrect status code recorded for RecordRequestLatency method, expected = %v, got %v", scenario.expectedRegisteredStatusCode, fakeAuthzMetrics.latencyCode)
}
})
}
}
type fakeAuthorizerMetrics struct {
totalCode string
latency float64
latencyCode string
}
func (f *fakeAuthorizerMetrics) RequestTotal(_ context.Context, code string) {
f.totalCode = code
}
func (f *fakeAuthorizerMetrics) RequestLatency(_ context.Context, code string, latency float64) {
f.latency = latency
f.latencyCode = code
}

View File

@ -21,10 +21,9 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"k8s.io/klog/v2"
authorizationv1 "k8s.io/api/authorization/v1"
authorizationv1beta1 "k8s.io/api/authorization/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -38,6 +37,8 @@ import (
"k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/kubernetes/scheme"
authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
)
const (
@ -55,7 +56,7 @@ func DefaultRetryBackoff() *wait.Backoff {
var _ authorizer.Authorizer = (*WebhookAuthorizer)(nil)
type subjectAccessReviewer interface {
Create(context.Context, *authorizationv1.SubjectAccessReview, metav1.CreateOptions) (*authorizationv1.SubjectAccessReview, error)
Create(context.Context, *authorizationv1.SubjectAccessReview, metav1.CreateOptions) (*authorizationv1.SubjectAccessReview, int, error)
}
type WebhookAuthorizer struct {
@ -65,11 +66,12 @@ type WebhookAuthorizer struct {
unauthorizedTTL time.Duration
retryBackoff wait.Backoff
decisionOnError authorizer.Decision
metrics AuthorizerMetrics
}
// NewFromInterface creates a WebhookAuthorizer using the given subjectAccessReview client
func NewFromInterface(subjectAccessReview authorizationv1client.SubjectAccessReviewInterface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff) (*WebhookAuthorizer, error) {
return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff)
func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, metrics)
}
// New creates a new WebhookAuthorizer from the provided kubeconfig file.
@ -96,11 +98,14 @@ func New(kubeConfigFile string, version string, authorizedTTL, unauthorizedTTL t
if err != nil {
return nil, err
}
return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff)
return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, AuthorizerMetrics{
RecordRequestTotal: noopMetrics{}.RecordRequestTotal,
RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
})
}
// newWithBackoff allows tests to skip the sleep.
func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff) (*WebhookAuthorizer, error) {
func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
return &WebhookAuthorizer{
subjectAccessReview: subjectAccessReview,
responseCache: cache.NewLRUExpireCache(8192),
@ -108,6 +113,7 @@ func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, un
unauthorizedTTL: unauthorizedTTL,
retryBackoff: retryBackoff,
decisionOnError: authorizer.DecisionNoOpinion,
metrics: metrics,
}, nil
}
@ -196,7 +202,23 @@ func (w *WebhookAuthorizer) Authorize(ctx context.Context, attr authorizer.Attri
// WithExponentialBackoff will return SAR create error (sarErr) if any.
if err := webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error {
var sarErr error
result, sarErr = w.subjectAccessReview.Create(ctx, r, metav1.CreateOptions{})
var statusCode int
start := time.Now()
result, statusCode, sarErr = w.subjectAccessReview.Create(ctx, r, metav1.CreateOptions{})
latency := time.Now().Sub(start)
if statusCode != 0 {
w.metrics.RecordRequestTotal(ctx, strconv.Itoa(statusCode))
w.metrics.RecordRequestLatency(ctx, strconv.Itoa(statusCode), latency.Seconds())
return sarErr
}
if sarErr != nil {
w.metrics.RecordRequestTotal(ctx, "<error>")
w.metrics.RecordRequestLatency(ctx, "<error>", latency.Seconds())
}
return sarErr
}, webhook.DefaultShouldRetry); err != nil {
klog.Errorf("Failed to make webhook authorizer request: %v", err)
@ -266,7 +288,7 @@ func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version s
if err != nil {
return nil, err
}
return &subjectAccessReviewV1Client{gw}, nil
return &subjectAccessReviewV1ClientGW{gw.RestClient}, nil
case authorizationv1beta1.SchemeGroupVersion.Version:
groupVersions := []schema.GroupVersion{authorizationv1beta1.SchemeGroupVersion}
@ -277,7 +299,7 @@ func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version s
if err != nil {
return nil, err
}
return &subjectAccessReviewV1beta1Client{gw}, nil
return &subjectAccessReviewV1beta1ClientGW{gw.RestClient}, nil
default:
return nil, fmt.Errorf(
@ -290,27 +312,58 @@ func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version s
}
type subjectAccessReviewV1Client struct {
w *webhook.GenericWebhook
client rest.Interface
}
func (t *subjectAccessReviewV1Client) Create(ctx context.Context, subjectAccessReview *authorizationv1.SubjectAccessReview, _ metav1.CreateOptions) (*authorizationv1.SubjectAccessReview, error) {
func (t *subjectAccessReviewV1Client) Create(ctx context.Context, subjectAccessReview *authorizationv1.SubjectAccessReview, opts metav1.CreateOptions) (result *authorizationv1.SubjectAccessReview, statusCode int, err error) {
result = &authorizationv1.SubjectAccessReview{}
restResult := t.client.Post().
Resource("subjectaccessreviews").
VersionedParams(&opts, scheme.ParameterCodec).
Body(subjectAccessReview).
Do(ctx)
restResult.StatusCode(&statusCode)
err = restResult.Into(result)
return
}
// subjectAccessReviewV1ClientGW used by the generic webhook, doesn't specify GVR.
type subjectAccessReviewV1ClientGW struct {
client rest.Interface
}
func (t *subjectAccessReviewV1ClientGW) Create(ctx context.Context, subjectAccessReview *authorizationv1.SubjectAccessReview, _ metav1.CreateOptions) (*authorizationv1.SubjectAccessReview, int, error) {
var statusCode int
result := &authorizationv1.SubjectAccessReview{}
err := t.w.RestClient.Post().Body(subjectAccessReview).Do(ctx).Into(result)
return result, err
restResult := t.client.Post().Body(subjectAccessReview).Do(ctx)
restResult.StatusCode(&statusCode)
err := restResult.Into(result)
return result, statusCode, err
}
type subjectAccessReviewV1beta1Client struct {
w *webhook.GenericWebhook
// subjectAccessReviewV1beta1ClientGW used by the generic webhook, doesn't specify GVR.
type subjectAccessReviewV1beta1ClientGW struct {
client rest.Interface
}
func (t *subjectAccessReviewV1beta1Client) Create(ctx context.Context, subjectAccessReview *authorizationv1.SubjectAccessReview, _ metav1.CreateOptions) (*authorizationv1.SubjectAccessReview, error) {
func (t *subjectAccessReviewV1beta1ClientGW) Create(ctx context.Context, subjectAccessReview *authorizationv1.SubjectAccessReview, _ metav1.CreateOptions) (*authorizationv1.SubjectAccessReview, int, error) {
var statusCode int
v1beta1Review := &authorizationv1beta1.SubjectAccessReview{Spec: v1SpecToV1beta1Spec(&subjectAccessReview.Spec)}
v1beta1Result := &authorizationv1beta1.SubjectAccessReview{}
err := t.w.RestClient.Post().Body(v1beta1Review).Do(ctx).Into(v1beta1Result)
restResult := t.client.Post().Body(v1beta1Review).Do(ctx)
restResult.StatusCode(&statusCode)
err := restResult.Into(v1beta1Result)
if err == nil {
subjectAccessReview.Status = v1beta1StatusToV1Status(&v1beta1Result.Status)
}
return subjectAccessReview, err
return subjectAccessReview, statusCode, err
}
// shouldCache determines whether it is safe to cache the given request attributes. If the

View File

@ -198,7 +198,7 @@ current-context: default
if err != nil {
return fmt.Errorf("error building sar client: %v", err)
}
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff)
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, noopAuthorizerMetrics())
return err
}()
if err != nil && !tt.wantErr {
@ -311,7 +311,7 @@ func (m *mockV1Service) HTTPStatusCode() int { return m.statusCode }
// newV1Authorizer creates a temporary kubeconfig file from the provided arguments and attempts to load
// a new WebhookAuthorizer from it.
func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration) (*WebhookAuthorizer, error) {
func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
tempfile, err := ioutil.TempFile("", "")
if err != nil {
return nil, err
@ -337,7 +337,7 @@ func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cache
if err != nil {
return nil, fmt.Errorf("error building sar client: %v", err)
}
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff)
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, metrics)
}
func TestV1TLSConfig(t *testing.T) {
@ -396,7 +396,7 @@ func TestV1TLSConfig(t *testing.T) {
}
defer server.Close()
wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0)
wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics())
if err != nil {
t.Errorf("%s: failed to create client: %v", tt.test, err)
return
@ -461,7 +461,7 @@ func TestV1Webhook(t *testing.T) {
}
defer s.Close()
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0)
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics())
if err != nil {
t.Fatal(err)
}
@ -563,7 +563,7 @@ func TestV1WebhookCache(t *testing.T) {
defer s.Close()
// Create an authorizer that caches successful responses "forever" (100 days).
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour)
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics())
if err != nil {
t.Fatal(err)
}
@ -653,3 +653,10 @@ func TestV1WebhookCache(t *testing.T) {
})
}
}
func noopAuthorizerMetrics() AuthorizerMetrics {
return AuthorizerMetrics{
RecordRequestTotal: noopMetrics{}.RecordRequestTotal,
RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
}
}

View File

@ -190,7 +190,7 @@ current-context: default
if err != nil {
return fmt.Errorf("error building sar client: %v", err)
}
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff)
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, noopAuthorizerMetrics())
return err
}()
if err != nil && !tt.wantErr {
@ -329,7 +329,7 @@ func newV1beta1Authorizer(callbackURL string, clientCert, clientKey, ca []byte,
if err != nil {
return nil, fmt.Errorf("error building sar client: %v", err)
}
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff)
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, noopAuthorizerMetrics())
}
func TestV1beta1TLSConfig(t *testing.T) {