add authz webhook matchcondition metrics

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>
Signed-off-by: Jordan Liggitt <liggitt@google.com>
Co-authored-by: Jordan Liggitt <liggitt@google.com>

pkg/kubeapiserver/authorizer/reload.go

@@ -32,11 +32,13 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/authorization/cel"
 	authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
 	"k8s.io/apiserver/pkg/authorization/union"
 	"k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
 	"k8s.io/apiserver/plugin/pkg/authorizer/webhook"
+	webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/auth/authorizer/abac"
 	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
@@ -142,6 +144,8 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
 				*r.initialConfig.WebhookRetryBackoff,
 				decisionOnError,
 				configuredAuthorizer.Webhook.MatchConditions,
+				configuredAuthorizer.Name,
+				kubeapiserverWebhookMetrics{MatcherMetrics: cel.NewMatcherMetrics()},
 			)
 			if err != nil {
 				return nil, nil, err
@@ -162,6 +166,13 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
 	return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil
 }
 
+type kubeapiserverWebhookMetrics struct {
+	// kube-apiserver doesn't report request metrics
+	webhookmetrics.NoopRequestMetrics
+	// kube-apiserver does report matchCondition metrics
+	cel.MatcherMetrics
+}
+
 // runReload starts checking the config file for changes and reloads the authorizer when it changes.
 // Blocks until ctx is complete.
 func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) {

staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go

@@ -26,7 +26,7 @@ import (
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1"
 )
 
-// DelegatingAuthorizerConfig is the minimal configuration needed to create an authenticator
+// DelegatingAuthorizerConfig is the minimal configuration needed to create an authorizer
 // built to delegate authorization to a kube API server
 type DelegatingAuthorizerConfig struct {
 	SubjectAccessReviewClient authorizationclient.AuthorizationV1Interface
@@ -55,9 +55,6 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
 		c.DenyCacheTTL,
 		*c.WebhookRetryBackoff,
 		authorizer.DecisionNoOpinion,
-		webhook.AuthorizerMetrics{
+		NewDelegatingAuthorizerMetrics(),
-			RecordRequestTotal:   RecordRequestTotal,
-			RecordRequestLatency: RecordRequestLatency,
-		},
 	)
 }

staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/metrics.go

@@ -18,18 +18,22 @@ package authorizerfactory
 
 import (
 	"context"
+	"sync"
 
+	celmetrics "k8s.io/apiserver/pkg/authorization/cel"
+	webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	compbasemetrics "k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
-type registerables []compbasemetrics.Registerable
+var registerMetrics sync.Once
 
-// init registers all metrics
+// RegisterMetrics registers authorizer metrics.
-func init() {
+func RegisterMetrics() {
-	for _, metric := range metrics {
+	registerMetrics.Do(func() {
-		legacyregistry.MustRegister(metric)
+		legacyregistry.MustRegister(requestTotal)
-	}
+		legacyregistry.MustRegister(requestLatency)
+	})
 }
 
 var (
@@ -51,19 +55,26 @@ var (
 		},
 		[]string{"code"},
 	)
-
-	metrics = registerables{
-		requestTotal,
-		requestLatency,
-	}
 )
 
+var _ = webhookmetrics.AuthorizerMetrics(delegatingAuthorizerMetrics{})
+
+type delegatingAuthorizerMetrics struct {
+	// no-op for matchCondition metrics for now, delegating authorization doesn't configure match conditions
+	celmetrics.NoopMatcherMetrics
+}
+
+func NewDelegatingAuthorizerMetrics() delegatingAuthorizerMetrics {
+	RegisterMetrics()
+	return delegatingAuthorizerMetrics{}
+}
+
 // RecordRequestTotal increments the total number of requests for the delegated authorization.
-func RecordRequestTotal(ctx context.Context, code string) {
+func (delegatingAuthorizerMetrics) RecordRequestTotal(ctx context.Context, code string) {
 	requestTotal.WithContext(ctx).WithLabelValues(code).Add(1)
 }
 
 // RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code.
-func RecordRequestLatency(ctx context.Context, code string, latency float64) {
+func (delegatingAuthorizerMetrics) RecordRequestLatency(ctx context.Context, code string, latency float64) {
 	requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency)
 }

staging/src/k8s.io/apiserver/pkg/authorization/cel/matcher.go

@@ -19,6 +19,7 @@ package cel
 import (
 	"context"
 	"fmt"
+	"time"
 
 	celgo "github.com/google/cel-go/cel"
 
@@ -28,11 +29,29 @@ import (
 
 type CELMatcher struct {
 	CompilationResults []CompilationResult
+
+	// These are optional fields which can be populated if metrics reporting is desired
+	Metrics        MatcherMetrics
+	AuthorizerType string
+	AuthorizerName string
 }
 
 // eval evaluates the given SubjectAccessReview against all cel matchCondition expression
 func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessReview) (bool, error) {
 	var evalErrors []error
+
+	metrics := c.Metrics
+	if metrics == nil {
+		metrics = NoopMatcherMetrics{}
+	}
+	start := time.Now()
+	defer func() {
+		metrics.RecordAuthorizationMatchConditionEvaluation(ctx, c.AuthorizerType, c.AuthorizerName, time.Since(start))
+		if len(evalErrors) > 0 {
+			metrics.RecordAuthorizationMatchConditionEvaluationFailure(ctx, c.AuthorizerType, c.AuthorizerName)
+		}
+	}()
+
 	va := map[string]interface{}{
 		"request": convertObjectToUnstructured(&r.Spec),
 	}
@@ -54,6 +73,7 @@ func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessR
 		// If at least one matchCondition successfully evaluates to FALSE,
 		// return early
 		if !match {
+			metrics.RecordAuthorizationMatchConditionExclusion(ctx, c.AuthorizerType, c.AuthorizerName)
 			return false, nil
 		}
 	}

staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// MatcherMetrics defines methods for reporting matchCondition metrics
+type MatcherMetrics interface {
+	// RecordAuthorizationMatchConditionEvaluation records the total time taken to evaluate matchConditions for an Authorize() call to the given authorizer
+	RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration)
+	// RecordAuthorizationMatchConditionEvaluationFailure increments if any evaluation error was encountered evaluating matchConditions for an Authorize() call to the given authorizer
+	RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string)
+	// RecordAuthorizationMatchConditionExclusion records increments when at least one matchCondition evaluates to false and excludes an Authorize() call to the given authorizer
+	RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string)
+}
+
+type NoopMatcherMetrics struct{}
+
+func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
+}
+func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
+}
+func (NoopMatcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
+}
+
+type matcherMetrics struct{}
+
+func NewMatcherMetrics() MatcherMetrics {
+	RegisterMetrics()
+	return matcherMetrics{}
+}
+
+const (
+	namespace = "apiserver"
+	subsystem = "authorization"
+)
+
+var (
+	authorizationMatchConditionEvaluationErrorsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_evaluation_errors_total",
+			Help:           "Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name"},
+	)
+	authorizationMatchConditionExclusionsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_exclusions_total",
+			Help:           "Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name"},
+	)
+	authorizationMatchConditionEvaluationSeconds = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_evaluation_seconds",
+			Help:           "Authorization match condition evaluation time in seconds, split by authorizer type and name.",
+			Buckets:        []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name"},
+	)
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(authorizationMatchConditionEvaluationErrorsTotal)
+		legacyregistry.MustRegister(authorizationMatchConditionExclusionsTotal)
+		legacyregistry.MustRegister(authorizationMatchConditionEvaluationSeconds)
+	})
+}
+
+func ResetMetricsForTest() {
+	authorizationMatchConditionEvaluationErrorsTotal.Reset()
+	authorizationMatchConditionExclusionsTotal.Reset()
+	authorizationMatchConditionEvaluationSeconds.Reset()
+}
+
+func (matcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
+	authorizationMatchConditionEvaluationErrorsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
+}
+
+func (matcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
+	authorizationMatchConditionExclusionsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
+}
+
+func (matcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
+	elapsedSeconds := elapsed.Seconds()
+	authorizationMatchConditionEvaluationSeconds.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Observe(elapsedSeconds)
+}

staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics_test.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+func TestRecordAuthorizationMatchConditionEvaluationFailure(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		metrics   []string
+		name      string
+		authztype string
+		want      string
+	}{
+		{
+			desc: "evaluation failure total",
+			metrics: []string{
+				"apiserver_authorization_match_condition_evaluation_errors_total",
+				"apiserver_authorization_match_condition_exclusions_total",
+				"apiserver_authorization_match_condition_evaluation_seconds",
+			},
+			name:      "wh1.example.com",
+			authztype: "Webhook",
+			want: `
+			# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
+			# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
+			apiserver_authorization_match_condition_evaluation_errors_total{name="wh1.example.com",type="Webhook"} 1
+			# HELP apiserver_authorization_match_condition_evaluation_seconds [ALPHA] Authorization match condition evaluation time in seconds, split by authorizer type and name.
+        	# TYPE apiserver_authorization_match_condition_evaluation_seconds histogram
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.001"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.005"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.01"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.025"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.1"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.2"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.25"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="+Inf"} 1
+        	apiserver_authorization_match_condition_evaluation_seconds_sum{name="wh1.example.com",type="Webhook"} 1
+        	apiserver_authorization_match_condition_evaluation_seconds_count{name="wh1.example.com",type="Webhook"} 1
+			# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
+			# TYPE apiserver_authorization_match_condition_exclusions_total counter
+			apiserver_authorization_match_condition_exclusions_total{name="wh1.example.com",type="Webhook"} 1
+			`,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			ResetMetricsForTest()
+			m := NewMatcherMetrics()
+			m.RecordAuthorizationMatchConditionEvaluationFailure(context.Background(), tt.authztype, tt.name)
+			m.RecordAuthorizationMatchConditionExclusion(context.Background(), tt.authztype, tt.name)
+			m.RecordAuthorizationMatchConditionEvaluation(context.Background(), tt.authztype, tt.name, time.Duration(1*time.Second))
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics/metrics.go

@@ -14,22 +14,36 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package webhook
+package metrics
 
 import (
 	"context"
+
+	"k8s.io/apiserver/pkg/authorization/cel"
 )
 
 // AuthorizerMetrics specifies a set of methods that are used to register various metrics for the webhook authorizer
-type AuthorizerMetrics struct {
+type AuthorizerMetrics interface {
-	// RecordRequestTotal increments the total number of requests for the webhook authorizer
+	// Request total and latency metrics
-	RecordRequestTotal func(ctx context.Context, code string)
+	RequestMetrics
-
+	// match condition metrics
-	// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
+	cel.MatcherMetrics
-	RecordRequestLatency func(ctx context.Context, code string, latency float64)
 }
 
-type noopMetrics struct{}
+type NoopAuthorizerMetrics struct {
+	NoopRequestMetrics
+	cel.NoopMatcherMetrics
+}
 
-func (noopMetrics) RecordRequestTotal(context.Context, string)            {}
+type RequestMetrics interface {
-func (noopMetrics) RecordRequestLatency(context.Context, string, float64) {}
+	// RecordRequestTotal increments the total number of requests for the webhook authorizer
+	RecordRequestTotal(ctx context.Context, code string)
+
+	// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
+	RecordRequestLatency(ctx context.Context, code string, latency float64)
+}
+
+type NoopRequestMetrics struct{}
+
+func (NoopRequestMetrics) RecordRequestTotal(context.Context, string)            {}
+func (NoopRequestMetrics) RecordRequestLatency(context.Context, string, float64) {}

staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/cel"
 )
 
 func TestAuthorizerMetrics(t *testing.T) {
@@ -76,11 +77,7 @@ func TestAuthorizerMetrics(t *testing.T) {
 			defer server.Close()
 
 			fakeAuthzMetrics := &fakeAuthorizerMetrics{}
-			authzMetrics := AuthorizerMetrics{
+			wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, fakeAuthzMetrics, []apiserver.WebhookMatchCondition{}, "")
-				RecordRequestTotal:   fakeAuthzMetrics.RequestTotal,
-				RecordRequestLatency: fakeAuthzMetrics.RequestLatency,
-			}
-			wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, authzMetrics, []apiserver.WebhookMatchCondition{})
 			if err != nil {
 				t.Error("failed to create client")
 				return
@@ -110,13 +107,15 @@ type fakeAuthorizerMetrics struct {
 
 	latency     float64
 	latencyCode string
+
+	cel.NoopMatcherMetrics
 }
 
-func (f *fakeAuthorizerMetrics) RequestTotal(_ context.Context, code string) {
+func (f *fakeAuthorizerMetrics) RecordRequestTotal(_ context.Context, code string) {
 	f.totalCode = code
 }
 
-func (f *fakeAuthorizerMetrics) RequestLatency(_ context.Context, code string, latency float64) {
+func (f *fakeAuthorizerMetrics) RecordRequestLatency(_ context.Context, code string, latency float64) {
 	f.latency = latency
 	f.latencyCode = code
 }

staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	"k8s.io/client-go/kubernetes/scheme"
 	authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1"
 	"k8s.io/client-go/rest"
@@ -70,13 +71,14 @@ type WebhookAuthorizer struct {
 	unauthorizedTTL     time.Duration
 	retryBackoff        wait.Backoff
 	decisionOnError     authorizer.Decision
-	metrics             AuthorizerMetrics
+	metrics             metrics.AuthorizerMetrics
 	celMatcher          *authorizationcel.CELMatcher
+	name                string
 }
 
 // NewFromInterface creates a WebhookAuthorizer using the given subjectAccessReview client
-func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
+func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) {
-	return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics)
+	return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics, "")
 }
 
 // New creates a new WebhookAuthorizer from the provided kubeconfig file.
@@ -98,24 +100,26 @@ func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1I
 //
 // For additional HTTP configuration, refer to the kubeconfig documentation
 // https://kubernetes.io/docs/user-guide/kubeconfig-file/.
-func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) {
+func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, name string, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) {
 	subjectAccessReview, err := subjectAccessReviewInterfaceFromConfig(config, version, retryBackoff)
 	if err != nil {
 		return nil, err
 	}
-	return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, AuthorizerMetrics{
+	return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, metrics, name)
-		RecordRequestTotal:   noopMetrics{}.RecordRequestTotal,
-		RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
-	})
 }
 
 // newWithBackoff allows tests to skip the sleep.
-func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
+func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, am metrics.AuthorizerMetrics, name string) (*WebhookAuthorizer, error) {
 	// compile all expressions once in validation and save the results to be used for eval later
 	cm, fieldErr := apiservervalidation.ValidateAndCompileMatchConditions(matchConditions)
 	if err := fieldErr.ToAggregate(); err != nil {
 		return nil, err
 	}
+	if cm != nil {
+		cm.AuthorizerType = "Webhook"
+		cm.AuthorizerName = name
+		cm.Metrics = am
+	}
 	return &WebhookAuthorizer{
 		subjectAccessReview: subjectAccessReview,
 		responseCache:       cache.NewLRUExpireCache(8192),
@@ -123,8 +127,9 @@ func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, un
 		unauthorizedTTL:     unauthorizedTTL,
 		retryBackoff:        retryBackoff,
 		decisionOnError:     decisionOnError,
-		metrics:             metrics,
+		metrics:             am,
 		celMatcher:          cm,
+		name:                name,
 	}, nil
 }
 

staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go

@@ -44,11 +44,15 @@ import (
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	celmetrics "k8s.io/apiserver/pkg/authorization/cel"
 	"k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	v1 "k8s.io/client-go/tools/clientcmd/api/v1"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
 )
 
 var testRetryBackoff = wait.Backoff{
@@ -210,7 +214,7 @@ current-context: default
 			if err != nil {
 				return fmt.Errorf("error building sar client: %v", err)
 			}
-			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics())
+			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
 			return err
 		}()
 		if err != nil && !tt.wantErr {
@@ -323,7 +327,7 @@ func (m *mockV1Service) HTTPStatusCode() int { return m.statusCode }
 
 // newV1Authorizer creates a temporary kubeconfig file from the provided arguments and attempts to load
 // a new WebhookAuthorizer from it.
-func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) {
+func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics metrics.AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition, authzName string) (*WebhookAuthorizer, error) {
 	tempfile, err := ioutil.TempFile("", "")
 	if err != nil {
 		return nil, err
@@ -353,7 +357,7 @@ func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cache
 	if err != nil {
 		return nil, fmt.Errorf("error building sar client: %v", err)
 	}
-	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics)
+	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics, authzName)
 }
 
 func TestV1TLSConfig(t *testing.T) {
@@ -412,7 +416,7 @@ func TestV1TLSConfig(t *testing.T) {
 			}
 			defer server.Close()
 
-			wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{})
+			wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "")
 			if err != nil {
 				t.Errorf("%s: failed to create client: %v", tt.test, err)
 				return
@@ -477,7 +481,7 @@ func TestV1Webhook(t *testing.T) {
 	}
 	defer s.Close()
 
-	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{})
+	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -584,7 +588,7 @@ func TestV1WebhookCache(t *testing.T) {
 		},
 	}
 	// Create an authorizer that caches successful responses "forever" (100 days).
-	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions)
+	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -760,7 +764,7 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) {
 	for i, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, test.featureEnabled)()
-			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions)
+			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "")
 			if test.expectedCompileErr && err == nil {
 				t.Fatalf("%d: Expected compile error", i)
 			} else if !test.expectedCompileErr && err != nil {
@@ -782,6 +786,112 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) {
 	}
 }
 
+func TestWebhookMetrics(t *testing.T) {
+	service := new(mockV1Service)
+	service.statusCode = 200
+	service.Allow()
+	s, err := NewV1TestServer(service, serverCert, serverKey, caCert)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)()
+
+	aliceAttr := authorizer.AttributesRecord{
+		User: &user.DefaultInfo{
+			Name: "alice",
+			UID:  "1",
+		},
+	}
+
+	testCases := []struct {
+		name         string
+		attr         authorizer.AttributesRecord
+		expressions1 []apiserver.WebhookMatchCondition
+		expressions2 []apiserver.WebhookMatchCondition
+		metrics      []string
+		want         string
+	}{
+		{
+			name: "should have one evaluation error from multiple failed match conditions",
+			attr: aliceAttr,
+			expressions1: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice'",
+				},
+				{
+					Expression: "request.resourceAttributes.verb == 'get'",
+				},
+				{
+					Expression: "request.resourceAttributes.namespace == 'kittensandponies'",
+				},
+			},
+			expressions2: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice'",
+				},
+			},
+			metrics: []string{
+				"apiserver_authorization_match_condition_evaluation_errors_total",
+			},
+			want: fmt.Sprintf(`
+					# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
+					# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
+					apiserver_authorization_match_condition_evaluation_errors_total{name="%s",type="%s"} 1
+					`, "wh1.example.com", "Webhook"),
+		},
+		{
+			name: "should have two webhook exclusions due to match condition",
+			attr: aliceAttr,
+			expressions1: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice2'",
+				},
+				{
+					Expression: "request.uid == '1'",
+				},
+			},
+			expressions2: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice1'",
+				},
+			},
+			metrics: []string{
+				"apiserver_authorization_match_condition_exclusions_total",
+			},
+			want: fmt.Sprintf(`
+					# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
+					# TYPE apiserver_authorization_match_condition_exclusions_total counter
+					apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1
+					apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1
+					`, "wh1.example.com", "Webhook", "wh2.example.com", "Webhook"),
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			celmetrics.ResetMetricsForTest()
+			defer celmetrics.ResetMetricsForTest()
+			wh1, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions1, "wh1.example.com")
+			if err != nil {
+				t.Fatal(err)
+			}
+			wh2, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions2, "wh2.example.com")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err == nil {
+				_, _, _ = wh1.Authorize(context.Background(), tt.attr)
+				_, _, _ = wh2.Authorize(context.Background(), tt.attr)
+			}
+
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
 func BenchmarkNoCELExpressionFeatureOff(b *testing.B) {
 	expressions := []apiserver.WebhookMatchCondition{}
 	b.Run("compile", func(b *testing.B) {
@@ -942,7 +1052,7 @@ func benchmarkNewWebhookAuthorizer(b *testing.B, expressions []apiserver.Webhook
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// Create an authorizer with or without expressions to compile
-		_, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions)
+		_, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "")
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -972,7 +1082,7 @@ func benchmarkWebhookAuthorize(b *testing.B, expressions []apiserver.WebhookMatc
 	defer s.Close()
 	defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, featureEnabled)()
 	// Create an authorizer with or without expressions to compile
-	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions)
+	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -1259,7 +1369,7 @@ func TestV1WebhookMatchConditions(t *testing.T) {
 
 	for i, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions)
+			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "")
 			if len(test.expectedCompileErr) > 0 && err == nil {
 				t.Fatalf("%d: Expected compile error", i)
 			} else if len(test.expectedCompileErr) == 0 && err != nil {
@@ -1292,9 +1402,17 @@ func TestV1WebhookMatchConditions(t *testing.T) {
 	}
 }
 
-func noopAuthorizerMetrics() AuthorizerMetrics {
+func noopAuthorizerMetrics() metrics.AuthorizerMetrics {
-	return AuthorizerMetrics{
+	return metrics.NoopAuthorizerMetrics{}
-		RecordRequestTotal:   noopMetrics{}.RecordRequestTotal,
+}
-		RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
+
+func celAuthorizerMetrics() metrics.AuthorizerMetrics {
+	return celAuthorizerMetricsType{
+		MatcherMetrics: celmetrics.NewMatcherMetrics(),
 	}
 }
+
+type celAuthorizerMetricsType struct {
+	metrics.NoopRequestMetrics
+	celmetrics.MatcherMetrics
+}

staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go

@@ -197,7 +197,7 @@ current-context: default
 			if err != nil {
 				return fmt.Errorf("error building sar client: %v", err)
 			}
-			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics())
+			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
 			return err
 		}()
 		if err != nil && !tt.wantErr {
@@ -340,7 +340,7 @@ func newV1beta1Authorizer(callbackURL string, clientCert, clientKey, ca []byte,
 	if err != nil {
 		return nil, fmt.Errorf("error building sar client: %v", err)
 	}
-	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics())
+	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
 }
 
 func TestV1beta1TLSConfig(t *testing.T) {

test/integration/auth/authz_config_test.go

@@ -36,6 +36,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	celmetrics "k8s.io/apiserver/pkg/authorization/cel"
 	authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
 	"k8s.io/apiserver/pkg/features"
 	authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
@@ -275,40 +276,50 @@ users:
 		serverAllowCalled.Store(0)
 		serverAllowReloadedCalled.Store(0)
 		authorizationmetrics.ResetMetricsForTest()
+		celmetrics.ResetMetricsForTest()
 	}
 	var adminClient *clientset.Clientset
-	assertCounts := func(errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount int32) {
+	type counts struct {
+		errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount, webhookExclusionCount, evalErrorsCount int32
+	}
+	assertCounts := func(c counts) {
 		t.Helper()
 		metrics, err := getMetrics(t, adminClient)
 		if err != nil {
-			t.Errorf("error getting metrics: %v", err)
+			t.Fatalf("error getting metrics: %v", err)
 		}
-		if e, a := errorCount, serverErrorCalled.Load(); e != a {
+		if e, a := c.errorCount, serverErrorCalled.Load(); e != a {
-			t.Errorf("expected fail webhook calls: %d, got %d", e, a)
+			t.Fatalf("expected fail webhook calls: %d, got %d", e, a)
 		}
-		if e, a := timeoutCount, serverTimeoutCalled.Load(); e != a {
+		if e, a := c.timeoutCount, serverTimeoutCalled.Load(); e != a {
-			t.Errorf("expected timeout webhook calls: %d, got %d", e, a)
+			t.Fatalf("expected timeout webhook calls: %d, got %d", e, a)
 		}
-		if e, a := denyCount, serverDenyCalled.Load(); e != a {
+		if e, a := c.denyCount, serverDenyCalled.Load(); e != a {
-			t.Errorf("expected deny webhook calls: %d, got %d", e, a)
+			t.Fatalf("expected deny webhook calls: %d, got %d", e, a)
 		}
-		if e, a := denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) {
+		if e, a := c.denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) {
-			t.Errorf("expected deny webhook denied metrics calls: %d, got %d", e, a)
+			t.Fatalf("expected deny webhook denied metrics calls: %d, got %d", e, a)
 		}
-		if e, a := noOpinionCount, serverNoOpinionCalled.Load(); e != a {
+		if e, a := c.noOpinionCount, serverNoOpinionCalled.Load(); e != a {
-			t.Errorf("expected noOpinion webhook calls: %d, got %d", e, a)
+			t.Fatalf("expected noOpinion webhook calls: %d, got %d", e, a)
 		}
-		if e, a := allowCount, serverAllowCalled.Load(); e != a {
+		if e, a := c.allowCount, serverAllowCalled.Load(); e != a {
-			t.Errorf("expected allow webhook calls: %d, got %d", e, a)
+			t.Fatalf("expected allow webhook calls: %d, got %d", e, a)
 		}
-		if e, a := allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) {
+		if e, a := c.allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) {
-			t.Errorf("expected allow webhook allowed metrics calls: %d, got %d", e, a)
+			t.Fatalf("expected allow webhook allowed metrics calls: %d, got %d", e, a)
 		}
-		if e, a := allowReloadedCount, serverAllowReloadedCalled.Load(); e != a {
+		if e, a := c.allowReloadedCount, serverAllowReloadedCalled.Load(); e != a {
-			t.Errorf("expected allowReloaded webhook calls: %d, got %d", e, a)
+			t.Fatalf("expected allowReloaded webhook calls: %d, got %d", e, a)
 		}
-		if e, a := allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) {
+		if e, a := c.allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) {
-			t.Errorf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a)
+			t.Fatalf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a)
+		}
+		if e, a := c.webhookExclusionCount, metrics.exclusions; e != int32(a) {
+			t.Fatalf("expected webhook exclusions due to match conditions: %d, got %d", e, a)
+		}
+		if e, a := c.evalErrorsCount, metrics.evalErrors; e != int32(a) {
+			t.Fatalf("expected webhook match condition eval errors: %d, got %d", e, a)
 		}
 		resetCounts()
 	}
@@ -348,7 +359,8 @@ authorizers:
       type: KubeConfigFile
       kubeConfigFile: `+serverTimeoutKubeconfigName+`
     matchConditions:
-    - expression: has(request.resourceAttributes)
+    # intentionally skip this check so we can trigger an eval error with a non-resource request
+    # - expression: has(request.resourceAttributes)
     - expression: 'request.resourceAttributes.namespace == "fail"'
     - expression: 'request.resourceAttributes.name == "timeout"'
 
@@ -423,7 +435,7 @@ authorizers:
 		t.Fatal("expected denied, got allowed")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(1, 0, 0, 0, 0, 0)
+		assertCounts(counts{errorCount: 1})
 	}
 
 	// timeout webhook short circuits
@@ -444,7 +456,7 @@ authorizers:
 		t.Fatal("expected denied, got allowed")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(0, 1, 0, 0, 0, 0)
+		assertCounts(counts{timeoutCount: 1, webhookExclusionCount: 1})
 	}
 
 	// deny webhook short circuits
@@ -465,7 +477,7 @@ authorizers:
 		t.Fatal("expected denied, got allowed")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(0, 0, 1, 0, 0, 0)
+		assertCounts(counts{denyCount: 1, webhookExclusionCount: 2})
 	}
 
 	// no-opinion webhook passes through, allow webhook allows
@@ -486,7 +498,26 @@ authorizers:
 		t.Fatal("expected allowed, got denied")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(0, 0, 0, 1, 1, 0)
+		assertCounts(counts{noOpinionCount: 1, allowCount: 1, webhookExclusionCount: 3})
+	}
+
+	// the timeout webhook results in match condition eval errors when evaluating a non-resource request
+	// failure policy is deny
+	t.Log("checking match condition eval error")
+	if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
+		User: "alice",
+		NonResourceAttributes: &authorizationv1.NonResourceAttributes{
+			Verb: "list",
+		},
+	}}, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	} else if result.Status.Allowed {
+		t.Fatal("expected denied, got allowed")
+	} else {
+		t.Log(result.Status.Reason)
+		// error webhook matchConditions skip non-resource request
+		// timeout webhook matchConditions error on non-resource request
+		assertCounts(counts{webhookExclusionCount: 1, evalErrorsCount: 1})
 	}
 
 	// check last loaded success/failure metric timestamps, ensure success is present, failure is not
@@ -550,7 +581,7 @@ authorizers:
 		t.Fatal("expected allowed, got denied")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(0, 0, 0, 1, 1, 0)
+		assertCounts(counts{noOpinionCount: 1, allowCount: 1, webhookExclusionCount: 3})
 	}
 
 	// write good config with different webhook
@@ -621,7 +652,7 @@ authorizers:
 		t.Fatal("expected allowed, got denied")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(0, 0, 0, 0, 0, 1)
+		assertCounts(counts{allowReloadedCount: 1})
 	}
 
 	// delete file (do this test last because it makes file watch fall back to one minute poll interval)
@@ -677,7 +708,7 @@ authorizers:
 		t.Fatal("expected allowed, got denied")
 	} else {
 		t.Log(result.Status.Reason)
-		assertCounts(0, 0, 0, 0, 0, 1)
+		assertCounts(counts{allowReloadedCount: 1})
 	}
 }
 
@@ -685,6 +716,8 @@ type metrics struct {
 	reloadSuccess *time.Time
 	reloadFailure *time.Time
 	decisions     map[authorizerKey]map[string]int
+	exclusions    int
+	evalErrors    int
 }
 type authorizerKey struct {
 	authorizerType string
@@ -692,6 +725,8 @@ type authorizerKey struct {
 }
 
 var decisionMetric = regexp.MustCompile(`apiserver_authorization_decisions_total\{decision="(.*?)",name="(.*?)",type="(.*?)"\} (\d+)`)
+var webhookExclusionMetric = regexp.MustCompile(`apiserver_authorization_match_condition_exclusions_total\{name="(.*?)",type="(.*?)"\} (\d+)`)
+var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authorization_match_condition_evaluation_errors_total\{name="(.*?)",type="(.*?)"\} (\d+)`)
 
 func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
 	data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
@@ -703,11 +738,13 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
 	//  apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1
 	//  apiserver_authorization_decisions_total{decision="denied",name="error.example.com",type="Webhook"} 1
 	//  apiserver_authorization_decisions_total{decision="denied",name="timeout.example.com",type="Webhook"} 1
+	//  apiserver_authorization_match_condition_exclusions_total{name="exclusion.example.com",type="webhook"} 1
 	if err != nil {
 		return nil, err
 	}
 
 	var m metrics
+	m.exclusions = 0
 	for _, line := range strings.Split(string(data), "\n") {
 		if matches := decisionMetric.FindStringSubmatch(line); matches != nil {
 			t.Log(line)
@@ -725,6 +762,24 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
 			m.decisions[key][matches[1]] = count
 
 		}
+		if matches := webhookExclusionMetric.FindStringSubmatch(line); matches != nil {
+			t.Log(matches)
+			count, err := strconv.Atoi(matches[3])
+			if err != nil {
+				return nil, err
+			}
+			t.Log(count)
+			m.exclusions += count
+		}
+		if matches := webhookMatchConditionEvalErrorMetric.FindStringSubmatch(line); matches != nil {
+			t.Log(matches)
+			count, err := strconv.Atoi(matches[3])
+			if err != nil {
+				return nil, err
+			}
+			t.Log(count)
+			m.evalErrors += count
+		}
 		if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") {
 			t.Log(line)
 			values := strings.Split(line, " ")