diff --git a/pkg/kubeapiserver/authorizer/reload.go b/pkg/kubeapiserver/authorizer/reload.go index a1cb36f62a9..d9b75e29450 100644 --- a/pkg/kubeapiserver/authorizer/reload.go +++ b/pkg/kubeapiserver/authorizer/reload.go @@ -32,11 +32,13 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" + "k8s.io/apiserver/pkg/authorization/cel" authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics" "k8s.io/apiserver/pkg/authorization/union" "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics" webhookutil "k8s.io/apiserver/pkg/util/webhook" "k8s.io/apiserver/plugin/pkg/authorizer/webhook" + webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/auth/authorizer/abac" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" @@ -142,6 +144,8 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut *r.initialConfig.WebhookRetryBackoff, decisionOnError, configuredAuthorizer.Webhook.MatchConditions, + configuredAuthorizer.Name, + kubeapiserverWebhookMetrics{MatcherMetrics: cel.NewMatcherMetrics()}, ) if err != nil { return nil, nil, err @@ -162,6 +166,13 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil } +type kubeapiserverWebhookMetrics struct { + // kube-apiserver doesn't report request metrics + webhookmetrics.NoopRequestMetrics + // kube-apiserver does report matchCondition metrics + cel.MatcherMetrics +} + // runReload starts checking the config file for changes and reloads the authorizer when it changes. // Blocks until ctx is complete. func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) { diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go index a8355ee6191..070189145af 100644 --- a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go +++ b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go @@ -26,7 +26,7 @@ import ( authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1" ) -// DelegatingAuthorizerConfig is the minimal configuration needed to create an authenticator +// DelegatingAuthorizerConfig is the minimal configuration needed to create an authorizer // built to delegate authorization to a kube API server type DelegatingAuthorizerConfig struct { SubjectAccessReviewClient authorizationclient.AuthorizationV1Interface @@ -55,9 +55,6 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) { c.DenyCacheTTL, *c.WebhookRetryBackoff, authorizer.DecisionNoOpinion, - webhook.AuthorizerMetrics{ - RecordRequestTotal: RecordRequestTotal, - RecordRequestLatency: RecordRequestLatency, - }, + NewDelegatingAuthorizerMetrics(), ) } diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/metrics.go b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/metrics.go index 08b3d54ab9e..df30479b722 100644 --- a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/metrics.go @@ -18,18 +18,22 @@ package authorizerfactory import ( "context" + "sync" + celmetrics "k8s.io/apiserver/pkg/authorization/cel" + webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics" compbasemetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) -type registerables []compbasemetrics.Registerable +var registerMetrics sync.Once -// init registers all metrics -func init() { - for _, metric := range metrics { - legacyregistry.MustRegister(metric) - } +// RegisterMetrics registers authorizer metrics. +func RegisterMetrics() { + registerMetrics.Do(func() { + legacyregistry.MustRegister(requestTotal) + legacyregistry.MustRegister(requestLatency) + }) } var ( @@ -51,19 +55,26 @@ var ( }, []string{"code"}, ) - - metrics = registerables{ - requestTotal, - requestLatency, - } ) +var _ = webhookmetrics.AuthorizerMetrics(delegatingAuthorizerMetrics{}) + +type delegatingAuthorizerMetrics struct { + // no-op for matchCondition metrics for now, delegating authorization doesn't configure match conditions + celmetrics.NoopMatcherMetrics +} + +func NewDelegatingAuthorizerMetrics() delegatingAuthorizerMetrics { + RegisterMetrics() + return delegatingAuthorizerMetrics{} +} + // RecordRequestTotal increments the total number of requests for the delegated authorization. -func RecordRequestTotal(ctx context.Context, code string) { +func (delegatingAuthorizerMetrics) RecordRequestTotal(ctx context.Context, code string) { requestTotal.WithContext(ctx).WithLabelValues(code).Add(1) } // RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code. -func RecordRequestLatency(ctx context.Context, code string, latency float64) { +func (delegatingAuthorizerMetrics) RecordRequestLatency(ctx context.Context, code string, latency float64) { requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency) } diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/cel/matcher.go b/staging/src/k8s.io/apiserver/pkg/authorization/cel/matcher.go index 30ce5b69c99..3202173a8dc 100644 --- a/staging/src/k8s.io/apiserver/pkg/authorization/cel/matcher.go +++ b/staging/src/k8s.io/apiserver/pkg/authorization/cel/matcher.go @@ -19,6 +19,7 @@ package cel import ( "context" "fmt" + "time" celgo "github.com/google/cel-go/cel" @@ -28,11 +29,29 @@ import ( type CELMatcher struct { CompilationResults []CompilationResult + + // These are optional fields which can be populated if metrics reporting is desired + Metrics MatcherMetrics + AuthorizerType string + AuthorizerName string } // eval evaluates the given SubjectAccessReview against all cel matchCondition expression func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessReview) (bool, error) { var evalErrors []error + + metrics := c.Metrics + if metrics == nil { + metrics = NoopMatcherMetrics{} + } + start := time.Now() + defer func() { + metrics.RecordAuthorizationMatchConditionEvaluation(ctx, c.AuthorizerType, c.AuthorizerName, time.Since(start)) + if len(evalErrors) > 0 { + metrics.RecordAuthorizationMatchConditionEvaluationFailure(ctx, c.AuthorizerType, c.AuthorizerName) + } + }() + va := map[string]interface{}{ "request": convertObjectToUnstructured(&r.Spec), } @@ -54,6 +73,7 @@ func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessR // If at least one matchCondition successfully evaluates to FALSE, // return early if !match { + metrics.RecordAuthorizationMatchConditionExclusion(ctx, c.AuthorizerType, c.AuthorizerName) return false, nil } } diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go b/staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go new file mode 100644 index 00000000000..c9431a705d4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go @@ -0,0 +1,120 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cel + +import ( + "context" + "sync" + "time" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +// MatcherMetrics defines methods for reporting matchCondition metrics +type MatcherMetrics interface { + // RecordAuthorizationMatchConditionEvaluation records the total time taken to evaluate matchConditions for an Authorize() call to the given authorizer + RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) + // RecordAuthorizationMatchConditionEvaluationFailure increments if any evaluation error was encountered evaluating matchConditions for an Authorize() call to the given authorizer + RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) + // RecordAuthorizationMatchConditionExclusion records increments when at least one matchCondition evaluates to false and excludes an Authorize() call to the given authorizer + RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) +} + +type NoopMatcherMetrics struct{} + +func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) { +} +func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) { +} +func (NoopMatcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) { +} + +type matcherMetrics struct{} + +func NewMatcherMetrics() MatcherMetrics { + RegisterMetrics() + return matcherMetrics{} +} + +const ( + namespace = "apiserver" + subsystem = "authorization" +) + +var ( + authorizationMatchConditionEvaluationErrorsTotal = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "match_condition_evaluation_errors_total", + Help: "Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.", + StabilityLevel: metrics.ALPHA, + }, + []string{"type", "name"}, + ) + authorizationMatchConditionExclusionsTotal = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "match_condition_exclusions_total", + Help: "Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.", + StabilityLevel: metrics.ALPHA, + }, + []string{"type", "name"}, + ) + authorizationMatchConditionEvaluationSeconds = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "match_condition_evaluation_seconds", + Help: "Authorization match condition evaluation time in seconds, split by authorizer type and name.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25}, + StabilityLevel: metrics.ALPHA, + }, + []string{"type", "name"}, + ) +) + +var registerMetrics sync.Once + +func RegisterMetrics() { + registerMetrics.Do(func() { + legacyregistry.MustRegister(authorizationMatchConditionEvaluationErrorsTotal) + legacyregistry.MustRegister(authorizationMatchConditionExclusionsTotal) + legacyregistry.MustRegister(authorizationMatchConditionEvaluationSeconds) + }) +} + +func ResetMetricsForTest() { + authorizationMatchConditionEvaluationErrorsTotal.Reset() + authorizationMatchConditionExclusionsTotal.Reset() + authorizationMatchConditionEvaluationSeconds.Reset() +} + +func (matcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) { + authorizationMatchConditionEvaluationErrorsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc() +} + +func (matcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) { + authorizationMatchConditionExclusionsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc() +} + +func (matcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) { + elapsedSeconds := elapsed.Seconds() + authorizationMatchConditionEvaluationSeconds.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Observe(elapsedSeconds) +} diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics_test.go new file mode 100644 index 00000000000..e900cdf402e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cel + +import ( + "context" + "strings" + "testing" + "time" + + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" +) + +func TestRecordAuthorizationMatchConditionEvaluationFailure(t *testing.T) { + testCases := []struct { + desc string + metrics []string + name string + authztype string + want string + }{ + { + desc: "evaluation failure total", + metrics: []string{ + "apiserver_authorization_match_condition_evaluation_errors_total", + "apiserver_authorization_match_condition_exclusions_total", + "apiserver_authorization_match_condition_evaluation_seconds", + }, + name: "wh1.example.com", + authztype: "Webhook", + want: ` + # HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name. + # TYPE apiserver_authorization_match_condition_evaluation_errors_total counter + apiserver_authorization_match_condition_evaluation_errors_total{name="wh1.example.com",type="Webhook"} 1 + # HELP apiserver_authorization_match_condition_evaluation_seconds [ALPHA] Authorization match condition evaluation time in seconds, split by authorizer type and name. + # TYPE apiserver_authorization_match_condition_evaluation_seconds histogram + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.001"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.005"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.01"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.025"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.1"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.2"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.25"} 0 + apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="+Inf"} 1 + apiserver_authorization_match_condition_evaluation_seconds_sum{name="wh1.example.com",type="Webhook"} 1 + apiserver_authorization_match_condition_evaluation_seconds_count{name="wh1.example.com",type="Webhook"} 1 + # HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it. + # TYPE apiserver_authorization_match_condition_exclusions_total counter + apiserver_authorization_match_condition_exclusions_total{name="wh1.example.com",type="Webhook"} 1 + `, + }, + } + + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + ResetMetricsForTest() + m := NewMatcherMetrics() + m.RecordAuthorizationMatchConditionEvaluationFailure(context.Background(), tt.authztype, tt.name) + m.RecordAuthorizationMatchConditionExclusion(context.Background(), tt.authztype, tt.name) + m.RecordAuthorizationMatchConditionEvaluation(context.Background(), tt.authztype, tt.name, time.Duration(1*time.Second)) + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics/metrics.go similarity index 58% rename from staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics.go rename to staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics/metrics.go index 0912378b17e..312f6ed8946 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics/metrics.go @@ -14,22 +14,36 @@ See the License for the specific language governing permissions and limitations under the License. */ -package webhook +package metrics import ( "context" + + "k8s.io/apiserver/pkg/authorization/cel" ) // AuthorizerMetrics specifies a set of methods that are used to register various metrics for the webhook authorizer -type AuthorizerMetrics struct { - // RecordRequestTotal increments the total number of requests for the webhook authorizer - RecordRequestTotal func(ctx context.Context, code string) - - // RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code. - RecordRequestLatency func(ctx context.Context, code string, latency float64) +type AuthorizerMetrics interface { + // Request total and latency metrics + RequestMetrics + // match condition metrics + cel.MatcherMetrics } -type noopMetrics struct{} +type NoopAuthorizerMetrics struct { + NoopRequestMetrics + cel.NoopMatcherMetrics +} -func (noopMetrics) RecordRequestTotal(context.Context, string) {} -func (noopMetrics) RecordRequestLatency(context.Context, string, float64) {} +type RequestMetrics interface { + // RecordRequestTotal increments the total number of requests for the webhook authorizer + RecordRequestTotal(ctx context.Context, code string) + + // RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code. + RecordRequestLatency(ctx context.Context, code string, latency float64) +} + +type NoopRequestMetrics struct{} + +func (NoopRequestMetrics) RecordRequestTotal(context.Context, string) {} +func (NoopRequestMetrics) RecordRequestLatency(context.Context, string, float64) {} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go index 6cdeb60f3be..d8385bb0a40 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/apiserver/pkg/apis/apiserver" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/cel" ) func TestAuthorizerMetrics(t *testing.T) { @@ -76,11 +77,7 @@ func TestAuthorizerMetrics(t *testing.T) { defer server.Close() fakeAuthzMetrics := &fakeAuthorizerMetrics{} - authzMetrics := AuthorizerMetrics{ - RecordRequestTotal: fakeAuthzMetrics.RequestTotal, - RecordRequestLatency: fakeAuthzMetrics.RequestLatency, - } - wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, authzMetrics, []apiserver.WebhookMatchCondition{}) + wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, fakeAuthzMetrics, []apiserver.WebhookMatchCondition{}, "") if err != nil { t.Error("failed to create client") return @@ -110,13 +107,15 @@ type fakeAuthorizerMetrics struct { latency float64 latencyCode string + + cel.NoopMatcherMetrics } -func (f *fakeAuthorizerMetrics) RequestTotal(_ context.Context, code string) { +func (f *fakeAuthorizerMetrics) RecordRequestTotal(_ context.Context, code string) { f.totalCode = code } -func (f *fakeAuthorizerMetrics) RequestLatency(_ context.Context, code string, latency float64) { +func (f *fakeAuthorizerMetrics) RecordRequestLatency(_ context.Context, code string, latency float64) { f.latency = latency f.latencyCode = code } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go index 29ee0e84d15..65c320f6d6c 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go @@ -39,6 +39,7 @@ import ( "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics" "k8s.io/client-go/kubernetes/scheme" authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1" "k8s.io/client-go/rest" @@ -70,13 +71,14 @@ type WebhookAuthorizer struct { unauthorizedTTL time.Duration retryBackoff wait.Backoff decisionOnError authorizer.Decision - metrics AuthorizerMetrics + metrics metrics.AuthorizerMetrics celMatcher *authorizationcel.CELMatcher + name string } // NewFromInterface creates a WebhookAuthorizer using the given subjectAccessReview client -func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) { - return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics) +func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) { + return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics, "") } // New creates a new WebhookAuthorizer from the provided kubeconfig file. @@ -98,24 +100,26 @@ func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1I // // For additional HTTP configuration, refer to the kubeconfig documentation // https://kubernetes.io/docs/user-guide/kubeconfig-file/. -func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) { +func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, name string, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) { subjectAccessReview, err := subjectAccessReviewInterfaceFromConfig(config, version, retryBackoff) if err != nil { return nil, err } - return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, AuthorizerMetrics{ - RecordRequestTotal: noopMetrics{}.RecordRequestTotal, - RecordRequestLatency: noopMetrics{}.RecordRequestLatency, - }) + return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, metrics, name) } // newWithBackoff allows tests to skip the sleep. -func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) { +func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, am metrics.AuthorizerMetrics, name string) (*WebhookAuthorizer, error) { // compile all expressions once in validation and save the results to be used for eval later cm, fieldErr := apiservervalidation.ValidateAndCompileMatchConditions(matchConditions) if err := fieldErr.ToAggregate(); err != nil { return nil, err } + if cm != nil { + cm.AuthorizerType = "Webhook" + cm.AuthorizerName = name + cm.Metrics = am + } return &WebhookAuthorizer{ subjectAccessReview: subjectAccessReview, responseCache: cache.NewLRUExpireCache(8192), @@ -123,8 +127,9 @@ func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, un unauthorizedTTL: unauthorizedTTL, retryBackoff: retryBackoff, decisionOnError: decisionOnError, - metrics: metrics, + metrics: am, celMatcher: cm, + name: name, }, nil } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go index e57dd05099a..a130a2f154b 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go @@ -44,11 +44,15 @@ import ( "k8s.io/apiserver/pkg/apis/apiserver" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" + celmetrics "k8s.io/apiserver/pkg/authorization/cel" "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" webhookutil "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics" v1 "k8s.io/client-go/tools/clientcmd/api/v1" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" ) var testRetryBackoff = wait.Backoff{ @@ -210,7 +214,7 @@ current-context: default if err != nil { return fmt.Errorf("error building sar client: %v", err) } - _, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics()) + _, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics(), "") return err }() if err != nil && !tt.wantErr { @@ -323,7 +327,7 @@ func (m *mockV1Service) HTTPStatusCode() int { return m.statusCode } // newV1Authorizer creates a temporary kubeconfig file from the provided arguments and attempts to load // a new WebhookAuthorizer from it. -func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) { +func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics metrics.AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition, authzName string) (*WebhookAuthorizer, error) { tempfile, err := ioutil.TempFile("", "") if err != nil { return nil, err @@ -353,7 +357,7 @@ func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cache if err != nil { return nil, fmt.Errorf("error building sar client: %v", err) } - return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics) + return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics, authzName) } func TestV1TLSConfig(t *testing.T) { @@ -412,7 +416,7 @@ func TestV1TLSConfig(t *testing.T) { } defer server.Close() - wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}) + wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "") if err != nil { t.Errorf("%s: failed to create client: %v", tt.test, err) return @@ -477,7 +481,7 @@ func TestV1Webhook(t *testing.T) { } defer s.Close() - wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}) + wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "") if err != nil { t.Fatal(err) } @@ -584,7 +588,7 @@ func TestV1WebhookCache(t *testing.T) { }, } // Create an authorizer that caches successful responses "forever" (100 days). - wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions) + wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions, "") if err != nil { t.Fatal(err) } @@ -760,7 +764,7 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) { for i, test := range tests { t.Run(test.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, test.featureEnabled)() - wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions) + wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "") if test.expectedCompileErr && err == nil { t.Fatalf("%d: Expected compile error", i) } else if !test.expectedCompileErr && err != nil { @@ -782,6 +786,112 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) { } } +func TestWebhookMetrics(t *testing.T) { + service := new(mockV1Service) + service.statusCode = 200 + service.Allow() + s, err := NewV1TestServer(service, serverCert, serverKey, caCert) + if err != nil { + t.Fatal(err) + } + defer s.Close() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)() + + aliceAttr := authorizer.AttributesRecord{ + User: &user.DefaultInfo{ + Name: "alice", + UID: "1", + }, + } + + testCases := []struct { + name string + attr authorizer.AttributesRecord + expressions1 []apiserver.WebhookMatchCondition + expressions2 []apiserver.WebhookMatchCondition + metrics []string + want string + }{ + { + name: "should have one evaluation error from multiple failed match conditions", + attr: aliceAttr, + expressions1: []apiserver.WebhookMatchCondition{ + { + Expression: "request.user == 'alice'", + }, + { + Expression: "request.resourceAttributes.verb == 'get'", + }, + { + Expression: "request.resourceAttributes.namespace == 'kittensandponies'", + }, + }, + expressions2: []apiserver.WebhookMatchCondition{ + { + Expression: "request.user == 'alice'", + }, + }, + metrics: []string{ + "apiserver_authorization_match_condition_evaluation_errors_total", + }, + want: fmt.Sprintf(` + # HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name. + # TYPE apiserver_authorization_match_condition_evaluation_errors_total counter + apiserver_authorization_match_condition_evaluation_errors_total{name="%s",type="%s"} 1 + `, "wh1.example.com", "Webhook"), + }, + { + name: "should have two webhook exclusions due to match condition", + attr: aliceAttr, + expressions1: []apiserver.WebhookMatchCondition{ + { + Expression: "request.user == 'alice2'", + }, + { + Expression: "request.uid == '1'", + }, + }, + expressions2: []apiserver.WebhookMatchCondition{ + { + Expression: "request.user == 'alice1'", + }, + }, + metrics: []string{ + "apiserver_authorization_match_condition_exclusions_total", + }, + want: fmt.Sprintf(` + # HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it. + # TYPE apiserver_authorization_match_condition_exclusions_total counter + apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1 + apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1 + `, "wh1.example.com", "Webhook", "wh2.example.com", "Webhook"), + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + celmetrics.ResetMetricsForTest() + defer celmetrics.ResetMetricsForTest() + wh1, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions1, "wh1.example.com") + if err != nil { + t.Fatal(err) + } + wh2, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions2, "wh2.example.com") + if err != nil { + t.Fatal(err) + } + if err == nil { + _, _, _ = wh1.Authorize(context.Background(), tt.attr) + _, _, _ = wh2.Authorize(context.Background(), tt.attr) + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { + t.Fatal(err) + } + }) + } +} + func BenchmarkNoCELExpressionFeatureOff(b *testing.B) { expressions := []apiserver.WebhookMatchCondition{} b.Run("compile", func(b *testing.B) { @@ -942,7 +1052,7 @@ func benchmarkNewWebhookAuthorizer(b *testing.B, expressions []apiserver.Webhook b.ResetTimer() for i := 0; i < b.N; i++ { // Create an authorizer with or without expressions to compile - _, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions) + _, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "") if err != nil { b.Fatal(err) } @@ -972,7 +1082,7 @@ func benchmarkWebhookAuthorize(b *testing.B, expressions []apiserver.WebhookMatc defer s.Close() defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, featureEnabled)() // Create an authorizer with or without expressions to compile - wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions) + wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "") if err != nil { b.Fatal(err) } @@ -1259,7 +1369,7 @@ func TestV1WebhookMatchConditions(t *testing.T) { for i, test := range tests { t.Run(test.name, func(t *testing.T) { - wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions) + wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "") if len(test.expectedCompileErr) > 0 && err == nil { t.Fatalf("%d: Expected compile error", i) } else if len(test.expectedCompileErr) == 0 && err != nil { @@ -1292,9 +1402,17 @@ func TestV1WebhookMatchConditions(t *testing.T) { } } -func noopAuthorizerMetrics() AuthorizerMetrics { - return AuthorizerMetrics{ - RecordRequestTotal: noopMetrics{}.RecordRequestTotal, - RecordRequestLatency: noopMetrics{}.RecordRequestLatency, +func noopAuthorizerMetrics() metrics.AuthorizerMetrics { + return metrics.NoopAuthorizerMetrics{} +} + +func celAuthorizerMetrics() metrics.AuthorizerMetrics { + return celAuthorizerMetricsType{ + MatcherMetrics: celmetrics.NewMatcherMetrics(), } } + +type celAuthorizerMetricsType struct { + metrics.NoopRequestMetrics + celmetrics.MatcherMetrics +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go index e2621c3ca75..3e7c9538e93 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go @@ -197,7 +197,7 @@ current-context: default if err != nil { return fmt.Errorf("error building sar client: %v", err) } - _, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics()) + _, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "") return err }() if err != nil && !tt.wantErr { @@ -340,7 +340,7 @@ func newV1beta1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, if err != nil { return nil, fmt.Errorf("error building sar client: %v", err) } - return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics()) + return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "") } func TestV1beta1TLSConfig(t *testing.T) { diff --git a/test/integration/auth/authz_config_test.go b/test/integration/auth/authz_config_test.go index 4cdfb458d1e..2ea7fe5fb1d 100644 --- a/test/integration/auth/authz_config_test.go +++ b/test/integration/auth/authz_config_test.go @@ -36,6 +36,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + celmetrics "k8s.io/apiserver/pkg/authorization/cel" authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics" "k8s.io/apiserver/pkg/features" authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics" @@ -275,40 +276,50 @@ users: serverAllowCalled.Store(0) serverAllowReloadedCalled.Store(0) authorizationmetrics.ResetMetricsForTest() + celmetrics.ResetMetricsForTest() } var adminClient *clientset.Clientset - assertCounts := func(errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount int32) { + type counts struct { + errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount, webhookExclusionCount, evalErrorsCount int32 + } + assertCounts := func(c counts) { t.Helper() metrics, err := getMetrics(t, adminClient) if err != nil { - t.Errorf("error getting metrics: %v", err) + t.Fatalf("error getting metrics: %v", err) } - if e, a := errorCount, serverErrorCalled.Load(); e != a { - t.Errorf("expected fail webhook calls: %d, got %d", e, a) + if e, a := c.errorCount, serverErrorCalled.Load(); e != a { + t.Fatalf("expected fail webhook calls: %d, got %d", e, a) } - if e, a := timeoutCount, serverTimeoutCalled.Load(); e != a { - t.Errorf("expected timeout webhook calls: %d, got %d", e, a) + if e, a := c.timeoutCount, serverTimeoutCalled.Load(); e != a { + t.Fatalf("expected timeout webhook calls: %d, got %d", e, a) } - if e, a := denyCount, serverDenyCalled.Load(); e != a { - t.Errorf("expected deny webhook calls: %d, got %d", e, a) + if e, a := c.denyCount, serverDenyCalled.Load(); e != a { + t.Fatalf("expected deny webhook calls: %d, got %d", e, a) } - if e, a := denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) { - t.Errorf("expected deny webhook denied metrics calls: %d, got %d", e, a) + if e, a := c.denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) { + t.Fatalf("expected deny webhook denied metrics calls: %d, got %d", e, a) } - if e, a := noOpinionCount, serverNoOpinionCalled.Load(); e != a { - t.Errorf("expected noOpinion webhook calls: %d, got %d", e, a) + if e, a := c.noOpinionCount, serverNoOpinionCalled.Load(); e != a { + t.Fatalf("expected noOpinion webhook calls: %d, got %d", e, a) } - if e, a := allowCount, serverAllowCalled.Load(); e != a { - t.Errorf("expected allow webhook calls: %d, got %d", e, a) + if e, a := c.allowCount, serverAllowCalled.Load(); e != a { + t.Fatalf("expected allow webhook calls: %d, got %d", e, a) } - if e, a := allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) { - t.Errorf("expected allow webhook allowed metrics calls: %d, got %d", e, a) + if e, a := c.allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) { + t.Fatalf("expected allow webhook allowed metrics calls: %d, got %d", e, a) } - if e, a := allowReloadedCount, serverAllowReloadedCalled.Load(); e != a { - t.Errorf("expected allowReloaded webhook calls: %d, got %d", e, a) + if e, a := c.allowReloadedCount, serverAllowReloadedCalled.Load(); e != a { + t.Fatalf("expected allowReloaded webhook calls: %d, got %d", e, a) } - if e, a := allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) { - t.Errorf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a) + if e, a := c.allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) { + t.Fatalf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a) + } + if e, a := c.webhookExclusionCount, metrics.exclusions; e != int32(a) { + t.Fatalf("expected webhook exclusions due to match conditions: %d, got %d", e, a) + } + if e, a := c.evalErrorsCount, metrics.evalErrors; e != int32(a) { + t.Fatalf("expected webhook match condition eval errors: %d, got %d", e, a) } resetCounts() } @@ -348,7 +359,8 @@ authorizers: type: KubeConfigFile kubeConfigFile: `+serverTimeoutKubeconfigName+` matchConditions: - - expression: has(request.resourceAttributes) + # intentionally skip this check so we can trigger an eval error with a non-resource request + # - expression: has(request.resourceAttributes) - expression: 'request.resourceAttributes.namespace == "fail"' - expression: 'request.resourceAttributes.name == "timeout"' @@ -423,7 +435,7 @@ authorizers: t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) - assertCounts(1, 0, 0, 0, 0, 0) + assertCounts(counts{errorCount: 1}) } // timeout webhook short circuits @@ -444,7 +456,7 @@ authorizers: t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) - assertCounts(0, 1, 0, 0, 0, 0) + assertCounts(counts{timeoutCount: 1, webhookExclusionCount: 1}) } // deny webhook short circuits @@ -465,7 +477,7 @@ authorizers: t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) - assertCounts(0, 0, 1, 0, 0, 0) + assertCounts(counts{denyCount: 1, webhookExclusionCount: 2}) } // no-opinion webhook passes through, allow webhook allows @@ -486,7 +498,26 @@ authorizers: t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) - assertCounts(0, 0, 0, 1, 1, 0) + assertCounts(counts{noOpinionCount: 1, allowCount: 1, webhookExclusionCount: 3}) + } + + // the timeout webhook results in match condition eval errors when evaluating a non-resource request + // failure policy is deny + t.Log("checking match condition eval error") + if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ + User: "alice", + NonResourceAttributes: &authorizationv1.NonResourceAttributes{ + Verb: "list", + }, + }}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } else if result.Status.Allowed { + t.Fatal("expected denied, got allowed") + } else { + t.Log(result.Status.Reason) + // error webhook matchConditions skip non-resource request + // timeout webhook matchConditions error on non-resource request + assertCounts(counts{webhookExclusionCount: 1, evalErrorsCount: 1}) } // check last loaded success/failure metric timestamps, ensure success is present, failure is not @@ -550,7 +581,7 @@ authorizers: t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) - assertCounts(0, 0, 0, 1, 1, 0) + assertCounts(counts{noOpinionCount: 1, allowCount: 1, webhookExclusionCount: 3}) } // write good config with different webhook @@ -621,7 +652,7 @@ authorizers: t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) - assertCounts(0, 0, 0, 0, 0, 1) + assertCounts(counts{allowReloadedCount: 1}) } // delete file (do this test last because it makes file watch fall back to one minute poll interval) @@ -677,7 +708,7 @@ authorizers: t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) - assertCounts(0, 0, 0, 0, 0, 1) + assertCounts(counts{allowReloadedCount: 1}) } } @@ -685,6 +716,8 @@ type metrics struct { reloadSuccess *time.Time reloadFailure *time.Time decisions map[authorizerKey]map[string]int + exclusions int + evalErrors int } type authorizerKey struct { authorizerType string @@ -692,6 +725,8 @@ type authorizerKey struct { } var decisionMetric = regexp.MustCompile(`apiserver_authorization_decisions_total\{decision="(.*?)",name="(.*?)",type="(.*?)"\} (\d+)`) +var webhookExclusionMetric = regexp.MustCompile(`apiserver_authorization_match_condition_exclusions_total\{name="(.*?)",type="(.*?)"\} (\d+)`) +var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authorization_match_condition_evaluation_errors_total\{name="(.*?)",type="(.*?)"\} (\d+)`) func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) { data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO()) @@ -703,11 +738,13 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) { // apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1 // apiserver_authorization_decisions_total{decision="denied",name="error.example.com",type="Webhook"} 1 // apiserver_authorization_decisions_total{decision="denied",name="timeout.example.com",type="Webhook"} 1 + // apiserver_authorization_match_condition_exclusions_total{name="exclusion.example.com",type="webhook"} 1 if err != nil { return nil, err } var m metrics + m.exclusions = 0 for _, line := range strings.Split(string(data), "\n") { if matches := decisionMetric.FindStringSubmatch(line); matches != nil { t.Log(line) @@ -725,6 +762,24 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) { m.decisions[key][matches[1]] = count } + if matches := webhookExclusionMetric.FindStringSubmatch(line); matches != nil { + t.Log(matches) + count, err := strconv.Atoi(matches[3]) + if err != nil { + return nil, err + } + t.Log(count) + m.exclusions += count + } + if matches := webhookMatchConditionEvalErrorMetric.FindStringSubmatch(line); matches != nil { + t.Log(matches) + count, err := strconv.Atoi(matches[3]) + if err != nil { + return nil, err + } + t.Log(count) + m.evalErrors += count + } if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") { t.Log(line) values := strings.Split(line, " ")