mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-04 09:49:50 +00:00
add authz webhook matchcondition metrics
Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> Signed-off-by: Jordan Liggitt <liggitt@google.com> Co-authored-by: Jordan Liggitt <liggitt@google.com>
This commit is contained in:
Rita Zhang
parent
8b8d133770
commit
e76fce7566
@ -32,11 +32,13 @@ import (
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
"k8s.io/apiserver/pkg/authorization/authorizer"
|
||||
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
|
||||
"k8s.io/apiserver/pkg/authorization/cel"
|
||||
authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
|
||||
"k8s.io/apiserver/pkg/authorization/union"
|
||||
"k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
|
||||
webhookutil "k8s.io/apiserver/pkg/util/webhook"
|
||||
"k8s.io/apiserver/plugin/pkg/authorizer/webhook"
|
||||
webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kubernetes/pkg/auth/authorizer/abac"
|
||||
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
||||
@ -142,6 +144,8 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
|
||||
*r.initialConfig.WebhookRetryBackoff,
|
||||
decisionOnError,
|
||||
configuredAuthorizer.Webhook.MatchConditions,
|
||||
configuredAuthorizer.Name,
|
||||
kubeapiserverWebhookMetrics{MatcherMetrics: cel.NewMatcherMetrics()},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@ -162,6 +166,13 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
|
||||
return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil
|
||||
}
|
||||
|
||||
type kubeapiserverWebhookMetrics struct {
|
||||
// kube-apiserver doesn't report request metrics
|
||||
webhookmetrics.NoopRequestMetrics
|
||||
// kube-apiserver does report matchCondition metrics
|
||||
cel.MatcherMetrics
|
||||
}
|
||||
|
||||
// runReload starts checking the config file for changes and reloads the authorizer when it changes.
|
||||
// Blocks until ctx is complete.
|
||||
func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) {
|
||||
|
||||
@ -26,7 +26,7 @@ import (
|
||||
authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1"
|
||||
)
|
||||
|
||||
// DelegatingAuthorizerConfig is the minimal configuration needed to create an authenticator
|
||||
// DelegatingAuthorizerConfig is the minimal configuration needed to create an authorizer
|
||||
// built to delegate authorization to a kube API server
|
||||
type DelegatingAuthorizerConfig struct {
|
||||
SubjectAccessReviewClient authorizationclient.AuthorizationV1Interface
|
||||
@ -55,9 +55,6 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
|
||||
c.DenyCacheTTL,
|
||||
*c.WebhookRetryBackoff,
|
||||
authorizer.DecisionNoOpinion,
|
||||
webhook.AuthorizerMetrics{
|
||||
RecordRequestTotal: RecordRequestTotal,
|
||||
RecordRequestLatency: RecordRequestLatency,
|
||||
},
|
||||
NewDelegatingAuthorizerMetrics(),
|
||||
)
|
||||
}
|
||||
|
||||
@ -18,18 +18,22 @@ package authorizerfactory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
celmetrics "k8s.io/apiserver/pkg/authorization/cel"
|
||||
webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
|
||||
compbasemetrics "k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
type registerables []compbasemetrics.Registerable
|
||||
var registerMetrics sync.Once
|
||||
|
||||
// init registers all metrics
|
||||
func init() {
|
||||
for _, metric := range metrics {
|
||||
legacyregistry.MustRegister(metric)
|
||||
}
|
||||
// RegisterMetrics registers authorizer metrics.
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
legacyregistry.MustRegister(requestTotal)
|
||||
legacyregistry.MustRegister(requestLatency)
|
||||
})
|
||||
}
|
||||
|
||||
var (
|
||||
@ -51,19 +55,26 @@ var (
|
||||
},
|
||||
[]string{"code"},
|
||||
)
|
||||
|
||||
metrics = registerables{
|
||||
requestTotal,
|
||||
requestLatency,
|
||||
}
|
||||
)
|
||||
|
||||
var _ = webhookmetrics.AuthorizerMetrics(delegatingAuthorizerMetrics{})
|
||||
|
||||
type delegatingAuthorizerMetrics struct {
|
||||
// no-op for matchCondition metrics for now, delegating authorization doesn't configure match conditions
|
||||
celmetrics.NoopMatcherMetrics
|
||||
}
|
||||
|
||||
func NewDelegatingAuthorizerMetrics() delegatingAuthorizerMetrics {
|
||||
RegisterMetrics()
|
||||
return delegatingAuthorizerMetrics{}
|
||||
}
|
||||
|
||||
// RecordRequestTotal increments the total number of requests for the delegated authorization.
|
||||
func RecordRequestTotal(ctx context.Context, code string) {
|
||||
func (delegatingAuthorizerMetrics) RecordRequestTotal(ctx context.Context, code string) {
|
||||
requestTotal.WithContext(ctx).WithLabelValues(code).Add(1)
|
||||
}
|
||||
|
||||
// RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code.
|
||||
func RecordRequestLatency(ctx context.Context, code string, latency float64) {
|
||||
func (delegatingAuthorizerMetrics) RecordRequestLatency(ctx context.Context, code string, latency float64) {
|
||||
requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency)
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ package cel
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
celgo "github.com/google/cel-go/cel"
|
||||
|
||||
@ -28,11 +29,29 @@ import (
|
||||
|
||||
type CELMatcher struct {
|
||||
CompilationResults []CompilationResult
|
||||
|
||||
// These are optional fields which can be populated if metrics reporting is desired
|
||||
Metrics MatcherMetrics
|
||||
AuthorizerType string
|
||||
AuthorizerName string
|
||||
}
|
||||
|
||||
// eval evaluates the given SubjectAccessReview against all cel matchCondition expression
|
||||
func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessReview) (bool, error) {
|
||||
var evalErrors []error
|
||||
|
||||
metrics := c.Metrics
|
||||
if metrics == nil {
|
||||
metrics = NoopMatcherMetrics{}
|
||||
}
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
metrics.RecordAuthorizationMatchConditionEvaluation(ctx, c.AuthorizerType, c.AuthorizerName, time.Since(start))
|
||||
if len(evalErrors) > 0 {
|
||||
metrics.RecordAuthorizationMatchConditionEvaluationFailure(ctx, c.AuthorizerType, c.AuthorizerName)
|
||||
}
|
||||
}()
|
||||
|
||||
va := map[string]interface{}{
|
||||
"request": convertObjectToUnstructured(&r.Spec),
|
||||
}
|
||||
@ -54,6 +73,7 @@ func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessR
|
||||
// If at least one matchCondition successfully evaluates to FALSE,
|
||||
// return early
|
||||
if !match {
|
||||
metrics.RecordAuthorizationMatchConditionExclusion(ctx, c.AuthorizerType, c.AuthorizerName)
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
120
staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go
Normal file
120
staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go
Normal file
@ -0,0 +1,120 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
// MatcherMetrics defines methods for reporting matchCondition metrics
|
||||
type MatcherMetrics interface {
|
||||
// RecordAuthorizationMatchConditionEvaluation records the total time taken to evaluate matchConditions for an Authorize() call to the given authorizer
|
||||
RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration)
|
||||
// RecordAuthorizationMatchConditionEvaluationFailure increments if any evaluation error was encountered evaluating matchConditions for an Authorize() call to the given authorizer
|
||||
RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string)
|
||||
// RecordAuthorizationMatchConditionExclusion records increments when at least one matchCondition evaluates to false and excludes an Authorize() call to the given authorizer
|
||||
RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string)
|
||||
}
|
||||
|
||||
type NoopMatcherMetrics struct{}
|
||||
|
||||
func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
|
||||
}
|
||||
func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
|
||||
}
|
||||
func (NoopMatcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
|
||||
}
|
||||
|
||||
type matcherMetrics struct{}
|
||||
|
||||
func NewMatcherMetrics() MatcherMetrics {
|
||||
RegisterMetrics()
|
||||
return matcherMetrics{}
|
||||
}
|
||||
|
||||
const (
|
||||
namespace = "apiserver"
|
||||
subsystem = "authorization"
|
||||
)
|
||||
|
||||
var (
|
||||
authorizationMatchConditionEvaluationErrorsTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "match_condition_evaluation_errors_total",
|
||||
Help: "Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"type", "name"},
|
||||
)
|
||||
authorizationMatchConditionExclusionsTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "match_condition_exclusions_total",
|
||||
Help: "Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"type", "name"},
|
||||
)
|
||||
authorizationMatchConditionEvaluationSeconds = metrics.NewHistogramVec(
|
||||
&metrics.HistogramOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "match_condition_evaluation_seconds",
|
||||
Help: "Authorization match condition evaluation time in seconds, split by authorizer type and name.",
|
||||
Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"type", "name"},
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
legacyregistry.MustRegister(authorizationMatchConditionEvaluationErrorsTotal)
|
||||
legacyregistry.MustRegister(authorizationMatchConditionExclusionsTotal)
|
||||
legacyregistry.MustRegister(authorizationMatchConditionEvaluationSeconds)
|
||||
})
|
||||
}
|
||||
|
||||
func ResetMetricsForTest() {
|
||||
authorizationMatchConditionEvaluationErrorsTotal.Reset()
|
||||
authorizationMatchConditionExclusionsTotal.Reset()
|
||||
authorizationMatchConditionEvaluationSeconds.Reset()
|
||||
}
|
||||
|
||||
func (matcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
|
||||
authorizationMatchConditionEvaluationErrorsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
|
||||
}
|
||||
|
||||
func (matcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
|
||||
authorizationMatchConditionExclusionsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
|
||||
}
|
||||
|
||||
func (matcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
|
||||
elapsedSeconds := elapsed.Seconds()
|
||||
authorizationMatchConditionEvaluationSeconds.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Observe(elapsedSeconds)
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
)
|
||||
|
||||
func TestRecordAuthorizationMatchConditionEvaluationFailure(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
metrics []string
|
||||
name string
|
||||
authztype string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
desc: "evaluation failure total",
|
||||
metrics: []string{
|
||||
"apiserver_authorization_match_condition_evaluation_errors_total",
|
||||
"apiserver_authorization_match_condition_exclusions_total",
|
||||
"apiserver_authorization_match_condition_evaluation_seconds",
|
||||
},
|
||||
name: "wh1.example.com",
|
||||
authztype: "Webhook",
|
||||
want: `
|
||||
# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
|
||||
# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
|
||||
apiserver_authorization_match_condition_evaluation_errors_total{name="wh1.example.com",type="Webhook"} 1
|
||||
# HELP apiserver_authorization_match_condition_evaluation_seconds [ALPHA] Authorization match condition evaluation time in seconds, split by authorizer type and name.
|
||||
# TYPE apiserver_authorization_match_condition_evaluation_seconds histogram
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.001"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.005"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.01"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.025"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.1"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.2"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.25"} 0
|
||||
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="+Inf"} 1
|
||||
apiserver_authorization_match_condition_evaluation_seconds_sum{name="wh1.example.com",type="Webhook"} 1
|
||||
apiserver_authorization_match_condition_evaluation_seconds_count{name="wh1.example.com",type="Webhook"} 1
|
||||
# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
|
||||
# TYPE apiserver_authorization_match_condition_exclusions_total counter
|
||||
apiserver_authorization_match_condition_exclusions_total{name="wh1.example.com",type="Webhook"} 1
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.desc, func(t *testing.T) {
|
||||
ResetMetricsForTest()
|
||||
m := NewMatcherMetrics()
|
||||
m.RecordAuthorizationMatchConditionEvaluationFailure(context.Background(), tt.authztype, tt.name)
|
||||
m.RecordAuthorizationMatchConditionExclusion(context.Background(), tt.authztype, tt.name)
|
||||
m.RecordAuthorizationMatchConditionEvaluation(context.Background(), tt.authztype, tt.name, time.Duration(1*time.Second))
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -14,22 +14,36 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package webhook
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"k8s.io/apiserver/pkg/authorization/cel"
|
||||
)
|
||||
|
||||
// AuthorizerMetrics specifies a set of methods that are used to register various metrics for the webhook authorizer
|
||||
type AuthorizerMetrics struct {
|
||||
// RecordRequestTotal increments the total number of requests for the webhook authorizer
|
||||
RecordRequestTotal func(ctx context.Context, code string)
|
||||
|
||||
// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
|
||||
RecordRequestLatency func(ctx context.Context, code string, latency float64)
|
||||
type AuthorizerMetrics interface {
|
||||
// Request total and latency metrics
|
||||
RequestMetrics
|
||||
// match condition metrics
|
||||
cel.MatcherMetrics
|
||||
}
|
||||
|
||||
type noopMetrics struct{}
|
||||
type NoopAuthorizerMetrics struct {
|
||||
NoopRequestMetrics
|
||||
cel.NoopMatcherMetrics
|
||||
}
|
||||
|
||||
func (noopMetrics) RecordRequestTotal(context.Context, string) {}
|
||||
func (noopMetrics) RecordRequestLatency(context.Context, string, float64) {}
|
||||
type RequestMetrics interface {
|
||||
// RecordRequestTotal increments the total number of requests for the webhook authorizer
|
||||
RecordRequestTotal(ctx context.Context, code string)
|
||||
|
||||
// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
|
||||
RecordRequestLatency(ctx context.Context, code string, latency float64)
|
||||
}
|
||||
|
||||
type NoopRequestMetrics struct{}
|
||||
|
||||
func (NoopRequestMetrics) RecordRequestTotal(context.Context, string) {}
|
||||
func (NoopRequestMetrics) RecordRequestLatency(context.Context, string, float64) {}
|
||||
@ -23,6 +23,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/apis/apiserver"
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
"k8s.io/apiserver/pkg/authorization/authorizer"
|
||||
"k8s.io/apiserver/pkg/authorization/cel"
|
||||
)
|
||||
|
||||
func TestAuthorizerMetrics(t *testing.T) {
|
||||
@ -76,11 +77,7 @@ func TestAuthorizerMetrics(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
fakeAuthzMetrics := &fakeAuthorizerMetrics{}
|
||||
authzMetrics := AuthorizerMetrics{
|
||||
RecordRequestTotal: fakeAuthzMetrics.RequestTotal,
|
||||
RecordRequestLatency: fakeAuthzMetrics.RequestLatency,
|
||||
}
|
||||
wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, authzMetrics, []apiserver.WebhookMatchCondition{})
|
||||
wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, fakeAuthzMetrics, []apiserver.WebhookMatchCondition{}, "")
|
||||
if err != nil {
|
||||
t.Error("failed to create client")
|
||||
return
|
||||
@ -110,13 +107,15 @@ type fakeAuthorizerMetrics struct {
|
||||
|
||||
latency float64
|
||||
latencyCode string
|
||||
|
||||
cel.NoopMatcherMetrics
|
||||
}
|
||||
|
||||
func (f *fakeAuthorizerMetrics) RequestTotal(_ context.Context, code string) {
|
||||
func (f *fakeAuthorizerMetrics) RecordRequestTotal(_ context.Context, code string) {
|
||||
f.totalCode = code
|
||||
}
|
||||
|
||||
func (f *fakeAuthorizerMetrics) RequestLatency(_ context.Context, code string, latency float64) {
|
||||
func (f *fakeAuthorizerMetrics) RecordRequestLatency(_ context.Context, code string, latency float64) {
|
||||
f.latency = latency
|
||||
f.latencyCode = code
|
||||
}
|
||||
|
||||
@ -39,6 +39,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/apiserver/pkg/util/webhook"
|
||||
"k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1"
|
||||
"k8s.io/client-go/rest"
|
||||
@ -70,13 +71,14 @@ type WebhookAuthorizer struct {
|
||||
unauthorizedTTL time.Duration
|
||||
retryBackoff wait.Backoff
|
||||
decisionOnError authorizer.Decision
|
||||
metrics AuthorizerMetrics
|
||||
metrics metrics.AuthorizerMetrics
|
||||
celMatcher *authorizationcel.CELMatcher
|
||||
name string
|
||||
}
|
||||
|
||||
// NewFromInterface creates a WebhookAuthorizer using the given subjectAccessReview client
|
||||
func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
|
||||
return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics)
|
||||
func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) {
|
||||
return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics, "")
|
||||
}
|
||||
|
||||
// New creates a new WebhookAuthorizer from the provided kubeconfig file.
|
||||
@ -98,24 +100,26 @@ func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1I
|
||||
//
|
||||
// For additional HTTP configuration, refer to the kubeconfig documentation
|
||||
// https://kubernetes.io/docs/user-guide/kubeconfig-file/.
|
||||
func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) {
|
||||
func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, name string, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) {
|
||||
subjectAccessReview, err := subjectAccessReviewInterfaceFromConfig(config, version, retryBackoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, AuthorizerMetrics{
|
||||
RecordRequestTotal: noopMetrics{}.RecordRequestTotal,
|
||||
RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
|
||||
})
|
||||
return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, metrics, name)
|
||||
}
|
||||
|
||||
// newWithBackoff allows tests to skip the sleep.
|
||||
func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
|
||||
func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, am metrics.AuthorizerMetrics, name string) (*WebhookAuthorizer, error) {
|
||||
// compile all expressions once in validation and save the results to be used for eval later
|
||||
cm, fieldErr := apiservervalidation.ValidateAndCompileMatchConditions(matchConditions)
|
||||
if err := fieldErr.ToAggregate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cm != nil {
|
||||
cm.AuthorizerType = "Webhook"
|
||||
cm.AuthorizerName = name
|
||||
cm.Metrics = am
|
||||
}
|
||||
return &WebhookAuthorizer{
|
||||
subjectAccessReview: subjectAccessReview,
|
||||
responseCache: cache.NewLRUExpireCache(8192),
|
||||
@ -123,8 +127,9 @@ func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, un
|
||||
unauthorizedTTL: unauthorizedTTL,
|
||||
retryBackoff: retryBackoff,
|
||||
decisionOnError: decisionOnError,
|
||||
metrics: metrics,
|
||||
metrics: am,
|
||||
celMatcher: cm,
|
||||
name: name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@ -44,11 +44,15 @@ import (
|
||||
"k8s.io/apiserver/pkg/apis/apiserver"
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
"k8s.io/apiserver/pkg/authorization/authorizer"
|
||||
celmetrics "k8s.io/apiserver/pkg/authorization/cel"
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
webhookutil "k8s.io/apiserver/pkg/util/webhook"
|
||||
"k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
|
||||
v1 "k8s.io/client-go/tools/clientcmd/api/v1"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
)
|
||||
|
||||
var testRetryBackoff = wait.Backoff{
|
||||
@ -210,7 +214,7 @@ current-context: default
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building sar client: %v", err)
|
||||
}
|
||||
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics())
|
||||
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
|
||||
return err
|
||||
}()
|
||||
if err != nil && !tt.wantErr {
|
||||
@ -323,7 +327,7 @@ func (m *mockV1Service) HTTPStatusCode() int { return m.statusCode }
|
||||
|
||||
// newV1Authorizer creates a temporary kubeconfig file from the provided arguments and attempts to load
|
||||
// a new WebhookAuthorizer from it.
|
||||
func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) {
|
||||
func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics metrics.AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition, authzName string) (*WebhookAuthorizer, error) {
|
||||
tempfile, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -353,7 +357,7 @@ func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cache
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error building sar client: %v", err)
|
||||
}
|
||||
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics)
|
||||
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics, authzName)
|
||||
}
|
||||
|
||||
func TestV1TLSConfig(t *testing.T) {
|
||||
@ -412,7 +416,7 @@ func TestV1TLSConfig(t *testing.T) {
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{})
|
||||
wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "")
|
||||
if err != nil {
|
||||
t.Errorf("%s: failed to create client: %v", tt.test, err)
|
||||
return
|
||||
@ -477,7 +481,7 @@ func TestV1Webhook(t *testing.T) {
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{})
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -584,7 +588,7 @@ func TestV1WebhookCache(t *testing.T) {
|
||||
},
|
||||
}
|
||||
// Create an authorizer that caches successful responses "forever" (100 days).
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions)
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -760,7 +764,7 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) {
|
||||
for i, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, test.featureEnabled)()
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions)
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "")
|
||||
if test.expectedCompileErr && err == nil {
|
||||
t.Fatalf("%d: Expected compile error", i)
|
||||
} else if !test.expectedCompileErr && err != nil {
|
||||
@ -782,6 +786,112 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookMetrics(t *testing.T) {
|
||||
service := new(mockV1Service)
|
||||
service.statusCode = 200
|
||||
service.Allow()
|
||||
s, err := NewV1TestServer(service, serverCert, serverKey, caCert)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)()
|
||||
|
||||
aliceAttr := authorizer.AttributesRecord{
|
||||
User: &user.DefaultInfo{
|
||||
Name: "alice",
|
||||
UID: "1",
|
||||
},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
attr authorizer.AttributesRecord
|
||||
expressions1 []apiserver.WebhookMatchCondition
|
||||
expressions2 []apiserver.WebhookMatchCondition
|
||||
metrics []string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "should have one evaluation error from multiple failed match conditions",
|
||||
attr: aliceAttr,
|
||||
expressions1: []apiserver.WebhookMatchCondition{
|
||||
{
|
||||
Expression: "request.user == 'alice'",
|
||||
},
|
||||
{
|
||||
Expression: "request.resourceAttributes.verb == 'get'",
|
||||
},
|
||||
{
|
||||
Expression: "request.resourceAttributes.namespace == 'kittensandponies'",
|
||||
},
|
||||
},
|
||||
expressions2: []apiserver.WebhookMatchCondition{
|
||||
{
|
||||
Expression: "request.user == 'alice'",
|
||||
},
|
||||
},
|
||||
metrics: []string{
|
||||
"apiserver_authorization_match_condition_evaluation_errors_total",
|
||||
},
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
|
||||
# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
|
||||
apiserver_authorization_match_condition_evaluation_errors_total{name="%s",type="%s"} 1
|
||||
`, "wh1.example.com", "Webhook"),
|
||||
},
|
||||
{
|
||||
name: "should have two webhook exclusions due to match condition",
|
||||
attr: aliceAttr,
|
||||
expressions1: []apiserver.WebhookMatchCondition{
|
||||
{
|
||||
Expression: "request.user == 'alice2'",
|
||||
},
|
||||
{
|
||||
Expression: "request.uid == '1'",
|
||||
},
|
||||
},
|
||||
expressions2: []apiserver.WebhookMatchCondition{
|
||||
{
|
||||
Expression: "request.user == 'alice1'",
|
||||
},
|
||||
},
|
||||
metrics: []string{
|
||||
"apiserver_authorization_match_condition_exclusions_total",
|
||||
},
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
|
||||
# TYPE apiserver_authorization_match_condition_exclusions_total counter
|
||||
apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1
|
||||
apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1
|
||||
`, "wh1.example.com", "Webhook", "wh2.example.com", "Webhook"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
celmetrics.ResetMetricsForTest()
|
||||
defer celmetrics.ResetMetricsForTest()
|
||||
wh1, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions1, "wh1.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wh2, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions2, "wh2.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err == nil {
|
||||
_, _, _ = wh1.Authorize(context.Background(), tt.attr)
|
||||
_, _, _ = wh2.Authorize(context.Background(), tt.attr)
|
||||
}
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkNoCELExpressionFeatureOff(b *testing.B) {
|
||||
expressions := []apiserver.WebhookMatchCondition{}
|
||||
b.Run("compile", func(b *testing.B) {
|
||||
@ -942,7 +1052,7 @@ func benchmarkNewWebhookAuthorizer(b *testing.B, expressions []apiserver.Webhook
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Create an authorizer with or without expressions to compile
|
||||
_, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions)
|
||||
_, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -972,7 +1082,7 @@ func benchmarkWebhookAuthorize(b *testing.B, expressions []apiserver.WebhookMatc
|
||||
defer s.Close()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, featureEnabled)()
|
||||
// Create an authorizer with or without expressions to compile
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions)
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -1259,7 +1369,7 @@ func TestV1WebhookMatchConditions(t *testing.T) {
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions)
|
||||
wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "")
|
||||
if len(test.expectedCompileErr) > 0 && err == nil {
|
||||
t.Fatalf("%d: Expected compile error", i)
|
||||
} else if len(test.expectedCompileErr) == 0 && err != nil {
|
||||
@ -1292,9 +1402,17 @@ func TestV1WebhookMatchConditions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func noopAuthorizerMetrics() AuthorizerMetrics {
|
||||
return AuthorizerMetrics{
|
||||
RecordRequestTotal: noopMetrics{}.RecordRequestTotal,
|
||||
RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
|
||||
func noopAuthorizerMetrics() metrics.AuthorizerMetrics {
|
||||
return metrics.NoopAuthorizerMetrics{}
|
||||
}
|
||||
|
||||
func celAuthorizerMetrics() metrics.AuthorizerMetrics {
|
||||
return celAuthorizerMetricsType{
|
||||
MatcherMetrics: celmetrics.NewMatcherMetrics(),
|
||||
}
|
||||
}
|
||||
|
||||
type celAuthorizerMetricsType struct {
|
||||
metrics.NoopRequestMetrics
|
||||
celmetrics.MatcherMetrics
|
||||
}
|
||||
|
||||
@ -197,7 +197,7 @@ current-context: default
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building sar client: %v", err)
|
||||
}
|
||||
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics())
|
||||
_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
|
||||
return err
|
||||
}()
|
||||
if err != nil && !tt.wantErr {
|
||||
@ -340,7 +340,7 @@ func newV1beta1Authorizer(callbackURL string, clientCert, clientKey, ca []byte,
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error building sar client: %v", err)
|
||||
}
|
||||
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics())
|
||||
return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
|
||||
}
|
||||
|
||||
func TestV1beta1TLSConfig(t *testing.T) {
|
||||
|
||||
@ -36,6 +36,7 @@ import (
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
celmetrics "k8s.io/apiserver/pkg/authorization/cel"
|
||||
authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
|
||||
@ -275,40 +276,50 @@ users:
|
||||
serverAllowCalled.Store(0)
|
||||
serverAllowReloadedCalled.Store(0)
|
||||
authorizationmetrics.ResetMetricsForTest()
|
||||
celmetrics.ResetMetricsForTest()
|
||||
}
|
||||
var adminClient *clientset.Clientset
|
||||
assertCounts := func(errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount int32) {
|
||||
type counts struct {
|
||||
errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount, webhookExclusionCount, evalErrorsCount int32
|
||||
}
|
||||
assertCounts := func(c counts) {
|
||||
t.Helper()
|
||||
metrics, err := getMetrics(t, adminClient)
|
||||
if err != nil {
|
||||
t.Errorf("error getting metrics: %v", err)
|
||||
t.Fatalf("error getting metrics: %v", err)
|
||||
}
|
||||
if e, a := errorCount, serverErrorCalled.Load(); e != a {
|
||||
t.Errorf("expected fail webhook calls: %d, got %d", e, a)
|
||||
if e, a := c.errorCount, serverErrorCalled.Load(); e != a {
|
||||
t.Fatalf("expected fail webhook calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := timeoutCount, serverTimeoutCalled.Load(); e != a {
|
||||
t.Errorf("expected timeout webhook calls: %d, got %d", e, a)
|
||||
if e, a := c.timeoutCount, serverTimeoutCalled.Load(); e != a {
|
||||
t.Fatalf("expected timeout webhook calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := denyCount, serverDenyCalled.Load(); e != a {
|
||||
t.Errorf("expected deny webhook calls: %d, got %d", e, a)
|
||||
if e, a := c.denyCount, serverDenyCalled.Load(); e != a {
|
||||
t.Fatalf("expected deny webhook calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) {
|
||||
t.Errorf("expected deny webhook denied metrics calls: %d, got %d", e, a)
|
||||
if e, a := c.denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) {
|
||||
t.Fatalf("expected deny webhook denied metrics calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := noOpinionCount, serverNoOpinionCalled.Load(); e != a {
|
||||
t.Errorf("expected noOpinion webhook calls: %d, got %d", e, a)
|
||||
if e, a := c.noOpinionCount, serverNoOpinionCalled.Load(); e != a {
|
||||
t.Fatalf("expected noOpinion webhook calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := allowCount, serverAllowCalled.Load(); e != a {
|
||||
t.Errorf("expected allow webhook calls: %d, got %d", e, a)
|
||||
if e, a := c.allowCount, serverAllowCalled.Load(); e != a {
|
||||
t.Fatalf("expected allow webhook calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) {
|
||||
t.Errorf("expected allow webhook allowed metrics calls: %d, got %d", e, a)
|
||||
if e, a := c.allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) {
|
||||
t.Fatalf("expected allow webhook allowed metrics calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := allowReloadedCount, serverAllowReloadedCalled.Load(); e != a {
|
||||
t.Errorf("expected allowReloaded webhook calls: %d, got %d", e, a)
|
||||
if e, a := c.allowReloadedCount, serverAllowReloadedCalled.Load(); e != a {
|
||||
t.Fatalf("expected allowReloaded webhook calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) {
|
||||
t.Errorf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a)
|
||||
if e, a := c.allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) {
|
||||
t.Fatalf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := c.webhookExclusionCount, metrics.exclusions; e != int32(a) {
|
||||
t.Fatalf("expected webhook exclusions due to match conditions: %d, got %d", e, a)
|
||||
}
|
||||
if e, a := c.evalErrorsCount, metrics.evalErrors; e != int32(a) {
|
||||
t.Fatalf("expected webhook match condition eval errors: %d, got %d", e, a)
|
||||
}
|
||||
resetCounts()
|
||||
}
|
||||
@ -348,7 +359,8 @@ authorizers:
|
||||
type: KubeConfigFile
|
||||
kubeConfigFile: `+serverTimeoutKubeconfigName+`
|
||||
matchConditions:
|
||||
- expression: has(request.resourceAttributes)
|
||||
# intentionally skip this check so we can trigger an eval error with a non-resource request
|
||||
# - expression: has(request.resourceAttributes)
|
||||
- expression: 'request.resourceAttributes.namespace == "fail"'
|
||||
- expression: 'request.resourceAttributes.name == "timeout"'
|
||||
|
||||
@ -423,7 +435,7 @@ authorizers:
|
||||
t.Fatal("expected denied, got allowed")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(1, 0, 0, 0, 0, 0)
|
||||
assertCounts(counts{errorCount: 1})
|
||||
}
|
||||
|
||||
// timeout webhook short circuits
|
||||
@ -444,7 +456,7 @@ authorizers:
|
||||
t.Fatal("expected denied, got allowed")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(0, 1, 0, 0, 0, 0)
|
||||
assertCounts(counts{timeoutCount: 1, webhookExclusionCount: 1})
|
||||
}
|
||||
|
||||
// deny webhook short circuits
|
||||
@ -465,7 +477,7 @@ authorizers:
|
||||
t.Fatal("expected denied, got allowed")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(0, 0, 1, 0, 0, 0)
|
||||
assertCounts(counts{denyCount: 1, webhookExclusionCount: 2})
|
||||
}
|
||||
|
||||
// no-opinion webhook passes through, allow webhook allows
|
||||
@ -486,7 +498,26 @@ authorizers:
|
||||
t.Fatal("expected allowed, got denied")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(0, 0, 0, 1, 1, 0)
|
||||
assertCounts(counts{noOpinionCount: 1, allowCount: 1, webhookExclusionCount: 3})
|
||||
}
|
||||
|
||||
// the timeout webhook results in match condition eval errors when evaluating a non-resource request
|
||||
// failure policy is deny
|
||||
t.Log("checking match condition eval error")
|
||||
if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
|
||||
User: "alice",
|
||||
NonResourceAttributes: &authorizationv1.NonResourceAttributes{
|
||||
Verb: "list",
|
||||
},
|
||||
}}, metav1.CreateOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if result.Status.Allowed {
|
||||
t.Fatal("expected denied, got allowed")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
// error webhook matchConditions skip non-resource request
|
||||
// timeout webhook matchConditions error on non-resource request
|
||||
assertCounts(counts{webhookExclusionCount: 1, evalErrorsCount: 1})
|
||||
}
|
||||
|
||||
// check last loaded success/failure metric timestamps, ensure success is present, failure is not
|
||||
@ -550,7 +581,7 @@ authorizers:
|
||||
t.Fatal("expected allowed, got denied")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(0, 0, 0, 1, 1, 0)
|
||||
assertCounts(counts{noOpinionCount: 1, allowCount: 1, webhookExclusionCount: 3})
|
||||
}
|
||||
|
||||
// write good config with different webhook
|
||||
@ -621,7 +652,7 @@ authorizers:
|
||||
t.Fatal("expected allowed, got denied")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(0, 0, 0, 0, 0, 1)
|
||||
assertCounts(counts{allowReloadedCount: 1})
|
||||
}
|
||||
|
||||
// delete file (do this test last because it makes file watch fall back to one minute poll interval)
|
||||
@ -677,7 +708,7 @@ authorizers:
|
||||
t.Fatal("expected allowed, got denied")
|
||||
} else {
|
||||
t.Log(result.Status.Reason)
|
||||
assertCounts(0, 0, 0, 0, 0, 1)
|
||||
assertCounts(counts{allowReloadedCount: 1})
|
||||
}
|
||||
}
|
||||
|
||||
@ -685,6 +716,8 @@ type metrics struct {
|
||||
reloadSuccess *time.Time
|
||||
reloadFailure *time.Time
|
||||
decisions map[authorizerKey]map[string]int
|
||||
exclusions int
|
||||
evalErrors int
|
||||
}
|
||||
type authorizerKey struct {
|
||||
authorizerType string
|
||||
@ -692,6 +725,8 @@ type authorizerKey struct {
|
||||
}
|
||||
|
||||
var decisionMetric = regexp.MustCompile(`apiserver_authorization_decisions_total\{decision="(.*?)",name="(.*?)",type="(.*?)"\} (\d+)`)
|
||||
var webhookExclusionMetric = regexp.MustCompile(`apiserver_authorization_match_condition_exclusions_total\{name="(.*?)",type="(.*?)"\} (\d+)`)
|
||||
var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authorization_match_condition_evaluation_errors_total\{name="(.*?)",type="(.*?)"\} (\d+)`)
|
||||
|
||||
func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
|
||||
data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
|
||||
@ -703,11 +738,13 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
|
||||
// apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1
|
||||
// apiserver_authorization_decisions_total{decision="denied",name="error.example.com",type="Webhook"} 1
|
||||
// apiserver_authorization_decisions_total{decision="denied",name="timeout.example.com",type="Webhook"} 1
|
||||
// apiserver_authorization_match_condition_exclusions_total{name="exclusion.example.com",type="webhook"} 1
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var m metrics
|
||||
m.exclusions = 0
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if matches := decisionMetric.FindStringSubmatch(line); matches != nil {
|
||||
t.Log(line)
|
||||
@ -725,6 +762,24 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
|
||||
m.decisions[key][matches[1]] = count
|
||||
|
||||
}
|
||||
if matches := webhookExclusionMetric.FindStringSubmatch(line); matches != nil {
|
||||
t.Log(matches)
|
||||
count, err := strconv.Atoi(matches[3])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.Log(count)
|
||||
m.exclusions += count
|
||||
}
|
||||
if matches := webhookMatchConditionEvalErrorMetric.FindStringSubmatch(line); matches != nil {
|
||||
t.Log(matches)
|
||||
count, err := strconv.Atoi(matches[3])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.Log(count)
|
||||
m.evalErrors += count
|
||||
}
|
||||
if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") {
|
||||
t.Log(line)
|
||||
values := strings.Split(line, " ")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user