[PodSecurity] Implement metricRecorder for admission (#104217)

* init

Signed-off-by: jyz0309 <45495947@qq.com>

go fmt

Signed-off-by: jyz0309 <45495947@qq.com>

remove useless code

Signed-off-by: jyz0309 <45495947@qq.com>

add metrics.Attributes interface

Signed-off-by: jyz0309 <45495947@qq.com>

address comment

Signed-off-by: jyz0309 <45495947@qq.com>

go fmt code

Signed-off-by: jyz0309 <45495947@qq.com>

resolve import cycle

Signed-off-by: jyz0309 <45495947@qq.com>

fix comment

Signed-off-by: jyz0309 <45495947@qq.com>

fix lints

Signed-off-by: jyz0309 <45495947@qq.com>

fix build error

Signed-off-by: jyz0309 <45495947@qq.com>

fix test

Signed-off-by: jyz0309 <45495947@qq.com>

try

Signed-off-by: jyz0309 <45495947@qq.com>

* try to compare version

Signed-off-by: jyz0309 <45495947@qq.com>

fix conflict

Signed-off-by: jyz0309 <45495947@qq.com>

remove unuse change

Signed-off-by: jyz0309 <45495947@qq.com>

* address comment

Signed-off-by: jyz0309 <45495947@qq.com>

* fix import error

Signed-off-by: jyz0309 <45495947@qq.com>

fix import

Signed-off-by: jyz0309 <45495947@qq.com>

address comment

Signed-off-by: jyz0309 <45495947@qq.com>

address comment

Signed-off-by: jyz0309 <45495947@qq.com>

* address comment

Signed-off-by: jyz0309 <45495947@qq.com>

* format code

Signed-off-by: jyz0309 <45495947@qq.com>

* remove exempt and error record

Signed-off-by: jyz0309 <45495947@qq.com>

* ignore pod

Signed-off-by: jyz0309 <45495947@qq.com>

* add decision default value

Signed-off-by: jyz0309 <45495947@qq.com>

* address comment

Signed-off-by: jyz0309 <45495947@qq.com>

* remore useless import

Signed-off-by: jyz0309 <45495947@qq.com>

* remove policy vaild check

Signed-off-by: jyz0309 <45495947@qq.com>

use init to register metric

Signed-off-by: jyz0309 <45495947@qq.com>

fix test

Signed-off-by: jyz0309 <45495947@qq.com>

remove check

Signed-off-by: jyz0309 <45495947@qq.com>

remove blank line

Signed-off-by: jyz0309 <45495947@qq.com>

add allowedImports

Signed-off-by: jyz0309 <45495947@qq.com>

Add mock recorder

Signed-off-by: jyz0309 <45495947@qq.com>

format code

Signed-off-by: jyz0309 <45495947@qq.com>

separe record into 3 function

Signed-off-by: jyz0309 <45495947@qq.com>

* fix comment

Signed-off-by: jyz0309 <45495947@qq.com>

plugin/pkg/admission/security/podsecurity/admission.go

@@ -51,6 +51,7 @@ import (
 	podsecurityadmission "k8s.io/pod-security-admission/admission"
 	podsecurityconfigloader "k8s.io/pod-security-admission/admission/api/load"
 	podsecurityadmissionapi "k8s.io/pod-security-admission/api"
+	podsecuritymetrics "k8s.io/pod-security-admission/metrics"
 	"k8s.io/pod-security-admission/policy"
 )
 
@@ -99,7 +100,7 @@ func newPlugin(reader io.Reader) (*Plugin, error) {
 		delegate: &podsecurityadmission.Admission{
 			Configuration:    config,
 			Evaluator:        evaluator,
-			Metrics:          nil, // TODO: wire to default prometheus metrics
+			Metrics:          podsecuritymetrics.NewPrometheusRecorder(podsecurityadmissionapi.GetAPIVersion()),
 			PodSpecExtractor: podsecurityadmission.DefaultPodSpecExtractor{},
 		},
 	}, nil

staging/publishing/import-restrictions.yaml

@@ -280,4 +280,6 @@
   - k8s.io/component-base/featuregate
   - k8s.io/component-base/logs
   - k8s.io/component-base/cli
+  - k8s.io/component-base/metrics
+  - k8s.io/component-base/version
   - k8s.io/utils

staging/src/k8s.io/pod-security-admission/admission/admission.go

@@ -53,7 +53,7 @@ type Admission struct {
 	Evaluator policy.Evaluator
 
 	// Metrics
-	Metrics metrics.EvaluationRecorder
+	Metrics metrics.Recorder
 
 	// Arbitrary object --> PodSpec
 	PodSpecExtractor PodSpecExtractor
@@ -172,7 +172,9 @@ func (a *Admission) ValidateConfiguration() error {
 			return fmt.Errorf("default policy does not match; CompleteConfiguration() was not called before ValidateConfiguration()")
 		}
 	}
-	// TODO: check metrics is non-nil?
+	if a.Metrics == nil {
+		return fmt.Errorf("Metrics recorder required")
+	}
 	if a.PodSpecExtractor == nil {
 		return fmt.Errorf("PodSpecExtractor required")
 	}
@@ -196,7 +198,7 @@ var (
 // Validate admits an API request.
 // The objects in admission attributes are expected to be external v1 objects that we care about.
 // The returned response may be shared and must not be mutated.
-func (a *Admission) Validate(ctx context.Context, attrs Attributes) *admissionv1.AdmissionResponse {
+func (a *Admission) Validate(ctx context.Context, attrs api.Attributes) *admissionv1.AdmissionResponse {
 	var response *admissionv1.AdmissionResponse
 	switch attrs.GetResource().GroupResource() {
 	case namespacesResource:
@@ -206,16 +208,13 @@ func (a *Admission) Validate(ctx context.Context, attrs Attributes) *admissionv1
 	default:
 		response = a.ValidatePodController(ctx, attrs)
 	}
-
-	// TODO: record metrics.
-
 	return response
 }
 
 // ValidateNamespace evaluates a namespace create or update request to ensure the pod security labels are valid,
 // and checks existing pods in the namespace for violations of the new policy when updating the enforce level on a namespace.
 // The returned response may be shared between evaluations and must not be mutated.
-func (a *Admission) ValidateNamespace(ctx context.Context, attrs Attributes) *admissionv1.AdmissionResponse {
+func (a *Admission) ValidateNamespace(ctx context.Context, attrs api.Attributes) *admissionv1.AdmissionResponse {
 	// short-circuit on subresources
 	if attrs.GetSubresource() != "" {
 		return sharedAllowedResponse()
@@ -303,7 +302,7 @@ var ignoredPodSubresources = map[string]bool{
 
 // ValidatePod evaluates a pod create or update request against the effective policy for the namespace.
 // The returned response may be shared between evaluations and must not be mutated.
-func (a *Admission) ValidatePod(ctx context.Context, attrs Attributes) *admissionv1.AdmissionResponse {
+func (a *Admission) ValidatePod(ctx context.Context, attrs api.Attributes) *admissionv1.AdmissionResponse {
 	// short-circuit on ignored subresources
 	if ignoredPodSubresources[attrs.GetSubresource()] {
 		return sharedAllowedResponse()
@@ -355,7 +354,7 @@ func (a *Admission) ValidatePod(ctx context.Context, attrs Attributes) *admissio
 
 // ValidatePodController evaluates a pod controller create or update request against the effective policy for the namespace.
 // The returned response may be shared between evaluations and must not be mutated.
-func (a *Admission) ValidatePodController(ctx context.Context, attrs Attributes) *admissionv1.AdmissionResponse {
+func (a *Admission) ValidatePodController(ctx context.Context, attrs api.Attributes) *admissionv1.AdmissionResponse {
 	// short-circuit on subresources
 	if attrs.GetSubresource() != "" {
 		return sharedAllowedResponse()
@@ -396,7 +395,7 @@ func (a *Admission) ValidatePodController(ctx context.Context, attrs Attributes)
 // EvaluatePod evaluates the given policy against the given pod(-like) object.
 // The enforce policy is only checked if enforce=true.
 // The returned response may be shared between evaluations and must not be mutated.
-func (a *Admission) EvaluatePod(ctx context.Context, nsPolicy api.Policy, nsPolicyErr error, podMetadata *metav1.ObjectMeta, podSpec *corev1.PodSpec, attrs Attributes, enforce bool) *admissionv1.AdmissionResponse {
+func (a *Admission) EvaluatePod(ctx context.Context, nsPolicy api.Policy, nsPolicyErr error, podMetadata *metav1.ObjectMeta, podSpec *corev1.PodSpec, attrs api.Attributes, enforce bool) *admissionv1.AdmissionResponse {
 	// short-circuit on exempt runtimeclass
 	if a.exemptRuntimeClass(podSpec.RuntimeClassName) {
 		return sharedAllowedResponse()
@@ -416,12 +415,16 @@ func (a *Admission) EvaluatePod(ctx context.Context, nsPolicy api.Policy, nsPoli
 	if enforce {
 		if result := policy.AggregateCheckResults(a.Evaluator.EvaluatePod(nsPolicy.Enforce, podMetadata, podSpec)); !result.Allowed {
 			response = forbiddenResponse(result.ForbiddenDetail())
+			a.Metrics.RecordEvaluation(metrics.DecisionDeny, nsPolicy.Enforce, metrics.ModeEnforce, attrs)
+		} else {
+			a.Metrics.RecordEvaluation(metrics.DecisionAllow, nsPolicy.Enforce, metrics.ModeEnforce, attrs)
 		}
 	}
 
 	// TODO: reuse previous evaluation if audit level+version is the same as enforce level+version
 	if result := policy.AggregateCheckResults(a.Evaluator.EvaluatePod(nsPolicy.Audit, podMetadata, podSpec)); !result.Allowed {
 		auditAnnotations["audit"] = result.ForbiddenDetail()
+		a.Metrics.RecordEvaluation(metrics.DecisionDeny, nsPolicy.Audit, metrics.ModeAudit, attrs)
 	}
 
 	// avoid adding warnings to a request we're already going to reject with an error
@@ -435,6 +438,7 @@ func (a *Admission) EvaluatePod(ctx context.Context, nsPolicy api.Policy, nsPoli
 				nsPolicy.Warn.Level,
 				result.ForbiddenDetail(),
 			))
+			a.Metrics.RecordEvaluation(metrics.DecisionDeny, nsPolicy.Warn, metrics.ModeWarn, attrs)
 		}
 	}
 

staging/src/k8s.io/pod-security-admission/admission/admission_test.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	admissionapi "k8s.io/pod-security-admission/admission/api"
 	"k8s.io/pod-security-admission/api"
+	"k8s.io/pod-security-admission/metrics"
 	"k8s.io/pod-security-admission/policy"
 	"k8s.io/utils/pointer"
 )
@@ -436,6 +437,7 @@ func TestValidateNamespace(t *testing.T) {
 						RuntimeClasses: tc.exemptRuntimeClasses,
 					},
 				},
+				Metrics:       NewMockRecorder(),
 				defaultPolicy: defaultPolicy,
 			}
 			result := a.ValidateNamespace(context.TODO(), attrs)
@@ -622,6 +624,7 @@ func TestValidatePodController(t *testing.T) {
 						Usernames:      tc.exemptUsers,
 					},
 				},
+				Metrics:         NewMockRecorder(),
 				defaultPolicy:   defaultPolicy,
 				NamespaceGetter: nsGetter,
 			}
@@ -640,3 +643,13 @@ func TestValidatePodController(t *testing.T) {
 		})
 	}
 }
+
+type MockRecorder struct {
+}
+
+func NewMockRecorder() *MockRecorder {
+	return &MockRecorder{}
+}
+
+func (r MockRecorder) RecordEvaluation(decision metrics.Decision, policy api.LevelVersion, evalMode metrics.Mode, attrs api.Attributes) {
+}

staging/src/k8s.io/pod-security-admission/admission/attributes.go

@@ -20,33 +20,9 @@ import (
 	admissionv1 "k8s.io/api/admission/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/pod-security-admission/api"
 )
 
-// Attributes exposes the admission request parameters consumed by the PodSecurity admission controller.
-type Attributes interface {
-	// GetName is the name of the object associated with the request.
-	GetName() string
-	// GetNamespace is the namespace associated with the request (if any)
-	GetNamespace() string
-	// GetResource is the name of the resource being requested.  This is not the kind.  For example: pods
-	GetResource() schema.GroupVersionResource
-	// GetSubresource is the name of the subresource being requested.  This is a different resource, scoped to the parent resource, but it may have a different kind.
-	// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
-	// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
-	GetSubresource() string
-	// GetOperation is the operation being performed
-	GetOperation() admissionv1.Operation
-
-	// GetObject returns the typed Object from incoming request.
-	// For objects in the core API group, the result must use the v1 API.
-	GetObject() (runtime.Object, error)
-	// GetOldObject returns the typed existing object. Only populated for UPDATE requests.
-	// For objects in the core API group, the result must use the v1 API.
-	GetOldObject() (runtime.Object, error)
-	// GetUserName is the requesting user's authenticated name.
-	GetUserName() string
-}
-
 // AttributesRecord is a simple struct implementing the Attributes interface.
 type AttributesRecord struct {
 	Name        string
@@ -85,7 +61,7 @@ func (a *AttributesRecord) GetOldObject() (runtime.Object, error) {
 }
 
 // RequestAttributes adapts an admission.Request to the Attributes interface.
-func RequestAttributes(request *admissionv1.AdmissionRequest, decoder runtime.Decoder) Attributes {
+func RequestAttributes(request *admissionv1.AdmissionRequest, decoder runtime.Decoder) api.Attributes {
 	return &attributes{
 		r:       request,
 		decoder: decoder,

staging/src/k8s.io/pod-security-admission/api/helpers.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/component-base/version"
 )
 
 type Version struct {
@@ -66,6 +67,23 @@ func MajorMinorVersion(major, minor int) Version {
 	return Version{major: major, minor: minor}
 }
 
+// GetAPIVersion get the version of apiServer and return the version major and minor
+func GetAPIVersion() Version {
+	var err error
+	v := Version{}
+	apiVersion := version.Get()
+	major, err := strconv.Atoi(apiVersion.Major)
+	if err != nil {
+		return v
+	}
+	minor, err := strconv.Atoi(apiVersion.Minor)
+	if err != nil {
+		return v
+	}
+	v = MajorMinorVersion(major, minor)
+	return v
+}
+
 func LatestVersion() Version {
 	return Version{latest: true}
 }

staging/src/k8s.io/pod-security-admission/api/interfaces.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	admissionv1 "k8s.io/api/admission/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// Attributes exposes the admission request parameters consumed by the PodSecurity admission controller.
+type Attributes interface {
+	// GetName is the name of the object associated with the request.
+	GetName() string
+	// GetNamespace is the namespace associated with the request (if any)
+	GetNamespace() string
+	// GetResource is the name of the resource being requested.  This is not the kind.  For example: pods
+	GetResource() schema.GroupVersionResource
+	// GetSubresource is the name of the subresource being requested.  This is a different resource, scoped to the parent resource, but it may have a different kind.
+	// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
+	// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
+	GetSubresource() string
+	// GetOperation is the operation being performed
+	GetOperation() admissionv1.Operation
+
+	// GetObject returns the typed Object from incoming request.
+	// For objects in the core API group, the result must use the v1 API.
+	GetObject() (runtime.Object, error)
+	// GetOldObject returns the typed existing object. Only populated for UPDATE requests.
+	// For objects in the core API group, the result must use the v1 API.
+	GetOldObject() (runtime.Object, error)
+	// GetUserName is the requesting user's authenticated name.
+	GetUserName() string
+}

staging/src/k8s.io/pod-security-admission/cmd/webhook/server/server.go

@@ -42,7 +42,9 @@ import (
 	"k8s.io/pod-security-admission/admission"
 	admissionapi "k8s.io/pod-security-admission/admission/api"
 	podsecurityconfigloader "k8s.io/pod-security-admission/admission/api/load"
+	"k8s.io/pod-security-admission/api"
 	"k8s.io/pod-security-admission/cmd/webhook/server/options"
+	"k8s.io/pod-security-admission/metrics"
 	"k8s.io/pod-security-admission/policy"
 )
 
@@ -265,7 +267,7 @@ func Setup(c *Config) (*Server, error) {
 	s.delegate = &admission.Admission{
 		Configuration:    c.PodSecurityConfig,
 		Evaluator:        evaluator,
-		Metrics:          nil, // TODO: wire to default prometheus metrics
+		Metrics:          metrics.NewPrometheusRecorder(api.GetAPIVersion()),
 		PodSpecExtractor: admission.DefaultPodSpecExtractor{},
 		PodLister:        admission.PodListerFromClient(client),
 		NamespaceGetter:  admission.NamespaceGetterFromListerAndClient(namespaceLister, client),

staging/src/k8s.io/pod-security-admission/metrics/metrics.go

@@ -16,9 +16,68 @@ limitations under the License.
 
 package metrics
 
-type EvaluationRecorder interface {
+import (
-	// TODO: fill in args required to record https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/2579-psp-replacement#monitoring
+	"k8s.io/component-base/metrics"
-	RecordEvaluation()
+	"k8s.io/pod-security-admission/api"
+)
+
+const (
+	ModeAudit     = "audit"
+	ModeEnforce   = "enforce"
+	ModeWarn      = "warn"
+	DecisionAllow = "allow" // Policy evaluated, request allowed
+	DecisionDeny  = "deny"  // Policy evaluated, request denied
+)
+
+var (
+	SecurityEvaluation = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "pod_security_evaluations_total",
+			Help:           "Counter of pod security evaluations.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"decision", "policy_level", "policy_version", "mode", "operation", "resource", "subresource"},
+	)
+
+	Registry = metrics.NewKubeRegistry()
+)
+
+type Decision string
+type Mode string
+
+type Recorder interface {
+	RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes)
 }
 
-// TODO: default prometheus-based implementation
+type PrometheusRecorder struct {
+	apiVersion api.Version
+}
+
+func init() {
+	Registry.MustRegister(SecurityEvaluation)
+}
+
+func NewPrometheusRecorder(version api.Version) *PrometheusRecorder {
+	return &PrometheusRecorder{apiVersion: version}
+}
+
+func (r PrometheusRecorder) RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes) {
+	dec := string(decision)
+	operation := string(attrs.GetOperation())
+	resource := attrs.GetResource().String()
+	subresource := attrs.GetSubresource()
+	var version string
+	if policy.Valid() {
+		if policy.Version.Latest() {
+			version = "latest"
+		} else {
+			if !r.apiVersion.Older(policy.Version) {
+				version = policy.Version.String()
+			} else {
+				version = "future"
+			}
+		}
+		SecurityEvaluation.WithLabelValues(dec, string(policy.Level),
+			version, string(evalMode), operation, resource, subresource).Inc()
+	}
+}