[PodSecurity] Fix up metrics & add tests

Update pod security metrics to match the spec in the KEP.

plugin/pkg/admission/security/podsecurity/admission.go

@@ -51,7 +51,7 @@ import (
 	podsecurityadmission "k8s.io/pod-security-admission/admission"
 	podsecurityconfigloader "k8s.io/pod-security-admission/admission/api/load"
 	podsecurityadmissionapi "k8s.io/pod-security-admission/api"
-	podsecuritymetrics "k8s.io/pod-security-admission/metrics"
+	"k8s.io/pod-security-admission/metrics"
 	"k8s.io/pod-security-admission/policy"
 )
 
@@ -94,13 +94,14 @@ func newPlugin(reader io.Reader) (*Plugin, error) {
 	if err != nil {
 		return nil, fmt.Errorf("could not create PodSecurityRegistry: %w", err)
 	}
+	metrics.LegacyMustRegister()
 
 	return &Plugin{
 		Handler: admission.NewHandler(admission.Create, admission.Update),
 		delegate: &podsecurityadmission.Admission{
 			Configuration:    config,
 			Evaluator:        evaluator,
-			Metrics:          podsecuritymetrics.NewPrometheusRecorder(podsecurityadmissionapi.GetAPIVersion()),
+			Metrics:          metrics.DefaultRecorder(),
 			PodSpecExtractor: podsecurityadmission.DefaultPodSpecExtractor{},
 		},
 	}, nil

staging/src/k8s.io/pod-security-admission/admission/admission_test.go

@@ -457,7 +457,7 @@ func TestValidateNamespace(t *testing.T) {
 				}
 			}
 
-			attrs := &AttributesRecord{
+			attrs := &api.AttributesRecord{
 				Object:      newObject,
 				OldObject:   oldObject,
 				Name:        newObject.Name,
@@ -508,7 +508,7 @@ func TestValidateNamespace(t *testing.T) {
 						RuntimeClasses: tc.exemptRuntimeClasses,
 					},
 				},
-				Metrics:       NewMockRecorder(),
+				Metrics:       &FakeRecorder{},
 				defaultPolicy: defaultPolicy,
 
 				namespacePodCheckTimeout: time.Second,
@@ -582,6 +582,7 @@ func TestValidatePodController(t *testing.T) {
 		api.WarnLevelLabel:    string(api.LevelBaseline),
 		api.AuditLevelLabel:   string(api.LevelBaseline),
 	}
+	nsLevelVersion := api.LevelVersion{api.LevelBaseline, api.LatestVersion()}
 
 	testCases := []struct {
 		desc                 string
@@ -671,7 +672,7 @@ func TestValidatePodController(t *testing.T) {
 				operation = admissionv1.Update
 			}
 
-			attrs := &AttributesRecord{
+			attrs := &api.AttributesRecord{
 				testName,
 				testNamespace,
 				tc.gvk,
@@ -700,6 +701,7 @@ func TestValidatePodController(t *testing.T) {
 						Labels:    nsLabels}},
 			}
 			PodSpecExtractor := &DefaultPodSpecExtractor{}
+			recorder := &FakeRecorder{}
 			a := &Admission{
 				PodLister:        podLister,
 				Evaluator:        evaluator,
@@ -711,7 +713,7 @@ func TestValidatePodController(t *testing.T) {
 						Usernames:      tc.exemptUsers,
 					},
 				},
-				Metrics:         NewMockRecorder(),
+				Metrics:         recorder,
 				defaultPolicy:   defaultPolicy,
 				NamespaceGetter: nsGetter,
 			}
@@ -727,16 +729,36 @@ func TestValidatePodController(t *testing.T) {
 			assert.Empty(t, resultError)
 			assert.Equal(t, tc.expectAuditAnnotations, result.AuditAnnotations, "unexpected AuditAnnotations")
 			assert.Equal(t, tc.expectWarnings, result.Warnings, "unexpected Warnings")
+
+			expectedEvaluations := []EvaluationRecord{}
+			if len(tc.expectAuditAnnotations) > 0 {
+				expectedEvaluations = append(expectedEvaluations, EvaluationRecord{testName, metrics.DecisionDeny, nsLevelVersion, metrics.ModeAudit})
+			}
+			if len(tc.expectWarnings) > 0 {
+				expectedEvaluations = append(expectedEvaluations, EvaluationRecord{testName, metrics.DecisionDeny, nsLevelVersion, metrics.ModeWarn})
+			}
+			recorder.ExpectEvaluations(t, expectedEvaluations)
 		})
 	}
 }
 
-type MockRecorder struct {
+type FakeRecorder struct {
+	evaluations []EvaluationRecord
 }
 
-func NewMockRecorder() *MockRecorder {
+type EvaluationRecord struct {
-	return &MockRecorder{}
+	ObjectName string
+	Decision   metrics.Decision
+	Policy     api.LevelVersion
+	Mode       metrics.Mode
 }
 
-func (r MockRecorder) RecordEvaluation(decision metrics.Decision, policy api.LevelVersion, evalMode metrics.Mode, attrs api.Attributes) {
+func (r *FakeRecorder) RecordEvaluation(decision metrics.Decision, policy api.LevelVersion, evalMode metrics.Mode, attrs api.Attributes) {
+	r.evaluations = append(r.evaluations, EvaluationRecord{attrs.GetName(), decision, policy, evalMode})
+}
+
+// ExpectEvaluation asserts that the evaluation was recorded, and clears the record.
+func (r *FakeRecorder) ExpectEvaluations(t *testing.T, expected []EvaluationRecord) {
+	t.Helper()
+	assert.ElementsMatch(t, expected, r.evaluations)
 }

staging/src/k8s.io/pod-security-admission/api/attributes.go

@@ -14,15 +14,41 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package admission
+package api
 
 import (
 	admissionv1 "k8s.io/api/admission/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/pod-security-admission/api"
 )
 
+// Attributes exposes the admission request parameters consumed by the PodSecurity admission controller.
+type Attributes interface {
+	// GetName is the name of the object associated with the request.
+	GetName() string
+	// GetNamespace is the namespace associated with the request (if any)
+	GetNamespace() string
+	// GetResource is the name of the resource being requested.  This is not the kind.  For example: pods
+	GetResource() schema.GroupVersionResource
+	// GetKind is the name of the kind being requested.  For example: Pod
+	GetKind() schema.GroupVersionKind
+	// GetSubresource is the name of the subresource being requested.  This is a different resource, scoped to the parent resource, but it may have a different kind.
+	// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
+	// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
+	GetSubresource() string
+	// GetOperation is the operation being performed
+	GetOperation() admissionv1.Operation
+
+	// GetObject returns the typed Object from incoming request.
+	// For objects in the core API group, the result must use the v1 API.
+	GetObject() (runtime.Object, error)
+	// GetOldObject returns the typed existing object. Only populated for UPDATE requests.
+	// For objects in the core API group, the result must use the v1 API.
+	GetOldObject() (runtime.Object, error)
+	// GetUserName is the requesting user's authenticated name.
+	GetUserName() string
+}
+
 // AttributesRecord is a simple struct implementing the Attributes interface.
 type AttributesRecord struct {
 	Name        string
@@ -64,8 +90,10 @@ func (a *AttributesRecord) GetOldObject() (runtime.Object, error) {
 	return a.OldObject, nil
 }
 
+var _ Attributes = &AttributesRecord{}
+
 // RequestAttributes adapts an admission.Request to the Attributes interface.
-func RequestAttributes(request *admissionv1.AdmissionRequest, decoder runtime.Decoder) api.Attributes {
+func RequestAttributes(request *admissionv1.AdmissionRequest, decoder runtime.Decoder) Attributes {
 	return &attributes{
 		r:       request,
 		decoder: decoder,
@@ -114,3 +142,5 @@ func (a *attributes) decode(in runtime.RawExtension) (runtime.Object, error) {
 	out, _, err := a.decoder.Decode(in.Raw, &gvk, nil)
 	return out, err
 }
+
+var _ Attributes = &attributes{}

staging/src/k8s.io/pod-security-admission/api/interfaces.go

@@ -1,50 +0,0 @@
-/*
-Copyright 2021 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
-	admissionv1 "k8s.io/api/admission/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-// Attributes exposes the admission request parameters consumed by the PodSecurity admission controller.
-type Attributes interface {
-	// GetName is the name of the object associated with the request.
-	GetName() string
-	// GetNamespace is the namespace associated with the request (if any)
-	GetNamespace() string
-	// GetResource is the name of the resource being requested.  This is not the kind.  For example: pods
-	GetResource() schema.GroupVersionResource
-	// GetKind is the name of the kind being requested.  For example: Pod
-	GetKind() schema.GroupVersionKind
-	// GetSubresource is the name of the subresource being requested.  This is a different resource, scoped to the parent resource, but it may have a different kind.
-	// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
-	// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
-	GetSubresource() string
-	// GetOperation is the operation being performed
-	GetOperation() admissionv1.Operation
-
-	// GetObject returns the typed Object from incoming request.
-	// For objects in the core API group, the result must use the v1 API.
-	GetObject() (runtime.Object, error)
-	// GetOldObject returns the typed existing object. Only populated for UPDATE requests.
-	// For objects in the core API group, the result must use the v1 API.
-	GetOldObject() (runtime.Object, error)
-	// GetUserName is the requesting user's authenticated name.
-	GetUserName() string
-}

staging/src/k8s.io/pod-security-admission/cmd/webhook/server/server.go

@@ -40,6 +40,8 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	compbasemetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/component-base/version/verflag"
 	"k8s.io/klog/v2"
 	"k8s.io/pod-security-admission/admission"
@@ -117,6 +119,11 @@ func (s *Server) Start(ctx context.Context) error {
 	// debugging or proxy purposes. The API server will not connect to an http webhook.
 	mux.HandleFunc("/", s.HandleValidate)
 
+	// Serve the global metrics registry.
+	metrics.LegacyMustRegister()
+	mux.Handle("/metrics",
+		compbasemetrics.HandlerFor(legacyregistry.DefaultGatherer, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}))
+
 	if s.insecureServing != nil {
 		if err := s.insecureServing.Serve(mux, 0, ctx.Done()); err != nil {
 			return fmt.Errorf("failed to start insecure server: %w", err)
@@ -206,7 +213,7 @@ func (s *Server) HandleValidate(w http.ResponseWriter, r *http.Request) {
 	}
 	klog.V(1).InfoS("received request", "UID", review.Request.UID, "kind", review.Request.Kind, "resource", review.Request.Resource)
 
-	attributes := admission.RequestAttributes(review.Request, codecs.UniversalDeserializer())
+	attributes := api.RequestAttributes(review.Request, codecs.UniversalDeserializer())
 	response := s.delegate.Validate(ctx, attributes)
 	response.UID = review.Request.UID // Response UID must match request UID
 	review.Response = response
@@ -276,7 +283,7 @@ func Setup(c *Config) (*Server, error) {
 	s.delegate = &admission.Admission{
 		Configuration:    c.PodSecurityConfig,
 		Evaluator:        evaluator,
-		Metrics:          metrics.NewPrometheusRecorder(api.GetAPIVersion()),
+		Metrics:          metrics.DefaultRecorder(),
 		PodSpecExtractor: admission.DefaultPodSpecExtractor{},
 		PodLister:        admission.PodListerFromClient(client),
 		NamespaceGetter:  admission.NamespaceGetterFromListerAndClient(namespaceLister, client),

staging/src/k8s.io/pod-security-admission/metrics/metrics.go

@@ -17,7 +17,15 @@ limitations under the License.
 package metrics
 
 import (
+	"strconv"
+	"strings"
+	"sync"
+
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/pod-security-admission/api"
 )
 
@@ -29,45 +37,67 @@ const (
 	DecisionDeny  = "deny"  // Policy evaluated, request denied
 )
 
-var (
-	SecurityEvaluation = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Name:           "pod_security_evaluations_total",
-			Help:           "Counter of pod security evaluations.",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"decision", "policy_level", "policy_version", "mode", "operation", "resource", "subresource"},
-	)
-
-	Registry = metrics.NewKubeRegistry()
-)
-
 type Decision string
 type Mode string
 
 type Recorder interface {
-	RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes)
+	RecordEvaluation(Decision, api.LevelVersion, Mode, api.Attributes)
+}
+
+var defaultRecorder = NewPrometheusRecorder(api.GetAPIVersion())
+
+func DefaultRecorder() Recorder {
+	return defaultRecorder
+}
+
+// MustRegister registers the global DefaultMetrics against the legacy registry.
+func LegacyMustRegister() {
+	defaultRecorder.MustRegister(legacyregistry.MustRegister)
 }
 
 type PrometheusRecorder struct {
 	apiVersion api.Version
+
+	evaluationsCounter *metrics.CounterVec
+
+	registerOnce sync.Once
 }
 
-func init() {
+var _ Recorder = &PrometheusRecorder{}
-	Registry.MustRegister(SecurityEvaluation)
-}
 
 func NewPrometheusRecorder(version api.Version) *PrometheusRecorder {
-	return &PrometheusRecorder{apiVersion: version}
+	evaluationsCounter := metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "pod_security_evaluations_total",
+			Help:           "Number of policy evaluations that occurred, not counting ignored or exempt requests.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"decision", "policy_level", "policy_version", "mode", "request_operation", "resource", "subresource"},
+	)
+
+	return &PrometheusRecorder{
+		apiVersion:         version,
+		evaluationsCounter: evaluationsCounter,
+	}
 }
 
-func (r PrometheusRecorder) RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes) {
+func (r *PrometheusRecorder) MustRegister(registerFunc func(...metrics.Registerable)) {
+	r.registerOnce.Do(func() {
+		registerFunc(r.evaluationsCounter)
+	})
+}
+
+func (r *PrometheusRecorder) Reset() {
+	r.evaluationsCounter.Reset()
+}
+
+func (r *PrometheusRecorder) RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes) {
 	dec := string(decision)
-	operation := string(attrs.GetOperation())
+	operation := operationLabel(attrs.GetOperation())
-	resource := attrs.GetResource().String()
+	resource := resourceLabel(attrs.GetResource())
 	subresource := attrs.GetSubresource()
+
 	var version string
-	if policy.Valid() {
 		if policy.Version.Latest() {
 			version = "latest"
 		} else {
@@ -77,7 +107,31 @@ func (r PrometheusRecorder) RecordEvaluation(decision Decision, policy api.Level
 				version = "future"
 			}
 		}
-		SecurityEvaluation.WithLabelValues(dec, string(policy.Level),
+
+		r.evaluationsCounter.WithLabelValues(dec, string(policy.Level),
 			version, string(evalMode), operation, resource, subresource).Inc()
+}
+
+func resourceLabel(resource schema.GroupVersionResource) string {
+	switch resource.GroupResource() {
+	case corev1.Resource("pods"):
+		return "pod"
+	case corev1.Resource("namespace"):
+		return "namespace"
+	default:
+		// Assume any other resource is a valid input to pod-security, and therefore a controller.
+		return "controller"
+	}
+}
+
+func operationLabel(op admissionv1.Operation) string {
+	switch op {
+	case admissionv1.Create:
+		return "create"
+	case admissionv1.Update:
+		return "update"
+	default:
+		// This is a slower operation, but never used in the default implementation.
+		return strings.ToLower(string(op))
 	}
 }

staging/src/k8s.io/pod-security-admission/metrics/metrics_test.go

@@ -15,3 +15,106 @@ limitations under the License.
 */
 
 package metrics
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	admissionv1 "k8s.io/api/admission/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/testutil"
+	"k8s.io/pod-security-admission/api"
+)
+
+var (
+	decisions  = []Decision{DecisionAllow, DecisionDeny}
+	modes      = []Mode{ModeEnforce, ModeAudit, ModeWarn}
+	operations = []admissionv1.Operation{admissionv1.Create, admissionv1.Update}
+	levels     = []api.Level{api.LevelPrivileged, api.LevelBaseline, api.LevelRestricted}
+
+	// Map of resource types to test to expected label value.
+	resourceExpectations = map[schema.GroupVersionResource]string{
+		corev1.SchemeGroupVersion.WithResource("pods"):        "pod",
+		appsv1.SchemeGroupVersion.WithResource("deployments"): "controller",
+		batchv1.SchemeGroupVersion.WithResource("cronjobs"):   "controller",
+	}
+
+	// Map of versions to expected label value (compared against testVersion).
+	versionExpectations = map[string]string{
+		"latest": "latest",
+		"v1.22":  "v1.22",
+		"v1.23":  "v1.23",
+		"v1.24":  "future",
+	}
+	testVersion = api.MajorMinorVersion(1, 23)
+)
+
+func TestRecordEvaluation(t *testing.T) {
+	recorder := NewPrometheusRecorder(testVersion)
+	registry := testutil.NewFakeKubeRegistry("1.23.0")
+	recorder.MustRegister(registry.MustRegister)
+
+	for _, decision := range decisions {
+		for _, mode := range modes {
+			for _, op := range operations {
+				for _, level := range levels {
+					for version, expectedVersion := range versionExpectations {
+						for resource, expectedResource := range resourceExpectations {
+							recorder.RecordEvaluation(decision, levelVersion(level, version), mode, &api.AttributesRecord{
+								Resource:  resource,
+								Operation: op,
+							})
+							expectedLabels := map[string]string{
+								"decision":          string(decision),
+								"policy_level":      string(level),
+								"policy_version":    expectedVersion,
+								"mode":              string(mode),
+								"request_operation": strings.ToLower(string(op)),
+								"resource":          expectedResource,
+								"subresource":       "",
+							}
+							val, err := testutil.GetCounterMetricValue(recorder.evaluationsCounter.With(expectedLabels))
+							require.NoError(t, err, expectedLabels)
+
+							if !assert.EqualValues(t, 1, val, expectedLabels) {
+								findMetric(t, registry, "pod_security_evaluations_total")
+							}
+
+							recorder.Reset()
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func levelVersion(level api.Level, version string) api.LevelVersion {
+	lv := api.LevelVersion{Level: level}
+	var err error
+	if lv.Version, err = api.ParseVersion(version); err != nil {
+		panic(err)
+	}
+	return lv
+}
+
+// findMetric dumps non-zero metric samples for the metric with the given name, to help with debugging.
+func findMetric(t *testing.T, gatherer metrics.Gatherer, metricName string) {
+	t.Helper()
+	m, _ := gatherer.Gather()
+	for _, mFamily := range m {
+		if mFamily.GetName() == metricName {
+			for _, metric := range mFamily.GetMetric() {
+				if metric.GetCounter().GetValue() > 0 {
+					t.Logf("Found metric: %s", metric.String())
+				}
+			}
+		}
+	}
+}

test/integration/auth/podsecurity_test.go

@@ -18,7 +18,9 @@ package auth
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -37,6 +39,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/component-base/featuregate"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/component-base/metrics/testutil"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/pkg/capabilities"
 	"k8s.io/kubernetes/pkg/features"
@@ -67,6 +70,8 @@ func TestPodSecurity(t *testing.T) {
 		ExemptRuntimeClasses: []string{},
 	}
 	podsecuritytest.Run(t, opts)
+
+	ValidatePluginMetrics(t, opts.ClientConfig)
 }
 
 // TestPodSecurityGAOnly ensures policies pass with only GA features enabled
@@ -88,6 +93,8 @@ func TestPodSecurityGAOnly(t *testing.T) {
 		Features: utilfeature.DefaultFeatureGate,
 	}
 	podsecuritytest.Run(t, opts)
+
+	ValidatePluginMetrics(t, opts.ClientConfig)
 }
 
 func TestPodSecurityWebhook(t *testing.T) {
@@ -125,6 +132,8 @@ func TestPodSecurityWebhook(t *testing.T) {
 		ExemptRuntimeClasses: []string{},
 	}
 	podsecuritytest.Run(t, opts)
+
+	ValidateWebhookMetrics(t, webhookAddr)
 }
 
 func startPodSecurityServer(t *testing.T) *kubeapiservertesting.TestServer {
@@ -285,3 +294,52 @@ func installWebhook(t *testing.T, clientConfig *rest.Config, addr string) error
 
 	return nil
 }
+
+func ValidatePluginMetrics(t *testing.T, clientConfig *rest.Config) {
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	ctx := context.Background()
+	data, err := client.CoreV1().RESTClient().Get().AbsPath("metrics").DoRaw(ctx)
+	if err != nil {
+		t.Fatalf("Failed to read metrics: %v", err)
+	}
+	validateMetrics(t, data)
+}
+
+func ValidateWebhookMetrics(t *testing.T, webhookAddr string) {
+	endpoint := &url.URL{
+		Scheme: "https",
+		Host:   webhookAddr,
+		Path:   "/metrics",
+	}
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}}
+	resp, err := client.Get(endpoint.String())
+	if err != nil {
+		t.Fatalf("Failed to fetch metrics from %s: %v", endpoint.String(), err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Non-200 response trying to scrape metrics from %s: %v", endpoint.String(), resp)
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("Unable to read metrics response: %v", err)
+	}
+	validateMetrics(t, data)
+}
+
+func validateMetrics(t *testing.T, rawMetrics []byte) {
+	metrics := testutil.NewMetrics()
+	if err := testutil.ParseMetrics(string(rawMetrics), &metrics); err != nil {
+		t.Fatalf("Failed to parse metrics: %v", err)
+	}
+
+	if err := testutil.ValidateMetrics(metrics, "pod_security_evaluations_total",
+		"decision", "policy_level", "policy_version", "mode", "request_operation", "resource", "subresource"); err != nil {
+		t.Fatalf("Metric validation failed: %v", err)
+	}
+}