github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

[PodSecurity] Fix up metrics & add tests

Browse Source 
Update pod security metrics to match the spec in the KEP.
This commit is contained in:
Tim Allclair 2021-10-25 21:43:15 -07:00
parent ac2d872ed9
commit e46928c0b1
8 changed files with 313 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
plugin/pkg/admission/security/podsecurity/admission.go
View File

@ -51,7 +51,7 @@ import (
podsecurityadmission "k8s.io/pod-security-admission/admission"
podsecurityconfigloader "k8s.io/pod-security-admission/admission/api/load"
podsecurityadmissionapi "k8s.io/pod-security-admission/api"
podsecuritymetrics "k8s.io/pod-security-admission/metrics"
"k8s.io/pod-security-admission/metrics"
"k8s.io/pod-security-admission/policy"
)
@ -94,13 +94,14 @@ func newPlugin(reader io.Reader) (*Plugin, error) {
if err != nil {
return nil, fmt.Errorf("could not create PodSecurityRegistry: %w", err)
}
metrics.LegacyMustRegister()
return &Plugin{
Handler: admission.NewHandler(admission.Create, admission.Update),
delegate: &podsecurityadmission.Admission{
Configuration: config,
Evaluator: evaluator,
Metrics: podsecuritymetrics.NewPrometheusRecorder(podsecurityadmissionapi.GetAPIVersion()),
Metrics: metrics.DefaultRecorder(),
PodSpecExtractor: podsecurityadmission.DefaultPodSpecExtractor{},
},
}, nil

38
staging/src/k8s.io/pod-security-admission/admission/admission_test.go
View File

@ -457,7 +457,7 @@ func TestValidateNamespace(t *testing.T) {
}
}
attrs := &AttributesRecord{
attrs := &api.AttributesRecord{
Object: newObject,
OldObject: oldObject,
Name: newObject.Name,
@ -508,7 +508,7 @@ func TestValidateNamespace(t *testing.T) {
RuntimeClasses: tc.exemptRuntimeClasses,
},
},
Metrics: NewMockRecorder(),
Metrics: &FakeRecorder{},
defaultPolicy: defaultPolicy,
namespacePodCheckTimeout: time.Second,
@ -582,6 +582,7 @@ func TestValidatePodController(t *testing.T) {
api.WarnLevelLabel: string(api.LevelBaseline),
api.AuditLevelLabel: string(api.LevelBaseline),
}
nsLevelVersion := api.LevelVersion{api.LevelBaseline, api.LatestVersion()}
testCases := []struct {
desc string
@ -671,7 +672,7 @@ func TestValidatePodController(t *testing.T) {
operation = admissionv1.Update
}
attrs := &AttributesRecord{
attrs := &api.AttributesRecord{
testName,
testNamespace,
tc.gvk,
@ -700,6 +701,7 @@ func TestValidatePodController(t *testing.T) {
Labels: nsLabels}},
}
PodSpecExtractor := &DefaultPodSpecExtractor{}
recorder := &FakeRecorder{}
a := &Admission{
PodLister: podLister,
Evaluator: evaluator,
@ -711,7 +713,7 @@ func TestValidatePodController(t *testing.T) {
Usernames: tc.exemptUsers,
},
},
Metrics: NewMockRecorder(),
Metrics: recorder,
defaultPolicy: defaultPolicy,
NamespaceGetter: nsGetter,
}
@ -727,16 +729,36 @@ func TestValidatePodController(t *testing.T) {
assert.Empty(t, resultError)
assert.Equal(t, tc.expectAuditAnnotations, result.AuditAnnotations, "unexpected AuditAnnotations")
assert.Equal(t, tc.expectWarnings, result.Warnings, "unexpected Warnings")
expectedEvaluations := []EvaluationRecord{}
if len(tc.expectAuditAnnotations) > 0 {
expectedEvaluations = append(expectedEvaluations, EvaluationRecord{testName, metrics.DecisionDeny, nsLevelVersion, metrics.ModeAudit})
}
if len(tc.expectWarnings) > 0 {
expectedEvaluations = append(expectedEvaluations, EvaluationRecord{testName, metrics.DecisionDeny, nsLevelVersion, metrics.ModeWarn})
}
recorder.ExpectEvaluations(t, expectedEvaluations)
})
}
}
type MockRecorder struct {
type FakeRecorder struct {
evaluations []EvaluationRecord
}
func NewMockRecorder() *MockRecorder {
return &MockRecorder{}
type EvaluationRecord struct {
ObjectName string
Decision metrics.Decision
Policy api.LevelVersion
Mode metrics.Mode
}
func (r MockRecorder) RecordEvaluation(decision metrics.Decision, policy api.LevelVersion, evalMode metrics.Mode, attrs api.Attributes) {
func (r *FakeRecorder) RecordEvaluation(decision metrics.Decision, policy api.LevelVersion, evalMode metrics.Mode, attrs api.Attributes) {
r.evaluations = append(r.evaluations, EvaluationRecord{attrs.GetName(), decision, policy, evalMode})
}
// ExpectEvaluation asserts that the evaluation was recorded, and clears the record.
func (r *FakeRecorder) ExpectEvaluations(t *testing.T, expected []EvaluationRecord) {
t.Helper()
assert.ElementsMatch(t, expected, r.evaluations)
}

36
staging/src/k8s.io/pod-security-admission/admission/attributes.go → staging/src/k8s.io/pod-security-admission/api/attributes.go
View File

@ -14,15 +14,41 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
package api
import (
admissionv1 "k8s.io/api/admission/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/pod-security-admission/api"
)
// Attributes exposes the admission request parameters consumed by the PodSecurity admission controller.
type Attributes interface {
// GetName is the name of the object associated with the request.
GetName() string
// GetNamespace is the namespace associated with the request (if any)
GetNamespace() string
// GetResource is the name of the resource being requested. This is not the kind. For example: pods
GetResource() schema.GroupVersionResource
// GetKind is the name of the kind being requested. For example: Pod
GetKind() schema.GroupVersionKind
// GetSubresource is the name of the subresource being requested. This is a different resource, scoped to the parent resource, but it may have a different kind.
// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
GetSubresource() string
// GetOperation is the operation being performed
GetOperation() admissionv1.Operation
// GetObject returns the typed Object from incoming request.
// For objects in the core API group, the result must use the v1 API.
GetObject() (runtime.Object, error)
// GetOldObject returns the typed existing object. Only populated for UPDATE requests.
// For objects in the core API group, the result must use the v1 API.
GetOldObject() (runtime.Object, error)
// GetUserName is the requesting user's authenticated name.
GetUserName() string
}
// AttributesRecord is a simple struct implementing the Attributes interface.
type AttributesRecord struct {
Name string
@ -64,8 +90,10 @@ func (a *AttributesRecord) GetOldObject() (runtime.Object, error) {
return a.OldObject, nil
}
var _ Attributes = &AttributesRecord{}
// RequestAttributes adapts an admission.Request to the Attributes interface.
func RequestAttributes(request *admissionv1.AdmissionRequest, decoder runtime.Decoder) api.Attributes {
func RequestAttributes(request *admissionv1.AdmissionRequest, decoder runtime.Decoder) Attributes {
return &attributes{
r: request,
decoder: decoder,
@ -114,3 +142,5 @@ func (a *attributes) decode(in runtime.RawExtension) (runtime.Object, error) {
out, _, err := a.decoder.Decode(in.Raw, &gvk, nil)
return out, err
}
var _ Attributes = &attributes{}

50
staging/src/k8s.io/pod-security-admission/api/interfaces.go
View File

@ -1,50 +0,0 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
admissionv1 "k8s.io/api/admission/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// Attributes exposes the admission request parameters consumed by the PodSecurity admission controller.
type Attributes interface {
// GetName is the name of the object associated with the request.
GetName() string
// GetNamespace is the namespace associated with the request (if any)
GetNamespace() string
// GetResource is the name of the resource being requested. This is not the kind. For example: pods
GetResource() schema.GroupVersionResource
// GetKind is the name of the kind being requested. For example: Pod
GetKind() schema.GroupVersionKind
// GetSubresource is the name of the subresource being requested. This is a different resource, scoped to the parent resource, but it may have a different kind.
// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
GetSubresource() string
// GetOperation is the operation being performed
GetOperation() admissionv1.Operation
// GetObject returns the typed Object from incoming request.
// For objects in the core API group, the result must use the v1 API.
GetObject() (runtime.Object, error)
// GetOldObject returns the typed existing object. Only populated for UPDATE requests.
// For objects in the core API group, the result must use the v1 API.
GetOldObject() (runtime.Object, error)
// GetUserName is the requesting user's authenticated name.
GetUserName() string
}

11
staging/src/k8s.io/pod-security-admission/cmd/webhook/server/server.go
View File

@ -40,6 +40,8 @@ import (
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/version/verflag"
"k8s.io/klog/v2"
"k8s.io/pod-security-admission/admission"
@ -117,6 +119,11 @@ func (s *Server) Start(ctx context.Context) error {
// debugging or proxy purposes. The API server will not connect to an http webhook.
mux.HandleFunc("/", s.HandleValidate)
// Serve the global metrics registry.
metrics.LegacyMustRegister()
mux.Handle("/metrics",
compbasemetrics.HandlerFor(legacyregistry.DefaultGatherer, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}))
if s.insecureServing != nil {
if err := s.insecureServing.Serve(mux, 0, ctx.Done()); err != nil {
return fmt.Errorf("failed to start insecure server: %w", err)
@ -206,7 +213,7 @@ func (s *Server) HandleValidate(w http.ResponseWriter, r *http.Request) {
}
klog.V(1).InfoS("received request", "UID", review.Request.UID, "kind", review.Request.Kind, "resource", review.Request.Resource)
attributes := admission.RequestAttributes(review.Request, codecs.UniversalDeserializer())
attributes := api.RequestAttributes(review.Request, codecs.UniversalDeserializer())
response := s.delegate.Validate(ctx, attributes)
response.UID = review.Request.UID // Response UID must match request UID
review.Response = response
@ -276,7 +283,7 @@ func Setup(c *Config) (*Server, error) {
s.delegate = &admission.Admission{
Configuration: c.PodSecurityConfig,
Evaluator: evaluator,
Metrics: metrics.NewPrometheusRecorder(api.GetAPIVersion()),
Metrics: metrics.DefaultRecorder(),
PodSpecExtractor: admission.DefaultPodSpecExtractor{},
PodLister: admission.PodListerFromClient(client),
NamespaceGetter: admission.NamespaceGetterFromListerAndClient(namespaceLister, client),

100
staging/src/k8s.io/pod-security-admission/metrics/metrics.go
View File

@ -17,7 +17,15 @@ limitations under the License.
package metrics
import (
"strconv"
"strings"
"sync"
admissionv1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/pod-security-admission/api"
)
@ -29,45 +37,67 @@ const (
DecisionDeny = "deny" // Policy evaluated, request denied
)
var (
SecurityEvaluation = metrics.NewCounterVec(
&metrics.CounterOpts{
Name: "pod_security_evaluations_total",
Help: "Counter of pod security evaluations.",
StabilityLevel: metrics.ALPHA,
},
[]string{"decision", "policy_level", "policy_version", "mode", "operation", "resource", "subresource"},
)
Registry = metrics.NewKubeRegistry()
)
type Decision string
type Mode string
type Recorder interface {
RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes)
RecordEvaluation(Decision, api.LevelVersion, Mode, api.Attributes)
}
var defaultRecorder = NewPrometheusRecorder(api.GetAPIVersion())
func DefaultRecorder() Recorder {
return defaultRecorder
}
// MustRegister registers the global DefaultMetrics against the legacy registry.
func LegacyMustRegister() {
defaultRecorder.MustRegister(legacyregistry.MustRegister)
}
type PrometheusRecorder struct {
apiVersion api.Version
evaluationsCounter *metrics.CounterVec
registerOnce sync.Once
}
func init() {
Registry.MustRegister(SecurityEvaluation)
}
var _ Recorder = &PrometheusRecorder{}
func NewPrometheusRecorder(version api.Version) *PrometheusRecorder {
return &PrometheusRecorder{apiVersion: version}
evaluationsCounter := metrics.NewCounterVec(
&metrics.CounterOpts{
Name: "pod_security_evaluations_total",
Help: "Number of policy evaluations that occurred, not counting ignored or exempt requests.",
StabilityLevel: metrics.ALPHA,
},
[]string{"decision", "policy_level", "policy_version", "mode", "request_operation", "resource", "subresource"},
)
return &PrometheusRecorder{
apiVersion: version,
evaluationsCounter: evaluationsCounter,
}
}
func (r PrometheusRecorder) RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes) {
func (r *PrometheusRecorder) MustRegister(registerFunc func(...metrics.Registerable)) {
r.registerOnce.Do(func() {
registerFunc(r.evaluationsCounter)
})
}
func (r *PrometheusRecorder) Reset() {
r.evaluationsCounter.Reset()
}
func (r *PrometheusRecorder) RecordEvaluation(decision Decision, policy api.LevelVersion, evalMode Mode, attrs api.Attributes) {
dec := string(decision)
operation := string(attrs.GetOperation())
resource := attrs.GetResource().String()
operation := operationLabel(attrs.GetOperation())
resource := resourceLabel(attrs.GetResource())
subresource := attrs.GetSubresource()
var version string
if policy.Valid() {
if policy.Version.Latest() {
version = "latest"
} else {
@ -77,7 +107,31 @@ func (r PrometheusRecorder) RecordEvaluation(decision Decision, policy api.Level
version = "future"
}
}
SecurityEvaluation.WithLabelValues(dec, string(policy.Level),
r.evaluationsCounter.WithLabelValues(dec, string(policy.Level),
version, string(evalMode), operation, resource, subresource).Inc()
}
func resourceLabel(resource schema.GroupVersionResource) string {
switch resource.GroupResource() {
case corev1.Resource("pods"):
return "pod"
case corev1.Resource("namespace"):
return "namespace"
default:
// Assume any other resource is a valid input to pod-security, and therefore a controller.
return "controller"
}
}
func operationLabel(op admissionv1.Operation) string {
switch op {
case admissionv1.Create:
return "create"
case admissionv1.Update:
return "update"
default:
// This is a slower operation, but never used in the default implementation.
return strings.ToLower(string(op))
}
}

103
staging/src/k8s.io/pod-security-admission/metrics/metrics_test.go
View File

@ -15,3 +15,106 @@ limitations under the License.
*/
package metrics
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
admissionv1 "k8s.io/api/admission/v1"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/testutil"
"k8s.io/pod-security-admission/api"
)
var (
decisions = []Decision{DecisionAllow, DecisionDeny}
modes = []Mode{ModeEnforce, ModeAudit, ModeWarn}
operations = []admissionv1.Operation{admissionv1.Create, admissionv1.Update}
levels = []api.Level{api.LevelPrivileged, api.LevelBaseline, api.LevelRestricted}
// Map of resource types to test to expected label value.
resourceExpectations = map[schema.GroupVersionResource]string{
corev1.SchemeGroupVersion.WithResource("pods"): "pod",
appsv1.SchemeGroupVersion.WithResource("deployments"): "controller",
batchv1.SchemeGroupVersion.WithResource("cronjobs"): "controller",
}
// Map of versions to expected label value (compared against testVersion).
versionExpectations = map[string]string{
"latest": "latest",
"v1.22": "v1.22",
"v1.23": "v1.23",
"v1.24": "future",
}
testVersion = api.MajorMinorVersion(1, 23)
)
func TestRecordEvaluation(t *testing.T) {
recorder := NewPrometheusRecorder(testVersion)
registry := testutil.NewFakeKubeRegistry("1.23.0")
recorder.MustRegister(registry.MustRegister)
for _, decision := range decisions {
for _, mode := range modes {
for _, op := range operations {
for _, level := range levels {
for version, expectedVersion := range versionExpectations {
for resource, expectedResource := range resourceExpectations {
recorder.RecordEvaluation(decision, levelVersion(level, version), mode, &api.AttributesRecord{
Resource: resource,
Operation: op,
})
expectedLabels := map[string]string{
"decision": string(decision),
"policy_level": string(level),
"policy_version": expectedVersion,
"mode": string(mode),
"request_operation": strings.ToLower(string(op)),
"resource": expectedResource,
"subresource": "",
}
val, err := testutil.GetCounterMetricValue(recorder.evaluationsCounter.With(expectedLabels))
require.NoError(t, err, expectedLabels)
if !assert.EqualValues(t, 1, val, expectedLabels) {
findMetric(t, registry, "pod_security_evaluations_total")
}
recorder.Reset()
}
}
}
}
}
}
}
func levelVersion(level api.Level, version string) api.LevelVersion {
lv := api.LevelVersion{Level: level}
var err error
if lv.Version, err = api.ParseVersion(version); err != nil {
panic(err)
}
return lv
}
// findMetric dumps non-zero metric samples for the metric with the given name, to help with debugging.
func findMetric(t *testing.T, gatherer metrics.Gatherer, metricName string) {
t.Helper()
m, _ := gatherer.Gather()
for _, mFamily := range m {
if mFamily.GetName() == metricName {
for _, metric := range mFamily.GetMetric() {
if metric.GetCounter().GetValue() > 0 {
t.Logf("Found metric: %s", metric.String())
}
}
}
}
}

58
test/integration/auth/podsecurity_test.go
View File

@ -18,7 +18,9 @@ package auth
import (
"context"
"crypto/tls"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
@ -37,6 +39,7 @@ import (
"k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/metrics/testutil"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/capabilities"
"k8s.io/kubernetes/pkg/features"
@ -67,6 +70,8 @@ func TestPodSecurity(t *testing.T) {
ExemptRuntimeClasses: []string{},
}
podsecuritytest.Run(t, opts)
ValidatePluginMetrics(t, opts.ClientConfig)
}
// TestPodSecurityGAOnly ensures policies pass with only GA features enabled
@ -88,6 +93,8 @@ func TestPodSecurityGAOnly(t *testing.T) {
Features: utilfeature.DefaultFeatureGate,
}
podsecuritytest.Run(t, opts)
ValidatePluginMetrics(t, opts.ClientConfig)
}
func TestPodSecurityWebhook(t *testing.T) {
@ -125,6 +132,8 @@ func TestPodSecurityWebhook(t *testing.T) {
ExemptRuntimeClasses: []string{},
}
podsecuritytest.Run(t, opts)
ValidateWebhookMetrics(t, webhookAddr)
}
func startPodSecurityServer(t *testing.T) *kubeapiservertesting.TestServer {
@ -285,3 +294,52 @@ func installWebhook(t *testing.T, clientConfig *rest.Config, addr string) error
return nil
}
func ValidatePluginMetrics(t *testing.T, clientConfig *rest.Config) {
client, err := kubernetes.NewForConfig(clientConfig)
if err != nil {
t.Fatalf("Error creating client: %v", err)
}
ctx := context.Background()
data, err := client.CoreV1().RESTClient().Get().AbsPath("metrics").DoRaw(ctx)
if err != nil {
t.Fatalf("Failed to read metrics: %v", err)
}
validateMetrics(t, data)
}
func ValidateWebhookMetrics(t *testing.T, webhookAddr string) {
endpoint := &url.URL{
Scheme: "https",
Host: webhookAddr,
Path: "/metrics",
}
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
resp, err := client.Get(endpoint.String())
if err != nil {
t.Fatalf("Failed to fetch metrics from %s: %v", endpoint.String(), err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("Non-200 response trying to scrape metrics from %s: %v", endpoint.String(), resp)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Unable to read metrics response: %v", err)
}
validateMetrics(t, data)
}
func validateMetrics(t *testing.T, rawMetrics []byte) {
metrics := testutil.NewMetrics()
if err := testutil.ParseMetrics(string(rawMetrics), &metrics); err != nil {
t.Fatalf("Failed to parse metrics: %v", err)
}
if err := testutil.ValidateMetrics(metrics, "pod_security_evaluations_total",
"decision", "policy_level", "policy_version", "mode", "request_operation", "resource", "subresource"); err != nil {
t.Fatalf("Metric validation failed: %v", err)
}
}