Merge pull request #58143 from CaoShuFeng/audit_annotation_another_version

Automatic merge from submit-queue (batch tested with PRs 61610, 64591, 58143, 63929). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add PodSecurityPolicy information to audit logs

Depends on: https://github.com/kubernetes/kubernetes/pull/58806
Fix #56209

**Release note**:
```release-note
PodSecurityPolicy admission information is added to audit logs
```
This commit is contained in:
Kubernetes Submit Queue 2018-06-04 12:26:21 -07:00 committed by GitHub
commit 08c15a6a38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 463 additions and 1 deletions

View File

@ -84,6 +84,7 @@ var _ admission.MutationInterface = &PodSecurityPolicyPlugin{}
var _ admission.ValidationInterface = &PodSecurityPolicyPlugin{}
var _ genericadmissioninit.WantsAuthorizer = &PodSecurityPolicyPlugin{}
var _ kubeapiserveradmission.WantsInternalKubeInformerFactory = &PodSecurityPolicyPlugin{}
var auditKeyPrefix = strings.ToLower(PluginName) + "." + policy.GroupName + ".k8s.io"
// newPlugin creates a new PSP admission plugin.
func newPlugin(strategyFactory psp.StrategyFactory, failOnNoPolicies bool) *PodSecurityPolicyPlugin {
@ -136,6 +137,10 @@ func (c *PodSecurityPolicyPlugin) Admit(a admission.Attributes) error {
pod.ObjectMeta.Annotations = map[string]string{}
}
pod.ObjectMeta.Annotations[psputil.ValidatedPSPAnnotation] = pspName
key := auditKeyPrefix + "/" + "admit-policy"
if err := a.AddAnnotation(key, pspName); err != nil {
glog.Warningf("failed to set admission audit annotation %s to %s: %v", key, pspName, err)
}
return nil
}
@ -154,11 +159,15 @@ func (c *PodSecurityPolicyPlugin) Validate(a admission.Attributes) error {
pod := a.GetObject().(*api.Pod)
// compute the context. Mutation is not allowed. ValidatedPSPAnnotation is used as a hint to gain same speed-up.
allowedPod, _, validationErrs, err := c.computeSecurityContext(a, pod, false, pod.ObjectMeta.Annotations[psputil.ValidatedPSPAnnotation])
allowedPod, pspName, validationErrs, err := c.computeSecurityContext(a, pod, false, pod.ObjectMeta.Annotations[psputil.ValidatedPSPAnnotation])
if err != nil {
return admission.NewForbidden(a, err)
}
if apiequality.Semantic.DeepEqual(pod, allowedPod) {
key := auditKeyPrefix + "/" + "validate-policy"
if err := a.AddAnnotation(key, pspName); err != nil {
glog.Warningf("failed to set admission audit annotation %s to %s: %v", key, pspName, err)
}
return nil
}

View File

@ -1798,11 +1798,24 @@ func testPSPAdmit(testCaseName string, psps []*policy.PodSecurityPolicy, pod *ka
testPSPAdmitAdvanced(testCaseName, kadmission.Create, psps, nil, &user.DefaultInfo{}, pod, nil, shouldPassAdmit, shouldPassValidate, true, expectedPSP, t)
}
// fakeAttributes decorate kadmission.Attributes. It's used to trace the added annotations.
type fakeAttributes struct {
kadmission.Attributes
annotations map[string]string
}
func (f fakeAttributes) AddAnnotation(k, v string) error {
f.annotations[k] = v
return f.Attributes.AddAnnotation(k, v)
}
func testPSPAdmitAdvanced(testCaseName string, op kadmission.Operation, psps []*policy.PodSecurityPolicy, authz authorizer.Authorizer, userInfo user.Info, pod, oldPod *kapi.Pod, shouldPassAdmit, shouldPassValidate bool, canMutate bool, expectedPSP string, t *testing.T) {
originalPod := pod.DeepCopy()
plugin := NewTestAdmission(psps, authz)
attrs := kadmission.NewAttributesRecord(pod, oldPod, kapi.Kind("Pod").WithVersion("version"), pod.Namespace, "", kapi.Resource("pods").WithVersion("version"), "", op, userInfo)
annotations := make(map[string]string)
attrs = &fakeAttributes{attrs, annotations}
err := plugin.Admit(attrs)
if shouldPassAdmit && err != nil {
@ -1832,11 +1845,27 @@ func testPSPAdmitAdvanced(testCaseName string, op kadmission.Operation, psps []*
}
err = plugin.Validate(attrs)
psp := ""
if shouldPassAdmit && op == kadmission.Create {
psp = expectedPSP
}
validateAuditAnnotation(t, testCaseName, annotations, "podsecuritypolicy.policy.k8s.io/admit-policy", psp)
if shouldPassValidate && err != nil {
t.Errorf("%s: expected no errors on Validate but received %v", testCaseName, err)
} else if !shouldPassValidate && err == nil {
t.Errorf("%s: expected errors on Validate but received none", testCaseName)
}
if shouldPassValidate {
validateAuditAnnotation(t, testCaseName, annotations, "podsecuritypolicy.policy.k8s.io/validate-policy", expectedPSP)
} else {
validateAuditAnnotation(t, testCaseName, annotations, "podsecuritypolicy.policy.k8s.io/validate-policy", "")
}
}
func validateAuditAnnotation(t *testing.T, testCaseName string, annotations map[string]string, key, value string) {
if annotations[key] != value {
t.Errorf("%s: expected to have annotations[%s] set to %q, got %q", testCaseName, key, value, annotations[key])
}
}
func TestAssignSecurityContext(t *testing.T) {

View File

@ -9,6 +9,8 @@ load(
go_test(
name = "go_default_test",
srcs = [
"attributes_test.go",
"audit_test.go",
"chain_test.go",
"config_test.go",
"errors_test.go",
@ -16,12 +18,15 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/json:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library",
],
)
@ -29,6 +34,7 @@ go_library(
name = "go_default_library",
srcs = [
"attributes.go",
"audit.go",
"chain.go",
"config.go",
"decorator.go",
@ -48,8 +54,11 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library",
"//vendor/k8s.io/apiserver/pkg/audit:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
],
)

View File

@ -17,8 +17,13 @@ limitations under the License.
package admission
import (
"fmt"
"strings"
"sync"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apiserver/pkg/authentication/user"
)
@ -32,6 +37,11 @@ type attributesRecord struct {
object runtime.Object
oldObject runtime.Object
userInfo user.Info
// other elements are always accessed in single goroutine.
// But ValidatingAdmissionWebhook add annotations concurrently.
annotations map[string]string
annotationsLock sync.RWMutex
}
func NewAttributesRecord(object runtime.Object, oldObject runtime.Object, kind schema.GroupVersionKind, namespace, name string, resource schema.GroupVersionResource, subresource string, operation Operation, userInfo user.Info) Attributes {
@ -83,3 +93,48 @@ func (record *attributesRecord) GetOldObject() runtime.Object {
func (record *attributesRecord) GetUserInfo() user.Info {
return record.userInfo
}
// getAnnotations implements privateAnnotationsGetter.It's a private method used
// by WithAudit decorator.
func (record *attributesRecord) getAnnotations() map[string]string {
record.annotationsLock.RLock()
defer record.annotationsLock.RUnlock()
if record.annotations == nil {
return nil
}
cp := make(map[string]string, len(record.annotations))
for key, value := range record.annotations {
cp[key] = value
}
return cp
}
func (record *attributesRecord) AddAnnotation(key, value string) error {
if err := checkKeyFormat(key); err != nil {
return err
}
record.annotationsLock.Lock()
defer record.annotationsLock.Unlock()
if record.annotations == nil {
record.annotations = make(map[string]string)
}
if v, ok := record.annotations[key]; ok && v != value {
return fmt.Errorf("admission annotations are not allowd to be overwritten, key:%q, old value: %q, new value:%q", key, record.annotations[key], value)
}
record.annotations[key] = value
return nil
}
func checkKeyFormat(key string) error {
parts := strings.Split(key, "/")
if len(parts) != 2 {
return fmt.Errorf("annotation key has invalid format, the right format is a DNS subdomain prefix and '/' and key name. (e.g. 'podsecuritypolicy.admission.k8s.io/admit-policy')")
}
if msgs := validation.IsQualifiedName(key); len(msgs) != 0 {
return fmt.Errorf("annotation key has invalid format %s. A qualified name like 'podsecuritypolicy.admission.k8s.io/admit-policy' is required.", strings.Join(msgs, ","))
}
return nil
}

View File

@ -0,0 +1,64 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestAddAnnotation(t *testing.T) {
attr := &attributesRecord{}
// test AddAnnotation
attr.AddAnnotation("podsecuritypolicy.admission.k8s.io/validate-policy", "privileged")
attr.AddAnnotation("podsecuritypolicy.admission.k8s.io/admit-policy", "privileged")
annotations := attr.getAnnotations()
assert.Equal(t, annotations["podsecuritypolicy.admission.k8s.io/validate-policy"], "privileged")
// test overwrite
assert.Error(t, attr.AddAnnotation("podsecuritypolicy.admission.k8s.io/validate-policy", "privileged-overwrite"),
"admission annotations should not be allowd to be overwritten")
annotations = attr.getAnnotations()
assert.Equal(t, annotations["podsecuritypolicy.admission.k8s.io/validate-policy"], "privileged", "admission annotations should not be overwritten")
// test invalid plugin names
var testCases map[string]string = map[string]string{
"invalid dns subdomain": "INVALID-DNS-Subdomain/policy",
"no plugin name": "policy",
"no key name": "podsecuritypolicy.admission.k8s.io",
"empty key": "",
}
for name, invalidKey := range testCases {
err := attr.AddAnnotation(invalidKey, "value-foo")
assert.Error(t, err)
annotations = attr.getAnnotations()
assert.Equal(t, annotations[invalidKey], "", name+": invalid pluginName is not allowed ")
}
// test all saved annotations
assert.Equal(
t,
annotations,
map[string]string{
"podsecuritypolicy.admission.k8s.io/validate-policy": "privileged",
"podsecuritypolicy.admission.k8s.io/admit-policy": "privileged",
},
"unexpected final annotations",
)
}

View File

@ -0,0 +1,95 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
import (
"fmt"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/audit"
)
// auditHandler logs annotations set by other admission handlers
type auditHandler struct {
Interface
ae *auditinternal.Event
}
var _ Interface = &auditHandler{}
var _ MutationInterface = &auditHandler{}
var _ ValidationInterface = &auditHandler{}
// WithAudit is a decorator for a admission phase. It saves annotations
// of attribute into the audit event. Attributes passed to the Admit and
// Validate function must be instance of privateAnnotationsGetter or
// AnnotationsGetter, otherwise an error is returned.
func WithAudit(i Interface, ae *auditinternal.Event) Interface {
if i == nil {
return i
}
return &auditHandler{i, ae}
}
func (handler auditHandler) Admit(a Attributes) error {
if !handler.Interface.Handles(a.GetOperation()) {
return nil
}
if err := ensureAnnotationGetter(a); err != nil {
return err
}
var err error
if mutator, ok := handler.Interface.(MutationInterface); ok {
err = mutator.Admit(a)
handler.logAnnotations(a)
}
return err
}
func (handler auditHandler) Validate(a Attributes) error {
if !handler.Interface.Handles(a.GetOperation()) {
return nil
}
if err := ensureAnnotationGetter(a); err != nil {
return err
}
var err error
if validator, ok := handler.Interface.(ValidationInterface); ok {
err = validator.Validate(a)
handler.logAnnotations(a)
}
return err
}
func ensureAnnotationGetter(a Attributes) error {
_, okPrivate := a.(privateAnnotationsGetter)
_, okPublic := a.(AnnotationsGetter)
if okPrivate || okPublic {
return nil
}
return fmt.Errorf("attributes must be an instance of privateAnnotationsGetter or AnnotationsGetter")
}
func (handler auditHandler) logAnnotations(a Attributes) {
switch a := a.(type) {
case privateAnnotationsGetter:
audit.LogAnnotations(handler.ae, a.getAnnotations())
case AnnotationsGetter:
audit.LogAnnotations(handler.ae, a.GetAnnotations())
default:
// this will never happen, because we have already checked it in ensureAnnotationGetter
}
}

View File

@ -0,0 +1,173 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
import (
"fmt"
"testing"
"k8s.io/apimachinery/pkg/runtime/schema"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeHandler implements Interface
type fakeHandler struct {
// return value of Admit()
admit error
// annotations add to attributesRecord during Admit() phase
admitAnnotations map[string]string
// return value of Validate()
validate error
// annotations add to attributesRecord during Validate() phase
validateAnnotations map[string]string
// return value of Handles()
handles bool
}
var _ Interface = &fakeHandler{}
var _ MutationInterface = &fakeHandler{}
var _ ValidationInterface = &fakeHandler{}
func (h fakeHandler) Admit(a Attributes) error {
for k, v := range h.admitAnnotations {
a.AddAnnotation(k, v)
}
return h.admit
}
func (h fakeHandler) Validate(a Attributes) error {
for k, v := range h.validateAnnotations {
a.AddAnnotation(k, v)
}
return h.validate
}
func (h fakeHandler) Handles(o Operation) bool {
return h.handles
}
func attributes() Attributes {
return NewAttributesRecord(nil, nil, schema.GroupVersionKind{}, "", "", schema.GroupVersionResource{}, "", "", nil)
}
func TestWithAudit(t *testing.T) {
var testCases = map[string]struct {
admit error
admitAnnotations map[string]string
validate error
validateAnnotations map[string]string
handles bool
}{
"not handle": {
nil,
nil,
nil,
nil,
false,
},
"allow": {
nil,
nil,
nil,
nil,
true,
},
"allow with annotations": {
nil,
map[string]string{
"plugin.example.com/foo": "bar",
},
nil,
nil,
true,
},
"allow with annotations overwrite": {
nil,
map[string]string{
"plugin.example.com/foo": "bar",
},
nil,
map[string]string{
"plugin.example.com/foo": "bar",
},
true,
},
"forbidden error": {
NewForbidden(attributes(), fmt.Errorf("quota exceeded")),
nil,
NewForbidden(attributes(), fmt.Errorf("quota exceeded")),
nil,
true,
},
"forbidden error with annotations": {
NewForbidden(attributes(), fmt.Errorf("quota exceeded")),
nil,
NewForbidden(attributes(), fmt.Errorf("quota exceeded")),
map[string]string{
"plugin.example.com/foo": "bar",
},
true,
},
"forbidden error with annotations overwrite": {
NewForbidden(attributes(), fmt.Errorf("quota exceeded")),
map[string]string{
"plugin.example.com/foo": "bar",
},
NewForbidden(attributes(), fmt.Errorf("quota exceeded")),
map[string]string{
"plugin.example.com/foo": "bar",
},
true,
},
}
for tcName, tc := range testCases {
var handler Interface = fakeHandler{tc.admit, tc.admitAnnotations, tc.validate, tc.validateAnnotations, tc.handles}
ae := &auditinternal.Event{Level: auditinternal.LevelMetadata}
auditHandler := WithAudit(handler, ae)
a := attributes()
assert.Equal(t, handler.Handles(Create), auditHandler.Handles(Create), tcName+": WithAudit decorator should not effect the return value")
mutator, ok := handler.(MutationInterface)
require.True(t, ok)
auditMutator, ok := auditHandler.(MutationInterface)
require.True(t, ok)
assert.Equal(t, mutator.Admit(a), auditMutator.Admit(a), tcName+": WithAudit decorator should not effect the return value")
validator, ok := handler.(ValidationInterface)
require.True(t, ok)
auditValidator, ok := auditHandler.(ValidationInterface)
require.True(t, ok)
assert.Equal(t, validator.Validate(a), auditValidator.Validate(a), tcName+": WithAudit decorator should not effect the return value")
annotations := make(map[string]string, len(tc.admitAnnotations)+len(tc.validateAnnotations))
for k, v := range tc.admitAnnotations {
annotations[k] = v
}
for k, v := range tc.validateAnnotations {
annotations[k] = v
}
if len(annotations) == 0 {
assert.Nil(t, ae.Annotations, tcName+": unexptected annotations set in audit event")
} else {
assert.Equal(t, annotations, ae.Annotations, tcName+": unexptected annotations set in audit event")
}
}
}

View File

@ -49,6 +49,23 @@ type Attributes interface {
GetKind() schema.GroupVersionKind
// GetUserInfo is information about the requesting user
GetUserInfo() user.Info
// AddAnnotation sets annotation according to key-value pair. The key should be qualified, e.g., podsecuritypolicy.admission.k8s.io/admit-policy, where
// "podsecuritypolicy" is the name of the plugin, "admission.k8s.io" is the name of the organization, "admit-policy" is the key name.
// An error is returned if the format of key is invalid. When trying to overwrite annotation with a new value, an error is returned.
// Both ValidationInterface and MutationInterface are allowed to add Annotations.
AddAnnotation(key, value string) error
}
// privateAnnotationsGetter is a private interface which allows users to get annotations from Attributes.
type privateAnnotationsGetter interface {
getAnnotations() map[string]string
}
// AnnotationsGetter allows users to get annotations from Attributes. An alternate Attribute should implement
// this interface.
type AnnotationsGetter interface {
GetAnnotations() map[string]string
}
// Interface is an abstract, pluggable interface for Admission Control decisions.

View File

@ -97,6 +97,7 @@ func createHandler(r rest.NamedCreater, scope RequestScope, admit admission.Inte
trace.Step("Conversion done")
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
userInfo, _ := request.UserFrom(ctx)

View File

@ -56,6 +56,8 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope RequestSco
}
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
options := &metav1.DeleteOptions{}
if allowsOptions {
@ -188,6 +190,8 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope RequestSco
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
if admit != nil && admit.Handles(admission.Delete) {
userInfo, _ := request.UserFrom(ctx)

View File

@ -89,6 +89,8 @@ func PatchResource(r rest.Patcher, scope RequestScope, admit admission.Interface
}
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
audit.LogRequestPatch(ae, patchJS)
trace.Step("Recorded the audit event")

View File

@ -103,6 +103,9 @@ func ConnectResource(connecter rest.Connecter, scope RequestScope, admit admissi
}
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
opts, subpath, subpathKey := connecter.NewConnectOptions()
if err := getRequestOptions(req, scope, opts, subpath, subpathKey, isSubresource); err != nil {
err = errors.NewBadRequest(err.Error())

View File

@ -86,6 +86,7 @@ func UpdateResource(r rest.Updater, scope RequestScope, admit admission.Interfac
ae := request.AuditEventFrom(ctx)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
admit = admission.WithAudit(admit, ae)
if err := checkName(obj, name, namespace, scope.Namer); err != nil {
scope.err(err, w, req)