Add integration tests

This commit is contained in:
Joe Betz 2023-03-06 17:30:48 -05:00
parent 932a4d9724
commit c2b3871502
3 changed files with 375 additions and 9 deletions

View File

@ -18,13 +18,19 @@ package cel
import (
"context"
"fmt"
"os"
"strconv"
"strings"
"sync"
"testing"
"time"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/rest"
@ -34,11 +40,13 @@ import (
"k8s.io/kubernetes/test/integration/authutil"
"k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework"
"k8s.io/kubernetes/test/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
@ -252,10 +260,10 @@ func Test_ValidateNamespace_NoParams(t *testing.T) {
err: "",
},
}
for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.ValidatingAdmissionPolicy, true)()
server, err := apiservertesting.StartTestServer(t, nil, []string{
"--enable-admission-plugins", "ValidatingAdmissionPolicy",
}, framework.SharedEtcd())
@ -285,6 +293,190 @@ func Test_ValidateNamespace_NoParams(t *testing.T) {
})
}
}
func Test_ValidateAnnotationsAndWarnings(t *testing.T) {
testcases := []struct {
name string
policy *admissionregistrationv1alpha1.ValidatingAdmissionPolicy
policyBinding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding
object *v1.ConfigMap
err string
failureReason metav1.StatusReason
auditAnnotations map[string]string
warnings sets.Set[string]
}{
{
name: "with audit annotations",
policy: withAuditAnnotations([]admissionregistrationv1alpha1.AuditAnnotation{
{
Key: "example-key",
ValueExpression: "'object name: ' + object.metadata.name",
},
{
Key: "exclude-key",
ValueExpression: "null",
},
}, withParams(configParamKind(), withFailurePolicy(admissionregistrationv1alpha1.Fail, withConfigMapMatch(makePolicy("validate-audit-annotations"))))),
policyBinding: makeBinding("validate-audit-annotations-binding", "validate-audit-annotations", ""),
object: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test1-k8s",
},
},
err: "",
auditAnnotations: map[string]string{
"validate-audit-annotations/example-key": `object name: test1-k8s`,
},
},
{
name: "with audit annotations with invalid expression",
policy: withAuditAnnotations([]admissionregistrationv1alpha1.AuditAnnotation{
{
Key: "example-key",
ValueExpression: "string(params.metadata.name)", // runtime error, params is null
},
}, withParams(configParamKind(), withFailurePolicy(admissionregistrationv1alpha1.Fail, withConfigMapMatch(makePolicy("validate-audit-annotations-invalid"))))),
policyBinding: makeBinding("validate-audit-annotations-invalid-binding", "validate-audit-annotations-invalid", ""),
object: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test2-k8s",
},
},
err: "configmaps \"test2-k8s\" is forbidden: ValidatingAdmissionPolicy 'validate-audit-annotations-invalid' with binding 'validate-audit-annotations-invalid-binding' denied request: expression 'string(params.metadata.name)' resulted in error: no such key: metadata",
failureReason: metav1.StatusReasonInvalid,
},
{
name: "with audit annotations with invalid expression and ignore failure policy",
policy: withAuditAnnotations([]admissionregistrationv1alpha1.AuditAnnotation{
{
Key: "example-key",
ValueExpression: "string(params.metadata.name)", // runtime error, params is null
},
}, withParams(configParamKind(), withFailurePolicy(admissionregistrationv1alpha1.Ignore, withConfigMapMatch(makePolicy("validate-audit-annotations-invalid-ignore"))))),
policyBinding: makeBinding("validate-audit-annotations-invalid-ignore-binding", "validate-audit-annotations-invalid-ignore", ""),
object: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test3-k8s",
},
},
err: "",
},
{
name: "with warn validationActions",
policy: withValidations([]admissionregistrationv1alpha1.Validation{
{
Expression: "object.metadata.name.endsWith('k8s')",
},
}, withParams(configParamKind(), withFailurePolicy(admissionregistrationv1alpha1.Fail, withConfigMapMatch(makePolicy("validate-actions-warn"))))),
policyBinding: withValidationActions([]admissionregistrationv1alpha1.ValidationAction{admissionregistrationv1alpha1.Warn}, makeBinding("validate-actions-warn-binding", "validate-actions-warn", "")),
object: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test4-nope",
},
},
warnings: sets.New("Validation failed for ValidatingAdmissionPolicy 'validate-actions-warn' with binding 'validate-actions-warn-binding': failed expression: object.metadata.name.endsWith('k8s')"),
},
{
name: "with audit validationActions",
policy: withValidations([]admissionregistrationv1alpha1.Validation{
{
Expression: "object.metadata.name.endsWith('k8s')",
},
}, withParams(configParamKind(), withFailurePolicy(admissionregistrationv1alpha1.Fail, withConfigMapMatch(makePolicy("validate-actions-audit"))))),
policyBinding: withValidationActions([]admissionregistrationv1alpha1.ValidationAction{admissionregistrationv1alpha1.Deny, admissionregistrationv1alpha1.Audit}, makeBinding("validate-actions-audit-binding", "validate-actions-audit", "")),
object: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test5-nope",
},
},
err: "configmaps \"test5-nope\" is forbidden: ValidatingAdmissionPolicy 'validate-actions-audit' with binding 'validate-actions-audit-binding' denied request: failed expression: object.metadata.name.endsWith('k8s')",
failureReason: metav1.StatusReasonInvalid,
auditAnnotations: map[string]string{
"validation.policy.admission.k8s.io/validation_failure": `[{"message":"failed expression: object.metadata.name.endsWith('k8s')","policy":"validate-actions-audit","binding":"validate-actions-audit-binding","expressionIndex":1,"validationActions":["Deny","Audit"]}]`,
},
},
}
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.ValidatingAdmissionPolicy, true)()
// prepare audit policy file
policyFile, err := os.CreateTemp("", "audit-policy.yaml")
if err != nil {
t.Fatalf("Failed to create audit policy file: %v", err)
}
defer os.Remove(policyFile.Name())
if _, err := policyFile.Write([]byte(auditPolicy)); err != nil {
t.Fatalf("Failed to write audit policy file: %v", err)
}
if err := policyFile.Close(); err != nil {
t.Fatalf("Failed to close audit policy file: %v", err)
}
// prepare audit log file
logFile, err := os.CreateTemp("", "audit.log")
if err != nil {
t.Fatalf("Failed to create audit log file: %v", err)
}
defer os.Remove(logFile.Name())
server, err := apiservertesting.StartTestServer(t, nil, []string{
"--enable-admission-plugins", "ValidatingAdmissionPolicy",
"--audit-policy-file", policyFile.Name(),
"--audit-log-version", "audit.k8s.io/v1",
"--audit-log-mode", "blocking",
"--audit-log-path", logFile.Name(),
}, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}
defer server.TearDownFn()
config := server.ClientConfig
warnHandler := newWarningHandler()
config.WarningHandler = warnHandler
config.Impersonate.UserName = testReinvocationClientUsername
client, err := clientset.NewForConfig(config)
if err != nil {
t.Fatal(err)
}
for i, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
testCaseID := strconv.Itoa(i)
ns := "auditannotations-" + testCaseID
_, err = client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}
policy := withWaitReadyConstraintAndExpression(testcase.policy)
if _, err := client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Create(context.TODO(), policy, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if err := createAndWaitReadyNamespacedWithWarnHandler(t, client, withMatchNamespace(testcase.policyBinding, ns), nil, ns, warnHandler); err != nil {
t.Fatal(err)
}
warnHandler.reset()
testcase.object.Namespace = ns
_, err = client.CoreV1().ConfigMaps(ns).Create(context.TODO(), testcase.object, metav1.CreateOptions{})
code := int32(201)
if testcase.err != "" {
code = 422
}
auditAnnotationFilter := func(key, val string) bool {
_, ok := testcase.auditAnnotations[key]
return ok
}
checkExpectedError(t, err, testcase.err)
checkFailureReason(t, err, testcase.failureReason)
checkExpectedWarnings(t, warnHandler, testcase.warnings)
checkAuditEvents(t, logFile, expectedAuditEvents(testcase.auditAnnotations, ns, code), auditAnnotationFilter)
})
}
}
// Test_ValidateNamespace_WithConfigMapParams tests a ValidatingAdmissionPolicy that validates creation of a Namespace,
// using ConfigMap as a param reference.
@ -2254,14 +2446,22 @@ func withWaitReadyConstraintAndExpression(policy *admissionregistrationv1alpha1.
}
func createAndWaitReady(t *testing.T, client *clientset.Clientset, binding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding, matchLabels map[string]string) error {
marker := &v1.Endpoints{ObjectMeta: metav1.ObjectMeta{Name: "test-marker", Namespace: "default", Labels: matchLabels}}
return createAndWaitReadyNamespaced(t, client, binding, matchLabels, "default")
}
func createAndWaitReadyNamespaced(t *testing.T, client *clientset.Clientset, binding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding, matchLabels map[string]string, ns string) error {
return createAndWaitReadyNamespacedWithWarnHandler(t, client, binding, matchLabels, ns, newWarningHandler())
}
func createAndWaitReadyNamespacedWithWarnHandler(t *testing.T, client *clientset.Clientset, binding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding, matchLabels map[string]string, ns string, handler *warningHandler) error {
marker := &v1.Endpoints{ObjectMeta: metav1.ObjectMeta{Name: "test-marker", Namespace: ns, Labels: matchLabels}}
defer func() {
err := client.CoreV1().Endpoints("default").Delete(context.TODO(), marker.Name, metav1.DeleteOptions{})
err := client.CoreV1().Endpoints(ns).Delete(context.TODO(), marker.Name, metav1.DeleteOptions{})
if err != nil {
t.Logf("error deleting marker: %v", err)
}
}()
marker, err := client.CoreV1().Endpoints("default").Create(context.TODO(), marker, metav1.CreateOptions{})
marker, err := client.CoreV1().Endpoints(ns).Create(context.TODO(), marker, metav1.CreateOptions{})
if err != nil {
return err
}
@ -2272,7 +2472,11 @@ func createAndWaitReady(t *testing.T, client *clientset.Clientset, binding *admi
}
if waitErr := wait.PollImmediate(time.Millisecond*5, wait.ForeverTestTimeout, func() (bool, error) {
_, err := client.CoreV1().Endpoints("default").Patch(context.TODO(), marker.Name, types.JSONPatchType, []byte("[]"), metav1.PatchOptions{})
handler.reset()
_, err := client.CoreV1().Endpoints(ns).Patch(context.TODO(), marker.Name, types.JSONPatchType, []byte("[]"), metav1.PatchOptions{})
if handler.hasObservedMarker() {
return true, nil
}
if err != nil && strings.Contains(err.Error(), "marker denied; policy is ready") {
return true, nil
} else if err != nil && strings.Contains(err.Error(), "not yet synced to use for admission") {
@ -2285,9 +2489,26 @@ func createAndWaitReady(t *testing.T, client *clientset.Clientset, binding *admi
}); waitErr != nil {
return waitErr
}
t.Logf("Marker ready: %v", marker)
handler.reset()
return nil
}
func withMatchNamespace(binding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding, ns string) *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding {
binding.Spec.MatchResources = &admissionregistrationv1alpha1.MatchResources{
NamespaceSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "kubernetes.io/metadata.name",
Operator: metav1.LabelSelectorOpIn,
Values: []string{ns},
},
},
},
}
return binding
}
func makePolicy(name string) *admissionregistrationv1alpha1.ValidatingAdmissionPolicy {
return &admissionregistrationv1alpha1.ValidatingAdmissionPolicy{
ObjectMeta: metav1.ObjectMeta{Name: name},
@ -2395,6 +2616,11 @@ func withValidations(validations []admissionregistrationv1alpha1.Validation, pol
return policy
}
func withAuditAnnotations(auditAnnotations []admissionregistrationv1alpha1.AuditAnnotation, policy *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) *admissionregistrationv1alpha1.ValidatingAdmissionPolicy {
policy.Spec.AuditAnnotations = auditAnnotations
return policy
}
func makeBinding(name, policyName, paramName string) *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding {
var paramRef *admissionregistrationv1alpha1.ParamRef
if paramName != "" {
@ -2406,12 +2632,18 @@ func makeBinding(name, policyName, paramName string) *admissionregistrationv1alp
return &admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding{
ObjectMeta: metav1.ObjectMeta{Name: name},
Spec: admissionregistrationv1alpha1.ValidatingAdmissionPolicyBindingSpec{
PolicyName: policyName,
ParamRef: paramRef,
PolicyName: policyName,
ParamRef: paramRef,
ValidationActions: []admissionregistrationv1alpha1.ValidationAction{admissionregistrationv1alpha1.Deny},
},
}
}
func withValidationActions(validationActions []admissionregistrationv1alpha1.ValidationAction, binding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding) *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding {
binding.Spec.ValidationActions = validationActions
return binding
}
func withBindingExistsLabels(labels []string, policy *admissionregistrationv1alpha1.ValidatingAdmissionPolicy, binding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding) *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding {
if policy != nil {
// shallow copy
@ -2468,6 +2700,36 @@ func checkFailureReason(t *testing.T, err error, expectedReason metav1.StatusRea
}
}
func checkExpectedWarnings(t *testing.T, recordedWarnings *warningHandler, expectedWarnings sets.Set[string]) {
if !recordedWarnings.equals(expectedWarnings) {
t.Errorf("Expected warnings '%v' but got '%v", expectedWarnings, recordedWarnings)
}
}
func checkAuditEvents(t *testing.T, logFile *os.File, auditEvents []utils.AuditEvent, filter utils.AuditAnnotationsFilter) {
stream, err := os.OpenFile(logFile.Name(), os.O_RDWR, 0600)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
defer stream.Close()
if auditEvents != nil {
missing, err := utils.CheckAuditLinesFiltered(stream, auditEvents, auditv1.SchemeGroupVersion, filter)
if err != nil {
t.Errorf("unexpected error checking audit lines: %v", err)
}
if len(missing.MissingEvents) > 0 {
t.Errorf("failed to get expected events -- missing: %s", missing)
}
}
if err := stream.Truncate(0); err != nil {
t.Errorf("unexpected error truncate file: %v", err)
}
if _, err := stream.Seek(0, 0); err != nil {
t.Errorf("unexpected error reset offset: %v", err)
}
}
func withCRDParamKind(kind, crdGroup, crdVersion string) *admissionregistrationv1alpha1.ParamKind {
return &admissionregistrationv1alpha1.ParamKind{
APIVersion: crdGroup + "/" + crdVersion,
@ -2544,3 +2806,82 @@ func versionedCustomResourceDefinition() *apiextensionsv1.CustomResourceDefiniti
},
}
}
type warningHandler struct {
lock sync.Mutex
warnings sets.Set[string]
observedMarker bool
}
func newWarningHandler() *warningHandler {
return &warningHandler{warnings: sets.New[string]()}
}
func (w *warningHandler) reset() {
w.lock.Lock()
defer w.lock.Unlock()
w.warnings = sets.New[string]()
w.observedMarker = false
}
func (w *warningHandler) equals(s sets.Set[string]) bool {
w.lock.Lock()
defer w.lock.Unlock()
return w.warnings.Equal(s)
}
func (w *warningHandler) hasObservedMarker() bool {
w.lock.Lock()
defer w.lock.Unlock()
return w.observedMarker
}
func (w *warningHandler) HandleWarningHeader(code int, _ string, message string) {
if strings.HasSuffix(message, "marker denied; policy is ready") {
func() {
w.lock.Lock()
defer w.lock.Unlock()
w.observedMarker = true
}()
}
if code != 299 || len(message) == 0 {
return
}
w.lock.Lock()
defer w.lock.Unlock()
w.warnings.Insert(message)
}
func expectedAuditEvents(auditAnnotations map[string]string, ns string, code int32) []utils.AuditEvent {
return []utils.AuditEvent{
{
Level: auditinternal.LevelRequest,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", ns),
Verb: "create",
Code: code,
User: "system:apiserver",
ImpersonatedUser: testReinvocationClientUsername,
ImpersonatedGroups: "system:authenticated",
Resource: "configmaps",
Namespace: ns,
AuthorizeDecision: "allow",
RequestObject: true,
ResponseObject: false,
CustomAuditAnnotations: auditAnnotations,
},
}
}
const (
testReinvocationClientUsername = "webhook-reinvocation-integration-client"
auditPolicy = `
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Request
resources:
- group: "" # core
resources: ["configmaps"]
`
)

View File

@ -21,6 +21,7 @@ import (
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/test/utils/image"
)
@ -388,7 +389,7 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes
ExpectedEtcdPath: "/registry/validatingadmissionpolicies/vap1",
},
gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicybindings"): {
Stub: `{"metadata":{"name":"pb1","creationTimestamp":null},"spec":{"policyName":"replicalimit-policy.example.com","paramRef":{"name":"replica-limit-test.example.com"}}}`,
Stub: `{"metadata":{"name":"pb1","creationTimestamp":null},"spec":{"policyName":"replicalimit-policy.example.com","paramRef":{"name":"replica-limit-test.example.com"},"validationActions":["Deny"]}}`,
ExpectedEtcdPath: "/registry/validatingadmissionpolicybindings/pb1",
},
// --

View File

@ -53,8 +53,13 @@ type AuditEvent struct {
// not reference these maps after calling the Check functions.
AdmissionWebhookMutationAnnotations map[string]string
AdmissionWebhookPatchAnnotations map[string]string
// Only populated when a filter is provided to testEventFromInternalFiltered
CustomAuditAnnotations map[string]string
}
type AuditAnnotationsFilter func(key, val string) bool
// MissingEventsReport provides an analysis if any events are missing
type MissingEventsReport struct {
FirstEventChecked *auditinternal.Event
@ -78,6 +83,13 @@ func (m *MissingEventsReport) String() string {
// CheckAuditLines searches the audit log for the expected audit lines.
func CheckAuditLines(stream io.Reader, expected []AuditEvent, version schema.GroupVersion) (missingReport *MissingEventsReport, err error) {
return CheckAuditLinesFiltered(stream, expected, version, nil)
}
// CheckAuditLinesFiltered searches the audit log for the expected audit lines, customAnnotationsFilter
// controls which audit annotations are added to AuditEvent.CustomAuditAnnotations.
// If the customAnnotationsFilter is nil, AuditEvent.CustomAuditAnnotations will be empty.
func CheckAuditLinesFiltered(stream io.Reader, expected []AuditEvent, version schema.GroupVersion, customAnnotationsFilter AuditAnnotationsFilter) (missingReport *MissingEventsReport, err error) {
expectations := newAuditEventTracker(expected)
scanner := bufio.NewScanner(stream)
@ -100,7 +112,7 @@ func CheckAuditLines(stream io.Reader, expected []AuditEvent, version schema.Gro
}
missingReport.LastEventChecked = e
event, err := testEventFromInternal(e)
event, err := testEventFromInternalFiltered(e, customAnnotationsFilter)
if err != nil {
return missingReport, err
}
@ -162,6 +174,13 @@ func CheckForDuplicates(el auditinternal.EventList) (auditinternal.EventList, er
// testEventFromInternal takes an internal audit event and returns a test event
func testEventFromInternal(e *auditinternal.Event) (AuditEvent, error) {
return testEventFromInternalFiltered(e, nil)
}
// testEventFromInternalFiltered takes an internal audit event and returns a test event, customAnnotationsFilter
// controls which audit annotations are added to AuditEvent.CustomAuditAnnotations.
// If the customAnnotationsFilter is nil, AuditEvent.CustomAuditAnnotations will be empty.
func testEventFromInternalFiltered(e *auditinternal.Event, customAnnotationsFilter AuditAnnotationsFilter) (AuditEvent, error) {
event := AuditEvent{
Level: e.Level,
Stage: e.Stage,
@ -199,6 +218,11 @@ func testEventFromInternal(e *auditinternal.Event) (AuditEvent, error) {
event.AdmissionWebhookMutationAnnotations = map[string]string{}
}
event.AdmissionWebhookMutationAnnotations[k] = v
} else if customAnnotationsFilter != nil && customAnnotationsFilter(k, v) {
if event.CustomAuditAnnotations == nil {
event.CustomAuditAnnotations = map[string]string{}
}
event.CustomAuditAnnotations[k] = v
}
}
return event, nil