From 858e4508c4e41a857573e26aeba75c5b306fb0f7 Mon Sep 17 00:00:00 2001 From: Cao Shufeng Date: Fri, 27 Jul 2018 18:07:27 +0800 Subject: [PATCH] add an integration test for advanced audit feature --- test/e2e/auth/BUILD | 2 + test/e2e/auth/audit.go | 923 ++++++++++++-------------- test/integration/master/BUILD | 7 + test/integration/master/audit_test.go | 264 ++++++++ test/utils/BUILD | 3 + test/utils/audit.go | 106 +++ 6 files changed, 806 insertions(+), 499 deletions(-) create mode 100644 test/integration/master/audit_test.go create mode 100644 test/utils/audit.go diff --git a/test/e2e/auth/BUILD b/test/e2e/auth/BUILD index a5afae44546..5fdff943cdb 100644 --- a/test/e2e/auth/BUILD +++ b/test/e2e/auth/BUILD @@ -41,6 +41,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", @@ -49,6 +50,7 @@ go_library( "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/utils:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", diff --git a/test/e2e/auth/audit.go b/test/e2e/auth/audit.go index 80705d12289..97aaf5c8295 100644 --- a/test/e2e/auth/audit.go +++ b/test/e2e/auth/audit.go @@ -17,7 +17,6 @@ limitations under the License. package auth import ( - "bufio" "encoding/json" "fmt" "strings" @@ -31,10 +30,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/apis/audit/v1beta1" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" "github.com/evanphx/json-patch" @@ -80,7 +81,7 @@ var _ = SIGDescribe("Advanced Audit", func() { testCases := []struct { action func() - events []auditEvent + events []utils.AuditEvent }{ // Create, get, update, patch, delete, list, watch pods. { @@ -118,103 +119,103 @@ var _ = SIGDescribe("Advanced Audit", func() { f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) }, - []auditEvent{ + []utils.AuditEvent{ { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), - "create", - 201, - auditTestUser, - "pods", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), - "get", - 200, - auditTestUser, - "pods", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "get", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), - "list", - 200, - auditTestUser, - "pods", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), + Verb: "list", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseStarted, - fmt.Sprintf("/api/v1/namespaces/%s/pods?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "pods", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseStarted, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "pods", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), - "update", - 200, - auditTestUser, - "pods", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "update", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), - "patch", - 200, - auditTestUser, - "pods", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "patch", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), - "delete", - 200, - auditTestUser, - "pods", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, }, }, @@ -247,103 +248,103 @@ var _ = SIGDescribe("Advanced Audit", func() { err = f.ClientSet.AppsV1().Deployments(namespace).Delete("audit-deployment", &metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete deployments") }, - []auditEvent{ + []utils.AuditEvent{ { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments", namespace), - "create", - 201, - auditTestUser, - "deployments", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments", namespace), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), - "get", - 200, - auditTestUser, - "deployments", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), + Verb: "get", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments", namespace), - "list", - 200, - auditTestUser, - "deployments", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments", namespace), + Verb: "list", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseStarted, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "deployments", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseStarted, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequest, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "deployments", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), - "update", - 200, - auditTestUser, - "deployments", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), + Verb: "update", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), - "patch", - 200, - auditTestUser, - "deployments", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), + Verb: "patch", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - v1beta1.LevelRequestResponse, - v1beta1.StageResponseComplete, - fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), - "delete", - 200, - auditTestUser, - "deployments", - namespace, - true, - true, - "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/audit-deployment", namespace), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "deployments", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, }, }, @@ -382,103 +383,103 @@ var _ = SIGDescribe("Advanced Audit", func() { err = f.ClientSet.CoreV1().ConfigMaps(namespace).Delete(configMap.Name, &metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete audit-configmap") }, - []auditEvent{ + []utils.AuditEvent{ { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace), - "create", - 201, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), - "get", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "get", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace), - "list", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace), + Verb: "list", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseStarted, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseStarted, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), - "update", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "update", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), - "patch", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "patch", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), - "delete", - 200, - auditTestUser, - "configmaps", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, }, }, @@ -516,103 +517,103 @@ var _ = SIGDescribe("Advanced Audit", func() { err = f.ClientSet.CoreV1().Secrets(namespace).Delete(secret.Name, &metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete audit-secret") }, - []auditEvent{ + []utils.AuditEvent{ { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets", namespace), - "create", - 201, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets", namespace), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), - "get", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), + Verb: "get", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets", namespace), - "list", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets", namespace), + Verb: "list", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseStarted, - fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseStarted, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), - "watch", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), - "update", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), + Verb: "update", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), - "patch", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), + Verb: "patch", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - v1beta1.LevelMetadata, - v1beta1.StageResponseComplete, - fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), - "delete", - 200, - auditTestUser, - "secrets", - namespace, - false, - false, - "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets/audit-secret", namespace), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "secrets", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, }, }, @@ -623,51 +624,51 @@ var _ = SIGDescribe("Advanced Audit", func() { framework.ExpectNoError(err, "failed to create custom resource definition") fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient) }, - []auditEvent{ + []utils.AuditEvent{ { - level: v1beta1.LevelRequestResponse, - stage: v1beta1.StageResponseComplete, - requestURI: "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions", - verb: "create", - code: 201, - user: auditTestUser, - resource: "customresourcedefinitions", - requestObject: true, - responseObject: true, - authorizeDecision: "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions", + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "customresourcedefinitions", + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - level: v1beta1.LevelMetadata, - stage: v1beta1.StageResponseComplete, - requestURI: fmt.Sprintf("/apis/%s/v1beta1/%s", crdNamespace, crdName), - verb: "create", - code: 201, - user: auditTestUser, - resource: crdName, - requestObject: false, - responseObject: false, - authorizeDecision: "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/%s/v1beta1/%s", crdNamespace, crdName), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: crdName, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, { - level: v1beta1.LevelRequestResponse, - stage: v1beta1.StageResponseComplete, - requestURI: fmt.Sprintf("/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/%s", crd.Name), - verb: "delete", - code: 200, - user: auditTestUser, - resource: "customresourcedefinitions", - requestObject: false, - responseObject: true, - authorizeDecision: "allow", + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/%s", crd.Name), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "customresourcedefinitions", + RequestObject: false, + ResponseObject: true, + AuthorizeDecision: "allow", }, { - level: v1beta1.LevelMetadata, - stage: v1beta1.StageResponseComplete, - requestURI: fmt.Sprintf("/apis/%s/v1beta1/%s/setup-instance", crdNamespace, crdName), - verb: "delete", - code: 200, - user: auditTestUser, - resource: crdName, - requestObject: false, - responseObject: false, - authorizeDecision: "allow", + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/apis/%s/v1beta1/%s/setup-instance", crdNamespace, crdName), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: crdName, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", }, }, }, @@ -676,7 +677,7 @@ var _ = SIGDescribe("Advanced Audit", func() { // test authorizer annotations, RBAC is required. annotationTestCases := []struct { action func() - events []auditEvent + events []utils.AuditEvent }{ // get a pod with unauthorized user @@ -685,19 +686,19 @@ var _ = SIGDescribe("Advanced Audit", func() { _, err := anonymousClient.CoreV1().Pods(namespace).Get("another-audit-pod", metav1.GetOptions{}) expectForbidden(err) }, - []auditEvent{ + []utils.AuditEvent{ { - level: v1beta1.LevelRequest, - stage: v1beta1.StageResponseComplete, - requestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace), - verb: "get", - code: 403, - user: auditTestUser, - resource: "pods", - namespace: namespace, - requestObject: false, - responseObject: false, - authorizeDecision: "forbid", + Level: auditinternal.LevelRequest, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace), + Verb: "get", + Code: 403, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "forbid", }, }, }, @@ -706,7 +707,7 @@ var _ = SIGDescribe("Advanced Audit", func() { if framework.IsRBACEnabled(f) { testCases = append(testCases, annotationTestCases...) } - expectedEvents := []auditEvent{} + expectedEvents := []utils.AuditEvent{} for _, t := range testCases { t.action() expectedEvents = append(expectedEvents, t.events...) @@ -717,96 +718,20 @@ var _ = SIGDescribe("Advanced Audit", func() { pollingInterval := 30 * time.Second pollingTimeout := 5 * time.Minute err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) { - ok, err := checkAuditLines(f, expectedEvents) + // Fetch the log stream. + stream, err := f.ClientSet.CoreV1().RESTClient().Get().AbsPath("/logs/kube-apiserver-audit.log").Stream() + if err != nil { + return false, err + } + defer stream.Close() + missing, err := utils.CheckAuditLines(stream, expectedEvents, v1beta1.SchemeGroupVersion) if err != nil { framework.Logf("Failed to observe audit events: %v", err) + } else if len(missing) > 0 { + framework.Logf("Events %#v not found!", missing) } - return ok, nil + return len(missing) == 0, nil }) framework.ExpectNoError(err, "after %v failed to observe audit events", pollingTimeout) }) }) - -type auditEvent struct { - level v1beta1.Level - stage v1beta1.Stage - requestURI string - verb string - code int32 - user string - resource string - namespace string - requestObject bool - responseObject bool - authorizeDecision string -} - -// Search the audit log for the expected audit lines. -func checkAuditLines(f *framework.Framework, expected []auditEvent) (bool, error) { - expectations := map[auditEvent]bool{} - for _, event := range expected { - expectations[event] = false - } - - // Fetch the log stream. - stream, err := f.ClientSet.CoreV1().RESTClient().Get().AbsPath("/logs/kube-apiserver-audit.log").Stream() - if err != nil { - return false, err - } - defer stream.Close() - - scanner := bufio.NewScanner(stream) - for scanner.Scan() { - line := scanner.Text() - event, err := parseAuditLine(line) - if err != nil { - return false, err - } - - // If the event was expected, mark it as found. - if _, found := expectations[event]; found { - expectations[event] = true - } - } - if err := scanner.Err(); err != nil { - return false, err - } - - noneMissing := true - for event, found := range expectations { - if !found { - framework.Logf("Event %#v not found!", event) - } - noneMissing = noneMissing && found - } - return noneMissing, nil -} - -func parseAuditLine(line string) (auditEvent, error) { - var e v1beta1.Event - if err := json.Unmarshal([]byte(line), &e); err != nil { - return auditEvent{}, err - } - event := auditEvent{ - level: e.Level, - stage: e.Stage, - requestURI: e.RequestURI, - verb: e.Verb, - user: e.User.Username, - } - if e.ObjectRef != nil { - event.namespace = e.ObjectRef.Namespace - event.resource = e.ObjectRef.Resource - } - if e.ResponseStatus != nil { - event.code = e.ResponseStatus.Code - } - if e.ResponseObject != nil { - event.responseObject = true - } - if e.RequestObject != nil { - event.requestObject = true - } - event.authorizeDecision = e.Annotations["authorization.k8s.io/decision"] - return event, nil -} diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index 96fe7cd49d4..fa012cd25ad 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -10,6 +10,7 @@ go_test( name = "go_default_test", size = "large", srcs = [ + "audit_test.go", "crd_test.go", "kms_transformation_test.go", "kube_apiserver_test.go", @@ -37,7 +38,11 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/group:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/bearertoken:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", @@ -58,6 +63,8 @@ go_test( "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", "//test/integration:go_default_library", "//test/integration/framework:go_default_library", + "//test/utils:go_default_library", + "//vendor/github.com/evanphx/json-patch:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ diff --git a/test/integration/master/audit_test.go b/test/integration/master/audit_test.go new file mode 100644 index 00000000000..35c0fccc234 --- /dev/null +++ b/test/integration/master/audit_test.go @@ -0,0 +1,264 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package master + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1" + "k8s.io/client-go/kubernetes" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/framework" + "k8s.io/kubernetes/test/utils" + + "github.com/evanphx/json-patch" +) + +var ( + auditPolicyPattern = ` +apiVersion: {version} +kind: Policy +rules: + - level: RequestResponse + resources: + - group: "" # core + resources: ["configmaps"] + +` + namespace = "default" + watchTestTimeout int64 = 1 + watchOptions = metav1.ListOptions{TimeoutSeconds: &watchTestTimeout} + patch, _ = json.Marshal(jsonpatch.Patch{}) + auditTestUser = "system:apiserver" + versions = map[string]schema.GroupVersion{ + "audit.k8s.io/v1": auditv1.SchemeGroupVersion, + "audit.k8s.io/v1beta1": auditv1beta1.SchemeGroupVersion, + } +) + +// TestAudit ensures that both v1beta1 and v1 version audit api could work. +func TestAudit(t *testing.T) { + for version := range versions { + testAudit(t, version) + } +} + +func testAudit(t *testing.T, version string) { + // prepare audit policy file + auditPolicy := []byte(strings.Replace(auditPolicyPattern, "{version}", version, 1)) + policyFile, err := ioutil.TempFile("", "audit-policy.yaml") + if err != nil { + t.Fatalf("Failed to create audit policy file: %v", err) + } + defer os.Remove(policyFile.Name()) + if _, err := policyFile.Write(auditPolicy); err != nil { + t.Fatalf("Failed to write audit policy file: %v", err) + } + if err := policyFile.Close(); err != nil { + t.Fatalf("Failed to close audit policy file: %v", err) + } + + // prepare audit log file + logFile, err := ioutil.TempFile("", "audit.log") + if err != nil { + t.Fatalf("Failed to create audit log file: %v", err) + } + defer os.Remove(logFile.Name()) + + // start api server + result := kubeapiservertesting.StartTestServerOrDie(t, nil, + []string{ + "--audit-policy-file", policyFile.Name(), + "--audit-log-version", version, + "--audit-log-mode", "blocking", + "--audit-log-path", logFile.Name()}, + framework.SharedEtcd()) + defer result.TearDownFn() + + kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + func() { + // create, get, watch, update, patch, list and delete configmap. + configMap := &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit-configmap", + }, + Data: map[string]string{ + "map-key": "map-value", + }, + } + + _, err := kubeclient.CoreV1().ConfigMaps(namespace).Create(configMap) + expectNoError(t, err, "failed to create audit-configmap") + + _, err = kubeclient.CoreV1().ConfigMaps(namespace).Get(configMap.Name, metav1.GetOptions{}) + expectNoError(t, err, "failed to get audit-configmap") + + configMapChan, err := kubeclient.CoreV1().ConfigMaps(namespace).Watch(watchOptions) + expectNoError(t, err, "failed to create watch for config maps") + for range configMapChan.ResultChan() { + // Block until watchOptions.TimeoutSeconds expires. + // If the test finishes before watchOptions.TimeoutSeconds expires, the watch audit + // event at stage ResponseComplete will not be generated. + } + + _, err = kubeclient.CoreV1().ConfigMaps(namespace).Update(configMap) + expectNoError(t, err, "failed to update audit-configmap") + + _, err = kubeclient.CoreV1().ConfigMaps(namespace).Patch(configMap.Name, types.JSONPatchType, patch) + expectNoError(t, err, "failed to patch configmap") + + _, err = kubeclient.CoreV1().ConfigMaps(namespace).List(metav1.ListOptions{}) + expectNoError(t, err, "failed to list config maps") + + err = kubeclient.CoreV1().ConfigMaps(namespace).Delete(configMap.Name, &metav1.DeleteOptions{}) + expectNoError(t, err, "failed to delete audit-configmap") + }() + + expectedEvents := []utils.AuditEvent{ + { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "get", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace), + Verb: "list", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseStarted, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "update", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "patch", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "configmaps", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, + } + + stream, err := os.Open(logFile.Name()) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer stream.Close() + missing, err := utils.CheckAuditLines(stream, expectedEvents, versions[version]) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(missing) > 0 { + t.Errorf("Failed to match all expected events, events %#v not found!", missing) + } +} + +func expectNoError(t *testing.T, err error, msg string) { + if err != nil { + t.Fatalf("%s: %v", msg, err) + } +} diff --git a/test/utils/BUILD b/test/utils/BUILD index eed466cc119..5a1023742f8 100644 --- a/test/utils/BUILD +++ b/test/utils/BUILD @@ -8,6 +8,7 @@ load( go_library( name = "go_default_library", srcs = [ + "audit.go", "conditions.go", "create_resources.go", "delete_resources.go", @@ -49,6 +50,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", diff --git a/test/utils/audit.go b/test/utils/audit.go new file mode 100644 index 00000000000..e4cd1b07722 --- /dev/null +++ b/test/utils/audit.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bufio" + "fmt" + "io" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/audit" +) + +type AuditEvent struct { + Level auditinternal.Level + Stage auditinternal.Stage + RequestURI string + Verb string + Code int32 + User string + Resource string + Namespace string + RequestObject bool + ResponseObject bool + AuthorizeDecision string +} + +// Search the audit log for the expected audit lines. +func CheckAuditLines(stream io.Reader, expected []AuditEvent, version schema.GroupVersion) (missing []AuditEvent, err error) { + expectations := map[AuditEvent]bool{} + for _, event := range expected { + expectations[event] = false + } + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + line := scanner.Text() + event, err := parseAuditLine(line, version) + if err != nil { + return expected, err + } + + // If the event was expected, mark it as found. + if _, found := expectations[event]; found { + expectations[event] = true + } + } + if err := scanner.Err(); err != nil { + return expected, err + } + + missing = make([]AuditEvent, 0) + for event, found := range expectations { + if !found { + missing = append(missing, event) + } + } + return missing, nil +} + +func parseAuditLine(line string, version schema.GroupVersion) (AuditEvent, error) { + e := &auditinternal.Event{} + decoder := audit.Codecs.UniversalDecoder(version) + if err := runtime.DecodeInto(decoder, []byte(line), e); err != nil { + return AuditEvent{}, fmt.Errorf("failed decoding buf: %s, apiVersion: %s", line, version) + } + + event := AuditEvent{ + Level: e.Level, + Stage: e.Stage, + RequestURI: e.RequestURI, + Verb: e.Verb, + User: e.User.Username, + } + if e.ObjectRef != nil { + event.Namespace = e.ObjectRef.Namespace + event.Resource = e.ObjectRef.Resource + } + if e.ResponseStatus != nil { + event.Code = e.ResponseStatus.Code + } + if e.ResponseObject != nil { + event.ResponseObject = true + } + if e.RequestObject != nil { + event.RequestObject = true + } + event.AuthorizeDecision = e.Annotations["authorization.k8s.io/decision"] + return event, nil +}