diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 7ff43a1b552..774154f6bd2 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -376,6 +376,8 @@ else ADMISSION_CONTROL=${KUBE_ADMISSION_CONTROL} fi +ENABLE_APISERVER_DYNAMIC_AUDIT="${ENABLE_APISERVER_DYNAMIC_AUDIT:-false}" + # Optional: if set to true kube-up will automatically check for existing resources and clean them up. KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false} diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 256f5b90f85..d3ac8d26d1c 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1709,6 +1709,11 @@ function start-kube-apiserver { fi fi + if [[ "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}" == "true" ]]; then + params+=" --audit-dynamic-configuration" + RUNTIME_CONFIG="${RUNTIME_CONFIG},auditconfiguration.k8s.io/v1alpha1=true" + fi + if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then params+=" --enable-logs-handler=false" fi diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index dbe053a89e7..24d4bdc8f0d 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1111,6 +1111,7 @@ MULTIZONE: $(yaml-quote ${MULTIZONE:-}) NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-}) ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-}) ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) +ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_DYNAMIC_AUDIT:-}) ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false}) ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote ${ENABLE_PATCH_CONVERSION_DETECTOR:-false}) ADVANCED_AUDIT_POLICY: $(yaml-quote ${ADVANCED_AUDIT_POLICY:-}) diff --git a/test/e2e/auth/BUILD b/test/e2e/auth/BUILD index e8b93c2865c..e55b179704d 100644 --- a/test/e2e/auth/BUILD +++ b/test/e2e/auth/BUILD @@ -9,6 +9,7 @@ go_library( name = "go_default_library", srcs = [ "audit.go", + "audit_dynamic.go", "certificates.go", "framework.go", "metadata_concealment.go", @@ -26,6 +27,7 @@ go_library( "//pkg/security/podsecuritypolicy/util:go_default_library", "//plugin/pkg/admission/serviceaccount:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/batch/v1:go_default_library", "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", @@ -39,6 +41,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/test/e2e/auth/audit_dynamic.go b/test/e2e/auth/audit_dynamic.go new file mode 100644 index 00000000000..e3145fe8c99 --- /dev/null +++ b/test/e2e/auth/audit_dynamic.go @@ -0,0 +1,381 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo" + + auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +var _ = SIGDescribe("[Feature:DynamicAudit]", func() { + f := framework.NewDefaultFramework("audit") + + It("should dynamically audit API calls", func() { + namespace := f.Namespace.Name + + By("Creating a kubernetes client that impersonates an unauthorized anonymous user") + config, err := framework.LoadConfig() + framework.ExpectNoError(err, "failed to fetch config") + + config.Impersonate = restclient.ImpersonationConfig{ + UserName: "system:anonymous", + Groups: []string{"system:unauthenticated"}, + } + anonymousClient, err := clientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create the anonymous client") + + _, err = f.ClientSet.CoreV1().Namespaces().Create(&apiv1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit", + }, + }) + framework.ExpectNoError(err, "failed to create namespace") + + _, err = f.ClientSet.CoreV1().Pods(namespace).Create(&apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit-proxy", + Labels: map[string]string{ + "app": "audit", + }, + }, + Spec: apiv1.PodSpec{ + Containers: []apiv1.Container{ + { + Name: "proxy", + Image: imageutils.GetE2EImage(imageutils.AuditProxy), + Ports: []apiv1.ContainerPort{ + { + ContainerPort: 8080, + }, + }, + }, + }, + }, + }) + framework.ExpectNoError(err, "failed to create proxy pod") + + _, err = f.ClientSet.CoreV1().Services(namespace).Create(&apiv1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit", + }, + Spec: apiv1.ServiceSpec{ + Ports: []apiv1.ServicePort{ + { + Port: 80, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + }, + }, + Selector: map[string]string{ + "app": "audit", + }, + }, + }) + framework.ExpectNoError(err, "failed to create proxy service") + + config, err = framework.LoadConfig() + framework.ExpectNoError(err, "failed to load config") + + var podIP string + // get pod ip + err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) { + p, err := f.ClientSet.CoreV1().Pods(namespace).Get("audit-proxy", metav1.GetOptions{}) + if errors.IsNotFound(err) { + framework.Logf("waiting for audit-proxy pod to be present") + return false, nil + } else if err != nil { + return false, err + } + podIP = p.Status.PodIP + if podIP == "" { + framework.Logf("waiting for audit-proxy pod IP to be ready") + return false, nil + } + return true, nil + }) + framework.ExpectNoError(err, "timed out waiting for audit-proxy pod to be ready") + + podURL := fmt.Sprintf("http://%s:8080", podIP) + // create audit sink + sink := auditregv1alpha1.AuditSink{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: auditregv1alpha1.Policy{ + Level: auditregv1alpha1.LevelRequestResponse, + Stages: []auditregv1alpha1.Stage{ + auditregv1alpha1.StageRequestReceived, + auditregv1alpha1.StageResponseStarted, + auditregv1alpha1.StageResponseComplete, + auditregv1alpha1.StagePanic, + }, + }, + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &podURL, + }, + }, + }, + } + + _, err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Create(&sink) + framework.ExpectNoError(err, "failed to create audit sink") + framework.Logf("created audit sink") + + // check that we are receiving logs in the proxy + err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) { + logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy") + if err != nil { + framework.Logf("waiting for audit-proxy pod logs to be available") + return false, nil + } + if logs == "" { + framework.Logf("waiting for audit-proxy pod logs to be non-empty") + return false, nil + } + return true, nil + }) + framework.ExpectNoError(err, "failed to get logs from audit-proxy pod") + + auditTestUser = "kubernetes-admin" + testCases := []struct { + action func() + events []utils.AuditEvent + }{ + // Create, get, update, patch, delete, list, watch pods. + // TODO(@pbarker): dedupe this with the main audit test once policy functionality is available + // https://github.com/kubernetes/kubernetes/issues/70818 + { + func() { + pod := &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit-pod", + }, + Spec: apiv1.PodSpec{ + Containers: []apiv1.Container{{ + Name: "pause", + Image: imageutils.GetPauseImageName(), + }}, + }, + } + updatePod := func(pod *apiv1.Pod) {} + + f.PodClient().CreateSync(pod) + + _, err := f.PodClient().Get(pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get audit-pod") + + podChan, err := f.PodClient().Watch(watchOptions) + framework.ExpectNoError(err, "failed to create watch for pods") + for range podChan.ResultChan() { + } + + f.PodClient().Update(pod.Name, updatePod) + + _, err = f.PodClient().List(metav1.ListOptions{}) + framework.ExpectNoError(err, "failed to list pods") + + _, err = f.PodClient().Patch(pod.Name, types.JSONPatchType, patch) + framework.ExpectNoError(err, "failed to patch pod") + + f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) + }, + []utils.AuditEvent{ + { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), + Verb: "create", + Code: 201, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "get", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), + Verb: "list", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseStarted, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), + Verb: "watch", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: false, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "update", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "patch", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace), + Verb: "delete", + Code: 200, + User: auditTestUser, + Resource: "pods", + Namespace: namespace, + RequestObject: true, + ResponseObject: true, + AuthorizeDecision: "allow", + }, + }, + }, + } + + // test authorizer annotations, RBAC is required. + annotationTestCases := []struct { + action func() + events []utils.AuditEvent + }{ + + // get a pod with unauthorized user + { + func() { + _, err := anonymousClient.CoreV1().Pods(namespace).Get("another-audit-pod", metav1.GetOptions{}) + expectForbidden(err) + }, + []utils.AuditEvent{ + { + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace), + Verb: "get", + Code: 403, + User: auditTestUser, + ImpersonatedUser: "system:anonymous", + ImpersonatedGroups: "system:unauthenticated", + Resource: "pods", + Namespace: namespace, + RequestObject: false, + ResponseObject: true, + AuthorizeDecision: "forbid", + }, + }, + }, + } + + if framework.IsRBACEnabled(f) { + testCases = append(testCases, annotationTestCases...) + } + expectedEvents := []utils.AuditEvent{} + for _, t := range testCases { + t.action() + expectedEvents = append(expectedEvents, t.events...) + } + + // The default flush timeout is 30 seconds, therefore it should be enough to retry once + // to find all expected events. However, we're waiting for 5 minutes to avoid flakes. + pollingInterval := 30 * time.Second + pollingTimeout := 5 * time.Minute + err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) { + // Fetch the logs + logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy") + if err != nil { + return false, err + } + reader := strings.NewReader(logs) + missingReport, err := utils.CheckAuditLines(reader, expectedEvents, auditv1.SchemeGroupVersion) + if err != nil { + framework.Logf("Failed to observe audit events: %v", err) + } else if len(missingReport.MissingEvents) > 0 { + framework.Logf(missingReport.String()) + } + return len(missingReport.MissingEvents) == 0, nil + }) + framework.ExpectNoError(err, "after %v failed to observe audit events", pollingTimeout) + err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Delete("test", &metav1.DeleteOptions{}) + framework.ExpectNoError(err, "could not delete audit configuration") + }) +}) diff --git a/test/e2e/common/util.go b/test/e2e/common/util.go index 113f1fffca1..9af3d2f2c93 100644 --- a/test/e2e/common/util.go +++ b/test/e2e/common/util.go @@ -48,6 +48,7 @@ var CurrentSuite Suite // only used by node e2e test. // TODO(random-liu): Change the image puller pod to use similar mechanism. var CommonImageWhiteList = sets.NewString( + imageutils.GetE2EImage(imageutils.AuditProxy), imageutils.GetE2EImage(imageutils.BusyBox), imageutils.GetE2EImage(imageutils.EntrypointTester), imageutils.GetE2EImage(imageutils.IpcUtils), diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index b51d0b9a479..26f3e4b1c97 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -105,6 +105,8 @@ const ( APIServer // AppArmorLoader image AppArmorLoader + // AuditProxy image + AuditProxy // BusyBox image BusyBox // CheckMetadataConcealment image @@ -196,6 +198,7 @@ func initImageConfigs() map[int]Config { configs[AdmissionWebhook] = Config{e2eRegistry, "webhook", "1.14v1"} configs[APIServer] = Config{e2eRegistry, "sample-apiserver", "1.10"} configs[AppArmorLoader] = Config{e2eRegistry, "apparmor-loader", "1.0"} + configs[AuditProxy] = Config{e2eRegistry, "audit-proxy", "1.0"} configs[BusyBox] = Config{dockerLibraryRegistry, "busybox", "1.29"} configs[CheckMetadataConcealment] = Config{e2eRegistry, "metadata-concealment", "1.2"} configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"}