diff --git a/cluster/common.sh b/cluster/common.sh index c535abf3571..57fd9b82519 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -656,6 +656,7 @@ NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-}) KUBE_UID: $(yaml-quote ${KUBE_UID:-}) ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-}) ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-}) +ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false}) EOF if [ -n "${KUBELET_PORT:-}" ]; then diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index bca8f376e83..5eb9a68af31 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -299,3 +299,9 @@ ENABLE_LEGACY_ABAC="${ENABLE_LEGACY_ABAC:-false}" # true, false # TODO(dawn1107): Remove this once the flag is built into CVM image. # Kernel panic upon soft lockup issue SOFTLOCKUP_PANIC="${SOFTLOCKUP_PANIC:-true}" # true, false + +# Enable a simple "AdvancedAuditing" setup for testing. +ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-true}" # true, false +if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT}" == "true" ]]; then + FEATURE_GATES="${FEATURE_GATES},AdvancedAuditing=true" +fi diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b4e5becf3bd..8215720b702 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -425,6 +425,20 @@ EOF fi } +function create-master-audit-policy { + # This is the config for the audit policy. + # TODO(timstclair): Provide a more thorough policy. + cat </etc/audit_policy.config +rules: + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + - level: Metadata +EOF +} + function create-kubelet-kubeconfig { echo "Creating kubelet kubeconfig file" cat </var/lib/kubelet/kubeconfig @@ -1053,6 +1067,8 @@ function start-kube-apiserver { params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}" fi + local audit_policy_config_mount="" + local audit_policy_config_volume="" if [[ "${ENABLE_APISERVER_BASIC_AUDIT:-}" == "true" ]]; then # We currently only support enabling with a fixed path and with built-in log # rotation "disabled" (large value) so it behaves like kube-apiserver.log. @@ -1066,6 +1082,27 @@ function start-kube-apiserver { # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver # never restarts. Please manually restart apiserver before this time. params+=" --audit-log-maxsize=2000000000" + elif [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then + # We currently only support enabling with a fixed path and with built-in log + # rotation "disabled" (large value) so it behaves like kube-apiserver.log. + # External log rotation should be set up the same as for kube-apiserver.log. + params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" + params+=" --audit-log-maxage=0" + params+=" --audit-log-maxbackup=0" + # Lumberjack doesn't offer any way to disable size-based rotation. It also + # has an in-memory counter that doesn't notice if you truncate the file. + # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log + # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver + # never restarts. Please manually restart apiserver before this time. + params+=" --audit-log-maxsize=2000000000" + + local audit_policy_file="/etc/audit_policy.config" + params+=" --audit-policy-file=${audit_policy_file}" + + # Create the audit policy file, and mount it into the apiserver pod. + create-master-audit-policy + audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": false}," + audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\"}}," fi if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then @@ -1174,6 +1211,8 @@ function start-kube-apiserver { sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" + sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}" sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 994c5e602ff..d8c7c96213e 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -176,8 +176,12 @@ {% endif -%} {% set audit_log = "" -%} +{% set audit_policy_config_mount = "" -%} +{% set audit_policy_config_volume = "" -%} {% if pillar['enable_apiserver_basic_audit'] is defined and pillar['enable_apiserver_basic_audit'] in ['true'] -%} {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%} +{% elif pillar['enable_apiserver_advanced_audit'] is defined and pillar['enable_apiserver_advanced_audit'] in ['true'] -%} + {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000 --audit-policy-file=/etc/audit_policy.config" -%} {% endif -%} {% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%} @@ -240,6 +244,7 @@ {{additional_cloud_config_mount}} {{webhook_config_mount}} {{webhook_authn_config_mount}} + {{audit_policy_config_mount}} {{admission_controller_config_mount}} {{image_policy_webhook_config_mount}} { "name": "srvkube", @@ -277,6 +282,7 @@ {{additional_cloud_config_volume}} {{webhook_config_volume}} {{webhook_authn_config_volume}} + {{audit_policy_config_volume}} {{admission_controller_config_volume}} {{image_policy_webhook_config_volume}} { "name": "srvkube", diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 2c96bf43e57..5155a8bd6f4 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -44,6 +44,7 @@ go_library( srcs = [ "addon_update.go", "apparmor.go", + "audit.go", "cadvisor.go", "certificates.go", "cluster_upgrade.go", diff --git a/test/e2e/audit.go b/test/e2e/audit.go new file mode 100644 index 00000000000..3536bcf39c1 --- /dev/null +++ b/test/e2e/audit.go @@ -0,0 +1,169 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bufio" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiv1 "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = framework.KubeDescribe("Advanced Audit [Feature:Audit]", func() { + f := framework.NewDefaultFramework("audit") + + It("should audit API calls", func() { + namespace := f.Namespace.Name + + // Create & Delete pod + pod := &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit-pod", + }, + Spec: apiv1.PodSpec{ + Containers: []apiv1.Container{{ + Name: "pause", + Image: framework.GetPauseImageName(f.ClientSet), + }}, + }, + } + f.PodClient().CreateSync(pod) + f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) + + // Create, Read, Delete secret + secret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "audit-secret", + }, + Data: map[string][]byte{ + "top-secret": []byte("foo-bar"), + }, + } + _, err := f.ClientSet.Core().Secrets(f.Namespace.Name).Create(secret) + framework.ExpectNoError(err, "failed to create audit-secret") + _, err = f.ClientSet.Core().Secrets(f.Namespace.Name).Get(secret.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get audit-secret") + err = f.ClientSet.Core().Secrets(f.Namespace.Name).Delete(secret.Name, &metav1.DeleteOptions{}) + framework.ExpectNoError(err, "failed to delete audit-secret") + + // /version should not be audited + _, err = f.ClientSet.Core().RESTClient().Get().AbsPath("/version").DoRaw() + framework.ExpectNoError(err, "failed to query version") + + expectedEvents := []auditEvent{{ + method: "create", + namespace: namespace, + uri: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace), + response: "201", + }, { + method: "delete", + namespace: namespace, + uri: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, pod.Name), + response: "200", + }, { + method: "create", + namespace: namespace, + uri: fmt.Sprintf("/api/v1/namespaces/%s/secrets", namespace), + response: "201", + }, { + method: "get", + namespace: namespace, + uri: fmt.Sprintf("/api/v1/namespaces/%s/secrets/%s", namespace, secret.Name), + response: "200", + }, { + method: "delete", + namespace: namespace, + uri: fmt.Sprintf("/api/v1/namespaces/%s/secrets/%s", namespace, secret.Name), + response: "200", + }} + expectAuditLines(f, expectedEvents) + }) +}) + +type auditEvent struct { + method, namespace, uri, response string +} + +// Search the audit log for the expected audit lines. +func expectAuditLines(f *framework.Framework, expected []auditEvent) { + expectations := map[auditEvent]bool{} + for _, event := range expected { + expectations[event] = false + } + + // Fetch the log stream. + stream, err := f.ClientSet.Core().RESTClient().Get().AbsPath("/logs/kube-apiserver-audit.log").Stream() + framework.ExpectNoError(err, "could not read audit log") + defer stream.Close() + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + line := scanner.Text() + event, err := parseAuditLine(line) + framework.ExpectNoError(err) + + // If the event was expected, mark it as found. + if _, found := expectations[event]; found { + expectations[event] = true + } + + // /version should not be audited (filtered in the policy). + Expect(event.uri).NotTo(HavePrefix("/version")) + } + framework.ExpectNoError(scanner.Err(), "error reading audit log") + + for event, found := range expectations { + Expect(found).To(BeTrue(), "Event %#v not found!", event) + } +} + +func parseAuditLine(line string) (auditEvent, error) { + fields := strings.Fields(line) + if len(fields) < 3 { + return auditEvent{}, fmt.Errorf("could not parse audit line: %s", line) + } + // Ignore first field (timestamp) + if fields[1] != "AUDIT:" { + return auditEvent{}, fmt.Errorf("unexpected audit line format: %s", line) + } + fields = fields[2:] + event := auditEvent{} + for _, f := range fields { + parts := strings.SplitN(f, "=", 2) + if len(parts) != 2 { + return auditEvent{}, fmt.Errorf("could not parse audit line (part: %q): %s", f, line) + } + value := strings.Trim(parts[1], "\"") + switch parts[0] { + case "method": + event.method = value + case "namespace": + event.namespace = value + case "uri": + event.uri = value + case "response": + event.response = value + } + } + return event, nil +}