Merge pull request #46557 from timstclair/audit-test

Automatic merge from submit-queue (batch tested with PRs 46648, 46500, 46238, 46668, 46557)

Add an e2e test for AdvancedAuditing

Enable a simple "advanced auditing" setup for e2e tests running on GCE, and add an e2e test that creates & deletes a pod, a secret, and verifies that they're audited.

Includes https://github.com/kubernetes/kubernetes/pull/46548

For https://github.com/kubernetes/features/issues/22

/cc @ericchiang @sttts @soltysh @ihmccreery

cluster/common.sh

@@ -656,6 +656,7 @@ NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-})
 KUBE_UID: $(yaml-quote ${KUBE_UID:-})
 ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-})
 ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-})
+ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-})
 ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false})
 EOF
   if [ -n "${KUBELET_PORT:-}" ]; then

cluster/gce/config-test.sh

@@ -299,3 +299,9 @@ ENABLE_LEGACY_ABAC="${ENABLE_LEGACY_ABAC:-false}" # true, false
 # TODO(dawn1107): Remove this once the flag is built into CVM image.
 # Kernel panic upon soft lockup issue
 SOFTLOCKUP_PANIC="${SOFTLOCKUP_PANIC:-true}" # true, false
+
+# Enable a simple "AdvancedAuditing" setup for testing.
+ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-true}" # true, false
+if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT}" == "true" ]]; then
+  FEATURE_GATES="${FEATURE_GATES},AdvancedAuditing=true"
+fi

cluster/gce/gci/configure-helper.sh

@@ -425,6 +425,20 @@ EOF
   fi
 }
 
+function create-master-audit-policy {
+  # This is the config for the audit policy.
+  # TODO(timstclair): Provide a more thorough policy.
+  cat <<EOF >/etc/audit_policy.config
+rules:
+  - level: None
+    nonResourceURLs:
+      - /healthz*
+      - /version
+      - /swagger*
+  - level: Metadata
+EOF
+}
+
 function create-kubelet-kubeconfig {
   echo "Creating kubelet kubeconfig file"
   cat <<EOF >/var/lib/kubelet/kubeconfig
@@ -1053,6 +1067,8 @@ function start-kube-apiserver {
     params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}"
   fi
 
+  local audit_policy_config_mount=""
+  local audit_policy_config_volume=""
   if [[ "${ENABLE_APISERVER_BASIC_AUDIT:-}" == "true" ]]; then
     # We currently only support enabling with a fixed path and with built-in log
     # rotation "disabled" (large value) so it behaves like kube-apiserver.log.
@@ -1066,6 +1082,27 @@ function start-kube-apiserver {
     # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver
     # never restarts. Please manually restart apiserver before this time.
     params+=" --audit-log-maxsize=2000000000"
+  elif [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
+    # We currently only support enabling with a fixed path and with built-in log
+    # rotation "disabled" (large value) so it behaves like kube-apiserver.log.
+    # External log rotation should be set up the same as for kube-apiserver.log.
+    params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
+    params+=" --audit-log-maxage=0"
+    params+=" --audit-log-maxbackup=0"
+    # Lumberjack doesn't offer any way to disable size-based rotation. It also
+    # has an in-memory counter that doesn't notice if you truncate the file.
+    # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log
+    # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver
+    # never restarts. Please manually restart apiserver before this time.
+    params+=" --audit-log-maxsize=2000000000"
+
+    local audit_policy_file="/etc/audit_policy.config"
+    params+=" --audit-policy-file=${audit_policy_file}"
+
+    # Create the audit policy file, and mount it into the apiserver pod.
+    create-master-audit-policy
+    audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": false},"
+    audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\"}},"
   fi
 
   if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then
@@ -1174,6 +1211,8 @@ function start-kube-apiserver {
   sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
   sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"
   sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}"
+  sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
+  sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
   sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}"
   sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}"
   sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -176,8 +176,12 @@
 {% endif -%}
 
 {% set audit_log = "" -%}
+{% set audit_policy_config_mount = "" -%}
+{% set audit_policy_config_volume = "" -%}
 {% if pillar['enable_apiserver_basic_audit'] is defined and pillar['enable_apiserver_basic_audit'] in ['true']  -%}
   {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%}
+{% elif pillar['enable_apiserver_advanced_audit'] is defined and pillar['enable_apiserver_advanced_audit'] in ['true']  -%}
+  {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000 --audit-policy-file=/etc/audit_policy.config" -%}
 {% endif -%}
 
 {% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%}
@@ -240,6 +244,7 @@
         {{additional_cloud_config_mount}}
         {{webhook_config_mount}}
         {{webhook_authn_config_mount}}
+        {{audit_policy_config_mount}}
         {{admission_controller_config_mount}}
         {{image_policy_webhook_config_mount}}
         { "name": "srvkube",
@@ -277,6 +282,7 @@
   {{additional_cloud_config_volume}}
   {{webhook_config_volume}}
   {{webhook_authn_config_volume}}
+  {{audit_policy_config_volume}}
   {{admission_controller_config_volume}}
   {{image_policy_webhook_config_volume}}
   { "name": "srvkube",

test/e2e/BUILD

@@ -44,6 +44,7 @@ go_library(
     srcs = [
         "addon_update.go",
         "apparmor.go",
+        "audit.go",
         "cadvisor.go",
         "certificates.go",
         "cluster_upgrade.go",

test/e2e/audit.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"bufio"
+	"fmt"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiv1 "k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = framework.KubeDescribe("Advanced Audit [Feature:Audit]", func() {
+	f := framework.NewDefaultFramework("audit")
+
+	It("should audit API calls", func() {
+		namespace := f.Namespace.Name
+
+		// Create & Delete pod
+		pod := &apiv1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "audit-pod",
+			},
+			Spec: apiv1.PodSpec{
+				Containers: []apiv1.Container{{
+					Name:  "pause",
+					Image: framework.GetPauseImageName(f.ClientSet),
+				}},
+			},
+		}
+		f.PodClient().CreateSync(pod)
+		f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
+
+		// Create, Read, Delete secret
+		secret := &apiv1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "audit-secret",
+			},
+			Data: map[string][]byte{
+				"top-secret": []byte("foo-bar"),
+			},
+		}
+		_, err := f.ClientSet.Core().Secrets(f.Namespace.Name).Create(secret)
+		framework.ExpectNoError(err, "failed to create audit-secret")
+		_, err = f.ClientSet.Core().Secrets(f.Namespace.Name).Get(secret.Name, metav1.GetOptions{})
+		framework.ExpectNoError(err, "failed to get audit-secret")
+		err = f.ClientSet.Core().Secrets(f.Namespace.Name).Delete(secret.Name, &metav1.DeleteOptions{})
+		framework.ExpectNoError(err, "failed to delete audit-secret")
+
+		// /version should not be audited
+		_, err = f.ClientSet.Core().RESTClient().Get().AbsPath("/version").DoRaw()
+		framework.ExpectNoError(err, "failed to query version")
+
+		expectedEvents := []auditEvent{{
+			method:    "create",
+			namespace: namespace,
+			uri:       fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
+			response:  "201",
+		}, {
+			method:    "delete",
+			namespace: namespace,
+			uri:       fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, pod.Name),
+			response:  "200",
+		}, {
+			method:    "create",
+			namespace: namespace,
+			uri:       fmt.Sprintf("/api/v1/namespaces/%s/secrets", namespace),
+			response:  "201",
+		}, {
+			method:    "get",
+			namespace: namespace,
+			uri:       fmt.Sprintf("/api/v1/namespaces/%s/secrets/%s", namespace, secret.Name),
+			response:  "200",
+		}, {
+			method:    "delete",
+			namespace: namespace,
+			uri:       fmt.Sprintf("/api/v1/namespaces/%s/secrets/%s", namespace, secret.Name),
+			response:  "200",
+		}}
+		expectAuditLines(f, expectedEvents)
+	})
+})
+
+type auditEvent struct {
+	method, namespace, uri, response string
+}
+
+// Search the audit log for the expected audit lines.
+func expectAuditLines(f *framework.Framework, expected []auditEvent) {
+	expectations := map[auditEvent]bool{}
+	for _, event := range expected {
+		expectations[event] = false
+	}
+
+	// Fetch the log stream.
+	stream, err := f.ClientSet.Core().RESTClient().Get().AbsPath("/logs/kube-apiserver-audit.log").Stream()
+	framework.ExpectNoError(err, "could not read audit log")
+	defer stream.Close()
+
+	scanner := bufio.NewScanner(stream)
+	for scanner.Scan() {
+		line := scanner.Text()
+		event, err := parseAuditLine(line)
+		framework.ExpectNoError(err)
+
+		// If the event was expected, mark it as found.
+		if _, found := expectations[event]; found {
+			expectations[event] = true
+		}
+
+		// /version should not be audited (filtered in the policy).
+		Expect(event.uri).NotTo(HavePrefix("/version"))
+	}
+	framework.ExpectNoError(scanner.Err(), "error reading audit log")
+
+	for event, found := range expectations {
+		Expect(found).To(BeTrue(), "Event %#v not found!", event)
+	}
+}
+
+func parseAuditLine(line string) (auditEvent, error) {
+	fields := strings.Fields(line)
+	if len(fields) < 3 {
+		return auditEvent{}, fmt.Errorf("could not parse audit line: %s", line)
+	}
+	// Ignore first field (timestamp)
+	if fields[1] != "AUDIT:" {
+		return auditEvent{}, fmt.Errorf("unexpected audit line format: %s", line)
+	}
+	fields = fields[2:]
+	event := auditEvent{}
+	for _, f := range fields {
+		parts := strings.SplitN(f, "=", 2)
+		if len(parts) != 2 {
+			return auditEvent{}, fmt.Errorf("could not parse audit line (part: %q): %s", f, line)
+		}
+		value := strings.Trim(parts[1], "\"")
+		switch parts[0] {
+		case "method":
+			event.method = value
+		case "namespace":
+			event.namespace = value
+		case "uri":
+			event.uri = value
+		case "response":
+			event.response = value
+		}
+	}
+	return event, nil
+}