Merge pull request #46897 from timstclair/audit-policy

Automatic merge from submit-queue (batch tested with PRs 46897, 46899, 46864, 46854, 46875)

Write audit policy file for GCE/GKE configuration

Setup the audit policy configuration for GCE & GKE. Here is the high level summary of the policy:

- Default logging everything at `Metadata`
- Known write APIs default to `RequestResponse`
- Known read-only APIs default to `Request`
- Except secrets & configmaps are logged at `Metadata`
- Don't log events
- Don't log `/version`, swagger or healthchecks

In addition to the above, I spent time analyzing the noisiest lines in the audit log from a cluster that soaked for 24 hours (and ran a batch of e2e tests). Of those top requests, those that were identified as low-risk (all read-only, except update kube-system endpoints by controllers) are dropped.

I suspect we'll want to tweak this a bit more once we've had a time to soak it on some real clusters.

For kubernetes/features#22

/cc @sttts @ericchiang

cluster/gce/gci/configure-helper.sh

@@ -425,16 +425,102 @@ EOF
   fi
 }
 
+# Write the config for the audit policy.
 function create-master-audit-policy {
-  # This is the config for the audit policy.
-  # TODO(timstclair): Provide a more thorough policy.
-  cat <<EOF >/etc/audit_policy.config
+  local -r path="${1}"
+
+  # Known api groups
+  local -r known_apis='
+      - group: "" # core
+      - group: "admissionregistration.k8s.io/v1alpha1"
+      - group: "apps/v1beta1"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io/v1beta1"
+      - group: "extensions/v1beta1"
+      - group: "networking.k8s.io/v1"
+      - group: "policy/v1beta1"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io/v1alpha1"
+      - group: "storage.k8s.io"'
+
+  cat <<EOF >"${path}"
 rules:
+  # The following requests were manually identified as high-volume and low-risk,
+  # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services"]
+  - level: None
+    # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
+    # TODO(#46983): Change this to the ingress controller service account.
+    users: ["system:unsecured"]
+    namespaces: ["kube-sytem"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["configmaps"]
+  - level: None
+    users: ["kubelet"] # legacy kubelet identity
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes"]
+  - level: None
+    groups: ["system:nodes"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes"]
+  - level: None
+    users:
+      - system:kube-controller-manager
+      - system:kube-scheduler
+      - system:serviceaccount:kube-system:endpoint-controller
+    verbs: ["get", "update"]
+    namespaces: ["kube-sytem"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints"]
+  - level: None
+    users: ["system:apiserver"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["namespaces"]
+
+  # Don't log these read-only URLs.
   - level: None
     nonResourceURLs:
       - /healthz*
       - /version
       - /swagger*
+
+  # Don't log events requests.
+  - level: None
+    resources:
+      - group: "" # core
+        resources: ["events"]
+
+  # Secrets & ConfigMaps can contain sensitive & binary data,
+  # so only log at the Metadata level.
+  - level: Metadata
+    resources:
+      - group: "" # core
+        resources: ["secrets", "configmaps"]
+  # Get repsonses can be large; skip them.
+  - level: Request
+    verbs: ["get", "list", "watch"]
+    resources: ${known_apis}
+  # Default level for known APIs
+  - level: RequestResponse
+    resources: ${known_apis}
+  # Default level for all other requests.
   - level: Metadata
 EOF
 }
@@ -1112,7 +1198,7 @@ function start-kube-apiserver {
     params+=" --audit-policy-file=${audit_policy_file}"
 
     # Create the audit policy file, and mount it into the apiserver pod.
-    create-master-audit-policy
+    create-master-audit-policy "${audit_policy_file}"
     audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": false},"
     audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\"}},"
   fi