Merge pull request #64721 from Cynerva/gkk/audit

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

juju: Add audit support to kubernetes-master charm

**What this PR does / why we need it**:

This PR adds audit configuration support to the kubernetes-master charm. Specifically, it enables audit logging by default, and adds two new config options: `audit-policy` and `audit-webhook-config`.

**Release note**:

```release-note
NONE
```

cluster/juju/layers/kubernetes-master/config.yaml

@@ -1,4 +1,48 @@
 options:
+  audit-policy:
+    type: string
+    default: |
+      apiVersion: audit.k8s.io/v1beta1
+      kind: Policy
+      rules:
+      # Don't log read-only requests from the apiserver
+      - level: None
+        users: ["system:apiserver"]
+        verbs: ["get", "list", "watch"]
+      # Don't log kube-proxy watches
+      - level: None
+        users: ["system:kube-proxy"]
+        verbs: ["watch"]
+        resources:
+        - resources: ["endpoints", "services"]
+      # Don't log nodes getting their own status
+      - level: None
+        userGroups: ["system:nodes"]
+        verbs: ["get"]
+        resources:
+        - resources: ["nodes"]
+      # Don't log kube-controller-manager and kube-scheduler getting endpoints
+      - level: None
+        users: ["system:unsecured"]
+        namespaces: ["kube-system"]
+        verbs: ["get"]
+        resources:
+        - resources: ["endpoints"]
+      # Log everything else at the Request level.
+      - level: Request
+        omitStages:
+        - RequestReceived
+    description: |
+      Audit policy passed to kube-apiserver via --audit-policy-file.
+      For more info, please refer to the upstream documentation at
+      https://kubernetes.io/docs/tasks/debug-application-cluster/audit/
+  audit-webhook-config:
+    type: string
+    default: ""
+    description: |
+      Audit webhook config passed to kube-apiserver via --audit-webhook-config-file.
+      For more info, please refer to the upstream documentation at
+      https://kubernetes.io/docs/tasks/debug-application-cluster/audit/
   enable-dashboard-addons:
     type: boolean
     default: True

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -508,7 +508,7 @@ def start_master(etcd):
     handle_etcd_relation(etcd)
 
     # Add CLI options to all components
-    configure_apiserver(etcd.get_connection_string(), getStorageBackend())
+    configure_apiserver(etcd.get_connection_string())
     configure_controller_manager()
     configure_scheduler()
     set_state('kubernetes-master.components.started')
@@ -905,13 +905,14 @@ def on_config_allow_privileged_change():
     remove_state('config.changed.allow-privileged')
 
 
-@when('config.changed.api-extra-args')
+@when_any('config.changed.api-extra-args',
+          'config.changed.audit-policy',
+          'config.changed.audit-webhook-config')
 @when('kubernetes-master.components.started')
 @when('leadership.set.auto_storage_backend')
 @when('etcd.available')
-def on_config_api_extra_args_change(etcd):
+def reconfigure_apiserver(etcd):
-    configure_apiserver(etcd.get_connection_string(),
+    configure_apiserver(etcd.get_connection_string())
-                        getStorageBackend())
 
 
 @when('config.changed.controller-manager-extra-args')
@@ -1145,7 +1146,20 @@ def configure_kubernetes_service(service, base_args, extra_args_key):
     db.set(prev_args_key, args)
 
 
-def configure_apiserver(etcd_connection_string, leader_etcd_version):
+def remove_if_exists(path):
+    try:
+        os.remove(path)
+    except FileNotFoundError:
+        pass
+
+
+def write_audit_config_file(path, contents):
+    with open(path, 'w') as f:
+        header = '# Autogenerated by kubernetes-master charm'
+        f.write(header + '\n' + contents)
+
+
+def configure_apiserver(etcd_connection_string):
     api_opts = {}
 
     # Get the tls paths from the layer data.
@@ -1183,8 +1197,9 @@ def configure_apiserver(etcd_connection_string, leader_etcd_version):
     api_opts['logtostderr'] = 'true'
     api_opts['insecure-bind-address'] = '127.0.0.1'
     api_opts['insecure-port'] = '8080'
-    api_opts['storage-backend'] = leader_etcd_version
+    api_opts['storage-backend'] = getStorageBackend()
     api_opts['basic-auth-file'] = '/root/cdk/basic_auth.csv'
+
     api_opts['token-auth-file'] = '/root/cdk/known_tokens.csv'
     api_opts['service-account-key-file'] = '/root/cdk/serviceaccount.key'
     api_opts['kubelet-preferred-address-types'] = \
@@ -1263,6 +1278,31 @@ def configure_apiserver(etcd_connection_string, leader_etcd_version):
         api_opts['cloud-provider'] = 'openstack'
         api_opts['cloud-config'] = str(cloud_config_path)
 
+    audit_root = '/root/cdk/audit'
+    os.makedirs(audit_root, exist_ok=True)
+
+    audit_log_path = audit_root + '/audit.log'
+    api_opts['audit-log-path'] = audit_log_path
+    api_opts['audit-log-maxsize'] = '100'
+    api_opts['audit-log-maxbackup'] = '9'
+
+    audit_policy_path = audit_root + '/audit-policy.yaml'
+    audit_policy = hookenv.config('audit-policy')
+    if audit_policy:
+        write_audit_config_file(audit_policy_path, audit_policy)
+        api_opts['audit-policy-file'] = audit_policy_path
+    else:
+        remove_if_exists(audit_policy_path)
+
+    audit_webhook_config_path = audit_root + '/audit-webhook-config.yaml'
+    audit_webhook_config = hookenv.config('audit-webhook-config')
+    if audit_webhook_config:
+        write_audit_config_file(audit_webhook_config_path,
+                                audit_webhook_config)
+        api_opts['audit-webhook-config-file'] = audit_webhook_config_path
+    else:
+        remove_if_exists(audit_webhook_config_path)
+
     configure_kubernetes_service('kube-apiserver', api_opts, 'api-extra-args')
     restart_apiserver()