From eadce7a1809ca09a0f440e2ecf97171b2178e1c4 Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Tue, 26 Sep 2017 15:26:02 +0200 Subject: [PATCH] Add audit-logging, feature-gates & few admission plugins to kubemark --- cluster/gce/config-test.sh | 3 - cluster/gce/gci/configure-helper.sh | 2 +- cluster/kubemark/gce/config-default.sh | 44 ++-- .../resources/manifests/kube-apiserver.yaml | 2 + .../resources/start-kubemark-master.sh | 203 +++++++++++++++++- test/kubemark/start-kubemark.sh | 8 +- 6 files changed, 244 insertions(+), 18 deletions(-) diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index f9caefec296..c5cd8ef7923 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -341,9 +341,6 @@ SOFTLOCKUP_PANIC="${SOFTLOCKUP_PANIC:-true}" # true, false # Enable a simple "AdvancedAuditing" setup for testing. ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-true}" # true, false -if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT}" == "true" ]]; then - FEATURE_GATES="${FEATURE_GATES},AdvancedAuditing=true" -fi ENABLE_BIG_CLUSTER_SUBNETS="${ENABLE_BIG_CLUSTER_SUBNETS:-false}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 3e2e55ca86e..31a379e4193 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -520,7 +520,7 @@ rules: - group: "" # core resources: ["endpoints", "services", "services/status"] - level: None - # Ingress controller reads `configmaps/ingress-uid` through the unsecured port. + # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. users: ["system:unsecured"] namespaces: ["kube-system"] diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index ee4b6f4e713..3b1c6eeddea 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -31,29 +31,40 @@ MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20GB} MASTER_ROOT_DISK_SIZE=${KUBEMARK_MASTER_ROOT_DISK_SIZE:-10GB} REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false} PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} +NODE_ACCELERATORS=${NODE_ACCELERATORS:-""} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci} MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-stable-60-9592-90-0} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud} +# GPUs supported in GCE do not have compatible drivers in Debian 7. +if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then + NODE_ACCELERATORS="" +fi + NETWORK=${KUBE_GCE_NETWORK:-e2e} INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}" MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master" AGGREGATOR_MASTER_NAME="${INSTANCE_PREFIX}-kubemark-aggregator" MASTER_TAG="kubemark-master" +ETCD_QUORUM_READ="${ENABLE_ETCD_QUORUM_READ:-false}" EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.224.0.0/11}" RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100} +KUBE_APISERVER_REQUEST_TIMEOUT=300 # Set etcd image (e.g. 3.0.17-alpha.1) and version (e.g. 3.0.17) if you need # non-default version. ETCD_IMAGE="${TEST_ETCD_IMAGE:-}" ETCD_VERSION="${TEST_ETCD_VERSION:-}" -# Storage backend. 'etcd2' supported, 'etcd3' experimental. + +# Storage backend. 'etcd2' and 'etcd3' are supported. STORAGE_BACKEND=${STORAGE_BACKEND:-} +# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported. +STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-} # Default Log level for all components in test clusters and variables to override it in specific components. TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=2}" @@ -67,23 +78,16 @@ TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}" TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:-}" -KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}" - # ContentType used by all components to communicate with apiserver. TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" -# ContentType used to store objects in underlying database. -TEST_CLUSTER_STORAGE_MEDIA_TYPE="" -if [ -n "${STORAGE_MEDIA_TYPE:-}" ]; then - TEST_CLUSTER_STORAGE_MEDIA_TYPE="--storage-media-type=${STORAGE_MEDIA_TYPE}" -fi -ENABLE_GARBAGE_COLLECTOR=${ENABLE_GARBAGE_COLLECTOR:-true} +KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}" -CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PodPreset,DefaultTolerationSeconds,NodeRestriction,ResourceQuota}" +CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota}" # Master components' test arguments. -APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_MEDIA_TYPE} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" -CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" +APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}" +CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" # Hollow-node components' test arguments. @@ -96,3 +100,19 @@ ALLOCATE_NODE_CIDRS=true # Optional: Enable cluster autoscaler. ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}" + +# Optional: set feature gates +FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" + +if [[ ! -z "${NODE_ACCELERATORS}" ]]; then + FEATURE_GATES="${FEATURE_GATES},Accelerators=true" +fi + +# Enable a simple "AdvancedAuditing" setup for testing. +ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-false}" + +# Optional: enable pod priority +ENABLE_POD_PRIORITY="${ENABLE_POD_PRIORITY:-}" +if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then + FEATURE_GATES="${FEATURE_GATES},PodPriority=true" +fi diff --git a/test/kubemark/resources/manifests/kube-apiserver.yaml b/test/kubemark/resources/manifests/kube-apiserver.yaml index af4d1cd357e..162d2144208 100644 --- a/test/kubemark/resources/manifests/kube-apiserver.yaml +++ b/test/kubemark/resources/manifests/kube-apiserver.yaml @@ -36,6 +36,7 @@ spec: hostPort: 8080 protocol: TCP volumeMounts: +{{audit_policy_config_mount}} - name: srvkube mountPath: /etc/srv/kubernetes readOnly: true @@ -50,6 +51,7 @@ spec: - name: srvsshproxy mountPath: /etc/srv/sshproxy volumes: +{{audit_policy_config_volume}} - name: srvkube hostPath: path: /etc/srv/kubernetes diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh index 4aa204881db..dccbe886b86 100755 --- a/test/kubemark/resources/start-kubemark-master.sh +++ b/test/kubemark/resources/start-kubemark-master.sh @@ -313,6 +313,156 @@ function setup-addon-manifests { chmod 644 "${dst_dir}"/* } +# Write the config for the audit policy. +# Note: This duplicates the function in cluster/gce/gci/configure-helper.sh. +# TODO: Get rid of this function when #53321 is fixed. +function create-master-audit-policy { + local -r path="${1}" + local -r policy="${2:-}" + + if [[ -n "${policy}" ]]; then + echo "${policy}" > "${path}" + return + fi + + # Known api groups + local -r known_apis=' + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io"' + + cat <"${path}" +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +rules: + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. + # TODO(#46983): Change this to the ingress controller service account. + users: ["system:unsecured"] + namespaces: ["kube-system"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["configmaps"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + # Get repsonses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: ${known_apis} + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: ${known_apis} + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" +EOF +} + # Computes command line arguments to be passed to etcd. function compute-etcd-params { local params="${ETCD_TEST_ARGS:-}" @@ -360,10 +510,40 @@ function compute-kube-apiserver-params { params+=" --secure-port=443" params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" params+=" --target-ram-mb=$((${NUM_NODES} * 60))" - params+=" --storage-backend=${STORAGE_BACKEND}" params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}" params+=" --authorization-mode=Node,RBAC" + params+=" --allow-privileged=true" + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + params+=" --storage-backend=${STORAGE_BACKEND}" + fi + if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then + params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" + fi + if [[ -n "${ETCD_QUORUM_READ:-}" ]]; then + params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}" + fi + if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then + params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" + fi + if [[ -n "${RUNTIME_CONFIG:-}" ]]; then + params+=" --runtime-config=${RUNTIME_CONFIG}" + fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then + # Create the audit policy file, and mount it into the apiserver pod. + create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}" + + # The config below matches the one in cluster/gce/gci/configure-helper.sh. + # TODO: Currently supporting just log backend. Support webhook if needed. + params+=" --audit-policy-file=${audit_policy_file}" + params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" + params+=" --audit-log-maxage=0" + params+=" --audit-log-maxbackup=0" + params+=" --audit-log-maxsize=2000000000" + fi echo "${params}" } @@ -424,6 +604,25 @@ function start-kubemaster-component() { else local -r component_docker_tag=$(cat ${KUBE_BINDIR}/${component}.docker_tag) sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}" + if [ "${component}" == "kube-apiserver" ]; then + local audit_policy_config_mount="" + local audit_policy_config_volume="" + if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then + read -d '' audit_policy_config_mount << EOF +- name: auditpolicyconfigmount + mountPath: ${audit_policy_file} + readOnly: true +EOF + read -d '' audit_policy_config_volume << EOF +- name: auditpolicyconfigmount + hostPath: + path: ${audit_policy_file} + type: FileOrCreate +EOF + fi + sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" + sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" + fi fi cp "${src_file}" /etc/kubernetes/manifests } @@ -497,6 +696,8 @@ assemble-docker-flags DOCKER_REGISTRY="gcr.io/google_containers" load-docker-images +readonly audit_policy_file="/etc/audit_policy.config" + # Start kubelet as a supervisord process and master components as pods. start-kubelet start-kubemaster-component "etcd" diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 626f713e86a..fdb841f1f4f 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -68,9 +68,15 @@ SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-}" # Apiserver related variables. APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-}" -STORAGE_BACKEND="${STORAGE_BACKEND:-}" +STORAGE_MEDIA_TYPE="${STORAGE_MEDIA_TYPE:-}" +STORAGE_BACKEND="${STORAGE_BACKEND:-etcd3}" +ETCD_QUORUM_READ="${ETCD_QUORUM_READ:-}" +RUNTIME_CONFIG="${RUNTIME_CONFIG:-}" NUM_NODES="${NUM_NODES:-}" CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-}" +FEATURE_GATES="${FEATURE_GATES:-}" +KUBE_APISERVER_REQUEST_TIMEOUT="${KUBE_APISERVER_REQUEST_TIMEOUT:-}" +ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-}" EOF echo "Created the environment file for master." }