mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Merge pull request #53053 from shyamjvs/enable-audit-logging-kubemark
Automatic merge from submit-queue (batch tested with PRs 51765, 53053, 52771, 52860, 53284). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add audit-logging, feature-gates & few admission plugins to kubemark To make kubemark match real cluster settings. Also includes a few other settings like request-timeout, etcd-quorum, etc. Fixes https://github.com/kubernetes/kubernetes/issues/53021 Related https://github.com/kubernetes/kubernetes/issues/51899 https://github.com/kubernetes/kubernetes/issues/44701 cc @kubernetes/sig-scalability-misc @wojtek-t @gmarek @smarterclayton
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
f48eccad9e
@ -341,9 +341,6 @@ SOFTLOCKUP_PANIC="${SOFTLOCKUP_PANIC:-true}" # true, false
|
||||
|
||||
# Enable a simple "AdvancedAuditing" setup for testing.
|
||||
ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-true}" # true, false
|
||||
if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT}" == "true" ]]; then
|
||||
FEATURE_GATES="${FEATURE_GATES},AdvancedAuditing=true"
|
||||
fi
|
||||
|
||||
if [[ "${ENABLE_APISERVER_BASIC_AUDIT:-}" == "true" ]]; then
|
||||
echo "Warning: Basic audit logging is deprecated and will be removed. Please use advanced auditing instead."
|
||||
|
||||
@ -526,7 +526,7 @@ rules:
|
||||
- group: "" # core
|
||||
resources: ["endpoints", "services", "services/status"]
|
||||
- level: None
|
||||
# Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
|
||||
# Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
|
||||
# TODO(#46983): Change this to the ingress controller service account.
|
||||
users: ["system:unsecured"]
|
||||
namespaces: ["kube-system"]
|
||||
|
||||
@ -31,29 +31,40 @@ MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20GB}
|
||||
MASTER_ROOT_DISK_SIZE=${KUBEMARK_MASTER_ROOT_DISK_SIZE:-10GB}
|
||||
REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
|
||||
PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
|
||||
NODE_ACCELERATORS=${NODE_ACCELERATORS:-""}
|
||||
|
||||
MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci}
|
||||
NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci}
|
||||
MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-stable-60-9592-90-0}
|
||||
MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
|
||||
|
||||
# GPUs supported in GCE do not have compatible drivers in Debian 7.
|
||||
if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
|
||||
NODE_ACCELERATORS=""
|
||||
fi
|
||||
|
||||
NETWORK=${KUBE_GCE_NETWORK:-e2e}
|
||||
INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}"
|
||||
MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
|
||||
AGGREGATOR_MASTER_NAME="${INSTANCE_PREFIX}-kubemark-aggregator"
|
||||
MASTER_TAG="kubemark-master"
|
||||
ETCD_QUORUM_READ="${ENABLE_ETCD_QUORUM_READ:-false}"
|
||||
EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
|
||||
MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
|
||||
CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.224.0.0/11}"
|
||||
RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
|
||||
TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
|
||||
KUBE_APISERVER_REQUEST_TIMEOUT=300
|
||||
|
||||
# Set etcd image (e.g. gcr.io/google_containers/etcd) and version (e.g. 3.1.10) if you need
|
||||
# non-default version.
|
||||
ETCD_IMAGE="${TEST_ETCD_IMAGE:-}"
|
||||
ETCD_VERSION="${TEST_ETCD_VERSION:-}"
|
||||
# Storage backend. 'etcd2' supported, 'etcd3' experimental.
|
||||
|
||||
# Storage backend. 'etcd2' and 'etcd3' are supported.
|
||||
STORAGE_BACKEND=${STORAGE_BACKEND:-}
|
||||
# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported.
|
||||
STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-}
|
||||
|
||||
# Default Log level for all components in test clusters and variables to override it in specific components.
|
||||
TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=2}"
|
||||
@ -67,23 +78,16 @@ TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS
|
||||
TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}"
|
||||
TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:-}"
|
||||
|
||||
KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
|
||||
|
||||
# ContentType used by all components to communicate with apiserver.
|
||||
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
|
||||
# ContentType used to store objects in underlying database.
|
||||
TEST_CLUSTER_STORAGE_MEDIA_TYPE=""
|
||||
if [ -n "${STORAGE_MEDIA_TYPE:-}" ]; then
|
||||
TEST_CLUSTER_STORAGE_MEDIA_TYPE="--storage-media-type=${STORAGE_MEDIA_TYPE}"
|
||||
fi
|
||||
|
||||
ENABLE_GARBAGE_COLLECTOR=${ENABLE_GARBAGE_COLLECTOR:-true}
|
||||
KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
|
||||
|
||||
CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PodPreset,DefaultTolerationSeconds,NodeRestriction,ResourceQuota}"
|
||||
CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota}"
|
||||
|
||||
# Master components' test arguments.
|
||||
APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_MEDIA_TYPE} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
|
||||
CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
|
||||
APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}"
|
||||
CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
|
||||
SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
|
||||
|
||||
# Hollow-node components' test arguments.
|
||||
@ -96,3 +100,19 @@ ALLOCATE_NODE_CIDRS=true
|
||||
|
||||
# Optional: Enable cluster autoscaler.
|
||||
ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}"
|
||||
|
||||
# Optional: set feature gates
|
||||
FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
|
||||
|
||||
if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
|
||||
FEATURE_GATES="${FEATURE_GATES},Accelerators=true"
|
||||
fi
|
||||
|
||||
# Enable a simple "AdvancedAuditing" setup for testing.
|
||||
ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-false}"
|
||||
|
||||
# Optional: enable pod priority
|
||||
ENABLE_POD_PRIORITY="${ENABLE_POD_PRIORITY:-}"
|
||||
if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then
|
||||
FEATURE_GATES="${FEATURE_GATES},PodPriority=true"
|
||||
fi
|
||||
|
||||
@ -36,6 +36,7 @@ spec:
|
||||
hostPort: 8080
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
{{audit_policy_config_mount}}
|
||||
- name: srvkube
|
||||
mountPath: /etc/srv/kubernetes
|
||||
readOnly: true
|
||||
@ -50,6 +51,7 @@ spec:
|
||||
- name: srvsshproxy
|
||||
mountPath: /etc/srv/sshproxy
|
||||
volumes:
|
||||
{{audit_policy_config_volume}}
|
||||
- name: srvkube
|
||||
hostPath:
|
||||
path: /etc/srv/kubernetes
|
||||
|
||||
@ -313,6 +313,156 @@ function setup-addon-manifests {
|
||||
chmod 644 "${dst_dir}"/*
|
||||
}
|
||||
|
||||
# Write the config for the audit policy.
|
||||
# Note: This duplicates the function in cluster/gce/gci/configure-helper.sh.
|
||||
# TODO: Get rid of this function when #53321 is fixed.
|
||||
function create-master-audit-policy {
|
||||
local -r path="${1}"
|
||||
local -r policy="${2:-}"
|
||||
|
||||
if [[ -n "${policy}" ]]; then
|
||||
echo "${policy}" > "${path}"
|
||||
return
|
||||
fi
|
||||
|
||||
# Known api groups
|
||||
local -r known_apis='
|
||||
- group: "" # core
|
||||
- group: "admissionregistration.k8s.io"
|
||||
- group: "apiextensions.k8s.io"
|
||||
- group: "apiregistration.k8s.io"
|
||||
- group: "apps"
|
||||
- group: "authentication.k8s.io"
|
||||
- group: "authorization.k8s.io"
|
||||
- group: "autoscaling"
|
||||
- group: "batch"
|
||||
- group: "certificates.k8s.io"
|
||||
- group: "extensions"
|
||||
- group: "metrics"
|
||||
- group: "networking.k8s.io"
|
||||
- group: "policy"
|
||||
- group: "rbac.authorization.k8s.io"
|
||||
- group: "settings.k8s.io"
|
||||
- group: "storage.k8s.io"'
|
||||
|
||||
cat <<EOF >"${path}"
|
||||
apiVersion: audit.k8s.io/v1beta1
|
||||
kind: Policy
|
||||
rules:
|
||||
# The following requests were manually identified as high-volume and low-risk,
|
||||
# so drop them.
|
||||
- level: None
|
||||
users: ["system:kube-proxy"]
|
||||
verbs: ["watch"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["endpoints", "services", "services/status"]
|
||||
- level: None
|
||||
# Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
|
||||
# TODO(#46983): Change this to the ingress controller service account.
|
||||
users: ["system:unsecured"]
|
||||
namespaces: ["kube-system"]
|
||||
verbs: ["get"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["configmaps"]
|
||||
- level: None
|
||||
users: ["kubelet"] # legacy kubelet identity
|
||||
verbs: ["get"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["nodes", "nodes/status"]
|
||||
- level: None
|
||||
userGroups: ["system:nodes"]
|
||||
verbs: ["get"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["nodes", "nodes/status"]
|
||||
- level: None
|
||||
users:
|
||||
- system:kube-controller-manager
|
||||
- system:kube-scheduler
|
||||
- system:serviceaccount:kube-system:endpoint-controller
|
||||
verbs: ["get", "update"]
|
||||
namespaces: ["kube-system"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["endpoints"]
|
||||
- level: None
|
||||
users: ["system:apiserver"]
|
||||
verbs: ["get"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
|
||||
# Don't log HPA fetching metrics.
|
||||
- level: None
|
||||
users:
|
||||
- system:kube-controller-manager
|
||||
verbs: ["get", "list"]
|
||||
resources:
|
||||
- group: "metrics"
|
||||
# Don't log these read-only URLs.
|
||||
- level: None
|
||||
nonResourceURLs:
|
||||
- /healthz*
|
||||
- /version
|
||||
- /swagger*
|
||||
# Don't log events requests.
|
||||
- level: None
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["events"]
|
||||
# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
|
||||
- level: Request
|
||||
users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
|
||||
verbs: ["update","patch"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["nodes/status", "pods/status"]
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
- level: Request
|
||||
userGroups: ["system:nodes"]
|
||||
verbs: ["update","patch"]
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["nodes/status", "pods/status"]
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
# deletecollection calls can be large, don't log responses for expected namespace deletions
|
||||
- level: Request
|
||||
users: ["system:serviceaccount:kube-system:namespace-controller"]
|
||||
verbs: ["deletecollection"]
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
# Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
|
||||
# so only log at the Metadata level.
|
||||
- level: Metadata
|
||||
resources:
|
||||
- group: "" # core
|
||||
resources: ["secrets", "configmaps"]
|
||||
- group: authentication.k8s.io
|
||||
resources: ["tokenreviews"]
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
# Get repsonses can be large; skip them.
|
||||
- level: Request
|
||||
verbs: ["get", "list", "watch"]
|
||||
resources: ${known_apis}
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
# Default level for known APIs
|
||||
- level: RequestResponse
|
||||
resources: ${known_apis}
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
# Default level for all other requests.
|
||||
- level: Metadata
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
EOF
|
||||
}
|
||||
|
||||
# Computes command line arguments to be passed to etcd.
|
||||
function compute-etcd-params {
|
||||
local params="${ETCD_TEST_ARGS:-}"
|
||||
@ -360,10 +510,40 @@ function compute-kube-apiserver-params {
|
||||
params+=" --secure-port=443"
|
||||
params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv"
|
||||
params+=" --target-ram-mb=$((${NUM_NODES} * 60))"
|
||||
params+=" --storage-backend=${STORAGE_BACKEND}"
|
||||
params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
|
||||
params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}"
|
||||
params+=" --authorization-mode=Node,RBAC"
|
||||
params+=" --allow-privileged=true"
|
||||
if [[ -n "${STORAGE_BACKEND:-}" ]]; then
|
||||
params+=" --storage-backend=${STORAGE_BACKEND}"
|
||||
fi
|
||||
if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then
|
||||
params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}"
|
||||
fi
|
||||
if [[ -n "${ETCD_QUORUM_READ:-}" ]]; then
|
||||
params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}"
|
||||
fi
|
||||
if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then
|
||||
params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
|
||||
fi
|
||||
if [[ -n "${RUNTIME_CONFIG:-}" ]]; then
|
||||
params+=" --runtime-config=${RUNTIME_CONFIG}"
|
||||
fi
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
params+=" --feature-gates=${FEATURE_GATES}"
|
||||
fi
|
||||
if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
|
||||
# Create the audit policy file, and mount it into the apiserver pod.
|
||||
create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}"
|
||||
|
||||
# The config below matches the one in cluster/gce/gci/configure-helper.sh.
|
||||
# TODO: Currently supporting just log backend. Support webhook if needed.
|
||||
params+=" --audit-policy-file=${audit_policy_file}"
|
||||
params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
|
||||
params+=" --audit-log-maxage=0"
|
||||
params+=" --audit-log-maxbackup=0"
|
||||
params+=" --audit-log-maxsize=2000000000"
|
||||
fi
|
||||
echo "${params}"
|
||||
}
|
||||
|
||||
@ -424,6 +604,25 @@ function start-kubemaster-component() {
|
||||
else
|
||||
local -r component_docker_tag=$(cat ${KUBE_BINDIR}/${component}.docker_tag)
|
||||
sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}"
|
||||
if [ "${component}" == "kube-apiserver" ]; then
|
||||
local audit_policy_config_mount=""
|
||||
local audit_policy_config_volume=""
|
||||
if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
|
||||
read -d '' audit_policy_config_mount << EOF
|
||||
- name: auditpolicyconfigmount
|
||||
mountPath: ${audit_policy_file}
|
||||
readOnly: true
|
||||
EOF
|
||||
read -d '' audit_policy_config_volume << EOF
|
||||
- name: auditpolicyconfigmount
|
||||
hostPath:
|
||||
path: ${audit_policy_file}
|
||||
type: FileOrCreate
|
||||
EOF
|
||||
fi
|
||||
sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
|
||||
sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
|
||||
fi
|
||||
fi
|
||||
cp "${src_file}" /etc/kubernetes/manifests
|
||||
}
|
||||
@ -497,6 +696,8 @@ assemble-docker-flags
|
||||
DOCKER_REGISTRY="gcr.io/google_containers"
|
||||
load-docker-images
|
||||
|
||||
readonly audit_policy_file="/etc/audit_policy.config"
|
||||
|
||||
# Start kubelet as a supervisord process and master components as pods.
|
||||
start-kubelet
|
||||
start-kubemaster-component "etcd"
|
||||
|
||||
@ -68,9 +68,15 @@ SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-}"
|
||||
|
||||
# Apiserver related variables.
|
||||
APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-}"
|
||||
STORAGE_BACKEND="${STORAGE_BACKEND:-}"
|
||||
STORAGE_MEDIA_TYPE="${STORAGE_MEDIA_TYPE:-}"
|
||||
STORAGE_BACKEND="${STORAGE_BACKEND:-etcd3}"
|
||||
ETCD_QUORUM_READ="${ETCD_QUORUM_READ:-}"
|
||||
RUNTIME_CONFIG="${RUNTIME_CONFIG:-}"
|
||||
NUM_NODES="${NUM_NODES:-}"
|
||||
CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-}"
|
||||
FEATURE_GATES="${FEATURE_GATES:-}"
|
||||
KUBE_APISERVER_REQUEST_TIMEOUT="${KUBE_APISERVER_REQUEST_TIMEOUT:-}"
|
||||
ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-}"
|
||||
EOF
|
||||
echo "Created the environment file for master."
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user