From b6b55519cad1001d31fdd91d9b2deba3569234e8 Mon Sep 17 00:00:00 2001 From: immutablet Date: Thu, 10 Oct 2019 16:07:46 -0700 Subject: [PATCH] Isolate the logic related to the configuration of kube-apiserver into a separate script. --- build/lib/release.sh | 1 + cluster/gce/gci/BUILD | 2 + cluster/gce/gci/apiserver_manifest_test.go | 7 +- cluster/gce/gci/audit_policy_test.go | 2 +- cluster/gce/gci/configure-helper.sh | 429 +------------------- cluster/gce/gci/configure-kubeapiserver.sh | 442 +++++++++++++++++++++ cluster/gce/gci/configure.sh | 1 + cluster/gce/gci/configure_helper_test.go | 11 +- 8 files changed, 458 insertions(+), 437 deletions(-) create mode 100644 cluster/gce/gci/configure-kubeapiserver.sh diff --git a/build/lib/release.sh b/build/lib/release.sh index 83378c25af1..f8d3a5e0dce 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -413,6 +413,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${src_dir}/${internal_manifest}" "${dst_dir}" done cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh" + cp "${KUBE_ROOT}/cluster/gce/gci/configure-kubeapiserver.sh" "${dst_dir}/configure-kubeapiserver.sh" if [[ -e "${KUBE_ROOT}/cluster/gce/gci/gke-internal-configure-helper.sh" ]]; then cp "${KUBE_ROOT}/cluster/gce/gci/gke-internal-configure-helper.sh" "${dst_dir}/" fi diff --git a/cluster/gce/gci/BUILD b/cluster/gce/gci/BUILD index 1c070a7335f..5d138710e05 100644 --- a/cluster/gce/gci/BUILD +++ b/cluster/gce/gci/BUILD @@ -50,6 +50,7 @@ pkg_tar( "//cluster/gce/gci/mounter": "gci-mounter", "configure-helper.sh": "gci-configure-helper.sh", "health-monitor.sh": "health-monitor.sh", + "configure-kubeapiserver.sh": "configure-kubeapiserver.sh", }, mode = "0755", strip_prefix = ".", @@ -77,5 +78,6 @@ filegroup( name = "scripts-test-data", srcs = [ "configure-helper.sh", + "configure-kubeapiserver.sh", ], ) diff --git a/cluster/gce/gci/apiserver_manifest_test.go b/cluster/gce/gci/apiserver_manifest_test.go index 6bbdfce9908..d1bbe9e18e0 100644 --- a/cluster/gce/gci/apiserver_manifest_test.go +++ b/cluster/gce/gci/apiserver_manifest_test.go @@ -27,7 +27,7 @@ import ( "strings" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) const ( @@ -67,6 +67,7 @@ readonly CLOUD_KMS_INTEGRATION=true {{end}} ` kubeAPIServerManifestFileName = "kube-apiserver.manifest" + kubeAPIServerConfigScriptName = "configure-kubeapiserver.sh" kubeAPIServerStartFuncName = "start-kube-apiserver" ) @@ -88,7 +89,7 @@ func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCa } func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) { - c.mustInvokeFunc(kubeEnv, e) + c.mustInvokeFunc(kubeEnv, kubeAPIServerConfigScriptName, e) c.mustLoadPodFromManifest() } @@ -159,7 +160,7 @@ func TestEncryptionProviderConfig(t *testing.T) { EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")), } - c.mustInvokeFunc(deployHelperEnv, e) + c.mustInvokeFunc(deployHelperEnv, kubeAPIServerConfigScriptName, e) if _, err := os.Stat(p); err != nil { c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err) diff --git a/cluster/gce/gci/audit_policy_test.go b/cluster/gce/gci/audit_policy_test.go index 99557ffffea..52efbb89241 100644 --- a/cluster/gce/gci/audit_policy_test.go +++ b/cluster/gce/gci/audit_policy_test.go @@ -53,7 +53,7 @@ func TestCreateMasterAuditPolicy(t *testing.T) { // Initialize required environment variables. const kubeEnvTmpl = `readonly KUBE_HOME={{.KubeHome}}` - c.mustInvokeFunc(kubeEnvTmpl, kubeAPIServerEnv{KubeHome: c.kubeHome}) + c.mustInvokeFunc(kubeEnvTmpl, "configure-helper.sh", kubeAPIServerEnv{KubeHome: c.kubeHome}) policy, err := auditpolicy.LoadPolicyFromFile(policyFile) require.NoError(t, err, "Failed to load generated policy.") diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 8e1ebd7a3a3..eabd82d9796 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1726,433 +1726,6 @@ function prepare-mounter-rootfs { cp /etc/resolv.conf "${CONTAINERIZED_MOUNTER_ROOTFS}/etc/" } -# Starts kubernetes apiserver. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in function compute-master-manifest-variables) -# CLOUD_CONFIG_OPT -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -# INSECURE_PORT_MAPPING -function start-kube-apiserver { - echo "Start kubernetes api-server" - prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" - prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}" - - # Calculate variables and assemble the command line. - local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" - params+=" --address=127.0.0.1" - params+=" --allow-privileged=true" - params+=" --cloud-provider=gce" - params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" - if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then - params+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}" - params+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" - params+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" - params+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" - elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then - params+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" - echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled." - else - echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential." - exit 1 - fi - if [[ -z "${ETCD_SERVERS:-}" ]]; then - params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}" - elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then - params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" - fi - params+=" --secure-port=443" - if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then - # Default is :8080 - params+=" --insecure-port=0" - fi - params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" - params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" - params+=" --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname" - if [[ -s "${REQUESTHEADER_CA_CERT_PATH:-}" ]]; then - params+=" --requestheader-client-ca-file=${REQUESTHEADER_CA_CERT_PATH}" - params+=" --requestheader-allowed-names=aggregator" - params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-" - params+=" --requestheader-group-headers=X-Remote-Group" - params+=" --requestheader-username-headers=X-Remote-User" - params+=" --proxy-client-cert-file=${PROXY_CLIENT_CERT_PATH}" - params+=" --proxy-client-key-file=${PROXY_CLIENT_KEY_PATH}" - fi - params+=" --enable-aggregator-routing=true" - if [[ -e "${APISERVER_CLIENT_CERT_PATH}" ]] && [[ -e "${APISERVER_CLIENT_KEY_PATH}" ]]; then - params+=" --kubelet-client-certificate=${APISERVER_CLIENT_CERT_PATH}" - params+=" --kubelet-client-key=${APISERVER_CLIENT_KEY_PATH}" - fi - if [[ -n "${SERVICEACCOUNT_CERT_PATH:-}" ]]; then - params+=" --service-account-key-file=${SERVICEACCOUNT_CERT_PATH}" - fi - params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" - if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then - params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" - fi - if [[ -n "${STORAGE_BACKEND:-}" ]]; then - params+=" --storage-backend=${STORAGE_BACKEND}" - fi - if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then - params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" - fi - if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then - params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" - fi - if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]]; then - params+=" --request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}s" - fi - if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then - params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" - fi - if [[ -n "${NUM_NODES:-}" ]]; then - # If the cluster is large, increase max-requests-inflight limit in apiserver. - if [[ "${NUM_NODES}" -gt 3000 ]]; then - params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000" - elif [[ "${NUM_NODES}" -gt 500 ]]; then - params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500" - fi - # Set amount of memory available for apiserver based on number of nodes. - # TODO: Once we start setting proper requests and limits for apiserver - # we should reuse the same logic here instead of current heuristic. - params+=" --target-ram-mb=$((${NUM_NODES} * 60))" - fi - if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then - params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - fi - params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}" - params+=" --service-account-api-audiences=${SERVICEACCOUNT_ISSUER}" - params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}" - - local audit_policy_config_mount="" - local audit_policy_config_volume="" - local audit_webhook_config_mount="" - local audit_webhook_config_volume="" - if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then - local -r audit_policy_file="/etc/audit_policy.config" - params+=" --audit-policy-file=${audit_policy_file}" - # Create the audit policy file, and mount it into the apiserver pod. - create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}" - audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true}," - audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\", \"type\": \"FileOrCreate\"}}," - - if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then - # The advanced audit log backend config matches the basic audit log config. - params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" - params+=" --audit-log-maxage=0" - params+=" --audit-log-maxbackup=0" - # Lumberjack doesn't offer any way to disable size-based rotation. It also - # has an in-memory counter that doesn't notice if you truncate the file. - # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log - # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver - # never restarts. Please manually restart apiserver before this time. - params+=" --audit-log-maxsize=2000000000" - - # Batching parameters - if [[ -n "${ADVANCED_AUDIT_LOG_MODE:-}" ]]; then - params+=" --audit-log-mode=${ADVANCED_AUDIT_LOG_MODE}" - fi - if [[ -n "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}" ]]; then - params+=" --audit-log-batch-buffer-size=${ADVANCED_AUDIT_LOG_BUFFER_SIZE}" - fi - if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}" ]]; then - params+=" --audit-log-batch-max-size=${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE}" - fi - if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}" ]]; then - params+=" --audit-log-batch-max-wait=${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT}" - fi - if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}" ]]; then - params+=" --audit-log-batch-throttle-qps=${ADVANCED_AUDIT_LOG_THROTTLE_QPS}" - fi - if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}" ]]; then - params+=" --audit-log-batch-throttle-burst=${ADVANCED_AUDIT_LOG_THROTTLE_BURST}" - fi - if [[ -n "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}" ]]; then - params+=" --audit-log-initial-backoff=${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF}" - fi - # Truncating backend parameters - if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then - params+=" --audit-log-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}" - fi - fi - if [[ "${ADVANCED_AUDIT_BACKEND:-}" == *"webhook"* ]]; then - # Create the audit webhook config file, and mount it into the apiserver pod. - local -r audit_webhook_config_file="/etc/audit_webhook.config" - params+=" --audit-webhook-config-file=${audit_webhook_config_file}" - create-master-audit-webhook-config "${audit_webhook_config_file}" - audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true}," - audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}}," - - # Batching parameters - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MODE:-}" ]]; then - params+=" --audit-webhook-mode=${ADVANCED_AUDIT_WEBHOOK_MODE}" - else - params+=" --audit-webhook-mode=batch" - fi - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}" ]]; then - params+=" --audit-webhook-batch-buffer-size=${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE}" - fi - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}" ]]; then - params+=" --audit-webhook-batch-max-size=${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE}" - fi - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}" ]]; then - params+=" --audit-webhook-batch-max-wait=${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT}" - fi - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}" ]]; then - params+=" --audit-webhook-batch-throttle-qps=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS}" - fi - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}" ]]; then - params+=" --audit-webhook-batch-throttle-burst=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST}" - fi - if [[ -n "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}" ]]; then - params+=" --audit-webhook-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}" - fi - # Truncating backend parameters - if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then - params+=" --audit-webhook-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}" - fi - fi - fi - - if [[ "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}" == "true" ]]; then - params+=" --audit-dynamic-configuration" - RUNTIME_CONFIG="${RUNTIME_CONFIG},auditconfiguration.k8s.io/v1alpha1=true" - fi - - if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then - params+=" --enable-logs-handler=false" - fi - if [[ "${APISERVER_SET_KUBELET_CA:-false}" == "true" ]]; then - params+=" --kubelet-certificate-authority=${CA_CERT_BUNDLE_PATH}" - fi - - if [[ -n "${ADMISSION_CONTROL:-}" ]]; then - params+=" --enable-admission-plugins=${ADMISSION_CONTROL}" - params+=" --admission-control-config-file=/etc/srv/kubernetes/admission_controller_config.yaml" - fi - - # If GKE exec auth support is requested for webhooks, then - # gke-exec-auth-plugin needs to be mounted into the kube-apiserver container. - local webhook_exec_auth_plugin_mount="" - local webhook_exec_auth_plugin_volume="" - if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then - webhook_exec_auth_plugin_mount='{"name": "gkeauth", "mountPath": "/usr/bin/gke-exec-auth-plugin", "readOnly": true},' - webhook_exec_auth_plugin_volume='{"name": "gkeauth", "hostPath": {"path": "/home/kubernetes/bin/gke-exec-auth-plugin", "type": "File"}},' - fi - - if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then - params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" - fi - if [[ -n "${RUNTIME_CONFIG:-}" ]]; then - params+=" --runtime-config=${RUNTIME_CONFIG}" - fi - if [[ -n "${FEATURE_GATES:-}" ]]; then - params+=" --feature-gates=${FEATURE_GATES}" - fi - if [[ "${FEATURE_GATES:-}" =~ "RuntimeClass=true" ]]; then - params+=" --runtime-config=node.k8s.io/v1alpha1=true" - fi - if [[ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]]; then - params+=" --advertise-address=${MASTER_ADVERTISE_ADDRESS}" - if [[ -n "${PROXY_SSH_USER:-}" ]]; then - params+=" --ssh-user=${PROXY_SSH_USER}" - params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" - fi - elif [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then - local -r vm_external_ip=$(get-metadata-value "instance/network-interfaces/0/access-configs/0/external-ip") - if [[ -n "${PROXY_SSH_USER:-}" ]]; then - params+=" --advertise-address=${vm_external_ip}" - params+=" --ssh-user=${PROXY_SSH_USER}" - params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" - fi - fi - - local webhook_authn_config_mount="" - local webhook_authn_config_volume="" - if [[ -n "${GCP_AUTHN_URL:-}" ]]; then - params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" - webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," - webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"FileOrCreate\"}}," - if [[ -n "${GCP_AUTHN_CACHE_TTL:-}" ]]; then - params+=" --authentication-token-webhook-cache-ttl=${GCP_AUTHN_CACHE_TTL}" - fi - fi - - - local authorization_mode="RBAC" - local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" - - # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false - if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then - echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this." - # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions) - if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then - local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" - if [[ -n "${KUBE_USER:-}" ]]; then - sed -i -e "s/{{kube_user}}/${KUBE_USER}/g" "${abac_policy_json}" - else - sed -i -e "/{{kube_user}}/d" "${abac_policy_json}" - fi - cp "${abac_policy_json}" /etc/srv/kubernetes/ - fi - - params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl" - authorization_mode+=",ABAC" - fi - - local webhook_config_mount="" - local webhook_config_volume="" - if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then - authorization_mode="${authorization_mode},Webhook" - params+=" --authorization-webhook-config-file=/etc/gcp_authz.config" - webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," - webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}}," - if [[ -n "${GCP_AUTHZ_CACHE_AUTHORIZED_TTL:-}" ]]; then - params+=" --authorization-webhook-cache-authorized-ttl=${GCP_AUTHZ_CACHE_AUTHORIZED_TTL}" - fi - if [[ -n "${GCP_AUTHZ_CACHE_UNAUTHORIZED_TTL:-}" ]]; then - params+=" --authorization-webhook-cache-unauthorized-ttl=${GCP_AUTHZ_CACHE_UNAUTHORIZED_TTL}" - fi - fi - authorization_mode="Node,${authorization_mode}" - params+=" --authorization-mode=${authorization_mode}" - - local csc_config_mount="" - local csc_config_volume="" - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then - # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. - csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," - csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," - params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" - fi - - local container_env="" - if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env+="{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}" - fi - if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then - if [[ -n "${container_env}" ]]; then - container_env="${container_env}, " - fi - container_env+="{\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\"}" - fi - if [[ -n "${container_env}" ]]; then - container_env="\"env\":[${container_env}]," - fi - - local -r src_file="${src_dir}/kube-apiserver.manifest" - - # params is passed by reference, so no "$" - setup-etcd-encryption "${src_file}" params - - # Evaluate variables. - local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} - sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" - sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" - sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}" - sed -i -e "s@{{secure_port}}@443@g" "${src_file}" - sed -i -e "s@{{insecure_port_mapping}}@${INSECURE_PORT_MAPPING}@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" - sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" - sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" - sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" - sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" - sed -i -e "s@{{csc_config_mount}}@${csc_config_mount}@g" "${src_file}" - sed -i -e "s@{{csc_config_volume}}@${csc_config_volume}@g" "${src_file}" - sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" - sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" - sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}" - sed -i -e "s@{{audit_webhook_config_volume}}@${audit_webhook_config_volume}@g" "${src_file}" - sed -i -e "s@{{webhook_exec_auth_plugin_mount}}@${webhook_exec_auth_plugin_mount}@g" "${src_file}" - sed -i -e "s@{{webhook_exec_auth_plugin_volume}}@${webhook_exec_auth_plugin_volume}@g" "${src_file}" - - cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" -} - -# Sets-up etcd encryption. -# Configuration of etcd level encryption consists of the following steps: -# 1. Writing encryption provider config to disk -# 2. Adding encryption-provider-config flag to kube-apiserver -# 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested) -# -# Expects parameters: -# $1 - path to kube-apiserver template -# $2 - kube-apiserver startup flags (must be passed by reference) -# -# Assumes vars (supplied via kube-env): -# ENCRYPTION_PROVIDER_CONFIG -# CLOUD_KMS_INTEGRATION -# ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml) -function setup-etcd-encryption { - local kube_apiserver_template_path - local -n kube_api_server_params - local default_encryption_provider_config_vol - local default_encryption_provider_config_vol_mnt - local encryption_provider_config_vol_mnt - local encryption_provider_config_vol - local default_kms_socket_dir - local default_kms_socket_vol_mnt - local default_kms_socket_vol - local kms_socket_vol_mnt - local kms_socket_vol - local encryption_provider_config_path - - kube_apiserver_template_path="$1" - if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then - sed -i -e " { - s@{{encryption_provider_mount}}@@ - s@{{encryption_provider_volume}}@@ - s@{{kms_socket_mount}}@@ - s@{{kms_socket_volume}}@@ - } " "${kube_apiserver_template_path}" - return - fi - - kube_api_server_params="$2" - encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml} - - echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" - kube_api_server_params+=" --encryption-provider-config=${encryption_provider_config_path}" - - default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n') - default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n') - - encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode) - encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode) - sed -i -e " { - s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@ - s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@ - } " "${kube_apiserver_template_path}" - - if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then - default_kms_socket_dir="/var/run/kmsplugin" - default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n') - default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n') - - kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode) - kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode) - sed -i -e " { - s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@ - s@{{kms_socket_volume}}@${kms_socket_vol},@ - } " "${kube_apiserver_template_path}" - else - sed -i -e " { - s@{{kms_socket_mount}}@@ - s@{{kms_socket_volume}}@@ - } " "${kube_apiserver_template_path}" - fi -} - # Updates node labels used by addons. function update-legacy-addon-node-labels() { # need kube-apiserver to be ready @@ -3143,6 +2716,7 @@ function main() { fi KUBE_HOME="/home/kubernetes" + KUBE_BIN=${KUBE_HOME}/bin CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter" PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml" @@ -3228,6 +2802,7 @@ function main() { start-etcd-servers start-etcd-empty-dir-cleanup-pod fi + source ${KUBE_BIN}/configure-kubeapiserver.sh start-kube-apiserver if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then start-konnectivity-server diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh new file mode 100644 index 00000000000..edc0af586c5 --- /dev/null +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -0,0 +1,442 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Starts kubernetes apiserver. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in function compute-master-manifest-variables) +# CLOUD_CONFIG_OPT +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +# INSECURE_PORT_MAPPING +function start-kube-apiserver { + echo "Start kubernetes api-server" + prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" + prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}" + + # Calculate variables and assemble the command line. + local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" + params+=" --address=127.0.0.1" + params+=" --allow-privileged=true" + params+=" --cloud-provider=gce" + params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" + if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then + params+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}" + params+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" + params+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" + params+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" + elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then + params+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" + echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled." + else + echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential." + exit 1 + fi + if [[ -z "${ETCD_SERVERS:-}" ]]; then + params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}" + elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then + params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" + fi + params+=" --secure-port=443" + if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then + # Default is :8080 + params+=" --insecure-port=0" + fi + params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" + params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" + params+=" --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname" + if [[ -s "${REQUESTHEADER_CA_CERT_PATH:-}" ]]; then + params+=" --requestheader-client-ca-file=${REQUESTHEADER_CA_CERT_PATH}" + params+=" --requestheader-allowed-names=aggregator" + params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-" + params+=" --requestheader-group-headers=X-Remote-Group" + params+=" --requestheader-username-headers=X-Remote-User" + params+=" --proxy-client-cert-file=${PROXY_CLIENT_CERT_PATH}" + params+=" --proxy-client-key-file=${PROXY_CLIENT_KEY_PATH}" + fi + params+=" --enable-aggregator-routing=true" + if [[ -e "${APISERVER_CLIENT_CERT_PATH}" ]] && [[ -e "${APISERVER_CLIENT_KEY_PATH}" ]]; then + params+=" --kubelet-client-certificate=${APISERVER_CLIENT_CERT_PATH}" + params+=" --kubelet-client-key=${APISERVER_CLIENT_KEY_PATH}" + fi + if [[ -n "${SERVICEACCOUNT_CERT_PATH:-}" ]]; then + params+=" --service-account-key-file=${SERVICEACCOUNT_CERT_PATH}" + fi + params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" + if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then + params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" + fi + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + params+=" --storage-backend=${STORAGE_BACKEND}" + fi + if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then + params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" + fi + if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then + params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" + fi + if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]]; then + params+=" --request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}s" + fi + if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then + params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" + fi + if [[ -n "${NUM_NODES:-}" ]]; then + # If the cluster is large, increase max-requests-inflight limit in apiserver. + if [[ "${NUM_NODES}" -gt 3000 ]]; then + params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000" + elif [[ "${NUM_NODES}" -gt 500 ]]; then + params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500" + fi + # Set amount of memory available for apiserver based on number of nodes. + # TODO: Once we start setting proper requests and limits for apiserver + # we should reuse the same logic here instead of current heuristic. + params+=" --target-ram-mb=$((NUM_NODES * 60))" + fi + if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then + params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + fi + params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}" + params+=" --service-account-api-audiences=${SERVICEACCOUNT_ISSUER}" + params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}" + + local audit_policy_config_mount="" + local audit_policy_config_volume="" + local audit_webhook_config_mount="" + local audit_webhook_config_volume="" + if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then + local -r audit_policy_file="/etc/audit_policy.config" + params+=" --audit-policy-file=${audit_policy_file}" + # Create the audit policy file, and mount it into the apiserver pod. + create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}" + audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true}," + audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\", \"type\": \"FileOrCreate\"}}," + + if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then + # The advanced audit log backend config matches the basic audit log config. + params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" + params+=" --audit-log-maxage=0" + params+=" --audit-log-maxbackup=0" + # Lumberjack doesn't offer any way to disable size-based rotation. It also + # has an in-memory counter that doesn't notice if you truncate the file. + # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log + # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver + # never restarts. Please manually restart apiserver before this time. + params+=" --audit-log-maxsize=2000000000" + + # Batching parameters + if [[ -n "${ADVANCED_AUDIT_LOG_MODE:-}" ]]; then + params+=" --audit-log-mode=${ADVANCED_AUDIT_LOG_MODE}" + fi + if [[ -n "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}" ]]; then + params+=" --audit-log-batch-buffer-size=${ADVANCED_AUDIT_LOG_BUFFER_SIZE}" + fi + if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}" ]]; then + params+=" --audit-log-batch-max-size=${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE}" + fi + if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}" ]]; then + params+=" --audit-log-batch-max-wait=${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT}" + fi + if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}" ]]; then + params+=" --audit-log-batch-throttle-qps=${ADVANCED_AUDIT_LOG_THROTTLE_QPS}" + fi + if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}" ]]; then + params+=" --audit-log-batch-throttle-burst=${ADVANCED_AUDIT_LOG_THROTTLE_BURST}" + fi + if [[ -n "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}" ]]; then + params+=" --audit-log-initial-backoff=${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF}" + fi + # Truncating backend parameters + if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then + params+=" --audit-log-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}" + fi + fi + if [[ "${ADVANCED_AUDIT_BACKEND:-}" == *"webhook"* ]]; then + # Create the audit webhook config file, and mount it into the apiserver pod. + local -r audit_webhook_config_file="/etc/audit_webhook.config" + params+=" --audit-webhook-config-file=${audit_webhook_config_file}" + create-master-audit-webhook-config "${audit_webhook_config_file}" + audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true}," + audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}}," + + # Batching parameters + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MODE:-}" ]]; then + params+=" --audit-webhook-mode=${ADVANCED_AUDIT_WEBHOOK_MODE}" + else + params+=" --audit-webhook-mode=batch" + fi + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}" ]]; then + params+=" --audit-webhook-batch-buffer-size=${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE}" + fi + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}" ]]; then + params+=" --audit-webhook-batch-max-size=${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE}" + fi + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}" ]]; then + params+=" --audit-webhook-batch-max-wait=${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT}" + fi + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}" ]]; then + params+=" --audit-webhook-batch-throttle-qps=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS}" + fi + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}" ]]; then + params+=" --audit-webhook-batch-throttle-burst=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST}" + fi + if [[ -n "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}" ]]; then + params+=" --audit-webhook-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}" + fi + # Truncating backend parameters + if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then + params+=" --audit-webhook-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}" + fi + fi + fi + + if [[ "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}" == "true" ]]; then + params+=" --audit-dynamic-configuration" + RUNTIME_CONFIG="${RUNTIME_CONFIG},auditconfiguration.k8s.io/v1alpha1=true" + fi + + if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then + params+=" --enable-logs-handler=false" + fi + if [[ "${APISERVER_SET_KUBELET_CA:-false}" == "true" ]]; then + params+=" --kubelet-certificate-authority=${CA_CERT_BUNDLE_PATH}" + fi + + if [[ -n "${ADMISSION_CONTROL:-}" ]]; then + params+=" --enable-admission-plugins=${ADMISSION_CONTROL}" + params+=" --admission-control-config-file=/etc/srv/kubernetes/admission_controller_config.yaml" + fi + + # If GKE exec auth support is requested for webhooks, then + # gke-exec-auth-plugin needs to be mounted into the kube-apiserver container. + local webhook_exec_auth_plugin_mount="" + local webhook_exec_auth_plugin_volume="" + if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then + webhook_exec_auth_plugin_mount='{"name": "gkeauth", "mountPath": "/usr/bin/gke-exec-auth-plugin", "readOnly": true},' + webhook_exec_auth_plugin_volume='{"name": "gkeauth", "hostPath": {"path": "/home/kubernetes/bin/gke-exec-auth-plugin", "type": "File"}},' + fi + + if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then + params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" + fi + if [[ -n "${RUNTIME_CONFIG:-}" ]]; then + params+=" --runtime-config=${RUNTIME_CONFIG}" + fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + if [[ "${FEATURE_GATES:-}" =~ "RuntimeClass=true" ]]; then + params+=" --runtime-config=node.k8s.io/v1alpha1=true" + fi + if [[ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]]; then + params+=" --advertise-address=${MASTER_ADVERTISE_ADDRESS}" + if [[ -n "${PROXY_SSH_USER:-}" ]]; then + params+=" --ssh-user=${PROXY_SSH_USER}" + params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" + fi + elif [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then + local -r vm_external_ip=$(get-metadata-value "instance/network-interfaces/0/access-configs/0/external-ip") + if [[ -n "${PROXY_SSH_USER:-}" ]]; then + params+=" --advertise-address=${vm_external_ip}" + params+=" --ssh-user=${PROXY_SSH_USER}" + params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" + fi + fi + + local webhook_authn_config_mount="" + local webhook_authn_config_volume="" + if [[ -n "${GCP_AUTHN_URL:-}" ]]; then + params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" + webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," + webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"FileOrCreate\"}}," + if [[ -n "${GCP_AUTHN_CACHE_TTL:-}" ]]; then + params+=" --authentication-token-webhook-cache-ttl=${GCP_AUTHN_CACHE_TTL}" + fi + fi + + + local authorization_mode="RBAC" + local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" + + # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false + if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then + echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this." + # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions) + if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then + local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" + if [[ -n "${KUBE_USER:-}" ]]; then + sed -i -e "s/{{kube_user}}/${KUBE_USER}/g" "${abac_policy_json}" + else + sed -i -e "/{{kube_user}}/d" "${abac_policy_json}" + fi + cp "${abac_policy_json}" /etc/srv/kubernetes/ + fi + + params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl" + authorization_mode+=",ABAC" + fi + + local webhook_config_mount="" + local webhook_config_volume="" + if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then + authorization_mode="${authorization_mode},Webhook" + params+=" --authorization-webhook-config-file=/etc/gcp_authz.config" + webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," + webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}}," + if [[ -n "${GCP_AUTHZ_CACHE_AUTHORIZED_TTL:-}" ]]; then + params+=" --authorization-webhook-cache-authorized-ttl=${GCP_AUTHZ_CACHE_AUTHORIZED_TTL}" + fi + if [[ -n "${GCP_AUTHZ_CACHE_UNAUTHORIZED_TTL:-}" ]]; then + params+=" --authorization-webhook-cache-unauthorized-ttl=${GCP_AUTHZ_CACHE_UNAUTHORIZED_TTL}" + fi + fi + authorization_mode="Node,${authorization_mode}" + params+=" --authorization-mode=${authorization_mode}" + + local csc_config_mount="" + local csc_config_volume="" + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. + csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," + csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," + params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" + fi + + local container_env="" + if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then + container_env+="{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}" + fi + if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then + if [[ -n "${container_env}" ]]; then + container_env="${container_env}, " + fi + container_env+="{\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\"}" + fi + if [[ -n "${container_env}" ]]; then + container_env="\"env\":[${container_env}]," + fi + + local -r src_file="${src_dir}/kube-apiserver.manifest" + + # params is passed by reference, so no "$" + setup-etcd-encryption "${src_file}" params + + # Evaluate variables. + local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}" + sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" + sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" + sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}" + sed -i -e "s@{{secure_port}}@443@g" "${src_file}" + sed -i -e "s@{{insecure_port_mapping}}@${INSECURE_PORT_MAPPING}@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" + sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" + sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{csc_config_mount}}@${csc_config_mount}@g" "${src_file}" + sed -i -e "s@{{csc_config_volume}}@${csc_config_volume}@g" "${src_file}" + sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" + sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" + sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{audit_webhook_config_volume}}@${audit_webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{webhook_exec_auth_plugin_mount}}@${webhook_exec_auth_plugin_mount}@g" "${src_file}" + sed -i -e "s@{{webhook_exec_auth_plugin_volume}}@${webhook_exec_auth_plugin_volume}@g" "${src_file}" + + cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" +} + + +# Sets-up etcd encryption. +# Configuration of etcd level encryption consists of the following steps: +# 1. Writing encryption provider config to disk +# 2. Adding encryption-provider-config flag to kube-apiserver +# 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested) +# +# Expects parameters: +# $1 - path to kube-apiserver template +# $2 - kube-apiserver startup flags (must be passed by reference) +# +# Assumes vars (supplied via kube-env): +# ENCRYPTION_PROVIDER_CONFIG +# CLOUD_KMS_INTEGRATION +# ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml) +function setup-etcd-encryption { + local kube_apiserver_template_path + local -n kube_api_server_params + local default_encryption_provider_config_vol + local default_encryption_provider_config_vol_mnt + local encryption_provider_config_vol_mnt + local encryption_provider_config_vol + local default_kms_socket_dir + local default_kms_socket_vol_mnt + local default_kms_socket_vol + local kms_socket_vol_mnt + local kms_socket_vol + local encryption_provider_config_path + + kube_apiserver_template_path="$1" + if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then + sed -i -e " { + s@{{encryption_provider_mount}}@@ + s@{{encryption_provider_volume}}@@ + s@{{kms_socket_mount}}@@ + s@{{kms_socket_volume}}@@ + } " "${kube_apiserver_template_path}" + return + fi + + kube_api_server_params="$2" + encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml} + + echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" + kube_api_server_params+=" --encryption-provider-config=${encryption_provider_config_path}" + + default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n') + default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n') + + encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode) + encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode) + sed -i -e " { + s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@ + s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@ + } " "${kube_apiserver_template_path}" + + if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then + default_kms_socket_dir="/var/run/kmsplugin" + default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n') + default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n') + + kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode) + kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode) + sed -i -e " { + s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@ + s@{{kms_socket_volume}}@${kms_socket_vol},@ + } " "${kube_apiserver_template_path}" + else + sed -i -e " { + s@{{kms_socket_mount}}@@ + s@{{kms_socket_volume}}@@ + } " "${kube_apiserver_template_path}" + fi +} diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index fb0fd5c7c30..4f38c61c2bf 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -336,6 +336,7 @@ function install-kube-manifests { xargs sed -ri "s@(image\":\s+\")k8s.gcr.io@\1${kube_addon_registry}@" fi cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_BIN}/configure-helper.sh" + cp "${dst_dir}/kubernetes/gci-trusty/configure-kubeapiserver.sh" "${KUBE_BIN}/configure-kubeapiserver.sh" if [[ -e "${dst_dir}/kubernetes/gci-trusty/gke-internal-configure-helper.sh" ]]; then cp "${dst_dir}/kubernetes/gci-trusty/gke-internal-configure-helper.sh" "${KUBE_BIN}/" fi diff --git a/cluster/gce/gci/configure_helper_test.go b/cluster/gce/gci/configure_helper_test.go index d4af0598bfd..ce587f6a18e 100644 --- a/cluster/gce/gci/configure_helper_test.go +++ b/cluster/gce/gci/configure_helper_test.go @@ -26,14 +26,13 @@ import ( "testing" "text/template" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" ) const ( - envScriptFileName = "kube-env" - configureHelperScriptName = "configure-helper.sh" + envScriptFileName = "kube-env" ) type ManifestTestCase struct { @@ -123,15 +122,15 @@ func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) st return f.Name() } -func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) { +func (c *ManifestTestCase) mustInvokeFunc(envTemplate, scriptName string, env interface{}) { envScriptPath := c.mustCreateEnv(envTemplate, env) - args := fmt.Sprintf("source %s ; source %s; %s", envScriptPath, configureHelperScriptName, c.manifestFuncName) + args := fmt.Sprintf("source %s ; source %s; %s", envScriptPath, scriptName, c.manifestFuncName) cmd := exec.Command("bash", "-c", args) bs, err := cmd.CombinedOutput() if err != nil { c.t.Logf("%s", bs) - c.t.Fatalf("Failed to run configure-helper.sh: %v", err) + c.t.Fatalf("Failed to run %s: %v", scriptName, err) } c.t.Logf("%s", string(bs)) }