diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 7b356b9e544..d4cd0de63b0 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1113,7 +1113,8 @@ function start-kubelet { echo "Using kubelet binary at ${kubelet_bin}" local -r kubelet_env_file="/etc/default/kubelet" - echo "KUBELET_OPTS=\"${KUBELET_ARGS}\"" > "${kubelet_env_file}" + local kubelet_opts="${KUBELET_ARGS} ${KUBELET_CONFIG_FILE_ARG:-}" + echo "KUBELET_OPTS=\"${kubelet_opts}\"" > "${kubelet_env_file}" # Write the systemd service file for kubelet. cat </etc/systemd/system/kubelet.service @@ -2483,6 +2484,12 @@ fi source "${KUBE_HOME}/kube-env" + +if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then + echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml" + KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml" +fi + if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then source "${KUBE_HOME}/kube-master-certs" fi diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 58a19815b07..dd6da16bc07 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -25,7 +25,7 @@ set -o pipefail ### Hardcoded constants DEFAULT_CNI_VERSION="v0.6.0" -DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f" +DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f" DEFAULT_NPD_VERSION="v0.4.1" DEFAULT_NPD_SHA1="a57a3fe64cab8a18ec654f5cef0aec59dae62568" DEFAULT_MOUNTER_TAR_SHA="8003b798cf33c7f91320cd6ee5cec4fa22244571" @@ -54,37 +54,59 @@ EOF function download-kube-env { # Fetch kube-env from GCE metadata server. - (umask 077; - local -r tmp_kube_env="/tmp/kube-env.yaml" - curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o "${tmp_kube_env}" \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Convert the yaml format file into a shell-style file. - eval $(python -c ''' + ( + umask 077 + local -r tmp_kube_env="/tmp/kube-env.yaml" + curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o "${tmp_kube_env}" \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Convert the yaml format file into a shell-style file. + eval $(python -c ''' import pipes,sys,yaml for k,v in yaml.load(sys.stdin).iteritems(): print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))) ''' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env") - rm -f "${tmp_kube_env}" + rm -f "${tmp_kube_env}" + ) +} + +function download-kubelet-config { + local -r dest="$1" + echo "Downloading Kubelet config file, if it exists" + # Fetch kubelet config file from GCE metadata server. + ( + umask 077 + local -r tmp_kubelet_config="/tmp/kubelet-config.yaml" + if curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o "${tmp_kubelet_config}" \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kubelet-config; then + # only write to the final location if curl succeeds + mv "${tmp_kubelet_config}" "${dest}" + elif [[ "${REQUIRE_METADATA_KUBELET_CONFIG_FILE:-false}" == "true" ]]; then + echo "== Failed to download required Kubelet config file from metadata server ==" + exit 1 + fi ) } function download-kube-master-certs { # Fetch kube-env from GCE metadata server. - (umask 077; - local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml" - curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o "${tmp_kube_master_certs}" \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs - # Convert the yaml format file into a shell-style file. - eval $(python -c ''' + ( + umask 077 + local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml" + curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o "${tmp_kube_master_certs}" \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs + # Convert the yaml format file into a shell-style file. + eval $(python -c ''' import pipes,sys,yaml for k,v in yaml.load(sys.stdin).iteritems(): print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))) ''' < "${tmp_kube_master_certs}" > "${KUBE_HOME}/kube-master-certs") - rm -f "${tmp_kube_master_certs}" + rm -f "${tmp_kube_master_certs}" ) } @@ -356,13 +378,24 @@ function install-kube-binary-config { ######### Main Function ########## echo "Start to install kubernetes files" +# if install fails, message-of-the-day (motd) will warn at login shell set-broken-motd + KUBE_HOME="/home/kubernetes" KUBE_BIN="${KUBE_HOME}/bin" + +# download and source kube-env download-kube-env source "${KUBE_HOME}/kube-env" + +download-kubelet-config "${KUBE_HOME}/kubelet-config.yaml" + +# master certs if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then download-kube-master-certs fi + +# binaries and kube-system manifests install-kube-binary-config + echo "Done for installing kubernetes files" diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh index d6c48add00a..a67f28e9e30 100755 --- a/cluster/gce/gci/master-helper.sh +++ b/cluster/gce/gci/master-helper.sh @@ -107,6 +107,7 @@ function create-master-instance-internal() { "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}") local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml" + metadata="${metadata},kubelet-config=${KUBE_TEMP}/master-kubelet-config.yaml" metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml" metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt" diff --git a/cluster/gce/gci/node-helper.sh b/cluster/gce/gci/node-helper.sh index 33030d3f6de..23ef330d539 100755 --- a/cluster/gce/gci/node-helper.sh +++ b/cluster/gce/gci/node-helper.sh @@ -20,6 +20,7 @@ source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" function get-node-instance-metadata { local metadata="" metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml," + metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml," metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml," metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh," metadata+="cluster-location=${KUBE_TEMP}/cluster-location.txt," diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 42034137645..fc5b832eff6 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -502,6 +502,7 @@ function write-master-env { construct-kubelet-flags true build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" + build-kubelet-config true "${KUBE_TEMP}/master-kubelet-config.yaml" build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml" } @@ -512,6 +513,7 @@ function write-node-env { construct-kubelet-flags false build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" + build-kubelet-config false "${KUBE_TEMP}/node-kubelet-config.yaml" } function build-node-labels { @@ -531,16 +533,89 @@ function build-node-labels { echo $node_labels } +# yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result +# under the provided name. If the encoded structure is empty, echoes nothing. +# 1: name to be output in yaml +# 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray) +# 3: key-value separator (defaults to ':') +# 4: item separator (defaults to ',') +function yaml-map-string-stringarray { + declare -r name="${1}" + declare -r encoded="${2}" + declare -r kv_sep="${3:-:}" + declare -r item_sep="${4:-,}" + + declare -a pairs # indexed array + declare -A map # associative array + IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep + for pair in "${pairs[@]}"; do + declare key + declare value + IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep + map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys + done + # only output if there is a non-empty map + if [[ ${#map[@]} -gt 0 ]]; then + echo "${name}:" + for k in "${!map[@]}"; do + echo " ${k}:" + declare -a values + IFS="${item_sep}" read -ra values <<<"${map[$k]}" + for val in "${values[@]}"; do + # declare across two lines so errexit can catch failures + declare v + v=$(yaml-quote "${val}") + echo " - ${v}" + done + done + fi +} + +# yaml-map-string-string converts the encoded structure to yaml format, and echoes the result +# under the provided name. If the encoded structure is empty, echoes nothing. +# 1: name to be output in yaml +# 2: encoded map-string-string (no duplicate keys) +# 3: bool, whether to yaml-quote the value string in the output (defaults to true) +# 4: key-value separator (defaults to ':') +# 5: item separator (defaults to ',') +function yaml-map-string-string { + declare -r name="${1}" + declare -r encoded="${2}" + declare -r quote_val_string="${3:-true}" + declare -r kv_sep="${4:-:}" + declare -r item_sep="${5:-,}" + + declare -a pairs # indexed array + declare -A map # associative array + IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too + for pair in "${pairs[@]}"; do + declare key + declare value + IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep + map[$key]="${value}" # add to associative array + done + # only output if there is a non-empty map + if [[ ${#map[@]} -gt 0 ]]; then + echo "${name}:" + for k in "${!map[@]}"; do + if [[ "${quote_val_string}" == "true" ]]; then + # declare across two lines so errexit can catch failures + declare v + v=$(yaml-quote "${map[$k]}") + echo " ${k}: ${v}" + else + echo " ${k}: ${map[$k]}" + fi + done + fi +} + # $1: if 'true', we're rendering flags for a master, else a node function construct-kubelet-flags { local master=$1 local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}" flags+=" --allow-privileged=true" - flags+=" --cgroup-root=/" flags+=" --cloud-provider=gce" - flags+=" --cluster-dns=${DNS_SERVER_IP}" - flags+=" --cluster-domain=${DNS_DOMAIN}" - flags+=" --pod-manifest-path=/etc/kubernetes/manifests" # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter" flags+=" --experimental-check-node-capabilities-before-mount=true" @@ -549,33 +624,17 @@ function construct-kubelet-flags { if [[ "${master}" == "true" ]]; then flags+=" ${MASTER_KUBELET_TEST_ARGS:-}" - flags+=" --enable-debugging-handlers=false" - flags+=" --hairpin-mode=none" if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then #TODO(mikedanese): allow static pods to start before creating a client #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" flags+=" --register-schedulable=false" - else - # Note: Standalone mode is used by GKE - flags+=" --pod-cidr=${MASTER_IP_RANGE}" fi else # For nodes flags+=" ${NODE_KUBELET_TEST_ARGS:-}" - flags+=" --enable-debugging-handlers=true" flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" - if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \ - [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \ - [[ "${HAIRPIN_MODE:-}" == "none" ]]; then - flags+=" --hairpin-mode=${HAIRPIN_MODE}" - fi - flags+=" --anonymous-auth=false" - flags+=" --authentication-token-webhook" - flags+=" --authorization-mode=Webhook" - # Keep client-ca-file in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh - flags+=" --client-ca-file=/etc/srv/kubernetes/pki/ca-certificates.crt" fi # Network plugin if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then @@ -597,11 +656,6 @@ function construct-kubelet-flags { flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" fi flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}" - # Note: ENABLE_MANIFEST_URL is used by GKE - if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then - flags+=" --manifest-url=${MANIFEST_URL}" - flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}" - fi if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi @@ -612,12 +666,6 @@ function construct-kubelet-flags { if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" fi - if [[ -n "${EVICTION_HARD:-}" ]]; then - flags+=" --eviction-hard=${EVICTION_HARD}" - fi - if [[ -n "${FEATURE_GATES:-}" ]]; then - flags+=" --feature-gates=${FEATURE_GATES}" - fi # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it? if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then flags+=" --rotate-certificates=true" @@ -633,6 +681,91 @@ function construct-kubelet-flags { KUBELET_ARGS="${flags}" } +# $1: if 'true', we're rendering config for a master, else a node +function build-kubelet-config { + local master=$1 + local file=$2 + + rm -f "${file}" + { + declare quoted_dns_server_ip + declare quoted_dns_domain + quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}") + quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}") + cat <