Merge pull request #62183 from mtaufen/gce-kc-metadata

Automatic merge from submit-queue (batch tested with PRs 62486, 62471, 62183). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

provision kubelet config file for GCE instead of deprecated flags

Many Kubelet flags are now deprecated in favor of the versioned config file format. This PR adopts the versioned config file format in our cluster turn-up scripts.

```release-note
cluster/kube-up.sh now provisions a Kubelet config file for GCE via the metadata server. This file is installed by the corresponding GCE init scripts.
```

cluster/gce/gci/configure-helper.sh

@@ -1113,7 +1113,8 @@ function start-kubelet {
   echo "Using kubelet binary at ${kubelet_bin}"
 
   local -r kubelet_env_file="/etc/default/kubelet"
-  echo "KUBELET_OPTS=\"${KUBELET_ARGS}\"" > "${kubelet_env_file}"
+  local kubelet_opts="${KUBELET_ARGS} ${KUBELET_CONFIG_FILE_ARG:-}"
+  echo "KUBELET_OPTS=\"${kubelet_opts}\"" > "${kubelet_env_file}"
 
   # Write the systemd service file for kubelet.
   cat <<EOF >/etc/systemd/system/kubelet.service
@@ -2483,6 +2484,12 @@ fi
 
 source "${KUBE_HOME}/kube-env"
 
+
+if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
+  echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
+  KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
+fi
+
 if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
   source "${KUBE_HOME}/kube-master-certs"
 fi

cluster/gce/gci/configure.sh

@@ -25,7 +25,7 @@ set -o pipefail
 
 ### Hardcoded constants
 DEFAULT_CNI_VERSION="v0.6.0"
-DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f" 
+DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f"
 DEFAULT_NPD_VERSION="v0.4.1"
 DEFAULT_NPD_SHA1="a57a3fe64cab8a18ec654f5cef0aec59dae62568"
 DEFAULT_MOUNTER_TAR_SHA="8003b798cf33c7f91320cd6ee5cec4fa22244571"
@@ -54,37 +54,59 @@ EOF
 
 function download-kube-env {
   # Fetch kube-env from GCE metadata server.
-  (umask 077;
-  local -r tmp_kube_env="/tmp/kube-env.yaml"
-  curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
-    -H "X-Google-Metadata-Request: True" \
-    -o "${tmp_kube_env}" \
-    http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
-  # Convert the yaml format file into a shell-style file.
-  eval $(python -c '''
+  (
+    umask 077
+    local -r tmp_kube_env="/tmp/kube-env.yaml"
+    curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
+      -H "X-Google-Metadata-Request: True" \
+      -o "${tmp_kube_env}" \
+      http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+    # Convert the yaml format file into a shell-style file.
+    eval $(python -c '''
 import pipes,sys,yaml
 for k,v in yaml.load(sys.stdin).iteritems():
   print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
 ''' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env")
-  rm -f "${tmp_kube_env}"
+    rm -f "${tmp_kube_env}"
+  )
+}
+
+function download-kubelet-config {
+  local -r dest="$1"
+  echo "Downloading Kubelet config file, if it exists"
+  # Fetch kubelet config file from GCE metadata server.
+  (
+    umask 077
+    local -r tmp_kubelet_config="/tmp/kubelet-config.yaml"
+    if curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o "${tmp_kubelet_config}" \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kubelet-config; then
+      # only write to the final location if curl succeeds
+      mv "${tmp_kubelet_config}" "${dest}"
+    elif [[ "${REQUIRE_METADATA_KUBELET_CONFIG_FILE:-false}" == "true" ]]; then
+      echo "== Failed to download required Kubelet config file from metadata server =="
+      exit 1
+    fi
   )
 }
 
 function download-kube-master-certs {
   # Fetch kube-env from GCE metadata server.
-  (umask 077;
-  local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml"
-  curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
-    -H "X-Google-Metadata-Request: True" \
-    -o "${tmp_kube_master_certs}" \
-    http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs
-  # Convert the yaml format file into a shell-style file.
-  eval $(python -c '''
+  (
+    umask 077
+    local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml"
+    curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
+      -H "X-Google-Metadata-Request: True" \
+      -o "${tmp_kube_master_certs}" \
+      http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs
+    # Convert the yaml format file into a shell-style file.
+    eval $(python -c '''
 import pipes,sys,yaml
 for k,v in yaml.load(sys.stdin).iteritems():
   print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
 ''' < "${tmp_kube_master_certs}" > "${KUBE_HOME}/kube-master-certs")
-  rm -f "${tmp_kube_master_certs}"
+    rm -f "${tmp_kube_master_certs}"
   )
 }
 
@@ -356,13 +378,24 @@ function install-kube-binary-config {
 
 ######### Main Function ##########
 echo "Start to install kubernetes files"
+# if install fails, message-of-the-day (motd) will warn at login shell
 set-broken-motd
+
 KUBE_HOME="/home/kubernetes"
 KUBE_BIN="${KUBE_HOME}/bin"
+
+# download and source kube-env
 download-kube-env
 source "${KUBE_HOME}/kube-env"
+
+download-kubelet-config "${KUBE_HOME}/kubelet-config.yaml"
+
+# master certs
 if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
   download-kube-master-certs
 fi
+
+# binaries and kube-system manifests
 install-kube-binary-config
+
 echo "Done for installing kubernetes files"

cluster/gce/gci/master-helper.sh

@@ -107,6 +107,7 @@ function create-master-instance-internal() {
     "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
 
   local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
+  metadata="${metadata},kubelet-config=${KUBE_TEMP}/master-kubelet-config.yaml"
   metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"
   metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh"
   metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt"

cluster/gce/gci/node-helper.sh

@@ -20,6 +20,7 @@ source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
 function get-node-instance-metadata {
   local metadata=""
   metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
+  metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml,"
   metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml,"
   metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,"
   metadata+="cluster-location=${KUBE_TEMP}/cluster-location.txt,"

cluster/gce/util.sh

@@ -502,6 +502,7 @@ function write-master-env {
 
   construct-kubelet-flags true
   build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
+  build-kubelet-config true "${KUBE_TEMP}/master-kubelet-config.yaml"
   build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml"
 }
 
@@ -512,6 +513,7 @@ function write-node-env {
 
   construct-kubelet-flags false
   build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
+  build-kubelet-config false "${KUBE_TEMP}/node-kubelet-config.yaml"
 }
 
 function build-node-labels {
@@ -531,16 +533,89 @@ function build-node-labels {
   echo $node_labels
 }
 
+# yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result
+# under the provided name. If the encoded structure is empty, echoes nothing.
+# 1: name to be output in yaml
+# 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray)
+# 3: key-value separator (defaults to ':')
+# 4: item separator (defaults to ',')
+function yaml-map-string-stringarray {
+  declare -r name="${1}"
+  declare -r encoded="${2}"
+  declare -r kv_sep="${3:-:}"
+  declare -r item_sep="${4:-,}"
+
+  declare -a pairs # indexed array
+  declare -A map # associative array
+  IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep
+  for pair in "${pairs[@]}"; do
+    declare key
+    declare value
+    IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
+    map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys
+  done
+  # only output if there is a non-empty map
+  if [[ ${#map[@]} -gt 0 ]]; then
+    echo "${name}:"
+    for k in "${!map[@]}"; do
+      echo "  ${k}:"
+      declare -a values
+      IFS="${item_sep}" read -ra values <<<"${map[$k]}"
+      for val in "${values[@]}"; do
+        # declare across two lines so errexit can catch failures
+        declare v
+        v=$(yaml-quote "${val}")
+        echo "    - ${v}"
+      done
+    done
+  fi
+}
+
+# yaml-map-string-string converts the encoded structure to yaml format, and echoes the result
+# under the provided name. If the encoded structure is empty, echoes nothing.
+# 1: name to be output in yaml
+# 2: encoded map-string-string (no duplicate keys)
+# 3: bool, whether to yaml-quote the value string in the output (defaults to true)
+# 4: key-value separator (defaults to ':')
+# 5: item separator (defaults to ',')
+function yaml-map-string-string {
+  declare -r name="${1}"
+  declare -r encoded="${2}"
+  declare -r quote_val_string="${3:-true}"
+  declare -r kv_sep="${4:-:}"
+  declare -r item_sep="${5:-,}"
+
+  declare -a pairs # indexed array
+  declare -A map # associative array
+  IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too
+  for pair in "${pairs[@]}"; do
+    declare key
+    declare value
+    IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
+    map[$key]="${value}" # add to associative array
+  done
+  # only output if there is a non-empty map
+  if [[ ${#map[@]} -gt 0 ]]; then
+    echo "${name}:"
+    for k in "${!map[@]}"; do
+      if [[ "${quote_val_string}" == "true" ]]; then
+        # declare across two lines so errexit can catch failures
+        declare v
+        v=$(yaml-quote "${map[$k]}")
+        echo "  ${k}: ${v}"
+      else
+        echo "  ${k}: ${map[$k]}"
+      fi
+    done
+  fi
+}
+
 # $1: if 'true', we're rendering flags for a master, else a node
 function construct-kubelet-flags {
   local master=$1
   local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
   flags+=" --allow-privileged=true"
-  flags+=" --cgroup-root=/"
   flags+=" --cloud-provider=gce"
-  flags+=" --cluster-dns=${DNS_SERVER_IP}"
-  flags+=" --cluster-domain=${DNS_DOMAIN}"
-  flags+=" --pod-manifest-path=/etc/kubernetes/manifests"
   # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh
   flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter"
   flags+=" --experimental-check-node-capabilities-before-mount=true"
@@ -549,33 +624,17 @@ function construct-kubelet-flags {
 
   if [[ "${master}" == "true" ]]; then
     flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
-    flags+=" --enable-debugging-handlers=false"
-    flags+=" --hairpin-mode=none"
     if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
       #TODO(mikedanese): allow static pods to start before creating a client
       #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
       #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
       flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
       flags+=" --register-schedulable=false"
-    else
-      # Note: Standalone mode is used by GKE
-      flags+=" --pod-cidr=${MASTER_IP_RANGE}"
     fi
   else # For nodes
     flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
-    flags+=" --enable-debugging-handlers=true"
     flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
     flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
-    if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
-       [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
-       [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
-      flags+=" --hairpin-mode=${HAIRPIN_MODE}"
-    fi
-    flags+=" --anonymous-auth=false"
-    flags+=" --authentication-token-webhook"
-    flags+=" --authorization-mode=Webhook"
-    # Keep client-ca-file in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
-    flags+=" --client-ca-file=/etc/srv/kubernetes/pki/ca-certificates.crt"
   fi
   # Network plugin
   if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then
@@ -597,11 +656,6 @@ function construct-kubelet-flags {
     flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
   fi
   flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
-  # Note: ENABLE_MANIFEST_URL is used by GKE
-  if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
-    flags+=" --manifest-url=${MANIFEST_URL}"
-    flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}"
-  fi
   if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then
     flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
   fi
@@ -612,12 +666,6 @@ function construct-kubelet-flags {
   if [[ -n "${NODE_TAINTS:-}" ]]; then
     flags+=" --register-with-taints=${NODE_TAINTS}"
   fi
-  if [[ -n "${EVICTION_HARD:-}" ]]; then
-    flags+=" --eviction-hard=${EVICTION_HARD}"
-  fi
-  if [[ -n "${FEATURE_GATES:-}" ]]; then
-    flags+=" --feature-gates=${FEATURE_GATES}"
-  fi
   # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it?
   if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
     flags+=" --rotate-certificates=true"
@@ -633,6 +681,91 @@ function construct-kubelet-flags {
   KUBELET_ARGS="${flags}"
 }
 
+# $1: if 'true', we're rendering config for a master, else a node
+function build-kubelet-config {
+  local master=$1
+  local file=$2
+
+  rm -f "${file}"
+  {
+    declare quoted_dns_server_ip
+    declare quoted_dns_domain
+    quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}")
+    quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}")
+    cat <<EOF
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupRoot: /
+clusterDNS:
+  - ${quoted_dns_server_ip}
+clusterDomain: ${quoted_dns_domain}
+staticPodPath: /etc/kubernetes/manifests
+readOnlyPort: 10255
+EOF
+
+    # --- begin master-specific config ---
+    if [[ "${master}" == "true" ]]; then
+        cat <<EOF
+enableDebuggingHandlers: false
+hairpinMode: none
+authentication:
+  webhook:
+    enabled: false
+  anonymous:
+    enabled: true
+authorization:
+  mode: AlwaysAllow
+EOF
+      if [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
+        # Note: Standalone mode is used by GKE
+        declare quoted_master_ip_range
+        quoted_master_ip_range=$(yaml-quote "${MASTER_IP_RANGE}")
+        cat <<EOF
+podCidr: ${quoted_master_ip_range}
+EOF
+      fi
+      # --- end master-specific config ---
+    else
+      # --- begin node-specific config ---
+      # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
+      cat <<EOF
+enableDebuggingHandlers: true
+authentication:
+  x509:
+    clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt
+EOF
+      if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
+         [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
+         [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
+         declare quoted_hairpin_mode
+         quoted_hairpin_mode=$(yaml-quote "${HAIRPIN_MODE}")
+         cat <<EOF
+hairpinMode: ${quoted_hairpin_mode}
+EOF
+      fi
+      # --- end node-specific config ---
+    fi
+
+    # Note: ENABLE_MANIFEST_URL is used by GKE
+    if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
+      declare quoted_manifest_url
+      quoted_manifest_url=$(yaml-quote "${MANIFEST_URL}")
+      cat <<EOF
+staticPodURL: ${quoted_manifest_url}
+EOF
+      yaml-map-string-stringarray 'staticPodURLHeader' "${MANIFEST_URL_HEADER}"
+    fi
+
+    if [[ -n "${EVICTION_HARD:-}" ]]; then
+      yaml-map-string-string 'evictionHard' "${EVICTION_HARD}" true '<'
+    fi
+
+    if [[ -n "${FEATURE_GATES:-}" ]]; then
+      yaml-map-string-string 'featureGates' "${FEATURE_GATES}" false '='
+    fi
+  } > "${file}"
+}
+
 function build-kube-master-certs {
   local file=$1
   rm -f ${file}
@@ -758,6 +891,7 @@ LOAD_IMAGE_COMMAND: $(yaml-quote ${LOAD_IMAGE_COMMAND:-})
 ZONE: $(yaml-quote ${ZONE})
 VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR})
 KUBELET_ARGS: $(yaml-quote ${KUBELET_ARGS})
+REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true)
 EOF
   if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \
      [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]  || \

cmd/kubelet/app/server.go

@@ -229,6 +229,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
 			}
 
 			// run the kubelet
+			glog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration)
 			if err := Run(kubeletServer, kubeletDeps); err != nil {
 				glog.Fatal(err)
 			}