diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml new file mode 100644 index 00000000000..34a1e42bf61 --- /dev/null +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -0,0 +1,78 @@ +# Please keep kube-proxy configuration in-sync with: +# cluster/saltbase/salt/kube-proxy/kube-proxy.manifest + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + addonmanager.kubernetes.io/mode: Reconcile + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 10% + template: + metadata: + labels: + k8s-app: kube-proxy + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/kube-proxy-ds-ready: "true" + initContainers: + - name: touch-lock + image: busybox + command: ['/bin/touch', '/run/xtables.lock'] + securityContext: + privileged: true + volumeMounts: + - mountPath: /run + name: run + readOnly: false + containers: + - name: kube-proxy + image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}} + resources: + requests: + cpu: {{ cpurequest }} + command: + - /bin/sh + - -c + - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 + {{container_env}} + {{kube_cache_mutation_detector_env_name}} + {{kube_cache_mutation_detector_env_value}} + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/log + name: varlog + readOnly: false + - mountPath: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + readOnly: false + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + volumes: + - name: varlog + hostPath: + path: /var/log + - name: kubeconfig + hostPath: + path: /var/lib/kube-proxy/kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + - name: run + hostPath: + path: /run + serviceAccountName: kube-proxy diff --git a/cluster/addons/kube-proxy/kube-proxy-rbac.yaml b/cluster/addons/kube-proxy/kube-proxy-rbac.yaml new file mode 100644 index 00000000000..a12ef9d3bfe --- /dev/null +++ b/cluster/addons/kube-proxy/kube-proxy-rbac.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-proxy + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: system:kube-proxy + labels: + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: kube-system +roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/common.sh b/cluster/common.sh index c6ac31d591d..669e520483d 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -638,6 +638,7 @@ DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote ${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}) KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) +KUBE_PROXY_DAEMONSET: $(yaml-quote ${KUBE_PROXY_DAEMONSET:-false}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote ${NODE_PROBLEM_DETECTOR_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 2e10b9b0fbe..f36066d0767 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -297,3 +297,7 @@ PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUE PROMETHEUS_TO_SD_ENDPOINT="${PROMETHEUS_TO_SD_ENDPOINT:-https://monitoring.googleapis.com/}" PROMETHEUS_TO_SD_PREFIX="${PROMETHEUS_TO_SD_PREFIX:-custom.googleapis.com}" ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-false}" + +# TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here. +# Optional: Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise. +KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 006ba5b9603..c8ef2496fb2 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -356,3 +356,7 @@ PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUE PROMETHEUS_TO_SD_ENDPOINT="${PROMETHEUS_TO_SD_ENDPOINT:-https://monitoring.googleapis.com/}" PROMETHEUS_TO_SD_PREFIX="${PROMETHEUS_TO_SD_PREFIX:-custom.googleapis.com}" ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-true}" + +# TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here. +# Optional: Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise. +KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index f2ec467e09a..0999eb11aae 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -448,6 +448,7 @@ initial_etcd_cluster_state: '$(echo "${INITIAL_ETCD_CLUSTER_STATE:-}" | sed -e " ca_cert_bundle_path: '$(echo "${CA_CERT_BUNDLE_PATH:-}" | sed -e "s/'/''/g")' hostname: $(hostname -s) enable_default_storage_class: '$(echo "$ENABLE_DEFAULT_STORAGE_CLASS" | sed -e "s/'/''/g")' +kube_proxy_daemonset: '$(echo "$KUBE_PROXY_DAEMONSET" | sed -e "s/'/''/g")' EOF if [ -n "${STORAGE_BACKEND:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls @@ -652,14 +653,15 @@ EOF # This should happen both on cluster initialization and node upgrades. # -# - Uses the CA_CERT and KUBE_PROXY_TOKEN to generate a kubeconfig file for -# the kube-proxy to securely connect to the apiserver. +# - When run as static pods, use the CA_CERT and KUBE_PROXY_TOKEN to generate a +# kubeconfig file for the kube-proxy to securely connect to the apiserver. +# - When run as a daemonset, generate a kubeconfig file specific to service account. function create-salt-kubeproxy-auth() { local -r kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig" + local kubeconfig_content="" if [ ! -e "${kube_proxy_kubeconfig_file}" ]; then - mkdir -p /srv/salt-overlay/salt/kube-proxy - (umask 077; - cat > "${kube_proxy_kubeconfig_file}" < "${kube_proxy_kubeconfig_file}" </var/lib/kube-proxy/kubeconfig apiVersion: v1 kind: Config @@ -415,6 +415,30 @@ current-context: service-account-context EOF } +function create-kubeproxy-serviceaccount-kubeconfig { + echo "Creating kube-proxy serviceaccount kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${KUBERNETES_MASTER_NAME} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +} + function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-controller-manager @@ -619,8 +643,17 @@ function start-kubelet { if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi + local node_labels="" + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + fi if [[ -n "${NODE_LABELS:-}" ]]; then - flags+=" --node-labels=${NODE_LABELS}" + node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" + fi + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" @@ -672,11 +705,11 @@ function prepare-log-file { chown root:root $1 } -# Starts kube-proxy pod. -function start-kube-proxy { - echo "Start kube-proxy pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" +# Prepares parameters for kube-proxy manifest. +# $1 source path of kube-proxy manifest. +function prepare-kube-proxy-manifest-variables { + local -r src_file=$1; + remove-salt-config-comments "${src_file}" local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" @@ -695,14 +728,20 @@ function start-kube-proxy { params+=" ${KUBEPROXY_TEST_ARGS}" fi local container_env="" + local kube_cache_mutation_detector_env_name="" + local kube_cache_mutation_detector_env_value="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:\n - name: KUBE_CACHE_MUTATION_DETECTOR\n value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" + container_env="env:" + kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" + kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then @@ -719,6 +758,14 @@ function start-kube-proxy { mount -o remount,rw /sys; " sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}" fi +} + +# Starts kube-proxy static pod. +function start-kube-proxy { + echo "Start kube-proxy static pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + prepare-kube-proxy-manifest-variables "$src_file" cp "${src_file}" /etc/kubernetes/manifests } @@ -1223,6 +1270,10 @@ function start-kube-addons { setup-addon-manifests "addons" "rbac" # Set up manifests of other addons. + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then + prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml" + setup-addon-manifests "addons" "kube-proxy" + fi if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \ @@ -1470,7 +1521,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then create-master-etcd-auth else create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}" - create-kubeproxy-kubeconfig + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + else + create-kubeproxy-serviceaccount-kubeconfig + fi fi if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then @@ -1498,7 +1553,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-lb-controller start-rescheduler else - start-kube-proxy + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi # Kube-registry-proxy. if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then start-kube-registry-proxy diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index eb3e3caa186..83d4ca54329 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -80,7 +80,6 @@ function get-calico-typha-cpu { echo "${typha_cpu}" } - function config-ip-firewall { echo "Configuring IP firewall rules" # The GCI image has host firewall which drop most inbound/forwarded packets. @@ -681,8 +680,8 @@ function create-master-kubelet-auth { fi } -function create-kubeproxy-kubeconfig { - echo "Creating kube-proxy kubeconfig file" +function create-kubeproxy-user-kubeconfig { + echo "Creating kube-proxy user kubeconfig file" cat </var/lib/kube-proxy/kubeconfig apiVersion: v1 kind: Config @@ -703,6 +702,30 @@ current-context: service-account-context EOF } +function create-kubeproxy-serviceaccount-kubeconfig { + echo "Creating kube-proxy serviceaccount kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${KUBERNETES_MASTER_NAME} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +} + function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-controller-manager @@ -927,8 +950,17 @@ function start-kubelet { if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi + local node_labels="" + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + fi if [[ -n "${NODE_LABELS:-}" ]]; then - flags+=" --node-labels=${NODE_LABELS}" + node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" + fi + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" @@ -1009,11 +1041,11 @@ function prepare-log-file { chown root:root $1 } -# Starts kube-proxy pod. -function start-kube-proxy { - echo "Start kube-proxy pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" +# Prepares parameters for kube-proxy manifest. +# $1 source path of kube-proxy manifest. +function prepare-kube-proxy-manifest-variables { + local -r src_file=$1; + remove-salt-config-comments "${src_file}" local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" @@ -1032,19 +1064,34 @@ function start-kube-proxy { params+=" ${KUBEPROXY_TEST_ARGS}" fi local container_env="" + local kube_cache_mutation_detector_env_name="" + local kube_cache_mutation_detector_env_value="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:\n - name: KUBE_CACHE_MUTATION_DETECTOR\n value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" + container_env="env:" + kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" + kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} fi +} + +# Starts kube-proxy static pod. +function start-kube-proxy { + echo "Start kube-proxy static pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + prepare-kube-proxy-manifest-variables "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests } @@ -1647,6 +1694,10 @@ function start-kube-addons { setup-addon-manifests "addons" "rbac" # Set up manifests of other addons. + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then + prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml" + setup-addon-manifests "addons" "kube-proxy" + fi if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \ @@ -1895,7 +1946,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then else create-node-pki create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME} - create-kubeproxy-kubeconfig + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + else + create-kubeproxy-serviceaccount-kubeconfig + fi if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then create-node-problem-detector-kubeconfig fi @@ -1918,7 +1973,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-lb-controller start-rescheduler else - start-kube-proxy + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi # Kube-registry-proxy. if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then start-kube-registry-proxy diff --git a/cluster/saltbase/salt/kube-proxy/init.sls b/cluster/saltbase/salt/kube-proxy/init.sls index 6d65c574c45..4ae6103dacd 100644 --- a/cluster/saltbase/salt/kube-proxy/init.sls +++ b/cluster/saltbase/salt/kube-proxy/init.sls @@ -7,6 +7,7 @@ - makedirs: true # kube-proxy in a static pod +{% if pillar.get('kube_proxy_daemonset', '').lower() != 'true' %} /etc/kubernetes/manifests/kube-proxy.manifest: file.managed: - source: salt://kube-proxy/kube-proxy.manifest @@ -24,6 +25,7 @@ - require: - service: docker - service: kubelet +{% endif %} /var/log/kube-proxy.log: file.managed: diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index efb20b086a6..37d96c9b6a0 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -1,3 +1,6 @@ +# Please keep kube-proxy configuration in-sync with: +# cluster/addons/kube-proxy/kube-proxy-ds.yaml + {% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%} {% if grains.api_servers is defined -%} {% set api_servers = "--master=https://" + grains.api_servers -%} @@ -35,6 +38,8 @@ {% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%} {% set container_env = "" -%} +{% set kube_cache_mutation_detector_env_name = "" -%} +{% set kube_cache_mutation_detector_env_value = "" -%} # kube-proxy podspec apiVersion: v1 @@ -75,6 +80,8 @@ spec: - -c - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 {{container_env}} + {{kube_cache_mutation_detector_env_name}} + {{kube_cache_mutation_detector_env_value}} securityContext: privileged: true volumeMounts: diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 470760795b8..846935c3d84 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -164,10 +164,16 @@ {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} {% endif -%} -{% set node_labels = "" %} -{% if pillar['node_labels'] is defined -%} - {% set node_labels="--node-labels=" + pillar['node_labels'] %} -{% endif -%} +{% set kube_proxy_ds_label = "" %} +{% if grains['roles'][0] != 'kubernetes-master' and pillar.get('kube_proxy_daemonset', '').lower() == 'true' %} + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + {% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %} +{% endif %} +{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %} +{% if node_labels != "" %} + {% set node_labels="--node-labels=" + node_labels %} +{% endif %} {% set node_taints = "" %} {% if pillar['node_taints'] is defined -%}