diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml index a84f838f912..34a1e42bf61 100644 --- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -48,6 +48,8 @@ spec: - -c - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 {{container_env}} + {{kube_cache_mutation_detector_env_name}} + {{kube_cache_mutation_detector_env_value}} securityContext: privileged: true volumeMounts: diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index a20b4be4f5f..0999eb11aae 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -653,14 +653,15 @@ EOF # This should happen both on cluster initialization and node upgrades. # -# - Uses the CA_CERT and KUBE_PROXY_TOKEN to generate a kubeconfig file for -# the kube-proxy to securely connect to the apiserver. +# - When run as static pods, use the CA_CERT and KUBE_PROXY_TOKEN to generate a +# kubeconfig file for the kube-proxy to securely connect to the apiserver. +# - When run as a daemonset, generate a kubeconfig file specific to service account. function create-salt-kubeproxy-auth() { local -r kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig" + local kubeconfig_content="" if [ ! -e "${kube_proxy_kubeconfig_file}" ]; then - mkdir -p /srv/salt-overlay/salt/kube-proxy - (umask 077; - cat > "${kube_proxy_kubeconfig_file}" < "${kube_proxy_kubeconfig_file}" </var/lib/kube-proxy/kubeconfig apiVersion: v1 kind: Config @@ -409,6 +409,30 @@ current-context: service-account-context EOF } +function create-kubeproxy-serviceaccount-kubeconfig { + echo "Creating kube-proxy serviceaccount kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${KUBERNETES_MASTER_NAME} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +} + function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-controller-manager @@ -613,8 +637,17 @@ function start-kubelet { if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi + local node_labels="" + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + fi if [[ -n "${NODE_LABELS:-}" ]]; then - flags+=" --node-labels=${NODE_LABELS}" + node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" + fi + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" @@ -666,11 +699,11 @@ function prepare-log-file { chown root:root $1 } -# Starts kube-proxy pod. -function start-kube-proxy { - echo "Start kube-proxy pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" +# Prepares parameters for kube-proxy manifest. +# $1 source path of kube-proxy manifest. +function prepare-kube-proxy-manifest-variables { + local -r src_file=$1; + remove-salt-config-comments "${src_file}" local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" @@ -689,14 +722,20 @@ function start-kube-proxy { params+=" ${KUBEPROXY_TEST_ARGS}" fi local container_env="" + local kube_cache_mutation_detector_env_name="" + local kube_cache_mutation_detector_env_value="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:\n - name: KUBE_CACHE_MUTATION_DETECTOR\n value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" + container_env="env:" + kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" + kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then @@ -713,6 +752,14 @@ function start-kube-proxy { mount -o remount,rw /sys; " sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}" fi +} + +# Starts kube-proxy static pod. +function start-kube-proxy { + echo "Start kube-proxy static pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + prepare-kube-proxy-manifest-variables "$src_file" cp "${src_file}" /etc/kubernetes/manifests } @@ -1217,6 +1264,10 @@ function start-kube-addons { setup-addon-manifests "addons" "rbac" # Set up manifests of other addons. + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then + prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml" + setup-addon-manifests "addons" "kube-proxy" + fi if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \ @@ -1464,7 +1515,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then create-master-etcd-auth else create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}" - create-kubeproxy-kubeconfig + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + else + create-kubeproxy-serviceaccount-kubeconfig + fi fi if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then @@ -1492,7 +1547,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-lb-controller start-rescheduler else - start-kube-proxy + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi # Kube-registry-proxy. if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then start-kube-registry-proxy diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 08958b93ff1..e07b9ae1cd6 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -80,7 +80,6 @@ function get-calico-typha-cpu { echo "${typha_cpu}" } - function config-ip-firewall { echo "Configuring IP firewall rules" # The GCI image has host firewall which drop most inbound/forwarded packets. @@ -675,8 +674,8 @@ function create-master-kubelet-auth { fi } -function create-kubeproxy-kubeconfig { - echo "Creating kube-proxy kubeconfig file" +function create-kubeproxy-user-kubeconfig { + echo "Creating kube-proxy user kubeconfig file" cat </var/lib/kube-proxy/kubeconfig apiVersion: v1 kind: Config @@ -697,6 +696,30 @@ current-context: service-account-context EOF } +function create-kubeproxy-serviceaccount-kubeconfig { + echo "Creating kube-proxy serviceaccount kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${KUBERNETES_MASTER_NAME} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +} + function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-controller-manager @@ -921,8 +944,17 @@ function start-kubelet { if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi + local node_labels="" + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + fi if [[ -n "${NODE_LABELS:-}" ]]; then - flags+=" --node-labels=${NODE_LABELS}" + node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" + fi + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" @@ -1003,11 +1035,11 @@ function prepare-log-file { chown root:root $1 } -# Starts kube-proxy pod. -function start-kube-proxy { - echo "Start kube-proxy pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" +# Prepares parameters for kube-proxy manifest. +# $1 source path of kube-proxy manifest. +function prepare-kube-proxy-manifest-variables { + local -r src_file=$1; + remove-salt-config-comments "${src_file}" local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" @@ -1026,19 +1058,34 @@ function start-kube-proxy { params+=" ${KUBEPROXY_TEST_ARGS}" fi local container_env="" + local kube_cache_mutation_detector_env_name="" + local kube_cache_mutation_detector_env_value="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:\n - name: KUBE_CACHE_MUTATION_DETECTOR\n value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" + container_env="env:" + kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" + kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} fi +} + +# Starts kube-proxy static pod. +function start-kube-proxy { + echo "Start kube-proxy static pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + prepare-kube-proxy-manifest-variables "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests } @@ -1641,6 +1688,10 @@ function start-kube-addons { setup-addon-manifests "addons" "rbac" # Set up manifests of other addons. + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then + prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml" + setup-addon-manifests "addons" "kube-proxy" + fi if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \ @@ -1889,7 +1940,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then else create-node-pki create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME} - create-kubeproxy-kubeconfig + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + else + create-kubeproxy-serviceaccount-kubeconfig + fi if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then create-node-problem-detector-kubeconfig fi @@ -1912,7 +1967,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-lb-controller start-rescheduler else - start-kube-proxy + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi # Kube-registry-proxy. if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then start-kube-registry-proxy diff --git a/cluster/saltbase/salt/kube-proxy/init.sls b/cluster/saltbase/salt/kube-proxy/init.sls index 6d65c574c45..4ae6103dacd 100644 --- a/cluster/saltbase/salt/kube-proxy/init.sls +++ b/cluster/saltbase/salt/kube-proxy/init.sls @@ -7,6 +7,7 @@ - makedirs: true # kube-proxy in a static pod +{% if pillar.get('kube_proxy_daemonset', '').lower() != 'true' %} /etc/kubernetes/manifests/kube-proxy.manifest: file.managed: - source: salt://kube-proxy/kube-proxy.manifest @@ -24,6 +25,7 @@ - require: - service: docker - service: kubelet +{% endif %} /var/log/kube-proxy.log: file.managed: diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index 0befd5b6457..079e868086a 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -38,6 +38,8 @@ {% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%} {% set container_env = "" -%} +{% set kube_cache_mutation_detector_env_name = "" -%} +{% set kube_cache_mutation_detector_env_value = "" -%} # kube-proxy podspec apiVersion: v1 @@ -78,6 +80,8 @@ spec: - -c - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 {{container_env}} + {{kube_cache_mutation_detector_env_name}} + {{kube_cache_mutation_detector_env_value}} securityContext: privileged: true volumeMounts: diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 470760795b8..846935c3d84 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -164,10 +164,16 @@ {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} {% endif -%} -{% set node_labels = "" %} -{% if pillar['node_labels'] is defined -%} - {% set node_labels="--node-labels=" + pillar['node_labels'] %} -{% endif -%} +{% set kube_proxy_ds_label = "" %} +{% if grains['roles'][0] != 'kubernetes-master' and pillar.get('kube_proxy_daemonset', '').lower() == 'true' %} + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + {% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %} +{% endif %} +{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %} +{% if node_labels != "" %} + {% set node_labels="--node-labels=" + node_labels %} +{% endif %} {% set node_taints = "" %} {% if pillar['node_taints'] is defined -%}