From 39cb8222c79e0b5b9e9a9b78087cf5b38fe4eb4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20=C5=81ukaszewicz?= Date: Mon, 2 Dec 2019 12:51:54 +0100 Subject: [PATCH] Add an option to specify kubelet flags for heapster node. Useful in scalability tests, where we don't want test pods (e.g. Kubemark hollow nodes) to be scheduled on heapster node. --- cluster/gce/gci/node-helper.sh | 4 +++- cluster/gce/util.sh | 27 ++++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cluster/gce/gci/node-helper.sh b/cluster/gce/gci/node-helper.sh index f15ea55aaf6..d6ccd25ca91 100755 --- a/cluster/gce/gci/node-helper.sh +++ b/cluster/gce/gci/node-helper.sh @@ -17,9 +17,11 @@ # A library of helper functions and constant for GCI distro source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" +# shellcheck disable=SC2120 function get-node-instance-metadata-from-file { + local kube_env=${1:-node-kube-env} # optional local metadata="" - metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml," + metadata+="kube-env=${KUBE_TEMP}/${kube_env}.yaml," metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml," metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml," metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh," diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index afc2b1284cf..4ac1f5bb64a 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -592,7 +592,7 @@ function write-master-env { KUBERNETES_MASTER_NAME="${MASTER_NAME}" fi - construct-linux-kubelet-flags true + construct-linux-kubelet-flags "master" build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml" build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml" @@ -603,7 +603,9 @@ function write-linux-node-env { KUBERNETES_MASTER_NAME="${MASTER_NAME}" fi - construct-linux-kubelet-flags false + construct-linux-kubelet-flags "heapster" + build-linux-kube-env false "${KUBE_TEMP}/heapster-kube-env.yaml" + construct-linux-kubelet-flags "node" build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml" } @@ -616,7 +618,7 @@ function write-windows-node-env { } function build-linux-node-labels { - local master=$1 + local node_type=$1 local node_labels="" if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${master}" != "true" ]]; then # Add kube-proxy daemonset label to node to avoid situation during cluster @@ -626,10 +628,10 @@ function build-linux-node-labels { if [[ -n "${NODE_LABELS:-}" ]]; then node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" fi - if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${master}" != "true" ]]; then + if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${node_type}" != "master" ]]; then node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}" fi - if [[ -n "${MASTER_NODE_LABELS:-}" && "${master}" == "true" ]]; then + if [[ -n "${MASTER_NODE_LABELS:-}" && "${node_type}" == "master" ]]; then node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}" fi echo $node_labels @@ -740,7 +742,7 @@ function construct-common-kubelet-flags { # Sets KUBELET_ARGS with the kubelet flags for Linux nodes. # $1: if 'true', we're rendering flags for a master, else a node function construct-linux-kubelet-flags { - local master="$1" + local node_type="$1" local flags="$(construct-common-kubelet-flags)" # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter" @@ -751,7 +753,7 @@ function construct-linux-kubelet-flags { flags+=" --dynamic-config-dir=/var/lib/kubelet/dynamic-config" - if [[ "${master}" == "true" ]]; then + if [[ "${node_type}" == "master" ]]; then flags+=" ${MASTER_KUBELET_TEST_ARGS:-}" if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then #TODO(mikedanese): allow static pods to start before creating a client @@ -765,6 +767,9 @@ function construct-linux-kubelet-flags { flags+=" ${NODE_KUBELET_TEST_ARGS:-}" flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" + if [[ "${node_type}" == "heapster" ]]; then + flags+=" ${HEAPSTER_KUBELET_TEST_ARGS:-}" + fi fi # Network plugin if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then @@ -772,7 +777,7 @@ function construct-linux-kubelet-flags { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" || "${ENABLE_NETD:-}" == "true" ]]; then # Calico uses CNI always. # Note that network policy won't work for master node. - if [[ "${master}" == "true" ]]; then + if [[ "${node_type}" == "master" ]]; then flags+=" --network-plugin=${NETWORK_PROVIDER}" else flags+=" --network-plugin=cni" @@ -787,7 +792,7 @@ function construct-linux-kubelet-flags { flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" fi flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}" - local node_labels="$(build-linux-node-labels ${master})" + local node_labels="$(build-linux-node-labels ${node_type})" if [[ -n "${node_labels:-}" ]]; then flags+=" --node-labels=${node_labels}" fi @@ -1492,7 +1497,7 @@ EOF # TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler, # which reads the kube-env to determine the shape of a node and was broken by #60020. # This should be removed as soon as a more reliable source of information is available! - local node_labels="$(build-linux-node-labels false)" + local node_labels="$(build-linux-node-labels node)" local node_taints="${NODE_TAINTS:-}" local autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}" cat >>$file <