Add an option to specify kubelet flags for heapster node.

Useful in scalability tests, where we don't want test pods (e.g. Kubemark hollow
nodes) to be scheduled on heapster node.
This commit is contained in:
Janek Łukaszewicz 2019-12-02 12:51:54 +01:00
parent c8b123e8e1
commit 39cb8222c7
2 changed files with 19 additions and 12 deletions

View File

@ -17,9 +17,11 @@
# A library of helper functions and constant for GCI distro # A library of helper functions and constant for GCI distro
source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
# shellcheck disable=SC2120
function get-node-instance-metadata-from-file { function get-node-instance-metadata-from-file {
local kube_env=${1:-node-kube-env} # optional
local metadata="" local metadata=""
metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml," metadata+="kube-env=${KUBE_TEMP}/${kube_env}.yaml,"
metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml," metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml,"
metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml," metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml,"
metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh," metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,"

View File

@ -592,7 +592,7 @@ function write-master-env {
KUBERNETES_MASTER_NAME="${MASTER_NAME}" KUBERNETES_MASTER_NAME="${MASTER_NAME}"
fi fi
construct-linux-kubelet-flags true construct-linux-kubelet-flags "master"
build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml" build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml"
build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml" build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml"
@ -603,7 +603,9 @@ function write-linux-node-env {
KUBERNETES_MASTER_NAME="${MASTER_NAME}" KUBERNETES_MASTER_NAME="${MASTER_NAME}"
fi fi
construct-linux-kubelet-flags false construct-linux-kubelet-flags "heapster"
build-linux-kube-env false "${KUBE_TEMP}/heapster-kube-env.yaml"
construct-linux-kubelet-flags "node"
build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml" build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml"
} }
@ -616,7 +618,7 @@ function write-windows-node-env {
} }
function build-linux-node-labels { function build-linux-node-labels {
local master=$1 local node_type=$1
local node_labels="" local node_labels=""
if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${master}" != "true" ]]; then if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${master}" != "true" ]]; then
# Add kube-proxy daemonset label to node to avoid situation during cluster # Add kube-proxy daemonset label to node to avoid situation during cluster
@ -626,10 +628,10 @@ function build-linux-node-labels {
if [[ -n "${NODE_LABELS:-}" ]]; then if [[ -n "${NODE_LABELS:-}" ]]; then
node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
fi fi
if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${master}" != "true" ]]; then if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${node_type}" != "master" ]]; then
node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}" node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}"
fi fi
if [[ -n "${MASTER_NODE_LABELS:-}" && "${master}" == "true" ]]; then if [[ -n "${MASTER_NODE_LABELS:-}" && "${node_type}" == "master" ]]; then
node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}" node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}"
fi fi
echo $node_labels echo $node_labels
@ -740,7 +742,7 @@ function construct-common-kubelet-flags {
# Sets KUBELET_ARGS with the kubelet flags for Linux nodes. # Sets KUBELET_ARGS with the kubelet flags for Linux nodes.
# $1: if 'true', we're rendering flags for a master, else a node # $1: if 'true', we're rendering flags for a master, else a node
function construct-linux-kubelet-flags { function construct-linux-kubelet-flags {
local master="$1" local node_type="$1"
local flags="$(construct-common-kubelet-flags)" local flags="$(construct-common-kubelet-flags)"
# Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh
flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter" flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter"
@ -751,7 +753,7 @@ function construct-linux-kubelet-flags {
flags+=" --dynamic-config-dir=/var/lib/kubelet/dynamic-config" flags+=" --dynamic-config-dir=/var/lib/kubelet/dynamic-config"
if [[ "${master}" == "true" ]]; then if [[ "${node_type}" == "master" ]]; then
flags+=" ${MASTER_KUBELET_TEST_ARGS:-}" flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
#TODO(mikedanese): allow static pods to start before creating a client #TODO(mikedanese): allow static pods to start before creating a client
@ -765,6 +767,9 @@ function construct-linux-kubelet-flags {
flags+=" ${NODE_KUBELET_TEST_ARGS:-}" flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
if [[ "${node_type}" == "heapster" ]]; then
flags+=" ${HEAPSTER_KUBELET_TEST_ARGS:-}"
fi
fi fi
# Network plugin # Network plugin
if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then
@ -772,7 +777,7 @@ function construct-linux-kubelet-flags {
if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" || "${ENABLE_NETD:-}" == "true" ]]; then if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" || "${ENABLE_NETD:-}" == "true" ]]; then
# Calico uses CNI always. # Calico uses CNI always.
# Note that network policy won't work for master node. # Note that network policy won't work for master node.
if [[ "${master}" == "true" ]]; then if [[ "${node_type}" == "master" ]]; then
flags+=" --network-plugin=${NETWORK_PROVIDER}" flags+=" --network-plugin=${NETWORK_PROVIDER}"
else else
flags+=" --network-plugin=cni" flags+=" --network-plugin=cni"
@ -787,7 +792,7 @@ function construct-linux-kubelet-flags {
flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
fi fi
flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}" flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
local node_labels="$(build-linux-node-labels ${master})" local node_labels="$(build-linux-node-labels ${node_type})"
if [[ -n "${node_labels:-}" ]]; then if [[ -n "${node_labels:-}" ]]; then
flags+=" --node-labels=${node_labels}" flags+=" --node-labels=${node_labels}"
fi fi
@ -1492,7 +1497,7 @@ EOF
# TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler, # TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler,
# which reads the kube-env to determine the shape of a node and was broken by #60020. # which reads the kube-env to determine the shape of a node and was broken by #60020.
# This should be removed as soon as a more reliable source of information is available! # This should be removed as soon as a more reliable source of information is available!
local node_labels="$(build-linux-node-labels false)" local node_labels="$(build-linux-node-labels node)"
local node_taints="${NODE_TAINTS:-}" local node_taints="${NODE_TAINTS:-}"
local autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}" local autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}"
cat >>$file <<EOF cat >>$file <<EOF
@ -3266,7 +3271,7 @@ function create-heapster-node() {
--tags "${NODE_TAG}" \ --tags "${NODE_TAG}" \
${network} \ ${network} \
$(get-scope-flags) \ $(get-scope-flags) \
--metadata-from-file "$(get-node-instance-metadata-from-file)" --metadata-from-file "$(get-node-instance-metadata-from-file "heapster-kube-env")"
} }
# Assumes: # Assumes: