diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 914154657ab..34610848525 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -180,6 +180,13 @@ ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}" # Useful for scheduling heapster in large clusters with nodes of small size. HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}" +# Optional: Additional nodes would be created if their type and number is specified. +# NUM_NODES would be lowered respectively. +# Useful for running cluster-level addons that needs more resources than would fit +# on small nodes, like network plugins. +NUM_ADDITIONAL_NODES="${NUM_ADDITIONAL_NODES:-}" +ADDITIONAL_MACHINE_TYPE="${ADDITIONAL_MACHINE_TYPE:-}" + MASTER_NODE_LABELS="${KUBE_MASTER_NODE_LABELS:-}" # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes. NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index f14d2d587c5..d43547d9ff2 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -192,6 +192,13 @@ ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}" # Useful for scheduling heapster in large clusters with nodes of small size. HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}" +# Optional: Additional nodes would be created if their type and number is specified. +# NUM_NODES would be lowered respectively. +# Useful for running cluster-level addons that needs more resources than would fit +# on small nodes, like network plugins. +NUM_ADDITIONAL_NODES="${NUM_ADDITIONAL_NODES:-}" +ADDITIONAL_MACHINE_TYPE="${ADDITIONAL_MACHINE_TYPE:-}" + # Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. 3.3.15-0) if you need # non-default version. ETCD_IMAGE="${TEST_ETCD_IMAGE:-}" diff --git a/cluster/gce/gci/node-helper.sh b/cluster/gce/gci/node-helper.sh index 6cd72ec8cfe..f15ea55aaf6 100755 --- a/cluster/gce/gci/node-helper.sh +++ b/cluster/gce/gci/node-helper.sh @@ -39,7 +39,9 @@ function get-node-instance-metadata-from-file { # $1: template name (required). function create-linux-node-instance-template { local template_name="$1" + local machine_type="${2:-$NODE_SIZE}" + ensure-gci-metadata-files # shellcheck disable=2154 # 'scope_flags' is assigned by upstream - create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux" + create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux" "${machine_type}" } diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 460e24d6772..09aced7e6ff 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -2130,6 +2130,7 @@ function create-node-template() { local template_name="$1" local metadata_values="$4" local os="$5" + local machine_type="$6" # First, ensure the template doesn't exist. # TODO(zmerlynn): To make this really robust, we need to parse the output and @@ -2216,7 +2217,7 @@ function create-node-template() { if ! ${gcloud} compute instance-templates create \ "${template_name}" \ --project "${PROJECT}" \ - --machine-type "${NODE_SIZE}" \ + --machine-type "${machine_type}" \ --boot-disk-type "${NODE_DISK_TYPE}" \ --boot-disk-size "${NODE_DISK_SIZE}" \ ${node_image_flags} \ @@ -3068,6 +3069,10 @@ function create-nodes-template() { local windows_template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template" create-linux-node-instance-template $linux_template_name create-windows-node-instance-template $windows_template_name "${scope_flags[*]}" + if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" ]]; then + local linux_extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template" + create-linux-node-instance-template $linux_extra_template_name "${ADDITIONAL_MACHINE_TYPE}" + fi } # Assumes: @@ -3096,13 +3101,38 @@ function set_num_migs() { # - ZONE function create-linux-nodes() { local template_name="${NODE_INSTANCE_PREFIX}-template" + local extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template" - if [[ -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then - local -r nodes="${NUM_NODES}" - else + local nodes="${NUM_NODES}" + if [[ ! -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}" create-heapster-node - local -r nodes=$(( NUM_NODES - 1 )) + nodes=$(( nodes - 1 )) + fi + + if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" && "${NUM_ADDITIONAL_NODES:-}" -gt 0 ]]; then + local num_additional="${NUM_ADDITIONAL_NODES}" + if [[ "${NUM_ADDITIONAL_NODES:-}" -gt "${nodes}" ]]; then + echo "Capping NUM_ADDITIONAL_NODES to ${nodes}" + num_additional="${nodes}" + fi + if [[ "${num_additional:-}" -gt 0 ]]; then + echo "Creating ${num_additional} special nodes with machine-type ${ADDITIONAL_MACHINE_TYPE}" + local extra_group_name="${NODE_INSTANCE_PREFIX}-extra" + gcloud compute instance-groups managed \ + create "${extra_group_name}" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --base-instance-name "${extra_group_name}" \ + --size "${num_additional}" \ + --template "${extra_template_name}" || true; + gcloud compute instance-groups managed wait-until-stable \ + "${extra_group_name}" \ + --zone "${ZONE}" \ + --project "${PROJECT}" \ + --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true + nodes=$(( nodes - $num_additional )) + fi fi local instances_left=${nodes} @@ -3414,13 +3444,15 @@ function kube-down() { local all_instance_groups=(${INSTANCE_GROUPS[@]:-} ${WINDOWS_INSTANCE_GROUPS[@]:-}) for group in ${all_instance_groups[@]:-}; do - if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then - gcloud compute instance-groups managed delete \ - --project "${PROJECT}" \ - --quiet \ - --zone "${ZONE}" \ - "${group}" & - fi + { + if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then + gcloud compute instance-groups managed delete \ + --project "${PROJECT}" \ + --quiet \ + --zone "${ZONE}" \ + "${group}" + fi + } & done # Wait for last batch of jobs @@ -3429,14 +3461,21 @@ function kube-down() { } for template in ${templates[@]:-}; do - if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then - gcloud compute instance-templates delete \ - --project "${PROJECT}" \ - --quiet \ - "${template}" - fi + { + if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then + gcloud compute instance-templates delete \ + --project "${PROJECT}" \ + --quiet \ + "${template}" + fi + } & done + # Wait for last batch of jobs + kube::util::wait-for-jobs || { + echo -e "Failed to delete instance template(s)." >&2 + } + # Delete the special heapster node (if it exists). if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster" @@ -3717,7 +3756,7 @@ function set-replica-name() { # # $1: project function get-template() { - local linux_filter="${NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?" + local linux_filter="${NODE_INSTANCE_PREFIX}-(extra-)?template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?" local windows_filter="${WINDOWS_NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?" gcloud compute instance-templates list \ diff --git a/cluster/gce/windows/node-helper.sh b/cluster/gce/windows/node-helper.sh index f1ad723bacf..98e2421c283 100755 --- a/cluster/gce/windows/node-helper.sh +++ b/cluster/gce/windows/node-helper.sh @@ -54,5 +54,5 @@ function get-windows-node-instance-metadata { function create-windows-node-instance-template { local template_name="$1" local scopes_flag="$2" - create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows" + create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows" "${NODE_SIZE}" }