github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 20:53:33 +00:00
Code Issues Projects Releases Wiki Activity

Revert "Allow creation of clusters larger than 500 nodes"

Browse Source
This commit is contained in:
Zach Loafman 2015-12-10 16:07:17 -08:00
parent aaa1fe67f6
commit 9492fd6a80
2 changed files with 57 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
cluster/gce/upgrade.sh
View File

@ -174,7 +174,6 @@ function upgrade-nodes() {
# #
# Vars set: # Vars set:
# SANITIZED_VERSION # SANITIZED_VERSION
# INSTANCE_GROUPS
# KUBELET_TOKEN # KUBELET_TOKEN
# KUBE_PROXY_TOKEN # KUBE_PROXY_TOKEN
# CA_CERT_BASE64 # CA_CERT_BASE64
@ -185,7 +184,7 @@ function prepare-node-upgrade() {
echo "== Preparing node upgrade (to ${KUBE_VERSION}). ==" >&2 echo "== Preparing node upgrade (to ${KUBE_VERSION}). ==" >&2
SANITIZED_VERSION=$(echo ${KUBE_VERSION} | sed 's/[\.\+]/-/g') SANITIZED_VERSION=$(echo ${KUBE_VERSION} | sed 's/[\.\+]/-/g')
detect-node-names # sets INSTANCE_GROUPS detect-node-names
# TODO(zmerlynn): Refactor setting scope flags. # TODO(zmerlynn): Refactor setting scope flags.
local scope_flags= local scope_flags=
@ -232,18 +231,16 @@ function do-node-upgrade() {
subgroup="alpha compute" subgroup="alpha compute"
fi fi
local template_name=$(get-template-name-from-version ${SANITIZED_VERSION}) local template_name=$(get-template-name-from-version ${SANITIZED_VERSION})
for group in ${INSTANCE_GROUPS[@]}; do gcloud ${subgroup} rolling-updates \
gcloud ${subgroup} rolling-updates \ --project="${PROJECT}" \
--project="${PROJECT}" \ --zone="${ZONE}" \
--zone="${ZONE}" \ start \
start \ --group="${NODE_INSTANCE_PREFIX}-group" \
--group="${group}" \ --template="${template_name}" \
--template="${template_name}" \ --instance-startup-timeout=300s \
--instance-startup-timeout=300s \ --max-num-concurrent-instances=1 \
--max-num-concurrent-instances=1 \ --max-num-failed-instances=0 \
--max-num-failed-instances=0 \ --min-instance-update-time=0s
--min-instance-update-time=0s
done
# TODO(zmerlynn): Wait for the rolling-update to finish. # TODO(zmerlynn): Wait for the rolling-update to finish.

143
cluster/gce/util.sh
View File

@ -253,24 +253,12 @@ function upload-server-tars() {
# NODE_INSTANCE_PREFIX # NODE_INSTANCE_PREFIX
# Vars set: # Vars set:
# NODE_NAMES # NODE_NAMES
# INSTANCE_GROUPS
function detect-node-names { function detect-node-names {
detect-project detect-project
INSTANCE_GROUPS=() NODE_NAMES=($(gcloud compute instance-groups managed list-instances \
INSTANCE_GROUPS+=($(gcloud compute instance-groups managed list --zone "${ZONE}" --project "${PROJECT}" | grep ${NODE_INSTANCE_PREFIX} | cut -f1 -d" ")) "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}" \
NODE_NAMES=() --format=yaml | grep instance: | cut -d ' ' -f 2))
if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then echo "NODE_NAMES=${NODE_NAMES[*]}" >&2
for group in "${INSTANCE_GROUPS[@]}"; do
NODE_NAMES+=($(gcloud compute instance-groups managed list-instances \
"${group}" --zone "${ZONE}" --project "${PROJECT}" \
--format=yaml | grep instance: | cut -d ' ' -f 2))
done
echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]}" >&2
echo "NODE_NAMES=${NODE_NAMES[*]}" >&2
else
echo "INSTANCE_GROUPS=" >&2
echo "NODE_NAMES=" >&2
fi
} }
# Detect the information about the minions # Detect the information about the minions
@ -725,43 +713,17 @@ function kube-up {
create-node-instance-template $template_name create-node-instance-template $template_name
local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-500}
if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then
echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 500"
defaulted_max_instances_per_mig=500
fi
local num_migs=$(((${NUM_NODES} + ${defaulted_max_instances_per_mig} - 1) / ${defaulted_max_instances_per_mig}))
local instances_per_mig=$(((${NUM_NODES} + ${num_migs} - 1) / ${num_migs}))
local last_mig_size=$((${NUM_NODES} - (${num_migs} - 1) * ${instances_per_mig}))
#TODO: parallelize this loop to speed up the process
for i in $(seq $((${num_migs} - 1))); do
gcloud compute instance-groups managed \
create "${NODE_INSTANCE_PREFIX}-group-$i" \
--project "${PROJECT}" \
--zone "${ZONE}" \
--base-instance-name "${NODE_INSTANCE_PREFIX}" \
--size "${instances_per_mig}" \
--template "$template_name" || true;
gcloud compute instance-groups managed wait-until-stable \
"${NODE_INSTANCE_PREFIX}-group-$i" \
--zone "${ZONE}" \
--project "${PROJECT}" || true;
done
gcloud compute instance-groups managed \ gcloud compute instance-groups managed \
create "${NODE_INSTANCE_PREFIX}-group-${num_migs}" \ create "${NODE_INSTANCE_PREFIX}-group" \
--project "${PROJECT}" \ --project "${PROJECT}" \
--zone "${ZONE}" \ --zone "${ZONE}" \
--base-instance-name "${NODE_INSTANCE_PREFIX}" \ --base-instance-name "${NODE_INSTANCE_PREFIX}" \
--size "${last_mig_size}" \ --size "${NUM_NODES}" \
--template "$template_name" || true; --template "$template_name" || true;
gcloud compute instance-groups managed wait-until-stable \ gcloud compute instance-groups managed wait-until-stable \
"${NODE_INSTANCE_PREFIX}-group-${num_migs}" \ "${NODE_INSTANCE_PREFIX}-group" \
--zone "${ZONE}" \ --zone "${ZONE}" \
--project "${PROJECT}" || true; --project "${PROJECT}" || true;
detect-node-names detect-node-names
detect-master detect-master
@ -780,12 +742,9 @@ function kube-up {
METRICS+="--custom-metric-utilization metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/memory/node_reservation," METRICS+="--custom-metric-utilization metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/memory/node_reservation,"
METRICS+="utilization-target=${TARGET_NODE_UTILIZATION},utilization-target-type=GAUGE " METRICS+="utilization-target=${TARGET_NODE_UTILIZATION},utilization-target-type=GAUGE "
echo "Creating node autoscalers." echo "Creating node autoscaler."
gcloud compute instance-groups managed set-autoscaling "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}" \
for i in $(seq ${num_migs}); do --min-num-replicas "${AUTOSCALER_MIN_NODES}" --max-num-replicas "${AUTOSCALER_MAX_NODES}" ${METRICS} || true
gcloud compute instance-groups managed set-autoscaling "${NODE_INSTANCE_PREFIX}-group-$i" --zone "${ZONE}" --project "${PROJECT}" \
--min-num-replicas "${AUTOSCALER_MIN_NODES}" --max-num-replicas "${AUTOSCALER_MAX_NODES}" ${METRICS} || true
done
fi fi
echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization." echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization."
@ -851,51 +810,46 @@ function kube-up {
# down the firewall rules and routes. # down the firewall rules and routes.
function kube-down { function kube-down {
detect-project detect-project
detect-node-names # For INSTANCE_GROUPS
echo "Bringing down cluster" echo "Bringing down cluster"
set +e # Do not stop on error set +e # Do not stop on error
# Delete autoscaler for nodes if present. We assume that all or none instance groups have an autoscaler # Delete autoscaler for nodes if present.
local autoscaler local autoscaler
autoscaler=( $(gcloud compute instance-groups managed list --zone "${ZONE}" --project "${PROJECT}" \ autoscaler=( $(gcloud compute instance-groups managed list --zone "${ZONE}" --project "${PROJECT}" \
| grep "${NODE_INSTANCE_PREFIX}-group-1" \ | grep "${NODE_INSTANCE_PREFIX}-group" \
| awk '{print $7}') ) | awk '{print $7}') )
if [[ "${autoscaler:-}" == "yes" ]]; then if [[ "${autoscaler:-}" == "yes" ]]; then
for group in ${INSTANCE_GROUPS[@]}; do gcloud compute instance-groups managed stop-autoscaling "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}"
gcloud compute instance-groups managed stop-autoscaling "${group}" --zone "${ZONE}" --project "${PROJECT}"
done
fi fi
# Get the name of the managed instance group template before we delete the # Get the name of the managed instance group template before we delete the
# managed instange group. (The name of the managed instnace group template may # managed instange group. (The name of the managed instnace group template may
# change during a cluster upgrade.) # change during a cluster upgrade.)
local template=$(get-template "${PROJECT}" "${ZONE}" "${NODE_INSTANCE_PREFIX}-group-1") local template=$(get-template "${PROJECT}" "${ZONE}" "${NODE_INSTANCE_PREFIX}-group")
# The gcloud APIs don't return machine parseable error codes/retry information. Therefore the best we can # The gcloud APIs don't return machine parseable error codes/retry information. Therefore the best we can
# do is parse the output and special case particular responses we are interested in. # do is parse the output and special case particular responses we are interested in.
for group in ${INSTANCE_GROUPS[@]}; do if gcloud compute instance-groups managed describe "${NODE_INSTANCE_PREFIX}-group" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then deleteCmdOutput=$(gcloud compute instance-groups managed delete --zone "${ZONE}" \
deleteCmdOutput=$(gcloud compute instance-groups managed delete --zone "${ZONE}" \ --project "${PROJECT}" \
--project "${PROJECT}" \ --quiet \
--quiet \ "${NODE_INSTANCE_PREFIX}-group")
"${group}") if [[ "$deleteCmdOutput" != "" ]]; then
if [[ "$deleteCmdOutput" != "" ]]; then # Managed instance group deletion is done asynchronously, we must wait for it to complete, or subsequent steps fail
# Managed instance group deletion is done asynchronously, we must wait for it to complete, or subsequent steps fail deleteCmdOperationId=$(echo $deleteCmdOutput | grep "Operation:" | sed "s/.*Operation:[[:space:]]*\([^[:space:]]*\).*/\1/g")
deleteCmdOperationId=$(echo $deleteCmdOutput | grep "Operation:" | sed "s/.*Operation:[[:space:]]*\([^[:space:]]*\).*/\1/g") if [[ "$deleteCmdOperationId" != "" ]]; then
if [[ "$deleteCmdOperationId" != "" ]]; then deleteCmdStatus="PENDING"
deleteCmdStatus="PENDING" while [[ "$deleteCmdStatus" != "DONE" ]]
while [[ "$deleteCmdStatus" != "DONE" ]] do
do sleep 5
sleep 5 deleteCmdOperationOutput=$(gcloud compute instance-groups managed --zone "${ZONE}" --project "${PROJECT}" get-operation $deleteCmdOperationId)
deleteCmdOperationOutput=$(gcloud compute instance-groups managed --zone "${ZONE}" --project "${PROJECT}" get-operation $deleteCmdOperationId) deleteCmdStatus=$(echo $deleteCmdOperationOutput | grep -i "status:" | sed "s/.*status:[[:space:]]*\([^[:space:]]*\).*/\1/g")
deleteCmdStatus=$(echo $deleteCmdOperationOutput | grep -i "status:" | sed "s/.*status:[[:space:]]*\([^[:space:]]*\).*/\1/g") echo "Waiting for MIG deletion to complete. Current status: " $deleteCmdStatus
echo "Waiting for MIG deletion to complete. Current status: " $deleteCmdStatus done
done
fi
fi fi
fi fi
done fi
if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
gcloud compute instance-templates delete \ gcloud compute instance-templates delete \
@ -1028,13 +982,12 @@ function get-template {
# KUBE_RESOURCE_FOUND # KUBE_RESOURCE_FOUND
function check-resources { function check-resources {
detect-project detect-project
detect-node-names
echo "Looking for already existing resources" echo "Looking for already existing resources"
KUBE_RESOURCE_FOUND="" KUBE_RESOURCE_FOUND=""
if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then if gcloud compute instance-groups managed describe --project "${PROJECT}" --zone "${ZONE}" "${NODE_INSTANCE_PREFIX}-group" &>/dev/null; then
KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[@]}" KUBE_RESOURCE_FOUND="Managed instance group ${NODE_INSTANCE_PREFIX}-group"
return 1 return 1
fi fi
@ -1137,13 +1090,11 @@ function prepare-push() {
create-node-instance-template $tmp_template_name create-node-instance-template $tmp_template_name
local template_name="${NODE_INSTANCE_PREFIX}-template" local template_name="${NODE_INSTANCE_PREFIX}-template"
for group in ${INSTANCE_GROUPS[@]}; do gcloud compute instance-groups managed \
gcloud compute instance-groups managed \ set-instance-template "${NODE_INSTANCE_PREFIX}-group" \
set-instance-template "${group}" \ --template "$tmp_template_name" \
--template "$tmp_template_name" \ --zone "${ZONE}" \
--zone "${ZONE}" \ --project "${PROJECT}" || true;
--project "${PROJECT}" || true;
done
gcloud compute instance-templates delete \ gcloud compute instance-templates delete \
--project "${PROJECT}" \ --project "${PROJECT}" \
@ -1152,13 +1103,11 @@ function prepare-push() {
create-node-instance-template "$template_name" create-node-instance-template "$template_name"
for group in ${INSTANCE_GROUPS[@]}; do gcloud compute instance-groups managed \
gcloud compute instance-groups managed \ set-instance-template "${NODE_INSTANCE_PREFIX}-group" \
set-instance-template "${group}" \ --template "$template_name" \
--template "$template_name" \ --zone "${ZONE}" \
--zone "${ZONE}" \ --project "${PROJECT}" || true;
--project "${PROJECT}" || true;
done
gcloud compute instance-templates delete \ gcloud compute instance-templates delete \
--project "${PROJECT}" \ --project "${PROJECT}" \