diff --git a/cluster/gce/upgrade.sh b/cluster/gce/upgrade.sh index 9f4746ca8cb..cb916af7df0 100755 --- a/cluster/gce/upgrade.sh +++ b/cluster/gce/upgrade.sh @@ -207,8 +207,6 @@ function prepare-node-upgrade() { source "${KUBE_ROOT}/cluster/gce/debian/helper.sh" fi create-node-instance-template "${template_name}" - # The following is echo'd so that callers can get the template name. - echo $template_name echo "== Finished preparing node upgrade (to ${KUBE_VERSION}). ==" >&2 } @@ -220,8 +218,15 @@ function do-node-upgrade() { # NOTE(zmerlynn): If you are changing this gcloud command, update # test/e2e/cluster_upgrade.go to match this EXACTLY. local template_name=$(get-template-name-from-version ${SANITIZED_VERSION}) + local old_templates=() + local updates=() for group in ${INSTANCE_GROUPS[@]}; do - gcloud alpha compute rolling-updates \ + old_templates+=($(gcloud compute instance-groups managed list \ + --project="${PROJECT}" \ + --zone="${ZONE}" \ + --regexp="${group}" \ + --format='value(instanceTemplate)' || true)) + update=$(gcloud alpha compute rolling-updates \ --project="${PROJECT}" \ --zone="${ZONE}" \ start \ @@ -230,10 +235,36 @@ function do-node-upgrade() { --instance-startup-timeout=300s \ --max-num-concurrent-instances=1 \ --max-num-failed-instances=0 \ - --min-instance-update-time=0s + --min-instance-update-time=0s 2>&1) + id=$(echo "${update}" | grep "Started" | cut -d '/' -f 11 | cut -d ']' -f 1) + updates+=("${id}") done - # TODO(zmerlynn): Wait for the rolling-update to finish. + # Wait until rolling updates are finished. + for update in ${updates[@]}; do + while true; do + result=$(gcloud alpha compute rolling-updates \ + --project="${PROJECT}" \ + --zone="${ZONE}" \ + describe \ + ${update} \ + --format='value(status)' || true) + if [ $result = "ROLLED_OUT" ]; then + echo "Rolling update ${update} is ${result} state and finished." + break + fi + echo "Rolling update ${update} is stil in ${result} state." + sleep 10 + done + done + + # Remove the old templates. + for tmpl in ${old_templates[@]}; do + gcloud compute instance-templates delete \ + --quiet \ + --project="${PROJECT}" \ + "${tmpl}" || true + done echo "== Finished upgrading nodes to ${KUBE_VERSION}. ==" >&2 }