Fix waiting for node upgrades

This commit is contained in:
Wojciech Tyczynski 2016-06-22 16:46:45 +02:00
parent 1de2bddae7
commit 5094333bc2

View File

@ -207,8 +207,6 @@ function prepare-node-upgrade() {
source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
fi
create-node-instance-template "${template_name}"
# The following is echo'd so that callers can get the template name.
echo $template_name
echo "== Finished preparing node upgrade (to ${KUBE_VERSION}). ==" >&2
}
@ -220,8 +218,15 @@ function do-node-upgrade() {
# NOTE(zmerlynn): If you are changing this gcloud command, update
# test/e2e/cluster_upgrade.go to match this EXACTLY.
local template_name=$(get-template-name-from-version ${SANITIZED_VERSION})
local old_templates=()
local updates=()
for group in ${INSTANCE_GROUPS[@]}; do
gcloud alpha compute rolling-updates \
old_templates+=($(gcloud compute instance-groups managed list \
--project="${PROJECT}" \
--zone="${ZONE}" \
--regexp="${group}" \
--format='value(instanceTemplate)' || true))
update=$(gcloud alpha compute rolling-updates \
--project="${PROJECT}" \
--zone="${ZONE}" \
start \
@ -230,10 +235,36 @@ function do-node-upgrade() {
--instance-startup-timeout=300s \
--max-num-concurrent-instances=1 \
--max-num-failed-instances=0 \
--min-instance-update-time=0s
--min-instance-update-time=0s 2>&1)
id=$(echo "${update}" | grep "Started" | cut -d '/' -f 11 | cut -d ']' -f 1)
updates+=("${id}")
done
# TODO(zmerlynn): Wait for the rolling-update to finish.
# Wait until rolling updates are finished.
for update in ${updates[@]}; do
while true; do
result=$(gcloud alpha compute rolling-updates \
--project="${PROJECT}" \
--zone="${ZONE}" \
describe \
${update} \
--format='value(status)' || true)
if [ $result = "ROLLED_OUT" ]; then
echo "Rolling update ${update} is ${result} state and finished."
break
fi
echo "Rolling update ${update} is stil in ${result} state."
sleep 10
done
done
# Remove the old templates.
for tmpl in ${old_templates[@]}; do
gcloud compute instance-templates delete \
--quiet \
--project="${PROJECT}" \
"${tmpl}" || true
done
echo "== Finished upgrading nodes to ${KUBE_VERSION}. ==" >&2
}