Add some more retry...

This commit is contained in:
Brendan Burns 2015-10-23 13:57:13 -07:00
parent 447fe209ab
commit bd65c733b3
2 changed files with 19 additions and 1 deletions

View File

@ -37,7 +37,22 @@ GREP_REGEX=""
function gcloud-compute-list() {
local -r resource=$1
echo -e "\n\n[ ${resource} ]"
gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"
local attempt=1
local result=""
while true; do
echo "Attempt ${attempt} to list ${resource} in GCE"
if result=$(gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"); then
echo ${result}
return
fi
echo -e "${color_yellow}Attempt ${attempt} failed to list ${resource}. Retrying.${color_norm}" >&2
attempt=$(($attempt+1))
if [[ ${attempt} > 5 ]]; then
echo -e "${color_red}List ${resource} failed!${color_norm}" >&2
exit 2
fi
sleep $((5*${attempt}))
done
}
echo "Project: ${PROJECT}"

View File

@ -393,6 +393,7 @@ function create-node-template {
fi
echo -e "${color_yellow}Attempt ${attempt} failed to create instance template $template_name. Retrying.${color_norm}" >&2
attempt=$(($attempt+1))
sleep $(($attempt * 5))
else
break
fi
@ -419,6 +420,7 @@ function add-instance-metadata {
fi
echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
attempt=$(($attempt+1))
sleep $((5 * $attempt))
else
break
fi
@ -446,6 +448,7 @@ function add-instance-metadata-from-file {
fi
echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
attempt=$(($attempt+1))
sleep $(($attempt * 5))
else
break
fi