Add some more retry...

cluster/gce/list-resources.sh

@@ -37,7 +37,22 @@ GREP_REGEX=""
 function gcloud-compute-list() {
   local -r resource=$1
   echo -e "\n\n[ ${resource} ]"
-  gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"
+  local attempt=1
+  local result=""
+  while true; do
+    echo "Attempt ${attempt} to list ${resource} in GCE"
+    if result=$(gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"); then
+      echo ${result}
+      return
+    fi
+    echo -e "${color_yellow}Attempt ${attempt} failed to list ${resource}. Retrying.${color_norm}" >&2
+    attempt=$(($attempt+1))
+    if [[ ${attempt} > 5 ]]; then
+      echo -e "${color_red}List ${resource} failed!${color_norm}" >&2
+      exit 2
+    fi
+    sleep $((5*${attempt}))
+  done
 }
 
 echo "Project: ${PROJECT}"

cluster/gce/util.sh

@@ -393,6 +393,7 @@ function create-node-template {
         fi
         echo -e "${color_yellow}Attempt ${attempt} failed to create instance template $template_name. Retrying.${color_norm}" >&2
         attempt=$(($attempt+1))
+        sleep $(($attempt * 5))
     else
         break
     fi
@@ -419,6 +420,7 @@ function add-instance-metadata {
         fi
         echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
         attempt=$(($attempt+1))
+        sleep $((5 * $attempt))
     else
         break
     fi
@@ -446,6 +448,7 @@ function add-instance-metadata-from-file {
         fi
         echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
         attempt=$(($attempt+1))
+        sleep $(($attempt * 5))
     else
         break
     fi