From bd65c733b3d5719421143d06f1162cdbf6e2a780 Mon Sep 17 00:00:00 2001
From: Brendan Burns <bburns@google.com>
Date: Fri, 23 Oct 2015 13:57:13 -0700
Subject: [PATCH] Add some more retry...

---
 cluster/gce/list-resources.sh | 17 ++++++++++++++++-
 cluster/gce/util.sh           |  3 +++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/cluster/gce/list-resources.sh b/cluster/gce/list-resources.sh
index 9c0d9c0420b..651438748a4 100755
--- a/cluster/gce/list-resources.sh
+++ b/cluster/gce/list-resources.sh
@@ -37,7 +37,22 @@ GREP_REGEX=""
 function gcloud-compute-list() {
   local -r resource=$1
   echo -e "\n\n[ ${resource} ]"
-  gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"
+  local attempt=1
+  local result=""
+  while true; do
+    echo "Attempt ${attempt} to list ${resource} in GCE"
+    if result=$(gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"); then
+      echo ${result}
+      return
+    fi
+    echo -e "${color_yellow}Attempt ${attempt} failed to list ${resource}. Retrying.${color_norm}" >&2
+    attempt=$(($attempt+1))
+    if [[ ${attempt} > 5 ]]; then
+      echo -e "${color_red}List ${resource} failed!${color_norm}" >&2
+      exit 2
+    fi
+    sleep $((5*${attempt}))
+  done
 }
 
 echo "Project: ${PROJECT}"
diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
index f7a5bafe038..11ce2600583 100755
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -393,6 +393,7 @@ function create-node-template {
         fi
         echo -e "${color_yellow}Attempt ${attempt} failed to create instance template $template_name. Retrying.${color_norm}" >&2
         attempt=$(($attempt+1))
+        sleep $(($attempt * 5))
     else
         break
     fi
@@ -419,6 +420,7 @@ function add-instance-metadata {
         fi
         echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
         attempt=$(($attempt+1))
+        sleep $((5 * $attempt))
     else
         break
     fi
@@ -446,6 +448,7 @@ function add-instance-metadata-from-file {
         fi
         echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in ${instance}. Retrying.${color_norm}" >&2
         attempt=$(($attempt+1))
+        sleep $(($attempt * 5))
     else
         break
     fi