From 2595f66466e7aa33cb182f93263fe67b4365fabd Mon Sep 17 00:00:00 2001
From: Shyam Jeedigunta <shyamjvs@google.com>
Date: Tue, 29 Aug 2017 17:02:56 +0200
Subject: [PATCH] Retry master instance creation in case of retriable error
 (with sleep)

---
 cluster/gce/container-linux/master-helper.sh | 42 ++++++++++++++------
 cluster/gce/gci/master-helper.sh             | 42 ++++++++++++++------
 2 files changed, 58 insertions(+), 26 deletions(-)

diff --git a/cluster/gce/container-linux/master-helper.sh b/cluster/gce/container-linux/master-helper.sh
index d4b0ec4c144..9358d3026bd 100755
--- a/cluster/gce/container-linux/master-helper.sh
+++ b/cluster/gce/container-linux/master-helper.sh
@@ -66,6 +66,7 @@ function replicate-master-instance() {
 
 function create-master-instance-internal() {
   local gcloud="gcloud"
+  local retries=5
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
     gcloud="gcloud beta"
   fi
@@ -93,19 +94,34 @@ function create-master-instance-internal() {
   disk="${disk},boot=no"
   disk="${disk},auto-delete=no"
 
-  ${gcloud} compute instances create "${master_name}" \
-    --project "${PROJECT}" \
-    --zone "${ZONE}" \
-    --machine-type "${MASTER_SIZE}" \
-    --image-project="${MASTER_IMAGE_PROJECT}" \
-    --image "${MASTER_IMAGE}" \
-    --tags "${MASTER_TAG}" \
-    --scopes "storage-ro,compute-rw,monitoring,logging-write" \
-    --metadata-from-file "${metadata}" \
-    --disk "${disk}" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \
-    ${preemptible_master} \
-    ${network}
+  for attempt in $(seq 1 ${retries}); do
+    if result=$(${gcloud} compute instances create "${master_name}" \
+      --project "${PROJECT}" \
+      --zone "${ZONE}" \
+      --machine-type "${MASTER_SIZE}" \
+      --image-project="${MASTER_IMAGE_PROJECT}" \
+      --image "${MASTER_IMAGE}" \
+      --tags "${MASTER_TAG}" \
+      --scopes "storage-ro,compute-rw,monitoring,logging-write" \
+      --metadata-from-file "${metadata}" \
+      --disk "${disk}" \
+      --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
+      ${preemptible_master} \
+      ${network} 2>&1); then
+      echo "${result}" >&2
+      return 0
+    else
+      echo "${result}" >&2
+      if [[ ! "${result}" =~ "try again later" ]]; then
+        echo "Failed to create master instance due to non-retryable error" >&2
+        return 1
+      fi
+      sleep 10
+    fi
+  done
+
+  echo "Failed to create master instance despite ${retries} attempts" >&2
+  return 1
 }
 
 function get-metadata() {
diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh
index f27a7b24720..c8a31336ee4 100755
--- a/cluster/gce/gci/master-helper.sh
+++ b/cluster/gce/gci/master-helper.sh
@@ -75,6 +75,7 @@ function replicate-master-instance() {
 
 function create-master-instance-internal() {
   local gcloud="gcloud"
+  local retries=5
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
     gcloud="gcloud beta"
   fi
@@ -106,19 +107,34 @@ function create-master-instance-internal() {
   disk="${disk},boot=no"
   disk="${disk},auto-delete=no"
 
-  ${gcloud} compute instances create "${master_name}" \
-    --project "${PROJECT}" \
-    --zone "${ZONE}" \
-    --machine-type "${MASTER_SIZE}" \
-    --image-project="${MASTER_IMAGE_PROJECT}" \
-    --image "${MASTER_IMAGE}" \
-    --tags "${MASTER_TAG}" \
-    --scopes "storage-ro,compute-rw,monitoring,logging-write" \
-    --metadata-from-file "${metadata}" \
-    --disk "${disk}" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
-    ${preemptible_master} \
-    ${network}
+  for attempt in $(seq 1 ${retries}); do
+    if result=$(${gcloud} compute instances create "${master_name}" \
+      --project "${PROJECT}" \
+      --zone "${ZONE}" \
+      --machine-type "${MASTER_SIZE}" \
+      --image-project="${MASTER_IMAGE_PROJECT}" \
+      --image "${MASTER_IMAGE}" \
+      --tags "${MASTER_TAG}" \
+      --scopes "storage-ro,compute-rw,monitoring,logging-write" \
+      --metadata-from-file "${metadata}" \
+      --disk "${disk}" \
+      --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
+      ${preemptible_master} \
+      ${network} 2>&1); then
+      echo "${result}" >&2
+      return 0
+    else
+      echo "${result}" >&2
+      if [[ ! "${result}" =~ "try again later" ]]; then
+        echo "Failed to create master instance due to non-retryable error" >&2
+        return 1
+      fi
+      sleep 10
+    fi
+  done
+
+  echo "Failed to create master instance despite ${retries} attempts" >&2
+  return 1
 }
 
 function get-metadata() {