From c5b1508774163920a70180e63414633569267bce Mon Sep 17 00:00:00 2001
From: saadali <saadali@google.com>
Date: Mon, 11 May 2015 18:51:59 -0700
Subject: [PATCH 1/2] Make MIG deletion during GCE kube down blocking, so that
 subseqent template deletion doesn't fail.

---
 cluster/gce/util.sh | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
index 44ba42c793f..575f7e078ba 100755
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -644,10 +644,24 @@ function kube-down {
 
   echo "Bringing down cluster"
 
-  gcloud preview managed-instance-groups --zone "${ZONE}" delete \
+  deleteCmdOutput=$(gcloud preview managed-instance-groups --zone "${ZONE}" delete \
     --project "${PROJECT}" \
     --quiet \
-    "${NODE_INSTANCE_PREFIX}-group" || true
+    "${NODE_INSTANCE_PREFIX}-group" || true)
+  if [[ "$deleteCmdOutput" != ""  ]]; then
+    # Managed instance group deletion is done asyncronously, we must wait for it to complete, or subsequent steps fail
+    deleteCmdOperationId=$(echo $deleteCmdOutput | grep "Operation:" | sed "s/.*Operation:\s//" | sed "s/\s.*//" | sed "s/ //g")
+    if [[ "$deleteCmdOperationId" != ""  ]]; then
+      deleteCmdStatus="PENDING"
+      while [[ "$deleteCmdStatus" != "DONE" ]]
+      do
+	sleep 5
+        deleteCmdOperationOutput=$(gcloud preview managed-instance-groups --zone "${ZONE}" get-operation $deleteCmdOperationId)
+        deleteCmdStatus=$(echo $deleteCmdOperationOutput | grep -i "status:" | sed "s/.*status:\s//" | sed "s/\s.*//" | sed "s/ //g")
+        echo "Waiting for MIG deletion to complete. Current status: " $deleteCmdStatus
+      done
+    fi
+  fi
 
   gcloud compute instance-templates delete \
     --project "${PROJECT}" \

From 932cdd954dea696ed1c77c12c2d90fc47e42ec59 Mon Sep 17 00:00:00 2001
From: saadali <saadali@google.com>
Date: Tue, 12 May 2015 16:12:15 -0700
Subject: [PATCH 2/2] Clean up GCE kube-down script by using set e

---
 cluster/gce/util.sh | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
index 575f7e078ba..327f64e0319 100755
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -643,11 +643,14 @@ function kube-down {
   detect-project
 
   echo "Bringing down cluster"
+  set +e  # Do not stop on error
 
+  # The gcloud APIs don't return machine parsable error codes/retry information. Therefore the best we can
+  # do is parse the output and special case particular responses we are interested in.
   deleteCmdOutput=$(gcloud preview managed-instance-groups --zone "${ZONE}" delete \
     --project "${PROJECT}" \
     --quiet \
-    "${NODE_INSTANCE_PREFIX}-group" || true)
+    "${NODE_INSTANCE_PREFIX}-group")
   if [[ "$deleteCmdOutput" != ""  ]]; then
     # Managed instance group deletion is done asyncronously, we must wait for it to complete, or subsequent steps fail
     deleteCmdOperationId=$(echo $deleteCmdOutput | grep "Operation:" | sed "s/.*Operation:\s//" | sed "s/\s.*//" | sed "s/ //g")
@@ -666,7 +669,7 @@ function kube-down {
   gcloud compute instance-templates delete \
     --project "${PROJECT}" \
     --quiet \
-    "${NODE_INSTANCE_PREFIX}-template" || true
+    "${NODE_INSTANCE_PREFIX}-template"
 
   # First delete the master (if it exists).
   gcloud compute instances delete \
@@ -674,14 +677,14 @@ function kube-down {
     --quiet \
     --delete-disks all \
     --zone "${ZONE}" \
-    "${MASTER_NAME}" || true
+    "${MASTER_NAME}"
 
   # Delete the master pd (possibly leaked by kube-up if master create failed)
   gcloud compute disks delete \
     --project "${PROJECT}" \
     --quiet \
     --zone "${ZONE}" \
-    "${MASTER_NAME}"-pd || true
+    "${MASTER_NAME}"-pd
 
   # Find out what minions are running.
   local -a minions
@@ -697,7 +700,7 @@ function kube-down {
       --quiet \
       --delete-disks boot \
       --zone "${ZONE}" \
-      "${minions[@]::10}" || true
+      "${minions[@]::10}"
     minions=( "${minions[@]:10}" )
   done
 
@@ -705,13 +708,13 @@ function kube-down {
   gcloud compute firewall-rules delete  \
     --project "${PROJECT}" \
     --quiet \
-    "${MASTER_NAME}-https" || true
+    "${MASTER_NAME}-https"
 
   # Delete firewall rule for minions.
   gcloud compute firewall-rules delete  \
     --project "${PROJECT}" \
     --quiet \
-    "${MINION_TAG}-all" || true
+    "${MINION_TAG}-all"
 
   # Delete routes.
   local -a routes
@@ -723,7 +726,7 @@ function kube-down {
     gcloud compute routes delete \
       --project "${PROJECT}" \
       --quiet \
-      "${routes[@]::10}" || true
+      "${routes[@]::10}"
     routes=( "${routes[@]:10}" )
   done
 
@@ -733,10 +736,11 @@ function kube-down {
     --project "${PROJECT}" \
     --region "${REGION}" \
     --quiet \
-    "${MASTER_NAME}-ip" || true
+    "${MASTER_NAME}-ip"
 
   export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
   clear-kubeconfig
+  set -e
 }
 
 # Update a kubernetes cluster with latest source