From c3d5113365d92fdb8113b9872ad0a8e7551c23c4 Mon Sep 17 00:00:00 2001
From: "Madhusudan.C.S" <madhusudancs@google.com>
Date: Sun, 4 Jun 2017 13:19:59 -0700
Subject: [PATCH] Delete cluster role and their bindings federated clusters.

This is part of the namespace deletion big hammer. `kubefed join` not
just creates federation-system namespace, but also cluster role and
cluster role bindings in the joining clusters. Sometimes unjoin fails
to delete them. So we use a big hammer here to delete them.

This smells like a real problem in kubefed and needs investigation.
This is a short term fix to unblock the submit queue.
---
 federation/cluster/common.sh          | 19 +++++++++++++++++--
 federation/cluster/federation-down.sh |  2 +-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh
index 49223f1a188..a13c69a9f41 100644
--- a/federation/cluster/common.sh
+++ b/federation/cluster/common.sh
@@ -431,13 +431,28 @@ function cleanup-federation-api-objects {
   # This is a big hammer. We get rid of federation-system namespace from
   # all the clusters
   for context in $(federation_cluster_contexts); do
-    kube::log::status "Removing namespace \"${FEDERATION_NAMESPACE}\" from \"${context}\""
     (
+      local -r role="federation-controller-manager:${FEDERATION_NAME}-${context}-${HOST_CLUSTER_CONTEXT}"
+      kube::log::status "Removing namespace \"${FEDERATION_NAMESPACE}\", cluster role \"${role}\" and cluster role binding \"${role}\" from \"${context}\""
       # Try deleting until the namespace is completely gone.
-      while $host_kubectl --context="${context}" delete namespace ${FEDERATION_NAMESPACE} >/dev/null 2>&1; do
+      while $host_kubectl --context="${context}" delete namespace "${FEDERATION_NAMESPACE}" >/dev/null 2>&1; do
+        # It is usually slower to remove a namespace because it involves
+        # performing a cascading deletion of all the resources in the
+        # namespace. So we sleep a little longer than other resources
+        # before retrying
         sleep 5
       done
       kube::log::status "Removed namespace \"${FEDERATION_NAMESPACE}\" from \"${context}\""
+
+      while $host_kubectl --context="${context}" delete clusterrole "${role}" >/dev/null 2>&1; do
+        sleep 2
+      done
+      kube::log::status "Removed cluster role \"${role}\" from \"${context}\""
+
+      while $host_kubectl --context="${context}" delete clusterrolebinding "${role}" >/dev/null 2>&1; do
+        sleep 2
+      done
+      kube::log::status "Removed cluster role binding \"${role}\" from \"${context}\""
     ) &
   done
   wait
diff --git a/federation/cluster/federation-down.sh b/federation/cluster/federation-down.sh
index be91e974f60..8aca4580399 100755
--- a/federation/cluster/federation-down.sh
+++ b/federation/cluster/federation-down.sh
@@ -27,7 +27,7 @@ source "${KUBE_ROOT}/federation/cluster/common.sh"
 
 # federation_clusters returns a list of all the clusters in
 # federation, if at all the federation control plane exists
-# and there are any clusters registerd.
+# and there are any clusters registered.
 function federation_clusters() {
   if clusters=$("${KUBE_ROOT}/cluster/kubectl.sh" \
       --context="${FEDERATION_KUBE_CONTEXT}" \