diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh
index 37fe14a6ec4..a13c69a9f41 100644
--- a/federation/cluster/common.sh
+++ b/federation/cluster/common.sh
@@ -411,15 +411,50 @@ function push-federation-images {
 }
 
 function cleanup-federation-api-objects {
+  # This is a cleanup function. We cannot stop on errors here. So disable
+  # errexit in this function.
+  set +o errexit
+
   echo "Cleaning Federation control plane objects"
   # Delete all resources with the federated-cluster label.
   $host_kubectl delete pods,svc,rc,deployment,secret -lapp=federated-cluster
-  # Delete all resources in FEDERATION_NAMESPACE.
-  $host_kubectl delete pvc,pv,pods,svc,rc,deployment,secret --namespace=${FEDERATION_NAMESPACE} --all
-  $host_kubectl delete ns ${FEDERATION_NAMESPACE}
 
-  # Poll until the namespace is completely gone.
-  while $host_kubectl get namespace ${FEDERATION_NAMESPACE} >/dev/null 2>&1; do
-    sleep 5
+  # Delete all PVs bound to PVCs in FEDERATION_NAMESPACE
+  pvs=$($host_kubectl get pvc --namespace=${FEDERATION_NAMESPACE} -o jsonpath='{.items[*].spec.volumeName}')
+  while $host_kubectl delete pv ${pvs} >/dev/null 2>&1; do
+    sleep 2
   done
+
+  # Delete all resources in FEDERATION_NAMESPACE.
+  $host_kubectl delete pvc,pods,svc,rc,deployment,secret --namespace=${FEDERATION_NAMESPACE} --all
+
+  # This is a big hammer. We get rid of federation-system namespace from
+  # all the clusters
+  for context in $(federation_cluster_contexts); do
+    (
+      local -r role="federation-controller-manager:${FEDERATION_NAME}-${context}-${HOST_CLUSTER_CONTEXT}"
+      kube::log::status "Removing namespace \"${FEDERATION_NAMESPACE}\", cluster role \"${role}\" and cluster role binding \"${role}\" from \"${context}\""
+      # Try deleting until the namespace is completely gone.
+      while $host_kubectl --context="${context}" delete namespace "${FEDERATION_NAMESPACE}" >/dev/null 2>&1; do
+        # It is usually slower to remove a namespace because it involves
+        # performing a cascading deletion of all the resources in the
+        # namespace. So we sleep a little longer than other resources
+        # before retrying
+        sleep 5
+      done
+      kube::log::status "Removed namespace \"${FEDERATION_NAMESPACE}\" from \"${context}\""
+
+      while $host_kubectl --context="${context}" delete clusterrole "${role}" >/dev/null 2>&1; do
+        sleep 2
+      done
+      kube::log::status "Removed cluster role \"${role}\" from \"${context}\""
+
+      while $host_kubectl --context="${context}" delete clusterrolebinding "${role}" >/dev/null 2>&1; do
+        sleep 2
+      done
+      kube::log::status "Removed cluster role binding \"${role}\" from \"${context}\""
+    ) &
+  done
+  wait
+  set -o errexit
 }
diff --git a/federation/cluster/federation-down.sh b/federation/cluster/federation-down.sh
index be91e974f60..8aca4580399 100755
--- a/federation/cluster/federation-down.sh
+++ b/federation/cluster/federation-down.sh
@@ -27,7 +27,7 @@ source "${KUBE_ROOT}/federation/cluster/common.sh"
 
 # federation_clusters returns a list of all the clusters in
 # federation, if at all the federation control plane exists
-# and there are any clusters registerd.
+# and there are any clusters registered.
 function federation_clusters() {
   if clusters=$("${KUBE_ROOT}/cluster/kubectl.sh" \
       --context="${FEDERATION_KUBE_CONTEXT}" \