From eeda67232cea8917b9ab38b0e21a605384f87a18 Mon Sep 17 00:00:00 2001
From: Jingfang Liu <jingfangliu@google.com>
Date: Thu, 22 Mar 2018 14:47:21 -0700
Subject: [PATCH] Add retry to AssertCleanup

---
 test/e2e/framework/util.go | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go
index d3018951697..66c7c2a1364 100644
--- a/test/e2e/framework/util.go
+++ b/test/e2e/framework/util.go
@@ -2066,15 +2066,32 @@ func AssertCleanup(ns string, selectors ...string) {
 	if ns != "" {
 		nsArg = fmt.Sprintf("--namespace=%s", ns)
 	}
-	for _, selector := range selectors {
-		resources := RunKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg)
-		if resources != "" {
-			Failf("Resources left running after stop:\n%s", resources)
-		}
-		pods := RunKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}")
-		if pods != "" {
-			Failf("Pods left unterminated after stop:\n%s", pods)
+
+	backoff := wait.Backoff{
+		Duration: 5 * time.Second,
+		Factor:   2,
+		Steps:    3,
+	}
+	var e error
+	verifyCleanupFunc := func() (bool, error) {
+		e = nil
+		for _, selector := range selectors {
+			resources := RunKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg)
+			if resources != "" {
+				e = fmt.Errorf("Resources left running after stop:\n%s", resources)
+				return false, nil
+			}
+			pods := RunKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}")
+			if pods != "" {
+				e = fmt.Errorf("Pods left unterminated after stop:\n%s", pods)
+				return false, nil
+			}
 		}
+		return true, nil
+	}
+	err := wait.ExponentialBackoff(backoff, verifyCleanupFunc)
+	if err != nil {
+		Failf(e.Error())
 	}
 }