Merge pull request #61565 from Liujingfang1/issue61484

Automatic merge from submit-queue (batch tested with PRs 61904, 61565, 61401, 61432, 61772). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add retry to AssertCleanup

**What this PR does / why we need it**:
Add retry in AssertCleanup for e2e kubectl test to remove the flakeness

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #61484

**Special notes for your reviewer**:
This function is only used in e2e kubectl test. It doesn't affect other tests' behavior.

**Release note**:

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2018-03-29 11:46:12 -07:00 committed by GitHub
commit 0838c9e990
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -2076,15 +2076,32 @@ func AssertCleanup(ns string, selectors ...string) {
if ns != "" {
nsArg = fmt.Sprintf("--namespace=%s", ns)
}
for _, selector := range selectors {
resources := RunKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg)
if resources != "" {
Failf("Resources left running after stop:\n%s", resources)
}
pods := RunKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}")
if pods != "" {
Failf("Pods left unterminated after stop:\n%s", pods)
backoff := wait.Backoff{
Duration: 5 * time.Second,
Factor: 2,
Steps: 3,
}
var e error
verifyCleanupFunc := func() (bool, error) {
e = nil
for _, selector := range selectors {
resources := RunKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg)
if resources != "" {
e = fmt.Errorf("Resources left running after stop:\n%s", resources)
return false, nil
}
pods := RunKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}")
if pods != "" {
e = fmt.Errorf("Pods left unterminated after stop:\n%s", pods)
return false, nil
}
}
return true, nil
}
err := wait.ExponentialBackoff(backoff, verifyCleanupFunc)
if err != nil {
Failf(e.Error())
}
}