From f6b460c9910a0579892395c4bdf4a3bfba85e71e Mon Sep 17 00:00:00 2001
From: Aldo Culquicondor <acondor@google.com>
Date: Wed, 13 Nov 2019 13:20:18 -0500
Subject: [PATCH] Wait for terminating pods to be deleted

To reduce flakiness of Scheduler Predicates E2E tests

Signed-off-by: Aldo Culquicondor <acondor@google.com>
---
 test/e2e/scheduling/framework.go  | 61 +++++++++++++++++++++++++------
 test/e2e/scheduling/predicates.go |  1 +
 2 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/test/e2e/scheduling/framework.go b/test/e2e/scheduling/framework.go
index fd439057855..6f4a553880d 100644
--- a/test/e2e/scheduling/framework.go
+++ b/test/e2e/scheduling/framework.go
@@ -17,6 +17,7 @@ limitations under the License.
 package scheduling
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/onsi/ginkgo"
@@ -29,6 +30,11 @@ import (
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 )
 
+var (
+	timeout  = 10 * time.Minute
+	waitTime = 2 * time.Second
+)
+
 // SIGDescribe annotates the test with the SIG label.
 func SIGDescribe(text string, body func()) bool {
 	return ginkgo.Describe("[sig-scheduling] "+text, body)
@@ -36,13 +42,11 @@ func SIGDescribe(text string, body func()) bool {
 
 // WaitForStableCluster waits until all existing pods are scheduled and returns their amount.
 func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int {
-	timeout := 10 * time.Minute
 	startTime := time.Now()
-
 	// Wait for all pods to be scheduled.
-	allScheduledPods, allNotScheduledPods := getFilteredPods(c, masterNodes, metav1.NamespaceAll)
+	allScheduledPods, allNotScheduledPods := getScheduledAndUnscheduledPods(c, masterNodes, metav1.NamespaceAll)
 	for len(allNotScheduledPods) != 0 {
-		time.Sleep(2 * time.Second)
+		time.Sleep(waitTime)
 		if startTime.Add(timeout).Before(time.Now()) {
 			framework.Logf("Timed out waiting for the following pods to schedule")
 			for _, p := range allNotScheduledPods {
@@ -51,22 +55,57 @@ func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int {
 			framework.Failf("Timed out after %v waiting for stable cluster.", timeout)
 			break
 		}
-		allScheduledPods, allNotScheduledPods = getFilteredPods(c, masterNodes, metav1.NamespaceAll)
+		allScheduledPods, allNotScheduledPods = getScheduledAndUnscheduledPods(c, masterNodes, metav1.NamespaceAll)
 	}
 	return len(allScheduledPods)
 }
 
-// getFilteredPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out.
-func getFilteredPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) {
+// WaitForPodsToBeDeleted waits until pods that are terminating to get deleted.
+func WaitForPodsToBeDeleted(c clientset.Interface) {
+	startTime := time.Now()
+	deleting := getDeletingPods(c, metav1.NamespaceAll)
+	for len(deleting) != 0 {
+		if startTime.Add(timeout).Before(time.Now()) {
+			framework.Logf("Pods still not deleted")
+			for _, p := range deleting {
+				framework.Logf("%v/%v", p.Namespace, p.Name)
+			}
+			framework.Failf("Timed out after %v waiting for pods to be deleted", timeout)
+			break
+		}
+		time.Sleep(waitTime)
+		deleting = getDeletingPods(c, metav1.NamespaceAll)
+	}
+}
+
+// getScheduledAndUnscheduledPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out.
+func getScheduledAndUnscheduledPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) {
 	pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{})
-	framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster")
+	framework.ExpectNoError(err, fmt.Sprintf("listing all pods in namespace %q while waiting for stable cluster", ns))
 	// API server returns also Pods that succeeded. We need to filter them out.
 	filteredPods := make([]v1.Pod, 0, len(pods.Items))
-	for _, pod := range pods.Items {
-		if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed {
-			filteredPods = append(filteredPods, pod)
+	for _, p := range pods.Items {
+		if !podTerminated(p) {
+			filteredPods = append(filteredPods, p)
 		}
 	}
 	pods.Items = filteredPods
 	return e2epod.GetPodsScheduled(masterNodes, pods)
 }
+
+// getDeletingPods returns whether there are any pods marked for deletion.
+func getDeletingPods(c clientset.Interface, ns string) []v1.Pod {
+	pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{})
+	framework.ExpectNoError(err, fmt.Sprintf("listing all pods in namespace %q while waiting for pods to terminate", ns))
+	var deleting []v1.Pod
+	for _, p := range pods.Items {
+		if p.ObjectMeta.DeletionTimestamp != nil && !podTerminated(p) {
+			deleting = append(deleting, p)
+		}
+	}
+	return deleting
+}
+
+func podTerminated(p v1.Pod) bool {
+	return p.Status.Phase == v1.PodSucceeded || p.Status.Phase == v1.PodFailed
+}
diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go
index c6913c85431..4203f0cd9bb 100644
--- a/test/e2e/scheduling/predicates.go
+++ b/test/e2e/scheduling/predicates.go
@@ -129,6 +129,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
 			totalPodCapacity += podCapacity.Value()
 		}
 
+		WaitForPodsToBeDeleted(cs)
 		currentlyScheduledPods := WaitForStableCluster(cs, masterNodes)
 		podsNeededForSaturation := int(totalPodCapacity) - currentlyScheduledPods