diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 8b8c6b0197a..a44c56d001e 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -2317,38 +2317,6 @@ func retryCmd(command string, args ...string) (string, string, error) { return stdout, stderr, err } -// WaitForStableCluster waits until all existing pods are scheduled and returns their amount. -func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int { - timeout := 10 * time.Minute - startTime := time.Now() - - allPods, err := c.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) - ExpectNoError(err) - // API server returns also Pods that succeeded. We need to filter them out. - currentPods := make([]v1.Pod, 0, len(allPods.Items)) - for _, pod := range allPods.Items { - if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed { - currentPods = append(currentPods, pod) - } - - } - allPods.Items = currentPods - scheduledPods, currentlyNotScheduledPods := e2epod.GetPodsScheduled(masterNodes, allPods) - for len(currentlyNotScheduledPods) != 0 { - time.Sleep(2 * time.Second) - - allPods, err := c.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) - ExpectNoError(err) - scheduledPods, currentlyNotScheduledPods = e2epod.GetPodsScheduled(masterNodes, allPods) - - if startTime.Add(timeout).Before(time.Now()) { - Failf("Timed out after %v waiting for stable cluster.", timeout) - break - } - } - return len(scheduledPods) -} - // E2ETestNodePreparer implements testutils.TestNodePreparer interface, which is used // to create/modify Nodes before running a test. type E2ETestNodePreparer struct { diff --git a/test/e2e/scheduling/framework.go b/test/e2e/scheduling/framework.go index b5a32c66b2d..e855f857b2e 100644 --- a/test/e2e/scheduling/framework.go +++ b/test/e2e/scheduling/framework.go @@ -16,9 +16,69 @@ limitations under the License. package scheduling -import "github.com/onsi/ginkgo" +import ( + "time" + + "github.com/onsi/ginkgo" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" +) // SIGDescribe annotates the test with the SIG label. func SIGDescribe(text string, body func()) bool { return ginkgo.Describe("[sig-scheduling] "+text, body) } + +// WaitForStableCluster waits until all existing pods are scheduled and returns their amount. +func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int { + timeout := 10 * time.Minute + startTime := time.Now() + + allPods := getAllPods(c) + scheduledSystemPods, currentlyNotScheduledSystemPods := getSystemPods(c) + systemPods := scheduledSystemPods + currentlyNotScheduledSystemPods + + // Wait for system pods to be scheduled, and for pods in all other namespaces to be deleted + for currentlyNotScheduledSystemPods != 0 || systemPods != allPods { + time.Sleep(2 * time.Second) + + scheduledSystemPods, currentlyNotScheduledSystemPods := getSystemPods(c) + systemPods = scheduledSystemPods + currentlyNotScheduledSystemPods + allPods = getAllPods(c) + + if startTime.Add(timeout).Before(time.Now()) { + framework.Failf("Timed out after %v waiting for stable cluster.", timeout) + break + } + } + return scheduledSystemPods +} + +// getAllPods lists all pods in the cluster, with succeeded and failed pods filtered out and returns the count +func getAllPods(c clientset.Interface) int { + allPods, err := c.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) + framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster") + // API server returns also Pods that succeeded. We need to filter them out. + currentPods := make([]v1.Pod, 0, len(allPods.Items)) + for _, pod := range allPods.Items { + if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed { + currentPods = append(currentPods, pod) + } + + } + allPods.Items = currentPods + return len(allPods.Items) +} + +// getSystemPods lists the pods in the kube-system namespace and returns the number of scheduled and unscheduled pods +func getSystemPods(c clientset.Interface) (int, int) { + systemPods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{}) + framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster") + scheduledSystemPods, currentlyNotScheduledSystemPods := e2epod.GetPodsScheduled(masterNodes, systemPods) + return len(scheduledSystemPods), len(currentlyNotScheduledSystemPods) +} diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 57aaa9395d4..c6913c85431 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -129,7 +129,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { totalPodCapacity += podCapacity.Value() } - currentlyScheduledPods := framework.WaitForStableCluster(cs, masterNodes) + currentlyScheduledPods := WaitForStableCluster(cs, masterNodes) podsNeededForSaturation := int(totalPodCapacity) - currentlyScheduledPods ginkgo.By(fmt.Sprintf("Starting additional %v Pods to fully saturate the cluster max pods and trying to start another one", podsNeededForSaturation)) @@ -171,7 +171,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { nodeMaxAllocatable = allocatable.Value() } } - framework.WaitForStableCluster(cs, masterNodes) + WaitForStableCluster(cs, masterNodes) pods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) framework.ExpectNoError(err) @@ -246,7 +246,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { Description: Scheduling Pods MUST fail if the resource limits exceed Machine capacity. */ framework.ConformanceIt("validates resource limits of pods that are allowed to run ", func() { - framework.WaitForStableCluster(cs, masterNodes) + WaitForStableCluster(cs, masterNodes) nodeMaxAllocatable := int64(0) nodeToAllocatableMap := make(map[string]int64) for _, node := range nodeList.Items { @@ -356,7 +356,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { ginkgo.By("Trying to schedule Pod with nonempty NodeSelector.") podName := "restricted-pod" - framework.WaitForStableCluster(cs, masterNodes) + WaitForStableCluster(cs, masterNodes) conf := pausePodConfig{ Name: podName, @@ -411,7 +411,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { ginkgo.By("Trying to schedule Pod with nonempty NodeSelector.") podName := "restricted-pod" - framework.WaitForStableCluster(cs, masterNodes) + WaitForStableCluster(cs, masterNodes) conf := pausePodConfig{ Name: podName,