From b8b18a0a05a252233269eb72d2c30b92415ab764 Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Wed, 8 Jun 2022 16:48:51 +0800 Subject: [PATCH] [e2e] Should spread Pods to schedulable cluster zones Some Nodes like control plane ones should not be considered to spread Pods. Signed-off-by: Zhecheng Li --- test/e2e/scheduling/ubernetes_lite.go | 33 ++++++++++----------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/test/e2e/scheduling/ubernetes_lite.go b/test/e2e/scheduling/ubernetes_lite.go index 29e0138cb6e..ed60e27dd36 100644 --- a/test/e2e/scheduling/ubernetes_lite.go +++ b/test/e2e/scheduling/ubernetes_lite.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -46,17 +47,20 @@ var _ = SIGDescribe("Multi-AZ Clusters", func() { var zoneCount int var err error var cleanUp func() + var zoneNames sets.String ginkgo.BeforeEach(func() { + cs := f.ClientSet + if zoneCount <= 0 { - zoneCount, err = getZoneCount(f.ClientSet) + zoneNames, err = e2enode.GetSchedulableClusterZones(cs) framework.ExpectNoError(err) + zoneCount = len(zoneNames) } - ginkgo.By(fmt.Sprintf("Checking for multi-zone cluster. Zone count = %d", zoneCount)) - msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount) + ginkgo.By(fmt.Sprintf("Checking for multi-zone cluster. Schedulable zone count = %d", zoneCount)) + msg := fmt.Sprintf("Schedulable zone count is %d, only run for multi-zone clusters, skipping test", zoneCount) e2eskipper.SkipUnlessAtLeast(zoneCount, 2, msg) // TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread - cs := f.ClientSet e2enode.WaitForTotalHealthy(cs, time.Minute) nodeList, err := e2enode.GetReadySchedulableNodes(cs) framework.ExpectNoError(err) @@ -71,17 +75,17 @@ var _ = SIGDescribe("Multi-AZ Clusters", func() { } }) ginkgo.It("should spread the pods of a service across zones [Serial]", func() { - SpreadServiceOrFail(f, 5*zoneCount, imageutils.GetPauseImageName()) + SpreadServiceOrFail(f, 5*zoneCount, zoneNames, imageutils.GetPauseImageName()) }) ginkgo.It("should spread the pods of a replication controller across zones [Serial]", func() { - SpreadRCOrFail(f, int32(5*zoneCount), framework.ServeHostnameImage, []string{"serve-hostname"}) + SpreadRCOrFail(f, int32(5*zoneCount), zoneNames, framework.ServeHostnameImage, []string{"serve-hostname"}) }) }) // SpreadServiceOrFail check that the pods comprising a service // get spread evenly across available zones -func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) { +func SpreadServiceOrFail(f *framework.Framework, replicaCount int, zoneNames sets.String, image string) { // First create the service serviceName := "test-service" serviceSpec := &v1.Service{ @@ -130,8 +134,6 @@ func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) framework.ExpectNoError(err) // Now make sure they're spread across zones - zoneNames, err := e2enode.GetClusterZones(f.ClientSet) - framework.ExpectNoError(err) checkZoneSpreading(f.ClientSet, pods, zoneNames.List()) } @@ -146,15 +148,6 @@ func getZoneNameForNode(node v1.Node) (string, error) { node.Name, v1.LabelFailureDomainBetaZone, v1.LabelTopologyZone) } -// Return the number of zones in which we have nodes in this cluster. -func getZoneCount(c clientset.Interface) (int, error) { - zoneNames, err := e2enode.GetClusterZones(c) - if err != nil { - return -1, err - } - return len(zoneNames), nil -} - // Find the name of the zone in which the pod is scheduled func getZoneNameForPod(c clientset.Interface, pod v1.Pod) (string, error) { ginkgo.By(fmt.Sprintf("Getting zone name for pod %s, on node %s", pod.Name, pod.Spec.NodeName)) @@ -195,7 +188,7 @@ func checkZoneSpreading(c clientset.Interface, pods *v1.PodList, zoneNames []str // SpreadRCOrFail Check that the pods comprising a replication // controller get spread evenly across available zones -func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string, args []string) { +func SpreadRCOrFail(f *framework.Framework, replicaCount int32, zoneNames sets.String, image string, args []string) { name := "ubelite-spread-rc-" + string(uuid.NewUUID()) ginkgo.By(fmt.Sprintf("Creating replication controller %s", name)) controller, err := f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Create(context.TODO(), &v1.ReplicationController{ @@ -244,7 +237,5 @@ func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string, ar framework.ExpectNoError(err) // Now make sure they're spread across zones - zoneNames, err := e2enode.GetClusterZones(f.ClientSet) - framework.ExpectNoError(err) checkZoneSpreading(f.ClientSet, pods, zoneNames.List()) }