From 41afebd5d2854532bb0bc63f42037c8fff49161f Mon Sep 17 00:00:00 2001 From: Aleksandra Malinowska Date: Mon, 2 Oct 2017 13:10:08 +0200 Subject: [PATCH 1/2] make isAutoscalerEnabled check work with min size 0 --- test/e2e/autoscaling/cluster_size_autoscaling.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go index cf86cb6d709..269b732e1be 100644 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ b/test/e2e/autoscaling/cluster_size_autoscaling.go @@ -110,7 +110,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { Expect(nodeCount).Should(Equal(sum)) if framework.ProviderIs("gke") { - val, err := isAutoscalerEnabled(3) + val, err := isAutoscalerEnabled(5) framework.ExpectNoError(err) if !val { err = enableAutoscaler("default-pool", 3, 5) @@ -775,7 +775,7 @@ func getGKEClusterUrl() string { token) } -func isAutoscalerEnabled(expectedMinNodeCountInTargetPool int) (bool, error) { +func isAutoscalerEnabled(expectedMaxNodeCountInTargetPool int) (bool, error) { resp, err := http.Get(getGKEClusterUrl()) if err != nil { return false, err @@ -786,7 +786,7 @@ func isAutoscalerEnabled(expectedMinNodeCountInTargetPool int) (bool, error) { return false, err } strBody := string(body) - if strings.Contains(strBody, "\"minNodeCount\": "+strconv.Itoa(expectedMinNodeCountInTargetPool)) { + if strings.Contains(strBody, "\"maxNodeCount\": "+strconv.Itoa(expectedMaxNodeCountInTargetPool)) { return true, nil } return false, nil @@ -833,7 +833,7 @@ func enableAutoscaler(nodePool string, minCount, maxCount int) error { var finalErr error for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { - val, err := isAutoscalerEnabled(minCount) + val, err := isAutoscalerEnabled(maxCount) if err == nil && val { return nil } @@ -880,7 +880,7 @@ func disableAutoscaler(nodePool string, minCount, maxCount int) error { } for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { - if val, err := isAutoscalerEnabled(minCount); err == nil && !val { + if val, err := isAutoscalerEnabled(maxCount); err == nil && !val { return nil } } From 16c6337182efa8298cd1a6ab97123099675422e2 Mon Sep 17 00:00:00 2001 From: Aleksandra Malinowska Date: Mon, 2 Oct 2017 13:12:25 +0200 Subject: [PATCH 2/2] refactor test to work on GKE --- .../autoscaling/cluster_size_autoscaling.go | 121 ++++++++++++------ 1 file changed, 83 insertions(+), 38 deletions(-) diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go index 269b732e1be..e5f31db730b 100644 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ b/test/e2e/autoscaling/cluster_size_autoscaling.go @@ -64,8 +64,9 @@ const ( makeSchedulableDelay = 20 * time.Second freshStatusLimit = 20 * time.Second - gkeEndpoint = "https://test-container.sandbox.googleapis.com" - gkeUpdateTimeout = 15 * time.Minute + gkeEndpoint = "https://test-container.sandbox.googleapis.com" + gkeUpdateTimeout = 15 * time.Minute + gkeNodepoolNameKey = "cloud.google.com/gke-nodepool" disabledTaint = "DisabledForAutoscalingTest" criticalAddonsOnlyTaint = "CriticalAddonsOnly" @@ -619,21 +620,43 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) }) - It("Should be able to scale a node group down to 0[Feature:ClusterSizeAutoscalingScaleDown]", func() { - // Determine whether we want to run & adjust the setup if necessary - if len(originalSizes) < 2 { - if framework.ProviderIs("gke") { - By("Adding a new node pool") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "n1-standard-4", 1) - defer deleteNodePool(extraPoolName) - framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount+1, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 0, 1)) - } else { - framework.Skipf("At least 2 node groups are needed for scale-to-0 tests") - } - } + // Scale to 0 test is split into two functions (for GKE & GCE.) + // The reason for it is that scenario is exactly the same, + // but setup & verification use different APIs. + // + // Scenario: + // (GKE only) add an extra node pool with size 1 & enable autoscaling for it + // (GCE only) find the smallest MIG & resize it to 1 + // manually drain the single node from this node pool/MIG + // wait for cluster size to decrease + // verify the targeted node pool/MIG is of size 0 + gkeScaleToZero := func() { + // GKE-specific setup + By("Add a new node pool with 1 node and min size 0") + const extraPoolName = "extra-pool" + addNodePool(extraPoolName, "n1-standard-4", 1) + defer deleteNodePool(extraPoolName) + framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount+1, resizeTimeout)) + err := enableAutoscaler(extraPoolName, 0, 1) + framework.ExpectNoError(err) + ngNodes := getPoolNodes(f, extraPoolName) + Expect(len(ngNodes) == 1).To(BeTrue()) + node := ngNodes[0] + By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) + + // this part is identical + drainNode(f, node) + framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, + func(size int) bool { return size < nodeCount+1 }, scaleDownTimeout)) + + // GKE-specific check + newSize := getPoolSize(f, extraPoolName) + Expect(newSize).Should(Equal(0)) + } + + gceScaleToZero := func() { + // non-GKE only By("Find smallest node group and manually scale it to a single node") minMig := "" minSize := nodeCount @@ -646,35 +669,32 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { err := framework.ResizeGroup(minMig, int32(1)) framework.ExpectNoError(err) framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount-minSize+1, resizeTimeout)) - - By("Make the single node unschedulable") - allNodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) ngNodes, err := framework.GetGroupNodes(minMig) framework.ExpectNoError(err) - By(fmt.Sprintf("Target nodes for scale-down: %s", ngNodes)) Expect(len(ngNodes) == 1).To(BeTrue()) node, err := f.ClientSet.Core().Nodes().Get(ngNodes[0], metav1.GetOptions{}) + By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) framework.ExpectNoError(err) - makeNodeUnschedulable(f.ClientSet, node) - By("Manually drain the single node") - podOpts := metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, node.Name).String()} - pods, err := c.Core().Pods(metav1.NamespaceAll).List(podOpts) - framework.ExpectNoError(err) - for _, pod := range pods.Items { - err = f.ClientSet.Core().Pods(pod.Namespace).Delete(pod.Name, metav1.NewDeleteOptions(0)) - framework.ExpectNoError(err) - } - - By("The node should be removed") + // this part is identical + drainNode(f, node) framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, - func(size int) bool { return size < len(allNodes.Items) }, scaleDownTimeout)) - minSize, err = framework.GroupSize(minMig) + func(size int) bool { return size < nodeCount-minSize+1 }, scaleDownTimeout)) + + // non-GKE only + newSize, err := framework.GroupSize(minMig) framework.ExpectNoError(err) - Expect(minSize).Should(Equal(0)) + Expect(newSize).Should(Equal(0)) + } + + It("Should be able to scale a node group down to 0[Feature:ClusterSizeAutoscalingScaleDown]", func() { + if framework.ProviderIs("gke") { // In GKE, we can just add a node pool + gkeScaleToZero() + } else if len(originalSizes) < 2 { + gceScaleToZero() + } else { + framework.Skipf("At least 2 node groups are needed for scale-to-0 tests") + } }) It("Shouldn't perform scale up operation and should list unhealthy status if most of the cluster is broken[Feature:ClusterSizeAutoscalingScaleUp]", func() { @@ -914,13 +934,24 @@ func getPoolNodes(f *framework.Framework, poolName string) []*v1.Node { nodes := make([]*v1.Node, 0, 1) nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) for _, node := range nodeList.Items { - if poolLabel := node.Labels["cloud.google.com/gke-nodepool"]; poolLabel == poolName { + if node.Labels[gkeNodepoolNameKey] == poolName { nodes = append(nodes, &node) } } return nodes } +func getPoolSize(f *framework.Framework, poolName string) int { + size := 0 + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + for _, node := range nodeList.Items { + if node.Labels[gkeNodepoolNameKey] == poolName { + size++ + } + } + return size +} + func doPut(url, content string) (string, error) { req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(content))) if err != nil { @@ -1075,6 +1106,20 @@ func setMigSizes(sizes map[string]int) bool { return madeChanges } +func drainNode(f *framework.Framework, node *v1.Node) { + By("Make the single node unschedulable") + makeNodeUnschedulable(f.ClientSet, node) + + By("Manually drain the single node") + podOpts := metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, node.Name).String()} + pods, err := f.ClientSet.Core().Pods(metav1.NamespaceAll).List(podOpts) + framework.ExpectNoError(err) + for _, pod := range pods.Items { + err = f.ClientSet.Core().Pods(pod.Namespace).Delete(pod.Name, metav1.NewDeleteOptions(0)) + framework.ExpectNoError(err) + } +} + func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error { By(fmt.Sprintf("Taint node %s", node.Name)) for j := 0; j < 3; j++ {