From c5e68b54519c68fbf33abb0d4bb60df75208bc9d Mon Sep 17 00:00:00 2001 From: Zihong Zheng Date: Tue, 29 Nov 2016 19:40:02 -0800 Subject: [PATCH] Fixes flake: wait for dns pods terminating after test completed --- test/e2e/dns_autoscaling.go | 120 ++++++++++++++++++++++++++---------- test/e2e/framework/util.go | 26 ++++++++ 2 files changed, 112 insertions(+), 34 deletions(-) diff --git a/test/e2e/dns_autoscaling.go b/test/e2e/dns_autoscaling.go index 217883ea4be..4a8187c5b10 100644 --- a/test/e2e/dns_autoscaling.go +++ b/test/e2e/dns_autoscaling.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/v1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5" @@ -36,9 +37,8 @@ import ( const ( DNSdefaultTimeout = 5 * time.Minute - DNSNamespace = "kube-system" ClusterAddonLabelKey = "k8s-app" - KubeDNSLabelName = "kube-dns" + DNSLabelName = "kube-dns" DNSAutoscalerLabelName = "kube-dns-autoscaler" ) @@ -46,9 +46,18 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { f := framework.NewDefaultFramework("dns-autoscaling") var c clientset.Interface var previousParams map[string]string - DNSParams_1 := DNSParamsLinear{map[string]string{"linear": "{\"nodesPerReplica\": 1}"}, 1.0, 0.0} - DNSParams_2 := DNSParamsLinear{map[string]string{"linear": "{\"nodesPerReplica\": 2}"}, 2.0, 0.0} - DNSParams_3 := DNSParamsLinear{map[string]string{"linear": "{\"nodesPerReplica\": 3, \"coresPerReplica\": 3}"}, 3.0, 3.0} + var originDNSReplicasCount int + + DNSParams_1 := DNSParamsLinear{ + nodesPerReplica: 1, + } + DNSParams_2 := DNSParamsLinear{ + nodesPerReplica: 2, + } + DNSParams_3 := DNSParamsLinear{ + nodesPerReplica: 3, + coresPerReplica: 3, + } BeforeEach(func() { framework.SkipUnlessProviderIs("gce") @@ -56,24 +65,36 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { Expect(len(framework.GetReadySchedulableNodesOrDie(c).Items)).NotTo(BeZero()) + By("Collecting original replicas count and DNS scaling params") + var err error + originDNSReplicasCount, err = getDNSReplicas(c) + Expect(err).NotTo(HaveOccurred()) + pcm, err := fetchDNSScalingConfigMap(c) Expect(err).NotTo(HaveOccurred()) previousParams = pcm.Data + }) + + // This test is separated because it is slow and need to run serially. + // Will take around 5 minutes to run on a 4 nodes cluster. + It("[Serial] [Slow] kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func() { By("Replace the dns autoscaling parameters with testing parameters") - Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1.data))).NotTo(HaveOccurred()) + err := updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) + Expect(err).NotTo(HaveOccurred()) + defer func() { + By("Restoring intial dns autoscaling parameters") + Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))).NotTo(HaveOccurred()) + + By("Wait for number of running and ready kube-dns pods recover") + label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) + _, err := framework.WaitForPodsWithLabelRunningReady(c, api.NamespaceSystem, label, originDNSReplicasCount, DNSdefaultTimeout) + Expect(err).NotTo(HaveOccurred()) + }() By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) - }) - AfterEach(func() { - By("Restoring intial dns autoscaling parameters") - Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))).NotTo(HaveOccurred()) - }) - - // This test is separated because it is slow and need to run serially - It("[Serial] [Slow] kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func() { originalSizes := make(map[string]int) sum := 0 for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { @@ -96,11 +117,13 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { func(size int) bool { return size == increasedSize }, scaleUpTimeout)).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) By("Replace the dns autoscaling parameters with another testing parameters") - Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3.data))).NotTo(HaveOccurred()) + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_3))) + Expect(err).NotTo(HaveOccurred()) + By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_3) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) @@ -115,16 +138,28 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { + By("Replace the dns autoscaling parameters with testing parameters") + err := updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) + Expect(err).NotTo(HaveOccurred()) + defer func() { + By("Restoring intial dns autoscaling parameters") + Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))).NotTo(HaveOccurred()) + }() + By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) + Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) + By("--- Scenario: should scale kube-dns based on changed parameters ---") By("Replace the dns autoscaling parameters with another testing parameters") - Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3.data))).NotTo(HaveOccurred()) + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_3))) + Expect(err).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_3) + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_3) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") By("Delete the ConfigMap for autoscaler") - err := deleteDNSScalingConfigMap(c) + err = deleteDNSScalingConfigMap(c) Expect(err).NotTo(HaveOccurred()) By("Wait for the ConfigMap got re-created") @@ -135,7 +170,8 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { Expect(reflect.DeepEqual(previousParams, configMap.Data)).To(Equal(true)) By("Replace the dns autoscaling parameters with another testing parameters") - Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2.data))).NotTo(HaveOccurred()) + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_2))) + Expect(err).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_2) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) @@ -145,7 +181,8 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { Expect(deleteDNSAutoscalerPod(c)).NotTo(HaveOccurred()) By("Replace the dns autoscaling parameters with another testing parameters") - Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1.data))).NotTo(HaveOccurred()) + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) + Expect(err).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) @@ -153,9 +190,10 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { }) type DNSParamsLinear struct { - data map[string]string nodesPerReplica float64 coresPerReplica float64 + min int + max int } type getExpectReplicasFunc func(c clientset.Interface) int @@ -185,14 +223,14 @@ func getScheduableCores(nodes []v1.Node) int64 { scInt64, scOk := sc.AsInt64() if !scOk { - framework.Logf("unable to compute integer values of schedulable cores in the cluster") + framework.Logf("Unable to compute integer values of schedulable cores in the cluster") return 0 } return scInt64 } func fetchDNSScalingConfigMap(c clientset.Interface) (*v1.ConfigMap, error) { - cm, err := c.Core().ConfigMaps(DNSNamespace).Get(DNSAutoscalerLabelName) + cm, err := c.Core().ConfigMaps(api.NamespaceSystem).Get(DNSAutoscalerLabelName) if err != nil { return nil, err } @@ -200,23 +238,33 @@ func fetchDNSScalingConfigMap(c clientset.Interface) (*v1.ConfigMap, error) { } func deleteDNSScalingConfigMap(c clientset.Interface) error { - if err := c.Core().ConfigMaps(DNSNamespace).Delete(DNSAutoscalerLabelName, nil); err != nil { + if err := c.Core().ConfigMaps(api.NamespaceSystem).Delete(DNSAutoscalerLabelName, nil); err != nil { return err } framework.Logf("DNS autoscaling ConfigMap deleted.") return nil } +func packLinearParams(params *DNSParamsLinear) map[string]string { + paramsMap := make(map[string]string) + paramsMap["linear"] = fmt.Sprintf("{\"nodesPerReplica\": %v,\"coresPerReplica\": %v,\"min\": %v,\"max\": %v}", + params.nodesPerReplica, + params.coresPerReplica, + params.min, + params.max) + return paramsMap +} + func packDNSScalingConfigMap(params map[string]string) *v1.ConfigMap { configMap := v1.ConfigMap{} configMap.ObjectMeta.Name = DNSAutoscalerLabelName - configMap.ObjectMeta.Namespace = DNSNamespace + configMap.ObjectMeta.Namespace = api.NamespaceSystem configMap.Data = params return &configMap } func updateDNSScalingConfigMap(c clientset.Interface, configMap *v1.ConfigMap) error { - _, err := c.Core().ConfigMaps(DNSNamespace).Update(configMap) + _, err := c.Core().ConfigMaps(api.NamespaceSystem).Update(configMap) if err != nil { return err } @@ -225,13 +273,15 @@ func updateDNSScalingConfigMap(c clientset.Interface, configMap *v1.ConfigMap) e } func getDNSReplicas(c clientset.Interface) (int, error) { - label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: KubeDNSLabelName})) + label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) listOpts := v1.ListOptions{LabelSelector: label.String()} - deployments, err := c.Extensions().Deployments(DNSNamespace).List(listOpts) + deployments, err := c.Extensions().Deployments(api.NamespaceSystem).List(listOpts) if err != nil { return 0, err } - Expect(len(deployments.Items)).Should(Equal(1)) + if len(deployments.Items) != 1 { + return 0, fmt.Errorf("expected 1 DNS deployment, got %v", len(deployments.Items)) + } deployment := deployments.Items[0] return int(*(deployment.Spec.Replicas)), nil @@ -240,14 +290,16 @@ func getDNSReplicas(c clientset.Interface) (int, error) { func deleteDNSAutoscalerPod(c clientset.Interface) error { label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSAutoscalerLabelName})) listOpts := v1.ListOptions{LabelSelector: label.String()} - pods, err := c.Core().Pods(DNSNamespace).List(listOpts) + pods, err := c.Core().Pods(api.NamespaceSystem).List(listOpts) if err != nil { return err } - Expect(len(pods.Items)).Should(Equal(1)) + if len(pods.Items) != 1 { + return fmt.Errorf("expected 1 autoscaler pod, got %v", len(pods.Items)) + } podName := pods.Items[0].Name - if err := c.Core().Pods(DNSNamespace).Delete(podName, nil); err != nil { + if err := c.Core().Pods(api.NamespaceSystem).Delete(podName, nil); err != nil { return err } framework.Logf("DNS autoscaling pod %v deleted.", podName) @@ -265,7 +317,7 @@ func waitForDNSReplicasSatisfied(c clientset.Interface, getExpected getExpectRep } expected = getExpected(c) if current != expected { - framework.Logf("replicas not as expected: got %v, expected %v", current, expected) + framework.Logf("Replicas not as expected: got %v, expected %v", current, expected) return false, nil } return true, nil diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index a48b9cf7bcd..0f12abadf6e 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -2757,6 +2757,32 @@ func WaitForPodsWithLabel(c clientset.Interface, ns string, label labels.Selecto return } +// Wait for exact amount of matching pods to become running and ready. +// Return the list of matching pods. +func WaitForPodsWithLabelRunningReady(c clientset.Interface, ns string, label labels.Selector, num int, timeout time.Duration) (pods *v1.PodList, err error) { + var current int + err = wait.Poll(Poll, timeout, + func() (bool, error) { + pods, err := WaitForPodsWithLabel(c, ns, label) + if err != nil { + Logf("Failed to list pods: %v", err) + return false, nil + } + current = 0 + for _, pod := range pods.Items { + if flag, err := testutils.PodRunningReady(&pod); err == nil && flag == true { + current++ + } + } + if current != num { + Logf("Got %v pods running and ready, expect: %v", current, num) + return false, nil + } + return true, nil + }) + return pods, err +} + // DeleteRCAndPods a Replication Controller and all pods it spawned func DeleteRCAndPods(clientset clientset.Interface, internalClientset internalclientset.Interface, ns, name string) error { By(fmt.Sprintf("deleting replication controller %s in namespace %s", name, ns))