diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 1a8e3a2156b..d57bc1d35bf 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -112,7 +112,6 @@ go_library( "//pkg/client/clientset_generated/clientset/typed/extensions/v1beta1:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/cloudprovider:go_default_library", - "//pkg/cloudprovider/providers/aws:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/cloudprovider/providers/vsphere:go_default_library", "//pkg/controller:go_default_library", @@ -149,7 +148,6 @@ go_library( "//test/utils:go_default_library", "//vendor:github.com/aws/aws-sdk-go/aws", "//vendor:github.com/aws/aws-sdk-go/aws/session", - "//vendor:github.com/aws/aws-sdk-go/service/autoscaling", "//vendor:github.com/aws/aws-sdk-go/service/ec2", "//vendor:github.com/elazarl/goproxy", "//vendor:github.com/ghodss/yaml", @@ -211,6 +209,7 @@ go_test( "//pkg/api/v1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/metrics:go_default_library", + "//test/e2e/autoscaling:go_default_library", "//test/e2e/cluster-logging:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/perf:go_default_library", diff --git a/test/e2e/autoscaling/BUILD b/test/e2e/autoscaling/BUILD index 7124b4b6785..737b9316e36 100644 --- a/test/e2e/autoscaling/BUILD +++ b/test/e2e/autoscaling/BUILD @@ -18,7 +18,6 @@ go_library( deps = [ "//pkg/api/v1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", - "//test/e2e:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/scheduling:go_default_library", diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go index 652ee2481cb..155d64339ba 100644 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ b/test/e2e/autoscaling/cluster_size_autoscaling.go @@ -36,7 +36,6 @@ import ( policy "k8s.io/client-go/pkg/apis/policy/v1beta1" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" - "k8s.io/kubernetes/test/e2e" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/scheduling" testutils "k8s.io/kubernetes/test/utils" @@ -83,7 +82,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() { originalSizes = make(map[string]int) sum := 0 for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := e2e.GroupSize(mig) + size, err := framework.GroupSize(mig) framework.ExpectNoError(err) By(fmt.Sprintf("Initial size of %s: %d", mig, size)) originalSizes[mig] = size @@ -211,7 +210,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() { } } - nodes, err := e2e.GetGroupNodes(minMig) + nodes, err := framework.GetGroupNodes(minMig) framework.ExpectNoError(err) nodesSet := sets.NewString(nodes...) defer removeLabels(nodesSet) @@ -224,12 +223,12 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() { CreateNodeSelectorPods(f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false) By("Waiting for new node to appear and annotating it") - e2e.WaitForGroupSize(minMig, int32(minSize+1)) + framework.WaitForGroupSize(minMig, int32(minSize+1)) // Verify, that cluster size is increased framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout)) - newNodes, err := e2e.GetGroupNodes(minMig) + newNodes, err := framework.GetGroupNodes(minMig) framework.ExpectNoError(err) newNodesSet := sets.NewString(newNodes...) newNodesSet.Delete(nodes...) @@ -655,11 +654,11 @@ func waitForAllCaPodsReadyInNamespace(f *framework.Framework, c clientset.Interf func setMigSizes(sizes map[string]int) { for mig, desiredSize := range sizes { - currentSize, err := e2e.GroupSize(mig) + currentSize, err := framework.GroupSize(mig) framework.ExpectNoError(err) if desiredSize != currentSize { By(fmt.Sprintf("Setting size of %s to %d", mig, desiredSize)) - err = e2e.ResizeGroup(mig, int32(desiredSize)) + err = framework.ResizeGroup(mig, int32(desiredSize)) framework.ExpectNoError(err) } } diff --git a/test/e2e/autoscaling/dns_autoscaling.go b/test/e2e/autoscaling/dns_autoscaling.go index 4eee72dcbab..e4b95134fc2 100644 --- a/test/e2e/autoscaling/dns_autoscaling.go +++ b/test/e2e/autoscaling/dns_autoscaling.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" - "k8s.io/kubernetes/test/e2e" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -120,7 +119,7 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { originalSizes := make(map[string]int) sum := 0 for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := e2e.GroupSize(mig) + size, err := framework.GroupSize(mig) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Initial size of %s: %d", mig, size)) originalSizes[mig] = size diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 831dff0ac2f..1ca80f3183b 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -30,6 +30,7 @@ go_library( "pv_util.go", "resource_usage_gatherer.go", "service_util.go", + "size.go", "statefulset_utils.go", "test_context.go", "upgrade_util.go", @@ -88,6 +89,7 @@ go_library( "//vendor:github.com/aws/aws-sdk-go/aws", "//vendor:github.com/aws/aws-sdk-go/aws/awserr", "//vendor:github.com/aws/aws-sdk-go/aws/session", + "//vendor:github.com/aws/aws-sdk-go/service/autoscaling", "//vendor:github.com/aws/aws-sdk-go/service/ec2", "//vendor:github.com/golang/glog", "//vendor:github.com/google/cadvisor/info/v1", diff --git a/test/e2e/framework/size.go b/test/e2e/framework/size.go new file mode 100644 index 00000000000..7f53a8644b1 --- /dev/null +++ b/test/e2e/framework/size.go @@ -0,0 +1,128 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "os/exec" + "regexp" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/autoscaling" + awscloud "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" +) + +const ( + serveHostnameImage = "gcr.io/google_containers/serve_hostname:v1.4" + resizeNodeReadyTimeout = 2 * time.Minute + resizeNodeNotReadyTimeout = 2 * time.Minute + nodeReadinessTimeout = 3 * time.Minute + podNotReadyTimeout = 1 * time.Minute + podReadyTimeout = 2 * time.Minute + testPort = 9376 +) + +func ResizeGroup(group string, size int32) error { + if TestContext.ReportDir != "" { + CoreDump(TestContext.ReportDir) + defer CoreDump(TestContext.ReportDir) + } + if TestContext.Provider == "gce" || TestContext.Provider == "gke" { + // TODO: make this hit the compute API directly instead of shelling out to gcloud. + // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic + output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "resize", + group, fmt.Sprintf("--size=%v", size), + "--project="+TestContext.CloudConfig.ProjectID, "--zone="+TestContext.CloudConfig.Zone).CombinedOutput() + if err != nil { + Logf("Failed to resize node instance group: %v", string(output)) + } + return err + } else if TestContext.Provider == "aws" { + client := autoscaling.New(session.New()) + return awscloud.ResizeInstanceGroup(client, group, int(size)) + } else { + return fmt.Errorf("Provider does not support InstanceGroups") + } +} + +func GetGroupNodes(group string) ([]string, error) { + if TestContext.Provider == "gce" || TestContext.Provider == "gke" { + // TODO: make this hit the compute API directly instead of shelling out to gcloud. + // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic + output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", + "list-instances", group, "--project="+TestContext.CloudConfig.ProjectID, + "--zone="+TestContext.CloudConfig.Zone).CombinedOutput() + if err != nil { + return nil, err + } + re := regexp.MustCompile(".*RUNNING") + lines := re.FindAllString(string(output), -1) + for i, line := range lines { + lines[i] = line[:strings.Index(line, " ")] + } + return lines, nil + } else { + return nil, fmt.Errorf("provider does not support InstanceGroups") + } +} + +func GroupSize(group string) (int, error) { + if TestContext.Provider == "gce" || TestContext.Provider == "gke" { + // TODO: make this hit the compute API directly instead of shelling out to gcloud. + // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic + output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", + "list-instances", group, "--project="+TestContext.CloudConfig.ProjectID, + "--zone="+TestContext.CloudConfig.Zone).CombinedOutput() + if err != nil { + return -1, err + } + re := regexp.MustCompile("RUNNING") + return len(re.FindAllString(string(output), -1)), nil + } else if TestContext.Provider == "aws" { + client := autoscaling.New(session.New()) + instanceGroup, err := awscloud.DescribeInstanceGroup(client, group) + if err != nil { + return -1, fmt.Errorf("error describing instance group: %v", err) + } + if instanceGroup == nil { + return -1, fmt.Errorf("instance group not found: %s", group) + } + return instanceGroup.CurrentSize() + } else { + return -1, fmt.Errorf("provider does not support InstanceGroups") + } +} + +func WaitForGroupSize(group string, size int32) error { + timeout := 30 * time.Minute + for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) { + currentSize, err := GroupSize(group) + if err != nil { + Logf("Failed to get node instance group size: %v", err) + continue + } + if currentSize != int(size) { + Logf("Waiting for node instance group size %d, current size %d", size, currentSize) + continue + } + Logf("Node instance group has reached the desired size %d", size) + return nil + } + return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size) +} diff --git a/test/e2e/pd.go b/test/e2e/pd.go index 7485cb6074a..aafef7af3a6 100644 --- a/test/e2e/pd.go +++ b/test/e2e/pd.go @@ -413,7 +413,7 @@ var _ = framework.KubeDescribe("Pod Disks", func() { It("should be able to detach from a node which was deleted [Slow] [Disruptive] [Volume]", func() { framework.SkipUnlessProviderIs("gce") - initialGroupSize, err := GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup) + initialGroupSize, err := framework.GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup) framework.ExpectNoError(err, "Error getting group size") By("Creating a pd") @@ -461,13 +461,13 @@ var _ = framework.KubeDescribe("Pod Disks", func() { // The disk should be detached from host0 on it's deletion By("Waiting for pd to detach from host0") waitForPDDetach(diskName, host0Name) - framework.ExpectNoError(WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get back the cluster to inital size") + framework.ExpectNoError(framework.WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get back the cluster to inital size") return }) It("should be able to detach from a node whose api object was deleted [Slow] [Disruptive] [Volume]", func() { framework.SkipUnlessProviderIs("gce") - initialGroupSize, err := GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup) + initialGroupSize, err := framework.GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup) framework.ExpectNoError(err, "Error getting group size") By("Creating a pd") diskName, err := framework.CreatePDWithRetry() @@ -484,7 +484,7 @@ var _ = framework.KubeDescribe("Pod Disks", func() { // need to set the resource version or else the Create() fails _, err := nodeClient.Create(nodeToDelete) framework.ExpectNoError(err, "Unable to re-create the deleted node") - framework.ExpectNoError(WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get the node group back to the original size") + framework.ExpectNoError(framework.WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get the node group back to the original size") framework.WaitForNodeToBeReady(f.ClientSet, nodeToDelete.Name, nodeStatusTimeout) if len(nodes.Items) != originalCount { return fmt.Errorf("The node count is not back to original count") diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index 451893827b1..fa05b6a4763 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -18,8 +18,6 @@ package e2e import ( "fmt" - "os/exec" - "regexp" "strings" "time" @@ -29,11 +27,8 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/autoscaling" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - awscloud "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" ) const ( @@ -46,95 +41,6 @@ const ( testPort = 9376 ) -func ResizeGroup(group string, size int32) error { - if framework.TestContext.ReportDir != "" { - framework.CoreDump(framework.TestContext.ReportDir) - defer framework.CoreDump(framework.TestContext.ReportDir) - } - if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" { - // TODO: make this hit the compute API directly instead of shelling out to gcloud. - // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic - output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "resize", - group, fmt.Sprintf("--size=%v", size), - "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput() - if err != nil { - framework.Logf("Failed to resize node instance group: %v", string(output)) - } - return err - } else if framework.TestContext.Provider == "aws" { - client := autoscaling.New(session.New()) - return awscloud.ResizeInstanceGroup(client, group, int(size)) - } else { - return fmt.Errorf("Provider does not support InstanceGroups") - } -} - -func GetGroupNodes(group string) ([]string, error) { - if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" { - // TODO: make this hit the compute API directly instead of shelling out to gcloud. - // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic - output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", - "list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput() - if err != nil { - return nil, err - } - re := regexp.MustCompile(".*RUNNING") - lines := re.FindAllString(string(output), -1) - for i, line := range lines { - lines[i] = line[:strings.Index(line, " ")] - } - return lines, nil - } else { - return nil, fmt.Errorf("provider does not support InstanceGroups") - } -} - -func GroupSize(group string) (int, error) { - if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" { - // TODO: make this hit the compute API directly instead of shelling out to gcloud. - // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic - output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", - "list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput() - if err != nil { - return -1, err - } - re := regexp.MustCompile("RUNNING") - return len(re.FindAllString(string(output), -1)), nil - } else if framework.TestContext.Provider == "aws" { - client := autoscaling.New(session.New()) - instanceGroup, err := awscloud.DescribeInstanceGroup(client, group) - if err != nil { - return -1, fmt.Errorf("error describing instance group: %v", err) - } - if instanceGroup == nil { - return -1, fmt.Errorf("instance group not found: %s", group) - } - return instanceGroup.CurrentSize() - } else { - return -1, fmt.Errorf("provider does not support InstanceGroups") - } -} - -func WaitForGroupSize(group string, size int32) error { - timeout := 30 * time.Minute - for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) { - currentSize, err := GroupSize(group) - if err != nil { - framework.Logf("Failed to get node instance group size: %v", err) - continue - } - if currentSize != int(size) { - framework.Logf("Waiting for node instance group size %d, current size %d", size, currentSize) - continue - } - framework.Logf("Node instance group has reached the desired size %d", size) - return nil - } - return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size) -} - func svcByName(name string, port int) *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -213,7 +119,7 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() { } By("restoring the original node instance group size") - if err := ResizeGroup(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { + if err := framework.ResizeGroup(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { framework.Failf("Couldn't restore the original node instance group size: %v", err) } // In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a @@ -228,7 +134,7 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() { By("waiting 5 minutes for all dead tunnels to be dropped") time.Sleep(5 * time.Minute) } - if err := WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { + if err := framework.WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { framework.Failf("Couldn't restore the original node instance group size: %v", err) } if err := framework.WaitForClusterSize(c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute); err != nil { @@ -253,9 +159,9 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() { Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("decreasing cluster size to %d", replicas-1)) - err = ResizeGroup(group, replicas-1) + err = framework.ResizeGroup(group, replicas-1) Expect(err).NotTo(HaveOccurred()) - err = WaitForGroupSize(group, replicas-1) + err = framework.WaitForGroupSize(group, replicas-1) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForClusterSize(c, int(replicas-1), 10*time.Minute) Expect(err).NotTo(HaveOccurred()) @@ -281,9 +187,9 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() { Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("increasing cluster size to %d", replicas+1)) - err = ResizeGroup(group, replicas+1) + err = framework.ResizeGroup(group, replicas+1) Expect(err).NotTo(HaveOccurred()) - err = WaitForGroupSize(group, replicas+1) + err = framework.WaitForGroupSize(group, replicas+1) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForClusterSize(c, int(replicas+1), 10*time.Minute) Expect(err).NotTo(HaveOccurred())