From a55b4f2e77e611e8e585fc588a05cc8dfa86d973 Mon Sep 17 00:00:00 2001 From: "Tim St. Clair" Date: Wed, 6 Apr 2016 17:47:39 -0700 Subject: [PATCH] Move test/e2e framework & utility code to test/e2e/framework --- test/e2e/density.go | 36 +---- test/e2e/e2e.go | 72 ++-------- test/e2e/framework/cleanup.go | 61 +++++++++ test/e2e/{ => framework}/framework.go | 2 +- test/e2e/{ => framework}/kubelet_stats.go | 2 +- .../{ => framework}/log_size_monitoring.go | 2 +- test/e2e/{ => framework}/metrics_util.go | 30 ++++- test/e2e/{ => framework}/prompush.go | 2 +- .../resource_usage_gatherer.go | 2 +- test/e2e/framework/test_context.go | 126 ++++++++++++++++++ test/e2e/{ => framework}/util.go | 72 ++-------- test/e2e/monitoring.go | 7 - 12 files changed, 244 insertions(+), 170 deletions(-) create mode 100644 test/e2e/framework/cleanup.go rename test/e2e/{ => framework}/framework.go (99%) rename test/e2e/{ => framework}/kubelet_stats.go (99%) rename test/e2e/{ => framework}/log_size_monitoring.go (99%) rename test/e2e/{ => framework}/metrics_util.go (93%) rename test/e2e/{ => framework}/prompush.go (99%) rename test/e2e/{ => framework}/resource_usage_gatherer.go (99%) create mode 100644 test/e2e/framework/test_context.go rename test/e2e/{ => framework}/util.go (98%) diff --git a/test/e2e/density.go b/test/e2e/density.go index f81eaa26c79..c7b0183ae10 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -41,9 +41,7 @@ import ( . "github.com/onsi/gomega" ) -// NodeStartupThreshold is a rough estimate of the time allocated for a pod to start on a node. const ( - NodeStartupThreshold = 4 * time.Second MinSaturationThreshold = 2 * time.Minute MinPodsPerSecondThroughput = 8 ) @@ -51,30 +49,6 @@ const ( // Maximum container failures this test tolerates before failing. var MaxContainerFailures = 0 -// podLatencyData encapsulates pod startup latency information. -type podLatencyData struct { - // Name of the pod - Name string - // Node this pod was running on - Node string - // Latency information related to pod startuptime - Latency time.Duration -} - -type latencySlice []podLatencyData - -func (a latencySlice) Len() int { return len(a) } -func (a latencySlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a latencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency } - -func extractLatencyMetrics(latencies []podLatencyData) LatencyMetric { - length := len(latencies) - perc50 := latencies[int(math.Ceil(float64(length*50)/100))-1].Latency - perc90 := latencies[int(math.Ceil(float64(length*90)/100))-1].Latency - perc99 := latencies[int(math.Ceil(float64(length*99)/100))-1].Latency - return LatencyMetric{Perc50: perc50, Perc90: perc90, Perc99: perc99} -} - func density30AddonResourceVerifier() map[string]resourceConstraint { constraints := make(map[string]resourceConstraint) constraints["fluentd-elasticsearch"] = resourceConstraint{ @@ -526,15 +500,7 @@ var _ = KubeDescribe("Density", func() { podStartupLatency := PodStartupLatency{Latency: extractLatencyMetrics(e2eLag)} expectNoError(VerifyPodStartupLatency(podStartupLatency)) - // Log suspicious latency metrics/docker errors from all nodes that had slow startup times - for _, l := range startupLag { - if l.Latency > NodeStartupThreshold { - HighLatencyKubeletOperations(c, 1*time.Second, l.Node) - } - } - - Logf("Approx throughput: %v pods/min", - float64(nodeCount)/(e2eLag[len(e2eLag)-1].Latency.Minutes())) + logSuspiciousLatency(startupLag, e2eLag, nodeCount, c) } By("Deleting ReplicationController") diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 1ab91961b1b..7e34a3c0e46 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -17,7 +17,6 @@ limitations under the License. package e2e import ( - "flag" "fmt" "os" "path" @@ -34,7 +33,6 @@ import ( "golang.org/x/oauth2/google" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/runtime" @@ -48,78 +46,32 @@ const ( ) var ( - cloudConfig = &testContext.CloudConfig + cloudConfig = &framework.TestContext.CloudConfig ) -func RegisterFlags() { - // Turn on verbose by default to get spec names - config.DefaultReporterConfig.Verbose = true - - // Turn on EmitSpecProgress to get spec progress (especially on interrupt) - config.GinkgoConfig.EmitSpecProgress = true - - // Randomize specs as well as suites - config.GinkgoConfig.RandomizeAllSpecs = true - - flag.StringVar(&testContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") - flag.StringVar(&testContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") - flag.StringVar(&testContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") - flag.StringVar(&testContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") - flag.StringVar(&testContext.Host, "host", "", "The host, or apiserver, to connect to") - flag.StringVar(&testContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.") - flag.StringVar(&testContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, vagrant, etc.)") - flag.StringVar(&testContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") - flag.StringVar(&testContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.") - flag.StringVar(&testContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") - flag.StringVar(&testContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.") - flag.StringVar(&testContext.prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.") - flag.StringVar(&testContext.OSDistro, "os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).") - - // TODO: Flags per provider? Rename gce-project/gce-zone? - flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke") - flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable") - flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable") - flag.StringVar(&cloudConfig.ServiceAccount, "gce-service-account", "", "GCE service account to use for GCE API calls, if applicable") - flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable") - flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws") - flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster") - - flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources. Only required if provider is aws.") - flag.IntVar(&testContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.") - flag.StringVar(&testContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.") - flag.StringVar(&testContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.") - flag.BoolVar(&testContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.") - flag.BoolVar(&testContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.") - flag.BoolVar(&testContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.") - flag.BoolVar(&testContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", false, "If set to true framework will be monitoring resource usage of system add-ons in (some) e2e tests.") - flag.BoolVar(&testContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.") - flag.BoolVar(&testContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.") - flag.StringVar(&testContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.") -} - -// setupProviderConfig validates and sets up cloudConfig based on testContext.Provider. +// setupProviderConfig validates and sets up cloudConfig based on framework.TestContext.Provider. func setupProviderConfig() error { - switch testContext.Provider { + switch framework.TestContext.Provider { case "": glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.") case "gce", "gke": var err error - Logf("Fetching cloud provider for %q\r\n", testContext.Provider) + framework.Logf("Fetching cloud provider for %q\r\n", framework.TestContext.Provider) var tokenSource oauth2.TokenSource tokenSource = nil if cloudConfig.ServiceAccount != "" { // Use specified service account for auth - Logf("Using service account %q as token source.", cloudConfig.ServiceAccount) + framework.Logf("Using service account %q as token source.", cloudConfig.ServiceAccount) tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount) } - zone := testContext.CloudConfig.Zone + zone := framework.TestContext.CloudConfig.Zone region, err := gcecloud.GetGCERegion(zone) if err != nil { return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err) } managedZones := []string{zone} // Only single-zone for now - cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */) + cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */) if err != nil { return fmt.Errorf("Error building GCE/GKE provider: %v", err) } @@ -235,8 +187,8 @@ var _ = ginkgo.SynchronizedAfterSuite(func() { RunCleanupActions() }, func() { // Run only Ginkgo on node 1 - if testContext.ReportDir != "" { - CoreDump(testContext.ReportDir) + if framework.TestContext.ReportDir != "" { + CoreDump(framework.TestContext.ReportDir) } }) @@ -264,13 +216,13 @@ func RunE2ETests(t *testing.T) { // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins var r []ginkgo.Reporter - if testContext.ReportDir != "" { + if framework.TestContext.ReportDir != "" { // TODO: we should probably only be trying to create this directory once // rather than once-per-Ginkgo-node. - if err := os.MkdirAll(testContext.ReportDir, 0755); err != nil { + if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil { glog.Errorf("Failed creating report directory: %v", err) } else { - r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", testContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) + r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) } } glog.Infof("Starting e2e run %q on Ginkgo node %d", runId, config.GinkgoConfig.ParallelNode) diff --git a/test/e2e/framework/cleanup.go b/test/e2e/framework/cleanup.go new file mode 100644 index 00000000000..7d724850053 --- /dev/null +++ b/test/e2e/framework/cleanup.go @@ -0,0 +1,61 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import "sync" + +type CleanupActionHandle *int + +var cleanupActionsLock sync.Mutex +var cleanupActions = map[CleanupActionHandle]func(){} + +// AddCleanupAction installs a function that will be called in the event of the +// whole test being terminated. This allows arbitrary pieces of the overall +// test to hook into SynchronizedAfterSuite(). +func AddCleanupAction(fn func()) CleanupActionHandle { + p := CleanupActionHandle(new(int)) + cleanupActionsLock.Lock() + defer cleanupActionsLock.Unlock() + cleanupActions[p] = fn + return p +} + +// RemoveCleanupAction removes a function that was installed by +// AddCleanupAction. +func RemoveCleanupAction(p CleanupActionHandle) { + cleanupActionsLock.Lock() + defer cleanupActionsLock.Unlock() + delete(cleanupActions, p) +} + +// RunCleanupActions runs all functions installed by AddCleanupAction. It does +// not remove them (see RemoveCleanupAction) but it does run unlocked, so they +// may remove themselves. +func RunCleanupActions() { + list := []func(){} + func() { + cleanupActionsLock.Lock() + defer cleanupActionsLock.Unlock() + for _, fn := range cleanupActions { + list = append(list, fn) + } + }() + // Run unlocked. + for _, fn := range list { + fn() + } +} diff --git a/test/e2e/framework.go b/test/e2e/framework/framework.go similarity index 99% rename from test/e2e/framework.go rename to test/e2e/framework/framework.go index ae3d65223b7..ea2c2f00c97 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework/framework.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" diff --git a/test/e2e/kubelet_stats.go b/test/e2e/framework/kubelet_stats.go similarity index 99% rename from test/e2e/kubelet_stats.go rename to test/e2e/framework/kubelet_stats.go index e30e08fe9c4..bf768854dfe 100644 --- a/test/e2e/kubelet_stats.go +++ b/test/e2e/framework/kubelet_stats.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" diff --git a/test/e2e/log_size_monitoring.go b/test/e2e/framework/log_size_monitoring.go similarity index 99% rename from test/e2e/log_size_monitoring.go rename to test/e2e/framework/log_size_monitoring.go index 72290b93490..8c9394ec30d 100644 --- a/test/e2e/log_size_monitoring.go +++ b/test/e2e/framework/log_size_monitoring.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" diff --git a/test/e2e/metrics_util.go b/test/e2e/framework/metrics_util.go similarity index 93% rename from test/e2e/metrics_util.go rename to test/e2e/framework/metrics_util.go index ca6aff5a8c4..a83c7483edf 100644 --- a/test/e2e/metrics_util.go +++ b/test/e2e/framework/metrics_util.go @@ -14,13 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" "encoding/json" "fmt" "io" + "math" "sort" "strconv" "strings" @@ -37,6 +38,9 @@ import ( ) const ( + // NodeStartupThreshold is a rough estimate of the time allocated for a pod to start on a node. + NodeStartupThreshold = 4 * time.Second + podStartupThreshold time.Duration = 5 * time.Second listPodLatencySmallThreshold time.Duration = 1 * time.Second listPodLatencyMediumThreshold time.Duration = 1 * time.Second @@ -442,6 +446,30 @@ func extractMetricSamples(metricsBlob string) ([]*model.Sample, error) { } } +// podLatencyData encapsulates pod startup latency information. +type podLatencyData struct { + // Name of the pod + Name string + // Node this pod was running on + Node string + // Latency information related to pod startuptime + Latency time.Duration +} + +type latencySlice []podLatencyData + +func (a latencySlice) Len() int { return len(a) } +func (a latencySlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a latencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency } + +func extractLatencyMetrics(latencies []podLatencyData) LatencyMetric { + length := len(latencies) + perc50 := latencies[int(math.Ceil(float64(length*50)/100))-1].Latency + perc90 := latencies[int(math.Ceil(float64(length*90)/100))-1].Latency + perc99 := latencies[int(math.Ceil(float64(length*99)/100))-1].Latency + return LatencyMetric{Perc50: perc50, Perc90: perc90, Perc99: perc99} +} + // logSuspiciousLatency logs metrics/docker errors from all nodes that had slow startup times // If latencyDataLag is nil then it will be populated from latencyData func logSuspiciousLatency(latencyData []podLatencyData, latencyDataLag []podLatencyData, nodeCount int, c *client.Client) { diff --git a/test/e2e/prompush.go b/test/e2e/framework/prompush.go similarity index 99% rename from test/e2e/prompush.go rename to test/e2e/framework/prompush.go index a8af6faad7a..8f3a036718e 100644 --- a/test/e2e/prompush.go +++ b/test/e2e/framework/prompush.go @@ -15,7 +15,7 @@ limitations under the License. */ //This is a utility for prometheus pushing functionality. -package e2e +package framework import ( "fmt" diff --git a/test/e2e/resource_usage_gatherer.go b/test/e2e/framework/resource_usage_gatherer.go similarity index 99% rename from test/e2e/resource_usage_gatherer.go rename to test/e2e/framework/resource_usage_gatherer.go index 1091fa0feff..c4a6ead5332 100644 --- a/test/e2e/resource_usage_gatherer.go +++ b/test/e2e/framework/resource_usage_gatherer.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go new file mode 100644 index 00000000000..73c2d1e64fa --- /dev/null +++ b/test/e2e/framework/test_context.go @@ -0,0 +1,126 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "flag" + "os" + + "github.com/onsi/ginkgo/config" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + "k8s.io/kubernetes/pkg/cloudprovider" +) + +type TestContextType struct { + KubeConfig string + KubeContext string + KubeVolumeDir string + CertDir string + Host string + RepoRoot string + Provider string + CloudConfig CloudConfig + KubectlPath string + OutputDir string + ReportDir string + ReportPrefix string + Prefix string + MinStartupPods int + UpgradeTarget string + PrometheusPushGateway string + OSDistro string + VerifyServiceAccount bool + DeleteNamespace bool + CleanStart bool + // If set to true framework will start a goroutine monitoring resource usage of system add-ons. + // It will read the data every 30 seconds from all Nodes and print summary during afterEach. + GatherKubeSystemResourceUsageData bool + GatherLogsSizes bool + GatherMetricsAfterTest bool + // Currently supported values are 'hr' for human-readable and 'json'. It's a comma separated list. + OutputPrintType string + // CreateTestingNS is responsible for creating namespace used for executing e2e tests. + // It accepts namespace base name, which will be prepended with e2e prefix, kube client + // and labels to be applied to a namespace. + CreateTestingNS CreateTestingNSFn +} + +type CloudConfig struct { + ProjectID string + Zone string + Cluster string + MasterName string + NodeInstanceGroup string + NumNodes int + ClusterTag string + ServiceAccount string + + Provider cloudprovider.Interface +} + +var TestContext TestContextType + +func SetTestContext(t TestContextType) { + TestContext = t +} + +func RegisterFlags() { + // Turn on verbose by default to get spec names + config.DefaultReporterConfig.Verbose = true + + // Turn on EmitSpecProgress to get spec progress (especially on interrupt) + config.GinkgoConfig.EmitSpecProgress = true + + // Randomize specs as well as suites + config.GinkgoConfig.RandomizeAllSpecs = true + + flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") + flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") + flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") + flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") + flag.StringVar(&TestContext.Host, "host", "", "The host, or apiserver, to connect to") + flag.StringVar(&TestContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.") + flag.StringVar(&TestContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, vagrant, etc.)") + flag.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") + flag.StringVar(&TestContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.") + flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") + flag.StringVar(&testContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.") + flag.StringVar(&TestContext.Prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.") + flag.StringVar(&TestContext.OSDistro, "os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).") + + // TODO: Flags per provider? Rename gce-project/gce-zone? + cloudConfig := &TestContext.CloudConfig + flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke") + flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable") + flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable") + flag.StringVar(&cloudConfig.ServiceAccount, "gce-service-account", "", "GCE service account to use for GCE API calls, if applicable") + flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable") + flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws") + flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster") + + flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources. Only required if provider is aws.") + flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.") + flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.") + flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.") + flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.") + flag.BoolVar(&TestContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.") + flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.") + flag.BoolVar(&TestContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", false, "If set to true framework will be monitoring resource usage of system add-ons in (some) e2e tests.") + flag.BoolVar(&TestContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.") + flag.BoolVar(&TestContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.") + flag.StringVar(&TestContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.") +} diff --git a/test/e2e/util.go b/test/e2e/framework/util.go similarity index 98% rename from test/e2e/util.go rename to test/e2e/framework/util.go index 97a3a5968bf..8c8c6aa7269 100644 --- a/test/e2e/util.go +++ b/test/e2e/framework/util.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" @@ -50,7 +50,6 @@ import ( client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - "k8s.io/kubernetes/pkg/cloudprovider" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/kubectl" @@ -153,80 +152,29 @@ func getServicesProxyRequest(c *client.Client, request *restclient.Request) (*re return request.Prefix("proxy").Resource("services"), nil } -func GetServicesProxyRequest(c *client.Client, request *restclient.Request) (*restclient.Request, error) { - return getServicesProxyRequest(c, request) -} - -type CloudConfig struct { - ProjectID string - Zone string - Cluster string - MasterName string - NodeInstanceGroup string - NumNodes int - ClusterTag string - ServiceAccount string - - Provider cloudprovider.Interface -} - // unique identifier of the e2e run var runId = util.NewUUID() type CreateTestingNSFn func(baseName string, c *client.Client, labels map[string]string) (*api.Namespace, error) -type TestContextType struct { - KubeConfig string - KubeContext string - KubeVolumeDir string - CertDir string - Host string - RepoRoot string - Provider string - CloudConfig CloudConfig - KubectlPath string - OutputDir string - ReportDir string - ReportPrefix string - prefix string - MinStartupPods int - UpgradeTarget string - PrometheusPushGateway string - OSDistro string - VerifyServiceAccount bool - DeleteNamespace bool - CleanStart bool - // If set to true framework will start a goroutine monitoring resource usage of system add-ons. - // It will read the data every 30 seconds from all Nodes and print summary during afterEach. - GatherKubeSystemResourceUsageData bool - GatherLogsSizes bool - GatherMetricsAfterTest bool - // Currently supported values are 'hr' for human-readable and 'json'. It's a comma separated list. - OutputPrintType string - // CreateTestingNS is responsible for creating namespace used for executing e2e tests. - // It accepts namespace base name, which will be prepended with e2e prefix, kube client - // and labels to be applied to a namespace. - CreateTestingNS CreateTestingNSFn -} - -var testContext TestContextType - -func SetTestContext(t TestContextType) { - testContext = t -} - type ContainerFailures struct { status *api.ContainerStateTerminated - restarts int + Restarts int +} + +func GetMasterHost() string { + masterUrl, err := url.Parse(TestContext.Host) + ExpectNoError(err) + return masterUrl.Host } // Convenient wrapper around cache.Store that returns list of api.Pod instead of interface{}. -type podStore struct { +type PodStore struct { cache.Store stopCh chan struct{} } -func newPodStore(c *client.Client, namespace string, label labels.Selector, field fields.Selector) *podStore { +func NewPodStore(c *client.Client, namespace string, label labels.Selector, field fields.Selector) *PodStore { lw := &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { options.LabelSelector = label diff --git a/test/e2e/monitoring.go b/test/e2e/monitoring.go index 6d119547f81..e383fc9c85a 100644 --- a/test/e2e/monitoring.go +++ b/test/e2e/monitoring.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/json" "fmt" - "net/url" "time" influxdb "github.com/influxdb/influxdb/client" @@ -236,12 +235,6 @@ func validatePodsAndNodes(c *client.Client, expectedPods, expectedNodes []string return true } -func getMasterHost() string { - masterUrl, err := url.Parse(testContext.Host) - expectNoError(err) - return masterUrl.Host -} - func testMonitoringUsingHeapsterInfluxdb(c *client.Client) { // Check if heapster pods and services are up. expectedPods, err := verifyExpectedRcsExistAndGetExpectedPods(c)