diff --git a/test/e2e/framework/metrics_util.go b/test/e2e/framework/metrics_util.go index 66a6d30ee47..f21379552e4 100644 --- a/test/e2e/framework/metrics_util.go +++ b/test/e2e/framework/metrics_util.go @@ -134,9 +134,10 @@ var InterestingKubeletMetrics = []string{ // Dashboard metrics type LatencyMetric struct { - Perc50 time.Duration `json:"Perc50"` - Perc90 time.Duration `json:"Perc90"` - Perc99 time.Duration `json:"Perc99"` + Perc50 time.Duration `json:"Perc50"` + Perc90 time.Duration `json:"Perc90"` + Perc99 time.Duration `json:"Perc99"` + Perc100 time.Duration `json:"Perc100"` } type PodStartupLatency struct { @@ -450,7 +451,8 @@ func ExtractLatencyMetrics(latencies []PodLatencyData) LatencyMetric { perc50 := latencies[int(math.Ceil(float64(length*50)/100))-1].Latency perc90 := latencies[int(math.Ceil(float64(length*90)/100))-1].Latency perc99 := latencies[int(math.Ceil(float64(length*99)/100))-1].Latency - return LatencyMetric{Perc50: perc50, Perc90: perc90, Perc99: perc99} + perc100 := latencies[length-1].Latency + return LatencyMetric{Perc50: perc50, Perc90: perc90, Perc99: perc99, Perc100: perc100} } // LogSuspiciousLatency logs metrics/docker errors from all nodes that had slow startup times diff --git a/test/e2e_node/benchmark_util.go b/test/e2e_node/benchmark_util.go index eb404ec0ecc..919e79e94ee 100644 --- a/test/e2e_node/benchmark_util.go +++ b/test/e2e_node/benchmark_util.go @@ -27,7 +27,7 @@ import ( ) const ( - // TODO(coufon): be consistent with perf_util.go version (not exposed) + // TODO(coufon): be consistent with perf_util.go version currentTimeSeriesVersion = "v1" TimeSeriesTag = "[Result:TimeSeries]" TimeSeriesEnd = "[Finish:TimeSeries]" @@ -35,8 +35,8 @@ const ( type NodeTimeSeries struct { // value in OperationData is an array of timestamps - OperationData map[string][]int64 `json:"op_data,omitempty"` - ResourceData map[string]*ResourceSeries `json:"resource_data,omitempty"` + OperationData map[string][]int64 `json:"op_series,omitempty"` + ResourceData map[string]*ResourceSeries `json:"resource_series,omitempty"` Labels map[string]string `json:"labels"` Version string `json:"version"` } @@ -85,14 +85,15 @@ func getLatencyPerfData(latency framework.LatencyMetric, testName string) *perft DataItems: []perftype.DataItem{ { Data: map[string]float64{ - "Perc50": float64(latency.Perc50) / 1000000, - "Perc90": float64(latency.Perc90) / 1000000, - "Perc99": float64(latency.Perc99) / 1000000, + "Perc50": float64(latency.Perc50) / 1000000, + "Perc90": float64(latency.Perc90) / 1000000, + "Perc99": float64(latency.Perc99) / 1000000, + "Perc100": float64(latency.Perc100) / 1000000, }, Unit: "ms", Labels: map[string]string{ "datatype": "latency", - "latencytype": "test-e2e", + "latencytype": "create-pod", }, }, }, diff --git a/test/e2e_node/density_test.go b/test/e2e_node/density_test.go index 5135d5ab782..b9472eddd72 100644 --- a/test/e2e_node/density_test.go +++ b/test/e2e_node/density_test.go @@ -133,6 +133,30 @@ var _ = framework.KubeDescribe("Density [Serial] [Slow]", func() { podsNr: 105, interval: 0 * time.Millisecond, }, + { + podsNr: 10, + interval: 100 * time.Millisecond, + }, + { + podsNr: 35, + interval: 100 * time.Millisecond, + }, + { + podsNr: 105, + interval: 100 * time.Millisecond, + }, + { + podsNr: 10, + interval: 300 * time.Millisecond, + }, + { + podsNr: 35, + interval: 300 * time.Millisecond, + }, + { + podsNr: 105, + interval: 300 * time.Millisecond, + }, } for _, testArg := range dTests { @@ -348,7 +372,7 @@ func runDensitySeqTest(f *framework.Framework, rc *ResourceCollector, testArg de rc.Start() defer rc.Stop() - // create pods sequentially (back-to-back) + // Create pods sequentially (back-to-back). e2eLags have been sorted. batchlag, e2eLags := createBatchPodSequential(f, testPods) // Log throughput data. diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go index 2187a0e5021..a062d13fe8a 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/e2e_remote.go @@ -36,7 +36,6 @@ var sshOptions = flag.String("ssh-options", "", "Commandline options passed to s var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce") var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.") var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.") -var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.") var sshOptionsMap map[string]string @@ -148,7 +147,7 @@ func CreateTestArchive() (string, error) { } // Returns the command output, whether the exit was ok, and any errors -func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, setupNode bool, testArgs string) (string, bool, error) { +func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, setupNode bool, testArgs string, ginkgoFlags string) (string, bool, error) { if setupNode { uname, err := user.Current() if err != nil { @@ -216,7 +215,8 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), - fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", *testTimeoutSeconds, *ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs), + fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", + *testTimeoutSeconds, ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs), ) aggErrs := []error{} diff --git a/test/e2e_node/jenkins/benchmark/benchmark-config.yaml b/test/e2e_node/jenkins/benchmark/benchmark-config.yaml new file mode 100644 index 00000000000..00eb00a55d1 --- /dev/null +++ b/test/e2e_node/jenkins/benchmark/benchmark-config.yaml @@ -0,0 +1,50 @@ +--- +images: + containervm-density1: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 35 pods with 0s? interval \[Benchmark\]' + containervm-density2: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 105 pods with 0s? interval \[Benchmark\]' + containervm-density3: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-2 + tests: + - '.*create 105 pods with 0s? interval \[Benchmark\]' + containervm-density4: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 35 pods with 100ms interval \[Benchmark\]' + containervm-density5: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 105 pods with 100ms interval \[Benchmark\]' + containervm-density6: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-2 + tests: + - '.*create 105 pods with 100ms interval \[Benchmark\]' + containervm-density7: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 105 pods with 300ms interval \[Benchmark\]' + containervm-density8: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-2 + tests: + - '.*create 105 pods with 300ms interval \[Benchmark\]' diff --git a/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties b/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties new file mode 100644 index 00000000000..1fe8abd8566 --- /dev/null +++ b/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties @@ -0,0 +1,9 @@ +GCE_HOSTS= +GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/benchmark/benchmark-config.yaml +GCE_ZONE=us-central1-f +GCE_PROJECT=k8s-jkns-ci-node-e2e +CLEANUP=true +GINKGO_FLAGS='--skip="\[Flaky\]"' +SETUP_NODE=false +TEST_ARGS=--cgroups-per-qos=false +PARALLELISM=1 diff --git a/test/e2e_node/resource_collector.go b/test/e2e_node/resource_collector.go index 9d3a396646a..79285831742 100644 --- a/test/e2e_node/resource_collector.go +++ b/test/e2e_node/resource_collector.go @@ -206,7 +206,7 @@ func (r resourceUsageByCPU) Swap(i, j int) { r[i], r[j] = r[j], r[i] } func (r resourceUsageByCPU) Less(i, j int) bool { return r[i].CPUUsageInCores < r[j].CPUUsageInCores } // The percentiles to report. -var percentiles = [...]float64{0.05, 0.20, 0.50, 0.70, 0.90, 0.95, 0.99} +var percentiles = [...]float64{0.50, 0.90, 0.95, 0.99, 1.00} // GetBasicCPUStats returns the percentiles the cpu usage in cores for // containerName. This method examines all data currently in the buffer. diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index b603d275f88..b3464f803f7 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -41,7 +41,7 @@ import ( "github.com/pborman/uuid" "golang.org/x/oauth2" "golang.org/x/oauth2/google" - "google.golang.org/api/compute/v1" + compute "google.golang.org/api/compute/v1" ) var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") @@ -58,8 +58,16 @@ var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine") var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k 0 { + // Benchmark needs machine type non-empty. + if imageConfig.machine == "" { + imageConfig.machine = defaultMachine + } + // Use the Ginkgo focus in benchmark config. + ginkgoFlagsStr += (" " + testsToGinkgoFocus(imageConfig.tests)) + } + + host, err := createInstance(imageConfig) if *deleteInstances { - defer deleteInstance(image.image) + defer deleteInstance(host) } if err != nil { return &TestResult{ - err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", image.image, err), + err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", imageConfig.image, err), } } // Only delete the files if we are keeping the instance and want it cleaned up. // If we are going to delete the instance, don't bother with cleaning up the files deleteFiles := !*deleteInstances && *cleanup - return testHost(host, deleteFiles, junitFilePrefix, *setupNode) + return testHost(host, deleteFiles, junitFilePrefix, *setupNode, ginkgoFlagsStr) } // Provision a gce instance using image -func createInstance(image *internalGCEImage) (string, error) { - glog.V(1).Infof("Creating instance %+v", *image) - name := imageToInstanceName(image.image) +func createInstance(imageConfig *internalGCEImage) (string, error) { + glog.V(1).Infof("Creating instance %+v", *imageConfig) + name := imageToInstanceName(imageConfig) i := &compute.Instance{ Name: name, - MachineType: machineType(), + MachineType: machineType(imageConfig.machine), NetworkInterfaces: []*compute.NetworkInterface{ { AccessConfigs: []*compute.AccessConfig{ @@ -432,12 +467,12 @@ func createInstance(image *internalGCEImage) (string, error) { Boot: true, Type: "PERSISTENT", InitializeParams: &compute.AttachedDiskInitializeParams{ - SourceImage: sourceImage(image.image, image.project), + SourceImage: sourceImage(imageConfig.image, imageConfig.project), }, }, }, } - i.Metadata = image.metadata + i.Metadata = imageConfig.metadata op, err := computeService.Instances.Insert(*project, *zone, i).Do() if err != nil { return "", err @@ -520,12 +555,11 @@ func getComputeClient() (*compute.Service, error) { return nil, err } -func deleteInstance(image string) { - instanceName := imageToInstanceName(image) - glog.Infof("Deleting instance %q", instanceName) - _, err := computeService.Instances.Delete(*project, *zone, instanceName).Do() +func deleteInstance(host string) { + glog.Infof("Deleting instance %q", host) + _, err := computeService.Instances.Delete(*project, *zone, host).Do() if err != nil { - glog.Errorf("Error deleting instance %q: %v", instanceName, err) + glog.Errorf("Error deleting instance %q: %v", host, err) } } @@ -553,14 +587,32 @@ func parseInstanceMetadata(str string) map[string]string { return metadata } -func imageToInstanceName(image string) string { - return *instanceNamePrefix + "-" + image +func imageToInstanceName(imageConfig *internalGCEImage) string { + if imageConfig.machine == "" { + return *instanceNamePrefix + "-" + imageConfig.image + } + return imageConfig.machine + "-" + imageConfig.image + "-" + uuid.NewUUID().String()[:8] } func sourceImage(image, imageProject string) string { return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image) } -func machineType() string { - return fmt.Sprintf("zones/%s/machineTypes/n1-standard-1", *zone) +func machineType(machine string) string { + if machine == "" { + machine = defaultMachine + } + return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine) +} + +func testsToGinkgoFocus(tests []string) string { + focus := "--focus=\"" + for i, test := range tests { + if i == 0 { + focus += test + } else { + focus += ("|" + test) + } + } + return focus + "\"" }