From b8ab0c50c5dffb175964d663a8e5cf5fea2bb1cc Mon Sep 17 00:00:00 2001 From: Zhou Fang Date: Fri, 19 Aug 2016 14:40:23 -0700 Subject: [PATCH] add benchmark to jenkins --- test/e2e_node/e2e_remote.go | 6 +- .../jenkins/benchmark/benchmark-config.yaml | 50 ++++++++ .../benchmark/jenkins-benchmark.properties | 9 ++ test/e2e_node/runner/run_e2e.go | 110 +++++++++++++----- 4 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 test/e2e_node/jenkins/benchmark/benchmark-config.yaml create mode 100644 test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go index 2187a0e5021..a062d13fe8a 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/e2e_remote.go @@ -36,7 +36,6 @@ var sshOptions = flag.String("ssh-options", "", "Commandline options passed to s var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce") var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.") var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.") -var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.") var sshOptionsMap map[string]string @@ -148,7 +147,7 @@ func CreateTestArchive() (string, error) { } // Returns the command output, whether the exit was ok, and any errors -func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, setupNode bool, testArgs string) (string, bool, error) { +func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, setupNode bool, testArgs string, ginkgoFlags string) (string, bool, error) { if setupNode { uname, err := user.Current() if err != nil { @@ -216,7 +215,8 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), - fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", *testTimeoutSeconds, *ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs), + fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", + *testTimeoutSeconds, ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs), ) aggErrs := []error{} diff --git a/test/e2e_node/jenkins/benchmark/benchmark-config.yaml b/test/e2e_node/jenkins/benchmark/benchmark-config.yaml new file mode 100644 index 00000000000..00eb00a55d1 --- /dev/null +++ b/test/e2e_node/jenkins/benchmark/benchmark-config.yaml @@ -0,0 +1,50 @@ +--- +images: + containervm-density1: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 35 pods with 0s? interval \[Benchmark\]' + containervm-density2: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 105 pods with 0s? interval \[Benchmark\]' + containervm-density3: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-2 + tests: + - '.*create 105 pods with 0s? interval \[Benchmark\]' + containervm-density4: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 35 pods with 100ms interval \[Benchmark\]' + containervm-density5: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 105 pods with 100ms interval \[Benchmark\]' + containervm-density6: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-2 + tests: + - '.*create 105 pods with 100ms interval \[Benchmark\]' + containervm-density7: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-1 + tests: + - '.*create 105 pods with 300ms interval \[Benchmark\]' + containervm-density8: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images + machine: n1-standard-2 + tests: + - '.*create 105 pods with 300ms interval \[Benchmark\]' diff --git a/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties b/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties new file mode 100644 index 00000000000..1fe8abd8566 --- /dev/null +++ b/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties @@ -0,0 +1,9 @@ +GCE_HOSTS= +GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/benchmark/benchmark-config.yaml +GCE_ZONE=us-central1-f +GCE_PROJECT=k8s-jkns-ci-node-e2e +CLEANUP=true +GINKGO_FLAGS='--skip="\[Flaky\]"' +SETUP_NODE=false +TEST_ARGS=--cgroups-per-qos=false +PARALLELISM=1 diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index b603d275f88..b3464f803f7 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -41,7 +41,7 @@ import ( "github.com/pborman/uuid" "golang.org/x/oauth2" "golang.org/x/oauth2/google" - "google.golang.org/api/compute/v1" + compute "google.golang.org/api/compute/v1" ) var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") @@ -58,8 +58,16 @@ var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine") var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k 0 { + // Benchmark needs machine type non-empty. + if imageConfig.machine == "" { + imageConfig.machine = defaultMachine + } + // Use the Ginkgo focus in benchmark config. + ginkgoFlagsStr += (" " + testsToGinkgoFocus(imageConfig.tests)) + } + + host, err := createInstance(imageConfig) if *deleteInstances { - defer deleteInstance(image.image) + defer deleteInstance(host) } if err != nil { return &TestResult{ - err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", image.image, err), + err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", imageConfig.image, err), } } // Only delete the files if we are keeping the instance and want it cleaned up. // If we are going to delete the instance, don't bother with cleaning up the files deleteFiles := !*deleteInstances && *cleanup - return testHost(host, deleteFiles, junitFilePrefix, *setupNode) + return testHost(host, deleteFiles, junitFilePrefix, *setupNode, ginkgoFlagsStr) } // Provision a gce instance using image -func createInstance(image *internalGCEImage) (string, error) { - glog.V(1).Infof("Creating instance %+v", *image) - name := imageToInstanceName(image.image) +func createInstance(imageConfig *internalGCEImage) (string, error) { + glog.V(1).Infof("Creating instance %+v", *imageConfig) + name := imageToInstanceName(imageConfig) i := &compute.Instance{ Name: name, - MachineType: machineType(), + MachineType: machineType(imageConfig.machine), NetworkInterfaces: []*compute.NetworkInterface{ { AccessConfigs: []*compute.AccessConfig{ @@ -432,12 +467,12 @@ func createInstance(image *internalGCEImage) (string, error) { Boot: true, Type: "PERSISTENT", InitializeParams: &compute.AttachedDiskInitializeParams{ - SourceImage: sourceImage(image.image, image.project), + SourceImage: sourceImage(imageConfig.image, imageConfig.project), }, }, }, } - i.Metadata = image.metadata + i.Metadata = imageConfig.metadata op, err := computeService.Instances.Insert(*project, *zone, i).Do() if err != nil { return "", err @@ -520,12 +555,11 @@ func getComputeClient() (*compute.Service, error) { return nil, err } -func deleteInstance(image string) { - instanceName := imageToInstanceName(image) - glog.Infof("Deleting instance %q", instanceName) - _, err := computeService.Instances.Delete(*project, *zone, instanceName).Do() +func deleteInstance(host string) { + glog.Infof("Deleting instance %q", host) + _, err := computeService.Instances.Delete(*project, *zone, host).Do() if err != nil { - glog.Errorf("Error deleting instance %q: %v", instanceName, err) + glog.Errorf("Error deleting instance %q: %v", host, err) } } @@ -553,14 +587,32 @@ func parseInstanceMetadata(str string) map[string]string { return metadata } -func imageToInstanceName(image string) string { - return *instanceNamePrefix + "-" + image +func imageToInstanceName(imageConfig *internalGCEImage) string { + if imageConfig.machine == "" { + return *instanceNamePrefix + "-" + imageConfig.image + } + return imageConfig.machine + "-" + imageConfig.image + "-" + uuid.NewUUID().String()[:8] } func sourceImage(image, imageProject string) string { return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image) } -func machineType() string { - return fmt.Sprintf("zones/%s/machineTypes/n1-standard-1", *zone) +func machineType(machine string) string { + if machine == "" { + machine = defaultMachine + } + return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine) +} + +func testsToGinkgoFocus(tests []string) string { + focus := "--focus=\"" + for i, test := range tests { + if i == 0 { + focus += test + } else { + focus += ("|" + test) + } + } + return focus + "\"" }