mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-26 21:17:23 +00:00
add benchmark to jenkins
This commit is contained in:
parent
30eb6882f4
commit
b8ab0c50c5
@ -36,7 +36,6 @@ var sshOptions = flag.String("ssh-options", "", "Commandline options passed to s
|
|||||||
var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce")
|
var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce")
|
||||||
var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.")
|
var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.")
|
||||||
var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.")
|
var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.")
|
||||||
var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
|
|
||||||
|
|
||||||
var sshOptionsMap map[string]string
|
var sshOptionsMap map[string]string
|
||||||
|
|
||||||
@ -148,7 +147,7 @@ func CreateTestArchive() (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Returns the command output, whether the exit was ok, and any errors
|
// Returns the command output, whether the exit was ok, and any errors
|
||||||
func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, setupNode bool, testArgs string) (string, bool, error) {
|
func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, setupNode bool, testArgs string, ginkgoFlags string) (string, bool, error) {
|
||||||
if setupNode {
|
if setupNode {
|
||||||
uname, err := user.Current()
|
uname, err := user.Current()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -216,7 +215,8 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
|
|||||||
// Run the tests
|
// Run the tests
|
||||||
cmd = getSshCommand(" && ",
|
cmd = getSshCommand(" && ",
|
||||||
fmt.Sprintf("cd %s", tmp),
|
fmt.Sprintf("cd %s", tmp),
|
||||||
fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", *testTimeoutSeconds, *ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs),
|
fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
|
||||||
|
*testTimeoutSeconds, ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs),
|
||||||
)
|
)
|
||||||
aggErrs := []error{}
|
aggErrs := []error{}
|
||||||
|
|
||||||
|
50
test/e2e_node/jenkins/benchmark/benchmark-config.yaml
Normal file
50
test/e2e_node/jenkins/benchmark/benchmark-config.yaml
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
---
|
||||||
|
images:
|
||||||
|
containervm-density1:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-1
|
||||||
|
tests:
|
||||||
|
- '.*create 35 pods with 0s? interval \[Benchmark\]'
|
||||||
|
containervm-density2:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-1
|
||||||
|
tests:
|
||||||
|
- '.*create 105 pods with 0s? interval \[Benchmark\]'
|
||||||
|
containervm-density3:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-2
|
||||||
|
tests:
|
||||||
|
- '.*create 105 pods with 0s? interval \[Benchmark\]'
|
||||||
|
containervm-density4:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-1
|
||||||
|
tests:
|
||||||
|
- '.*create 35 pods with 100ms interval \[Benchmark\]'
|
||||||
|
containervm-density5:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-1
|
||||||
|
tests:
|
||||||
|
- '.*create 105 pods with 100ms interval \[Benchmark\]'
|
||||||
|
containervm-density6:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-2
|
||||||
|
tests:
|
||||||
|
- '.*create 105 pods with 100ms interval \[Benchmark\]'
|
||||||
|
containervm-density7:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-1
|
||||||
|
tests:
|
||||||
|
- '.*create 105 pods with 300ms interval \[Benchmark\]'
|
||||||
|
containervm-density8:
|
||||||
|
image: e2e-node-containervm-v20160321-image
|
||||||
|
project: kubernetes-node-e2e-images
|
||||||
|
machine: n1-standard-2
|
||||||
|
tests:
|
||||||
|
- '.*create 105 pods with 300ms interval \[Benchmark\]'
|
@ -0,0 +1,9 @@
|
|||||||
|
GCE_HOSTS=
|
||||||
|
GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/benchmark/benchmark-config.yaml
|
||||||
|
GCE_ZONE=us-central1-f
|
||||||
|
GCE_PROJECT=k8s-jkns-ci-node-e2e
|
||||||
|
CLEANUP=true
|
||||||
|
GINKGO_FLAGS='--skip="\[Flaky\]"'
|
||||||
|
SETUP_NODE=false
|
||||||
|
TEST_ARGS=--cgroups-per-qos=false
|
||||||
|
PARALLELISM=1
|
@ -41,7 +41,7 @@ import (
|
|||||||
"github.com/pborman/uuid"
|
"github.com/pborman/uuid"
|
||||||
"golang.org/x/oauth2"
|
"golang.org/x/oauth2"
|
||||||
"golang.org/x/oauth2/google"
|
"golang.org/x/oauth2/google"
|
||||||
"google.golang.org/api/compute/v1"
|
compute "google.golang.org/api/compute/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
|
var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
|
||||||
@ -58,8 +58,16 @@ var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar
|
|||||||
var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine")
|
var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine")
|
||||||
var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
|
var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
|
||||||
var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
|
var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
|
||||||
|
var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
|
||||||
|
|
||||||
var computeService *compute.Service
|
const (
|
||||||
|
defaultMachine = "n1-standard-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
computeService *compute.Service
|
||||||
|
arc Archive
|
||||||
|
)
|
||||||
|
|
||||||
type Archive struct {
|
type Archive struct {
|
||||||
sync.Once
|
sync.Once
|
||||||
@ -67,8 +75,6 @@ type Archive struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
var arc Archive
|
|
||||||
|
|
||||||
type TestResult struct {
|
type TestResult struct {
|
||||||
output string
|
output string
|
||||||
err error
|
err error
|
||||||
@ -85,6 +91,11 @@ type TestResult struct {
|
|||||||
// short-name:
|
// short-name:
|
||||||
// image: gce-image-name
|
// image: gce-image-name
|
||||||
// project: gce-image-project
|
// project: gce-image-project
|
||||||
|
// machine: for benchmark only, the machine type (GCE instance) to run test
|
||||||
|
// tests: for benchmark only, a list of ginkgo focus strings to match tests
|
||||||
|
|
||||||
|
// TODO(coufon): replace 'image' with 'node' in configurations
|
||||||
|
// and we plan to support testing custom machines other than GCE by specifying host
|
||||||
type ImageConfig struct {
|
type ImageConfig struct {
|
||||||
Images map[string]GCEImage `json:"images"`
|
Images map[string]GCEImage `json:"images"`
|
||||||
}
|
}
|
||||||
@ -97,6 +108,10 @@ type GCEImage struct {
|
|||||||
// Defaults to using only the latest image. Acceptible values are [0, # of images that match the regex).
|
// Defaults to using only the latest image. Acceptible values are [0, # of images that match the regex).
|
||||||
// If the number of existing previous images is lesser than what is desired, the test will use that is available.
|
// If the number of existing previous images is lesser than what is desired, the test will use that is available.
|
||||||
PreviousImages int `json:"previous_images, omitempty"`
|
PreviousImages int `json:"previous_images, omitempty"`
|
||||||
|
|
||||||
|
Machine string `json:"machine, omitempty"`
|
||||||
|
// The test is regarded as 'benchmark' is Tests is non-empty
|
||||||
|
Tests []string `json:"tests, omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type internalImageConfig struct {
|
type internalImageConfig struct {
|
||||||
@ -107,6 +122,8 @@ type internalGCEImage struct {
|
|||||||
image string
|
image string
|
||||||
project string
|
project string
|
||||||
metadata *compute.Metadata
|
metadata *compute.Metadata
|
||||||
|
machine string
|
||||||
|
tests []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@ -141,9 +158,11 @@ func main() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Fatalf("Could not parse image config file: %v", err)
|
glog.Fatalf("Could not parse image config file: %v", err)
|
||||||
}
|
}
|
||||||
for _, imageConfig := range externalImageConfig.Images {
|
for shortName, imageConfig := range externalImageConfig.Images {
|
||||||
var images []string
|
var images []string
|
||||||
|
isRegex, name := false, shortName
|
||||||
if imageConfig.ImageRegex != "" && imageConfig.Image == "" {
|
if imageConfig.ImageRegex != "" && imageConfig.Image == "" {
|
||||||
|
isRegex = true
|
||||||
images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages)
|
images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err)
|
glog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err)
|
||||||
@ -156,8 +175,13 @@ func main() {
|
|||||||
image: image,
|
image: image,
|
||||||
project: imageConfig.Project,
|
project: imageConfig.Project,
|
||||||
metadata: getImageMetadata(imageConfig.Metadata),
|
metadata: getImageMetadata(imageConfig.Metadata),
|
||||||
|
machine: imageConfig.Machine,
|
||||||
|
tests: imageConfig.Tests,
|
||||||
}
|
}
|
||||||
gceImages.images[image] = gceImage
|
if isRegex {
|
||||||
|
name = shortName + "-" + image
|
||||||
|
}
|
||||||
|
gceImages.images[name] = gceImage
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -224,7 +248,7 @@ func main() {
|
|||||||
fmt.Printf("Initializing e2e tests using host %s.\n", host)
|
fmt.Printf("Initializing e2e tests using host %s.\n", host)
|
||||||
running++
|
running++
|
||||||
go func(host string, junitFilePrefix string) {
|
go func(host string, junitFilePrefix string) {
|
||||||
results <- testHost(host, *cleanup, junitFilePrefix, *setupNode)
|
results <- testHost(host, *cleanup, junitFilePrefix, *setupNode, *ginkgoFlags)
|
||||||
}(host, host)
|
}(host, host)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -303,7 +327,7 @@ func getImageMetadata(input string) *compute.Metadata {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Run tests in archive against host
|
// Run tests in archive against host
|
||||||
func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode bool) *TestResult {
|
func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode bool, ginkgoFlagsStr string) *TestResult {
|
||||||
instance, err := computeService.Instances.Get(*project, *zone, host).Do()
|
instance, err := computeService.Instances.Get(*project, *zone, host).Do()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &TestResult{
|
return &TestResult{
|
||||||
@ -333,7 +357,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs)
|
output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr)
|
||||||
return &TestResult{
|
return &TestResult{
|
||||||
output: output,
|
output: output,
|
||||||
err: err,
|
err: err,
|
||||||
@ -393,30 +417,41 @@ func getGCEImages(imageRegex, project string, previousImages int) ([]string, err
|
|||||||
|
|
||||||
// Provision a gce instance using image and run the tests in archive against the instance.
|
// Provision a gce instance using image and run the tests in archive against the instance.
|
||||||
// Delete the instance afterward.
|
// Delete the instance afterward.
|
||||||
func testImage(image *internalGCEImage, junitFilePrefix string) *TestResult {
|
func testImage(imageConfig *internalGCEImage, junitFilePrefix string) *TestResult {
|
||||||
host, err := createInstance(image)
|
ginkgoFlagsStr := *ginkgoFlags
|
||||||
|
// Check whether the test is for benchmark.
|
||||||
|
if len(imageConfig.tests) > 0 {
|
||||||
|
// Benchmark needs machine type non-empty.
|
||||||
|
if imageConfig.machine == "" {
|
||||||
|
imageConfig.machine = defaultMachine
|
||||||
|
}
|
||||||
|
// Use the Ginkgo focus in benchmark config.
|
||||||
|
ginkgoFlagsStr += (" " + testsToGinkgoFocus(imageConfig.tests))
|
||||||
|
}
|
||||||
|
|
||||||
|
host, err := createInstance(imageConfig)
|
||||||
if *deleteInstances {
|
if *deleteInstances {
|
||||||
defer deleteInstance(image.image)
|
defer deleteInstance(host)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &TestResult{
|
return &TestResult{
|
||||||
err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", image.image, err),
|
err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", imageConfig.image, err),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only delete the files if we are keeping the instance and want it cleaned up.
|
// Only delete the files if we are keeping the instance and want it cleaned up.
|
||||||
// If we are going to delete the instance, don't bother with cleaning up the files
|
// If we are going to delete the instance, don't bother with cleaning up the files
|
||||||
deleteFiles := !*deleteInstances && *cleanup
|
deleteFiles := !*deleteInstances && *cleanup
|
||||||
return testHost(host, deleteFiles, junitFilePrefix, *setupNode)
|
return testHost(host, deleteFiles, junitFilePrefix, *setupNode, ginkgoFlagsStr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Provision a gce instance using image
|
// Provision a gce instance using image
|
||||||
func createInstance(image *internalGCEImage) (string, error) {
|
func createInstance(imageConfig *internalGCEImage) (string, error) {
|
||||||
glog.V(1).Infof("Creating instance %+v", *image)
|
glog.V(1).Infof("Creating instance %+v", *imageConfig)
|
||||||
name := imageToInstanceName(image.image)
|
name := imageToInstanceName(imageConfig)
|
||||||
i := &compute.Instance{
|
i := &compute.Instance{
|
||||||
Name: name,
|
Name: name,
|
||||||
MachineType: machineType(),
|
MachineType: machineType(imageConfig.machine),
|
||||||
NetworkInterfaces: []*compute.NetworkInterface{
|
NetworkInterfaces: []*compute.NetworkInterface{
|
||||||
{
|
{
|
||||||
AccessConfigs: []*compute.AccessConfig{
|
AccessConfigs: []*compute.AccessConfig{
|
||||||
@ -432,12 +467,12 @@ func createInstance(image *internalGCEImage) (string, error) {
|
|||||||
Boot: true,
|
Boot: true,
|
||||||
Type: "PERSISTENT",
|
Type: "PERSISTENT",
|
||||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||||
SourceImage: sourceImage(image.image, image.project),
|
SourceImage: sourceImage(imageConfig.image, imageConfig.project),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
i.Metadata = image.metadata
|
i.Metadata = imageConfig.metadata
|
||||||
op, err := computeService.Instances.Insert(*project, *zone, i).Do()
|
op, err := computeService.Instances.Insert(*project, *zone, i).Do()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@ -520,12 +555,11 @@ func getComputeClient() (*compute.Service, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteInstance(image string) {
|
func deleteInstance(host string) {
|
||||||
instanceName := imageToInstanceName(image)
|
glog.Infof("Deleting instance %q", host)
|
||||||
glog.Infof("Deleting instance %q", instanceName)
|
_, err := computeService.Instances.Delete(*project, *zone, host).Do()
|
||||||
_, err := computeService.Instances.Delete(*project, *zone, instanceName).Do()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Error deleting instance %q: %v", instanceName, err)
|
glog.Errorf("Error deleting instance %q: %v", host, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -553,14 +587,32 @@ func parseInstanceMetadata(str string) map[string]string {
|
|||||||
return metadata
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func imageToInstanceName(image string) string {
|
func imageToInstanceName(imageConfig *internalGCEImage) string {
|
||||||
return *instanceNamePrefix + "-" + image
|
if imageConfig.machine == "" {
|
||||||
|
return *instanceNamePrefix + "-" + imageConfig.image
|
||||||
|
}
|
||||||
|
return imageConfig.machine + "-" + imageConfig.image + "-" + uuid.NewUUID().String()[:8]
|
||||||
}
|
}
|
||||||
|
|
||||||
func sourceImage(image, imageProject string) string {
|
func sourceImage(image, imageProject string) string {
|
||||||
return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image)
|
return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image)
|
||||||
}
|
}
|
||||||
|
|
||||||
func machineType() string {
|
func machineType(machine string) string {
|
||||||
return fmt.Sprintf("zones/%s/machineTypes/n1-standard-1", *zone)
|
if machine == "" {
|
||||||
|
machine = defaultMachine
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testsToGinkgoFocus(tests []string) string {
|
||||||
|
focus := "--focus=\""
|
||||||
|
for i, test := range tests {
|
||||||
|
if i == 0 {
|
||||||
|
focus += test
|
||||||
|
} else {
|
||||||
|
focus += ("|" + test)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return focus + "\""
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user