diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 2e775f22b98..70d7b6e43bf 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -203,6 +203,7 @@ http-check-frequency http-port ignore-daemonsets ignore-not-found +image-config-file image-gc-high-threshold image-gc-low-threshold image-project diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh index 15b7969d5cd..31bce7561ac 100755 --- a/test/e2e_node/jenkins/e2e-node-jenkins.sh +++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh @@ -35,7 +35,7 @@ ARTIFACTS=${WORKSPACE}/_artifacts mkdir -p ${ARTIFACTS} go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=2 --ssh-env="gce" \ - --zone="$GCE_ZONE" --project="$GCE_PROJECT" --image-project="$GCE_IMAGE_PROJECT" \ - --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" \ + --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \ + --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \ --results-dir="$ARTIFACTS" --ginkgo-flags="$GINKGO_FLAGS" \ --setup-node="$SETUP_NODE" --instance-metadata="$GCE_INSTANCE_METADATA" diff --git a/test/e2e_node/jenkins/image-config.yaml b/test/e2e_node/jenkins/image-config.yaml new file mode 100644 index 00000000000..d6936329cd7 --- /dev/null +++ b/test/e2e_node/jenkins/image-config.yaml @@ -0,0 +1,16 @@ +# To copy an image between projects: +# `gcloud compute --project disks create --image=https://www.googleapis.com/compute/v1/projects//global/images/` +# `gcloud compute --project images create --source-disk=` +images: + ubuntu-docker9: + image: e2e-node-ubuntu-trusty-docker9-v1-image + project: kubernetes-node-e2e-images + ubuntu-docker10: + image: e2e-node-ubuntu-trusty-docker10-v1-image + project: kubernetes-node-e2e-images + coreos-stable: + image: e2e-node-coreos-alpha-1068-20160707-image + project: kubernetes-node-e2e-images + containervm: + image: e2e-node-containervm-v20160321-image + project: kubernetes-node-e2e-images diff --git a/test/e2e_node/jenkins/jenkins-ci.properties b/test/e2e_node/jenkins/jenkins-ci.properties index 9ef951eb42e..8392dd6a6c8 100644 --- a/test/e2e_node/jenkins/jenkins-ci.properties +++ b/test/e2e_node/jenkins/jenkins-ci.properties @@ -1,12 +1,7 @@ GCE_HOSTS= -# Keep GCE_IMAGES consistent with those in jenkins-pull.properties. -# To copy an image between projects: -# `gcloud compute --project disks create --image=https://www.googleapis.com/compute/v1/projects//global/images/` -# `gcloud compute --project images create --source-disk=` -GCE_IMAGES=e2e-node-ubuntu-trusty-docker9-v1-image,e2e-node-ubuntu-trusty-docker10-v1-image,e2e-node-coreos-alpha-1068-20160707-image,e2e-node-containervm-v20160321-image +GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml GCE_ZONE=us-central1-f GCE_PROJECT=kubernetes-jenkins -GCE_IMAGE_PROJECT=kubernetes-node-e2e-images CLEANUP=true GINKGO_FLAGS=--skip=FLAKY SETUP_NODE=false diff --git a/test/e2e_node/jenkins/jenkins-pull.properties b/test/e2e_node/jenkins/jenkins-pull.properties index ac64c6f381d..47b7c1c51f0 100644 --- a/test/e2e_node/jenkins/jenkins-pull.properties +++ b/test/e2e_node/jenkins/jenkins-pull.properties @@ -1,12 +1,7 @@ GCE_HOSTS= -# Keep GCE_IMAGES consistent with those in jenkins-ci.properties -# To copy an image between projects: -# `gcloud compute --project disks create --image=https://www.googleapis.com/compute/v1/projects//global/images/` -# `gcloud compute --project images create --source-disk=` -GCE_IMAGES=e2e-node-ubuntu-trusty-docker9-v1-image,e2e-node-ubuntu-trusty-docker10-v1-image,e2e-node-coreos-alpha-1068-20160707-image,e2e-node-containervm-v20160321-image +GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml GCE_ZONE=us-central1-f GCE_PROJECT=kubernetes-jenkins-pull -GCE_IMAGE_PROJECT=kubernetes-node-e2e-images CLEANUP=true GINKGO_FLAGS=--skip=FLAKY SETUP_NODE=false diff --git a/test/e2e_node/jenkins/template.properties b/test/e2e_node/jenkins/template.properties index 9970021319b..1f2b7a2a007 100644 --- a/test/e2e_node/jenkins/template.properties +++ b/test/e2e_node/jenkins/template.properties @@ -1,17 +1,19 @@ # Copy this file to your home directory and modify -# Names of gce hosts to test against (must be resolvable) or empty (one or more of GCE_IMAGES, GCE_HOSTS is required) +# Path to a yaml or json file describing images to run or empty +GCE_IMAGE_CONFIG_PATH= +# Names of gce hosts to test against (must be resolvable) or empty GCE_HOSTS= -# Names of gce images to test or empty (one or more of GCE_IMAGES, GCE_HOSTS is required) +# Comma-separated names of gce images to test or empty (one or more of GCE_IMAGE_CONFIG_PATH, GCE_IMAGES, GCE_HOSTS is required) GCE_IMAGES= # Gce zone to use - required when using GCE_IMAGES GCE_ZONE= -# Gce project to use for creating instances -# required when using GCE_IMAGES +# Gce project to use for creating instances +# required when using GCE_IMAGES or GCE_IMAGE_CONFIG_PATH GCE_PROJECT= # Gce project to use for GCE_IMAGES # required when using GCE_IMAGES GCE_IMAGE_PROJECT= -# If true, delete instances created from GCE_IMAGES and files copied to GCE_HOSTS +# If true, delete instances created from GCE_IMAGES/GCE_IMAGE_CONFIG_PATH and files copied to GCE_HOSTS CLEANUP=true # If true, current user will be added to the docker group on test node SETUP_NODE=false diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index ce365e74b1f..4f599d83ed9 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/test/e2e_node" + "github.com/ghodss/yaml" "github.com/golang/glog" "github.com/pborman/uuid" "golang.org/x/oauth2" @@ -43,6 +44,7 @@ import ( var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names") var zone = flag.String("zone", "", "gce zone the hosts live in") var project = flag.String("project", "", "gce project the hosts live in") +var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run") var imageProject = flag.String("image-project", "", "gce project the hosts live in") var images = flag.String("images", "", "images to test") var hosts = flag.String("hosts", "", "hosts to test") @@ -69,6 +71,24 @@ type TestResult struct { exitOk bool } +// ImageConfig specifies what images should be run and how for these tests. +// It can be created via the `--images` and `--image-project` flags, or by +// specifying the `--image-config-file` flag, pointing to a json or yaml file +// of the form: +// +// images: +// short-name: +// image: gce-image-name +// project: gce-image-project +type ImageConfig struct { + Images map[string]GCEImage `json:"images"` +} + +type GCEImage struct { + Image string `json:"image"` + Project string `json:"project"` +} + func main() { flag.Parse() rand.Seed(time.Now().UTC().UnixNano()) @@ -78,18 +98,50 @@ func main() { return } - if *hosts == "" && *images == "" { - glog.Fatalf("Must specify one of --images or --hosts flag.") + if *hosts == "" && *imageConfigFile == "" && *images == "" { + glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.") } - if *images != "" && *zone == "" { - glog.Fatal("Must specify --zone flag") + gceImages := &ImageConfig{ + Images: make(map[string]GCEImage), } + if *imageConfigFile != "" { + // parse images + imageConfigData, err := ioutil.ReadFile(*imageConfigFile) + if err != nil { + glog.Fatalf("Could not read image config file provided: %v", err) + } + err = yaml.Unmarshal(imageConfigData, gceImages) + if err != nil { + glog.Fatalf("Could not parse image config file: %v", err) + } + } + + // Allow users to specify additional images via cli flags for local testing + // convenience; merge in with config file if *images != "" { if *imageProject == "" { - glog.Fatal("Must specify --image-project flag") + glog.Fatal("Must specify --image-project if you specify --images") } + cliImages := strings.Split(*images, ",") + for _, img := range cliImages { + gceImages.Images[img] = GCEImage{ + Image: img, + Project: *imageProject, + } + } + } + + if len(gceImages.Images) != 0 && *zone == "" { + glog.Fatal("Must specify --zone flag") + } + for shortName, image := range gceImages.Images { + if image.Project == "" { + glog.Fatalf("Invalid config for %v; must specify a project", shortName) + } + } + if len(gceImages.Images) != 0 { if *project == "" { - glog.Fatal("Must specify --project flag") + glog.Fatal("Must specify --project flag to launch images into") } } if *instanceNamePrefix == "" { @@ -117,12 +169,12 @@ func main() { results := make(chan *TestResult) running := 0 - if *images != "" { - for _, image := range strings.Split(*images, ",") { - running++ - fmt.Printf("Initializing e2e tests using image %s.\n", image) - go func(image string, junitFileNum int) { results <- testImage(image, junitFileNum) }(image, running) - } + for shortName, image := range gceImages.Images { + running++ + fmt.Printf("Initializing e2e tests using image %s.\n", shortName) + go func(image, imageProject string, junitFileNum int) { + results <- testImage(image, imageProject, junitFileNum) + }(image.Image, image.Project, running) } if *hosts != "" { for _, host := range strings.Split(*hosts, ",") { @@ -213,8 +265,8 @@ func testHost(host string, deleteFiles bool, junitFileNum int, setupNode bool) * // Provision a gce instance using image and run the tests in archive against the instance. // Delete the instance afterward. -func testImage(image string, junitFileNum int) *TestResult { - host, err := createInstance(image) +func testImage(image, imageProject string, junitFileNum int) *TestResult { + host, err := createInstance(image, imageProject) if *deleteInstances { defer deleteInstance(image) } @@ -231,7 +283,7 @@ func testImage(image string, junitFileNum int) *TestResult { } // Provision a gce instance using image -func createInstance(image string) (string, error) { +func createInstance(image, imageProject string) (string, error) { name := imageToInstanceName(image) i := &compute.Instance{ Name: name, @@ -251,7 +303,7 @@ func createInstance(image string) (string, error) { Boot: true, Type: "PERSISTENT", InitializeParams: &compute.AttachedDiskInitializeParams{ - SourceImage: sourceImage(image), + SourceImage: sourceImage(image, imageProject), }, }, }, @@ -386,8 +438,8 @@ func imageToInstanceName(image string) string { return *instanceNamePrefix + "-" + image } -func sourceImage(image string) string { - return fmt.Sprintf("projects/%s/global/images/%s", *imageProject, image) +func sourceImage(image, imageProject string) string { + return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image) } func machineType() string {