Merge pull request #28708 from euank/node-e2e-image-conf

Automatic merge from submit-queue

node_e2e: configure gce images via config file

This file provides the abiliy to specify image project on a per-image
basis and is more extensible for future changes.

For backwards compatibility and local development convenience, the
existing flags are kept and should work.

The eventual goal is to be able to source some images, such as the CoreOS one (and possibly containervm one) from their upstream projects and do all new configuration changes via a cloud-init key added to the image config.

This PR is a first step there. A following PR will add a config key of `cloud-init` or `user-data` and migrate the CoreOS e2e to use that.

This motivation is driven by the fact that currently the changes needed for the CoreOS image can all be done quickly in cloud-init and this will make it much easier to update the image and ensure that changes are applied consistently.

/cc @timstclair @vishh @yifan-gu @pwittrock

hack/verify-flags/known-flags.txt

@@ -203,6 +203,7 @@ http-check-frequency
 http-port
 ignore-daemonsets
 ignore-not-found
+image-config-file
 image-gc-high-threshold
 image-gc-low-threshold
 image-project

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -35,7 +35,7 @@ ARTIFACTS=${WORKSPACE}/_artifacts
 
 mkdir -p ${ARTIFACTS}
 go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
-  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --image-project="$GCE_IMAGE_PROJECT" \
-  --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" \
+  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
+  --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
   --results-dir="$ARTIFACTS" --ginkgo-flags="$GINKGO_FLAGS" \
   --setup-node="$SETUP_NODE" --instance-metadata="$GCE_INSTANCE_METADATA"

test/e2e_node/jenkins/image-config.yaml

@@ -0,0 +1,16 @@
+# To copy an image between projects:
+# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
+# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
+images:
+  ubuntu-docker9:
+    image: e2e-node-ubuntu-trusty-docker9-v1-image
+    project: kubernetes-node-e2e-images
+  ubuntu-docker10:
+    image: e2e-node-ubuntu-trusty-docker10-v1-image
+    project: kubernetes-node-e2e-images
+  coreos-stable:
+    image: e2e-node-coreos-alpha-1068-20160707-image
+    project: kubernetes-node-e2e-images
+  containervm:
+    image: e2e-node-containervm-v20160321-image
+    project: kubernetes-node-e2e-images

test/e2e_node/jenkins/jenkins-ci.properties

@@ -1,12 +1,7 @@
 GCE_HOSTS=
-# Keep GCE_IMAGES consistent with those in jenkins-pull.properties.
-# To copy an image between projects:
-# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
-# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
-GCE_IMAGES=e2e-node-ubuntu-trusty-docker9-v1-image,e2e-node-ubuntu-trusty-docker10-v1-image,e2e-node-coreos-alpha-1068-20160707-image,e2e-node-containervm-v20160321-image
+GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
 GCE_ZONE=us-central1-f
 GCE_PROJECT=kubernetes-jenkins
-GCE_IMAGE_PROJECT=kubernetes-node-e2e-images
 CLEANUP=true
 GINKGO_FLAGS=--skip=FLAKY
 SETUP_NODE=false

test/e2e_node/jenkins/jenkins-pull.properties

@@ -1,12 +1,7 @@
 GCE_HOSTS=
-# Keep GCE_IMAGES consistent with those in jenkins-ci.properties
-# To copy an image between projects:
-# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
-# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
-GCE_IMAGES=e2e-node-ubuntu-trusty-docker9-v1-image,e2e-node-ubuntu-trusty-docker10-v1-image,e2e-node-coreos-alpha-1068-20160707-image,e2e-node-containervm-v20160321-image
+GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
 GCE_ZONE=us-central1-f
 GCE_PROJECT=kubernetes-jenkins-pull
-GCE_IMAGE_PROJECT=kubernetes-node-e2e-images
 CLEANUP=true
 GINKGO_FLAGS=--skip=FLAKY
 SETUP_NODE=false

test/e2e_node/jenkins/template.properties

@@ -1,17 +1,19 @@
 # Copy this file to your home directory and modify
-# Names of gce hosts to test against (must be resolvable) or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
+# Path to a yaml or json file describing images to run or empty
+GCE_IMAGE_CONFIG_PATH=
+# Names of gce hosts to test against (must be resolvable) or empty
 GCE_HOSTS=
-# Names of gce images to test or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
+# Comma-separated names of gce images to test or empty (one or more of GCE_IMAGE_CONFIG_PATH, GCE_IMAGES, GCE_HOSTS is required)
 GCE_IMAGES=
 # Gce zone to use - required when using GCE_IMAGES
 GCE_ZONE=
-# Gce project to use for creating instances 
-# required when using GCE_IMAGES
+# Gce project to use for creating instances
+# required when using GCE_IMAGES or GCE_IMAGE_CONFIG_PATH
 GCE_PROJECT=
 # Gce project to use for GCE_IMAGES
 # required when using GCE_IMAGES
 GCE_IMAGE_PROJECT=
-# If true, delete instances created from GCE_IMAGES and files copied to GCE_HOSTS
+# If true, delete instances created from GCE_IMAGES/GCE_IMAGE_CONFIG_PATH and files copied to GCE_HOSTS
 CLEANUP=true
 # If true, current user will be added to the docker group on test node
 SETUP_NODE=false

test/e2e_node/runner/run_e2e.go

@@ -33,6 +33,7 @@ import (
 
 	"k8s.io/kubernetes/test/e2e_node"
 
+	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	"github.com/pborman/uuid"
 	"golang.org/x/oauth2"
@@ -43,6 +44,7 @@ import (
 var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
 var zone = flag.String("zone", "", "gce zone the hosts live in")
 var project = flag.String("project", "", "gce project the hosts live in")
+var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run")
 var imageProject = flag.String("image-project", "", "gce project the hosts live in")
 var images = flag.String("images", "", "images to test")
 var hosts = flag.String("hosts", "", "hosts to test")
@@ -69,6 +71,24 @@ type TestResult struct {
 	exitOk bool
 }
 
+// ImageConfig specifies what images should be run and how for these tests.
+// It can be created via the `--images` and `--image-project` flags, or by
+// specifying the `--image-config-file` flag, pointing to a json or yaml file
+// of the form:
+//
+//     images:
+//       short-name:
+//         image: gce-image-name
+//         project: gce-image-project
+type ImageConfig struct {
+	Images map[string]GCEImage `json:"images"`
+}
+
+type GCEImage struct {
+	Image   string `json:"image"`
+	Project string `json:"project"`
+}
+
 func main() {
 	flag.Parse()
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -78,18 +98,50 @@ func main() {
 		return
 	}
 
-	if *hosts == "" && *images == "" {
-		glog.Fatalf("Must specify one of --images or --hosts flag.")
+	if *hosts == "" && *imageConfigFile == "" && *images == "" {
+		glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
 	}
-	if *images != "" && *zone == "" {
-		glog.Fatal("Must specify --zone flag")
+	gceImages := &ImageConfig{
+		Images: make(map[string]GCEImage),
 	}
+	if *imageConfigFile != "" {
+		// parse images
+		imageConfigData, err := ioutil.ReadFile(*imageConfigFile)
+		if err != nil {
+			glog.Fatalf("Could not read image config file provided: %v", err)
+		}
+		err = yaml.Unmarshal(imageConfigData, gceImages)
+		if err != nil {
+			glog.Fatalf("Could not parse image config file: %v", err)
+		}
+	}
+
+	// Allow users to specify additional images via cli flags for local testing
+	// convenience; merge in with config file
 	if *images != "" {
 		if *imageProject == "" {
-			glog.Fatal("Must specify --image-project flag")
+			glog.Fatal("Must specify --image-project if you specify --images")
 		}
+		cliImages := strings.Split(*images, ",")
+		for _, img := range cliImages {
+			gceImages.Images[img] = GCEImage{
+				Image:   img,
+				Project: *imageProject,
+			}
+		}
+	}
+
+	if len(gceImages.Images) != 0 && *zone == "" {
+		glog.Fatal("Must specify --zone flag")
+	}
+	for shortName, image := range gceImages.Images {
+		if image.Project == "" {
+			glog.Fatalf("Invalid config for %v; must specify a project", shortName)
+		}
+	}
+	if len(gceImages.Images) != 0 {
 		if *project == "" {
-			glog.Fatal("Must specify --project flag")
+			glog.Fatal("Must specify --project flag to launch images into")
 		}
 	}
 	if *instanceNamePrefix == "" {
@@ -117,12 +169,12 @@ func main() {
 
 	results := make(chan *TestResult)
 	running := 0
-	if *images != "" {
-		for _, image := range strings.Split(*images, ",") {
-			running++
-			fmt.Printf("Initializing e2e tests using image %s.\n", image)
-			go func(image string, junitFileNum int) { results <- testImage(image, junitFileNum) }(image, running)
-		}
+	for shortName, image := range gceImages.Images {
+		running++
+		fmt.Printf("Initializing e2e tests using image %s.\n", shortName)
+		go func(image, imageProject string, junitFileNum int) {
+			results <- testImage(image, imageProject, junitFileNum)
+		}(image.Image, image.Project, running)
 	}
 	if *hosts != "" {
 		for _, host := range strings.Split(*hosts, ",") {
@@ -213,8 +265,8 @@ func testHost(host string, deleteFiles bool, junitFileNum int, setupNode bool) *
 
 // Provision a gce instance using image and run the tests in archive against the instance.
 // Delete the instance afterward.
-func testImage(image string, junitFileNum int) *TestResult {
-	host, err := createInstance(image)
+func testImage(image, imageProject string, junitFileNum int) *TestResult {
+	host, err := createInstance(image, imageProject)
 	if *deleteInstances {
 		defer deleteInstance(image)
 	}
@@ -231,7 +283,7 @@ func testImage(image string, junitFileNum int) *TestResult {
 }
 
 // Provision a gce instance using image
-func createInstance(image string) (string, error) {
+func createInstance(image, imageProject string) (string, error) {
 	name := imageToInstanceName(image)
 	i := &compute.Instance{
 		Name:        name,
@@ -251,7 +303,7 @@ func createInstance(image string) (string, error) {
 				Boot:       true,
 				Type:       "PERSISTENT",
 				InitializeParams: &compute.AttachedDiskInitializeParams{
-					SourceImage: sourceImage(image),
+					SourceImage: sourceImage(image, imageProject),
 				},
 			},
 		},
@@ -386,8 +438,8 @@ func imageToInstanceName(image string) string {
 	return *instanceNamePrefix + "-" + image
 }
 
-func sourceImage(image string) string {
-	return fmt.Sprintf("projects/%s/global/images/%s", *imageProject, image)
+func sourceImage(image, imageProject string) string {
+	return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image)
 }
 
 func machineType() string {