Merge pull request #29815 from vishh/node-e2e-latest-gci

Automatic merge from submit-queue

Use latest GCI image based on a regex in Node e2e

This PR also makes it possible to run node e2e against multiple previous images, sorted by creation time. A regex for the image name can be used to instruct node e2e to identify test images.

Depends on #29577

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.kubernetes.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.kubernetes.io/reviews/kubernetes/kubernetes/29815)
<!-- Reviewable:end -->

cluster/gce/util.sh

@@ -39,7 +39,6 @@ fi
 
 function get_latest_gci_image() {
   # GCI milestone to use
-  # Update the GCI image in test/e2e_node/jenkins/image-config.yaml before updating the milestone here.
   GCI_MILESTONE="53"
 
   # First try to find an active (non-deprecated) image on this milestone.

hack/make-rules/test-e2e-node.sh

@@ -31,17 +31,20 @@ artifacts=${ARTIFACTS:-"/tmp/_artifacts"}
 remote=${REMOTE:-"false"}
 images=${IMAGES:-""}
 hosts=${HOSTS:-""}
+metadata=${INSTANCE_METADATA:-""}
+gci_image=$(gcloud compute images list --project google-containers \
+    --no-standard-images --regexp="gci-dev.*" --format="table[no-heading](name)")
 if [[ $hosts == "" && $images == "" ]]; then
-  images="e2e-node-containervm-v20160321-image"
+  images=$gci_image
+  metadata="user-data<${KUBE_ROOT}/test/e2e_node/jenkins/gci-init.yaml"
 fi
-image_project=${IMAGE_PROJECT:-"kubernetes-node-e2e-images"}
+image_project=${IMAGE_PROJECT:-"google-containers"}
 instance_prefix=${INSTANCE_PREFIX:-"test"}
 cleanup=${CLEANUP:-"true"}
 delete_instances=${DELETE_INSTANCES:-"false"}
 run_until_failure=${RUN_UNTIL_FAILURE:-"false"}
 list_images=${LIST_IMAGES:-"false"}
 test_args=${TEST_ARGS:-""}
-metadata=${INSTANCE_METADATA:-""}
 
 if  [[ $list_images == "true" ]]; then
   gcloud compute images list --project="${image_project}" | grep "e2e-node"
@@ -126,7 +129,7 @@ if [ $remote = true ] ; then
   echo "Ginkgo Flags: $ginkgoflags"
   echo "Instance Metadata: $metadata"
   # Invoke the runner
-  go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
+  go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
     --zone="$zone" --project="$project"  \
     --hosts="$hosts" --images="$images" --cleanup="$cleanup" \
     --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \

test/e2e_node/image_list.go

@@ -18,6 +18,7 @@ package e2e_node
 
 import (
 	"os/exec"
+	"os/user"
 	"time"
 
 	"github.com/golang/glog"
@@ -66,6 +67,10 @@ var NoPullImageRegistry = map[int]string{
 
 // Pre-fetch all images tests depend on so that we don't fail in an actual test
 func PrePullAllImages() error {
+	usr, err := user.Current()
+	if err != nil {
+		return err
+	}
 	for _, image := range ImageRegistry {
 		var (
 			err    error
@@ -78,8 +83,8 @@ func PrePullAllImages() error {
 			if output, err = exec.Command("docker", "pull", image).CombinedOutput(); err == nil {
 				break
 			}
-			glog.Warningf("Failed to pull %s, retrying in %s (%d of %d): %v",
+			glog.Warningf("Failed to pull %s as user %q, retrying in %s (%d of %d): %v",
-				image, imagePullRetryDelay.String(), i+1, maxImagePullRetries, err)
+				image, usr.Username, imagePullRetryDelay.String(), i+1, maxImagePullRetries, err)
 		}
 		if err != nil {
 			glog.Warningf("Could not pre-pull image %s %v output:  %s", image, err, output)

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -40,7 +40,7 @@ WORKSPACE=${WORKSPACE:-"/tmp/"}
 ARTIFACTS=${WORKSPACE}/_artifacts
 
 mkdir -p ${ARTIFACTS}
-go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
+go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
   --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
   --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
   --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \

test/e2e_node/jenkins/gci-init.yaml

@@ -2,9 +2,8 @@
 
 runcmd:
   - mount /tmp /tmp -o remount,exec,suid
-  - ETCD_VERSION=v3.0.3
+  - etcd_version=v2.2.5
-  - curl -L https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd.tar.gz
+  - curl -L https://github.com/coreos/etcd/releases/download/${etcd_version}/etcd-${etcd_version}-linux-amd64.tar.gz -o /tmp/etcd.tar.gz
   - tar xzvf /tmp/etcd.tar.gz -C /tmp
-  - cp /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcd* /tmp/
+  - cp  /tmp/etcd-${etcd_version}-linux-amd64/etcd* /tmp/
-  - rm -rf /tmp/etcd-${ETCD_VERSION}-linux-amd64/
+  - rm -rf /tmp/etcd-${etcd_version}-linux-amd64/
-  - usermod -a -G docker jenkins

test/e2e_node/jenkins/image-config.yaml

@@ -15,7 +15,7 @@ images:
   containervm:
     image: e2e-node-containervm-v20160321-image
     project: kubernetes-node-e2e-images
-  gci-dev:
+  gci-family:
-    image: gci-dev-53-8530-29-0
+    image_regex: gci-dev-.*
     project: google-containers
     metadata: "user-data<test/e2e_node/jenkins/gci-init.yaml"

test/e2e_node/runner/run_e2e.go

@@ -27,6 +27,8 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"regexp"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -86,9 +88,13 @@ type ImageConfig struct {
 }
 
 type GCEImage struct {
-	Image    string `json:"image"`
+	Image      string `json:"image, omitempty"`
-	Project  string `json:"project"`
+	Project    string `json:"project"`
-	Metadata string `json:"metadata"`
+	Metadata   string `json:"metadata"`
+	ImageRegex string `json:"image_regex, omitempty"`
+	// Defaults to using only the latest image. Acceptible values are [0, # of images that match the regex).
+	// If the number of existing previous images is lesser than what is desired, the test will use that is available.
+	PreviousImages int `json:"previous_images, omitempty"`
 }
 
 type internalImageConfig struct {
@@ -113,6 +119,12 @@ func main() {
 	if *hosts == "" && *imageConfigFile == "" && *images == "" {
 		glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
 	}
+	var err error
+	computeService, err = getComputeClient()
+	if err != nil {
+		glog.Fatalf("Unable to create gcloud compute service using defaults.  Make sure you are authenticated. %v", err)
+	}
+
 	gceImages := &internalImageConfig{
 		images: make(map[string]internalGCEImage),
 	}
@@ -127,13 +139,24 @@ func main() {
 		if err != nil {
 			glog.Fatalf("Could not parse image config file: %v", err)
 		}
-		for key, imageConfig := range externalImageConfig.Images {
+		for _, imageConfig := range externalImageConfig.Images {
-			gceImage := internalGCEImage{
+			var images []string
-				image:    imageConfig.Image,
+			if imageConfig.ImageRegex != "" && imageConfig.Image == "" {
-				project:  imageConfig.Project,
+				images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages)
-				metadata: getImageMetadata(imageConfig.Metadata),
+				if err != nil {
+					glog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err)
+				}
+			} else {
+				images = []string{imageConfig.Image}
+			}
+			for _, image := range images {
+				gceImage := internalGCEImage{
+					image:    image,
+					project:  imageConfig.Project,
+					metadata: getImageMetadata(imageConfig.Metadata),
+				}
+				gceImages.images[image] = gceImage
 			}
-			gceImages.images[key] = gceImage
 		}
 	}
 
@@ -184,12 +207,6 @@ func main() {
 	go arc.getArchive()
 	defer arc.deleteArchive()
 
-	var err error
-	computeService, err = getComputeClient()
-	if err != nil {
-		glog.Fatalf("Unable to create gcloud compute service using defaults.  Make sure you are authenticated. %v", err)
-	}
-
 	results := make(chan *TestResult)
 	running := 0
 	for shortName := range gceImages.images {
@@ -306,6 +323,55 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b
 	}
 }
 
+type imageObj struct {
+	creationTime time.Time
+	name         string
+}
+
+func (io imageObj) string() string {
+	return fmt.Sprintf("%q created %q", io.name, io.creationTime.String())
+}
+
+type byCreationTime []imageObj
+
+func (a byCreationTime) Len() int           { return len(a) }
+func (a byCreationTime) Less(i, j int) bool { return a[i].creationTime.After(a[j].creationTime) }
+func (a byCreationTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+// Returns a list of image names based on regex and number of previous images requested.
+func getGCEImages(imageRegex, project string, previousImages int) ([]string, error) {
+	ilc, err := computeService.Images.List(project).Do()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to list images in project %q and zone %q", project, zone)
+	}
+	imageObjs := []imageObj{}
+	imageRe := regexp.MustCompile(imageRegex)
+	for _, instance := range ilc.Items {
+		if imageRe.MatchString(instance.Name) {
+			creationTime, err := time.Parse(time.RFC3339, instance.CreationTimestamp)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to parse instance creation timestamp %q: %v", instance.CreationTimestamp, err)
+			}
+			io := imageObj{
+				creationTime: creationTime,
+				name:         instance.Name,
+			}
+			glog.V(4).Infof("Found image %q based on regex %q in project %q", io.string(), imageRegex, project)
+			imageObjs = append(imageObjs, io)
+		}
+	}
+	sort.Sort(byCreationTime(imageObjs))
+	images := []string{}
+	for _, imageObj := range imageObjs {
+		images = append(images, imageObj.name)
+		previousImages--
+		if previousImages < 0 {
+			break
+		}
+	}
+	return images, nil
+}
+
 // Provision a gce instance using image and run the tests in archive against the instance.
 // Delete the instance afterward.
 func testImage(image *internalGCEImage, junitFilePrefix string) *TestResult {
@@ -327,6 +393,7 @@ func testImage(image *internalGCEImage, junitFilePrefix string) *TestResult {
 
 // Provision a gce instance using image
 func createInstance(image *internalGCEImage) (string, error) {
+	glog.V(1).Infof("Creating instance %+v", *image)
 	name := imageToInstanceName(image.image)
 	i := &compute.Instance{
 		Name:        name,