Merge pull request #28213 from Random-Liu/docker-validation-node-e2e

Automatic merge from submit-queue

Node E2E: Prep for continuous Docker validation node e2e test

Based on https://github.com/kubernetes/kubernetes/pull/28516, for https://github.com/kubernetes/kubernetes/issues/25215.

https://github.com/kubernetes/kubernetes/pull/26813 added support to run e2e test on gci preview image and newest docker version.
This PR added the same support to node e2e test.

The main dependencies of node e2e test are `docker`, `kubelet`, `etcd` and `apiserver`.
Currently, node e2e test builds `kubelet` and `apiserver` locally, and copies them into `/tmp` directory in VM instance. GCI also has built-in `docker`. So the only dependency missing is `etcd`.

This PR injected a simple cloud-init script when creating instance to install `etcd` during node startup.

@andyzheng0831 for the cloud init script.
@wonderfly for the gci instance setup.
@pwittrock for the node e2e test change.

/cc @dchen1107 

[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/.github/PULL_REQUEST_TEMPLATE.md?pixel)]()

hack/verify-flags/known-flags.txt

@@ -214,6 +214,7 @@ input-dirs
 insecure-bind-address
 insecure-port
 insecure-skip-tls-verify
+instance-metadata
 instance-name-prefix
 iptables-masquerade-bit
 iptables-sync-period

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -38,4 +38,4 @@ go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="g
   --zone="$GCE_ZONE" --project="$GCE_PROJECT" --image-project="$GCE_IMAGE_PROJECT" \
   --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" \
   --results-dir="$ARTIFACTS" --ginkgo-flags="$GINKGO_FLAGS" \
-  --setup-node="$SETUP_NODE"
+  --setup-node="$SETUP_NODE" --instance-metadata="$GCE_INSTANCE_METADATA"

test/e2e_node/jenkins/gci-init.yaml

@@ -0,0 +1,9 @@
+#cloud-config
+
+runcmd:
+  - ETCD_VERSION=v2.2.5
+  - curl -L https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd.tar.gz
+  - tar xzvf /tmp/etcd.tar.gz -C /tmp
+  - sudo mv /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcd* /usr/local/bin/
+  - sudo chown root:root /usr/local/bin/etcd*
+  - rm -r /tmp/etcd*

test/e2e_node/jenkins/jenkins-docker-validation.properties

@@ -0,0 +1,18 @@
+GCI_IMAGE_PROJECT=container-vm-image-staging
+GCI_IMAGE_FAMILY=gci-preview-test
+GCI_IMAGE=$(gcloud compute images describe-from-family ${GCI_IMAGE_FAMILY} --project=${GCI_IMAGE_PROJECT} --format="value(name)")
+DOCKER_VERSION=$(curl -fsSL --retry 3 https://api.github.com/repos/docker/docker/releases | tac | tac | grep -m 1 "\"tag_name\"\:" | grep -Eo "[0-9\.rc-]+")
+GCI_CLOUD_INIT=test/e2e_node/jenkins/gci-init.yaml
+
+GCE_HOSTS=
+GCE_IMAGES=${GCI_IMAGE}
+GCE_IMAGE_PROJECT=${GCI_IMAGE_PROJECT}
+GCE_ZONE=us-central1-f
+GCE_PROJECT=kubernetes-jenkins
+# kubernetes-jenkins
+# user-data is the GCI cloud init config file.
+# gci-docker-version specifies docker version in GCI image.
+GCE_INSTANCE_METADATA="user-data<${GCI_CLOUD_INIT},gci-docker-version=${DOCKER_VERSION}"
+CLEANUP=true
+GINKGO_FLAGS=--skip=FLAKY
+SETUP_NODE=false

test/e2e_node/runner/run_e2e.go

@@ -23,6 +23,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"os"
@@ -49,6 +50,7 @@ var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts
 var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created")
 var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
 var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine")
+var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
 
 var computeService *compute.Service
 
@@ -254,6 +256,19 @@ func createInstance(image string) (string, error) {
 			},
 		},
 	}
+	if *instanceMetadata != "" {
+		raw := parseInstanceMetadata(*instanceMetadata)
+		i.Metadata = &compute.Metadata{}
+		metadata := []*compute.MetadataItems{}
+		for k, v := range raw {
+			val := v
+			metadata = append(metadata, &compute.MetadataItems{
+				Key:   k,
+				Value: &val,
+			})
+		}
+		i.Metadata.Items = metadata
+	}
 	op, err := computeService.Instances.Insert(*project, *zone, i).Do()
 	if err != nil {
 		return "", err
@@ -343,6 +358,30 @@ func deleteInstance(image string) {
 	}
 }
 
+func parseInstanceMetadata(str string) map[string]string {
+	metadata := make(map[string]string)
+	ss := strings.Split(str, ",")
+	for _, s := range ss {
+		kv := strings.Split(s, "=")
+		if len(kv) == 2 {
+			metadata[kv[0]] = kv[1]
+			continue
+		}
+		kp := strings.Split(s, "<")
+		if len(kp) != 2 {
+			glog.Errorf("Invalid instance metadata: %q", s)
+			continue
+		}
+		v, err := ioutil.ReadFile(kp[1])
+		if err != nil {
+			glog.Errorf("Failed to read metadata file %q: %v", kp[1], err)
+			continue
+		}
+		metadata[kp[0]] = string(v)
+	}
+	return metadata
+}
+
 func imageToInstanceName(image string) string {
 	return *instanceNamePrefix + "-" + image
 }