Merge pull request #48824 from yguo0905/docker-validation

Automatic merge from submit-queue (batch tested with PRs 48082, 48815, 48901, 48824)

Add test image name to the OS image field of the perf metrics

I'd like to add the resource usage benchmarks for COS m60 (docker 1.13.1) but don't want to remove the existing m59 (docker 1.11.2) [ones](https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/jenkins/benchmark/benchmark-config.yaml#L51-L71), in order to compare the results between the two docker versions.

The `image` reported in the metrics is from `Node.Status.NodeInfo.OSImage`, which is always "Container-Optimized OS from Google" (from `/etc/os-releases`) for COS. So there's no way to differentiate two milestones in the metrics.

This PR attaches the [image name](https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/jenkins/benchmark/benchmark-config.yaml#L52) to the `image` field of the metrics. So it will become "Container-Optimized OS from Google (cos-stable-59-9460-64-0)".

See the results of the test run:

[performance-memory-containervm-resource1-resource_0.json](https://storage.googleapis.com/ygg-gke-dev-bucket/e2e-node-test/ci-kubernetes-node-kubelet-benchmark/13/artifacts/performance-memory-containervm-resource1-resource_0.json)
[performance-memory-coreos-resource1-resource_0.json](https://storage.googleapis.com/ygg-gke-dev-bucket/e2e-node-test/ci-kubernetes-node-kubelet-benchmark/13/artifacts/performance-memory-coreos-resource1-resource_0.json)
[performance-memory-gci-resource1-resource_0.json](https://storage.googleapis.com/ygg-gke-dev-bucket/e2e-node-test/ci-kubernetes-node-kubelet-benchmark/13/artifacts/performance-memory-gci-resource1-resource_0.json)

**Release note**:
```
None
```

Ref: https://github.com/kubernetes/kubernetes/issues/42926

/sig node
/area node-e2e
/assign @dchen1107

hack/verify-flags/known-flags.txt

@@ -335,6 +335,7 @@ http-port
 ignore-daemonsets
 ignore-not-found
 image-config-file
+image-description
 image-gc-high-threshold
 image-gc-low-threshold
 image-project

test/e2e/framework/test_context.go

@@ -131,6 +131,8 @@ type NodeTestContextType struct {
 	PrepullImages bool
 	// KubeletConfig is the kubelet configuration the test is running against.
 	KubeletConfig componentconfig.KubeletConfiguration
+	// ImageDescription is the description of the image on which the test is running.
+	ImageDescription string
 }
 
 type CloudConfig struct {
@@ -250,6 +252,7 @@ func RegisterNodeFlags() {
 	// It is hard and unnecessary to deal with the complexity inside the test suite.
 	flag.BoolVar(&TestContext.NodeConformance, "conformance", false, "If true, the test suite will not start kubelet, and fetch system log (kernel, docker, kubelet log etc.) to the report directory.")
 	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
+	flag.StringVar(&TestContext.ImageDescription, "image-description", "", "The description of the image which the test will be running on.")
 }
 
 // ViperizeFlags sets up all flag and config processing. Future configuration info should be added to viper, not to flags.

test/e2e_node/benchmark_util.go

@@ -176,10 +176,14 @@ func getTestNodeInfo(f *framework.Framework, testName, testDesc string) map[stri
 		framework.Failf("Fail to fetch Memory capacity value as Int64.")
 	}
 
+	image := node.Status.NodeInfo.OSImage
+	if framework.TestContext.ImageDescription != "" {
+		image = fmt.Sprintf("%s (%s)", image, framework.TestContext.ImageDescription)
+	}
 	return map[string]string{
 		"node":    nodeName,
 		"test":    testName,
-		"image":   node.Status.NodeInfo.OSImage,
+		"image":   image,
 		"machine": fmt.Sprintf("cpu:%dcore,memory:%.1fGB", cpuValue, float32(memoryValue)/(1024*1024*1024)),
 		"desc":    testDesc,
 	}

test/e2e_node/remote/node_conformance.go

@@ -253,7 +253,7 @@ func stopKubelet(host, workspace string) error {
 }
 
 // RunTest runs test on the node.
-func (c *ConformanceRemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) {
+func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) {
 	// Install the cni plugins and add a basic CNI configuration.
 	if err := setupCNI(host, workspace); err != nil {
 		return "", err

test/e2e_node/remote/node_e2e.go

@@ -163,7 +163,7 @@ func updateOSSpecificKubeletFlags(args, host, workspace string) (string, error)
 }
 
 // RunTest runs test on the node.
-func (n *NodeE2ERemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) {
+func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) {
 	// Install the cni plugins and add a basic CNI configuration.
 	if err := setupCNI(host, workspace); err != nil {
 		return "", err
@@ -186,8 +186,8 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, junitFilePrefix, testA
 	glog.V(2).Infof("Starting tests on %q", host)
 	cmd := getSSHCommand(" && ",
 		fmt.Sprintf("cd %s", workspace),
-		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s %s",
-			timeout.Seconds(), ginkgoArgs, host, results, junitFilePrefix, testArgs),
+		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s",
+			timeout.Seconds(), ginkgoArgs, host, results, junitFilePrefix, imageDesc, testArgs),
 	)
 	return SSH(host, "sh", "-c", cmd)
 }

test/e2e_node/remote/remote.go

@@ -63,7 +63,7 @@ func CreateTestArchive(suite TestSuite) (string, error) {
 
 // Returns the command output, whether the exit was ok, and any errors
 // TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name.
-func RunRemote(suite TestSuite, archive string, host string, cleanup bool, junitFilePrefix string, testArgs string, ginkgoArgs string) (string, bool, error) {
+func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string) (string, bool, error) {
 	// Create the temp staging directory
 	glog.V(2).Infof("Staging test binaries on %q", host)
 	workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp())
@@ -108,7 +108,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, junit
 	}
 
 	glog.V(2).Infof("Running test on %q", host)
-	output, err := suite.RunTest(host, workspace, resultDir, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds)
+	output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds)
 
 	aggErrs := []error{}
 	// Do not log the output here, let the caller deal with the test output.

test/e2e_node/remote/types.go

@@ -34,12 +34,14 @@ type TestSuite interface {
 	// and test error if there is any.
 	// * host is the target node to run the test.
 	// * workspace is the directory on the testing host the test is running in. Note
-	// that the test package is unpacked in the workspace before running the test.
+	//   that the test package is unpacked in the workspace before running the test.
 	// * results is the directory the test should write result into. All logs should be
-	// saved as *.log, all junit file should start with junit*.
+	//   saved as *.log, all junit file should start with junit*.
+	// * imageDesc is the description of the image the test is running on.
+	//   It will be used for logging purpose only.
 	// * junitFilePrefix is the prefix of output junit file.
 	// * testArgs is the arguments passed to test.
 	// * ginkgoArgs is the arguments passed to ginkgo.
 	// * timeout is the test timeout.
-	RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error)
+	RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error)
 }

test/e2e_node/runner/remote/run_remote.go

@@ -282,7 +282,7 @@ func main() {
 			fmt.Printf("Initializing e2e tests using host %s.\n", host)
 			running++
 			go func(host string, junitFilePrefix string) {
-				results <- testHost(host, *cleanup, junitFilePrefix, *ginkgoFlags)
+				results <- testHost(host, *cleanup, "", junitFilePrefix, *ginkgoFlags)
 			}(host, host)
 		}
 	}
@@ -368,7 +368,7 @@ func getImageMetadata(input string) *compute.Metadata {
 }
 
 // Run tests in archive against host
-func testHost(host string, deleteFiles bool, junitFilePrefix string, ginkgoFlagsStr string) *TestResult {
+func testHost(host string, deleteFiles bool, imageDesc, junitFilePrefix, ginkgoFlagsStr string) *TestResult {
 	instance, err := computeService.Instances.Get(*project, *zone, host).Do()
 	if err != nil {
 		return &TestResult{
@@ -398,7 +398,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, ginkgoFlags
 		}
 	}
 
-	output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, junitFilePrefix, *testArgs, ginkgoFlagsStr)
+	output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, imageDesc, junitFilePrefix, *testArgs, ginkgoFlagsStr)
 	return &TestResult{
 		output: output,
 		err:    err,
@@ -484,7 +484,7 @@ func testImage(imageConfig *internalGCEImage, junitFilePrefix string) *TestResul
 	// If we are going to delete the instance, don't bother with cleaning up the files
 	deleteFiles := !*deleteInstances && *cleanup
 
-	result := testHost(host, deleteFiles, junitFilePrefix, ginkgoFlagsStr)
+	result := testHost(host, deleteFiles, imageConfig.image, junitFilePrefix, ginkgoFlagsStr)
 	// This is a temporary solution to collect serial node serial log. Only port 1 contains useful information.
 	// TODO(random-liu): Extract out and unify log collection logic with cluste e2e.
 	serialPortOutput, err := computeService.Instances.GetSerialPortOutput(*project, *zone, host).Port(1).Do()