Add node conformance ci test.

test/e2e_node/jenkins/conformance/conformance-jenkins.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script executed by jenkins to run node conformance test against gce
+# Usage: test/e2e_node/jenkins/conformance-node-jenkins.sh <path to properties>
+
+set -e
+set -x
+
+: "${1:?Usage test/e2e_node/jenkins/conformance-node-jenkins.sh <path to properties>}"
+
+. $1
+
+make generated_files
+
+WORKSPACE=${WORKSPACE:-"/tmp/"}
+ARTIFACTS=${WORKSPACE}/_artifacts
+TIMEOUT=${TIMEOUT:-"45m"}
+
+mkdir -p ${ARTIFACTS}
+
+go run test/e2e_node/runner/remote/run_remote.go  conformance \
+  --logtostderr --vmodule=*=4 --ssh-env="gce" \
+  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
+  --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
+  --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
+  --results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \
+  --test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \
+  --instance-metadata="$GCE_INSTANCE_METADATA"

test/e2e_node/jenkins/conformance/jenkins-conformance.properties

@@ -0,0 +1,6 @@
+GCE_HOSTS=
+GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
+GCE_ZONE=us-central1-f
+GCE_PROJECT=k8s-jkns-ci-node-e2e
+CLEANUP=true
+KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/'

test/e2e_node/remote/node_conformance.go

@@ -0,0 +1,298 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/test/e2e_node/builder"
+)
+
+// ConformanceRemote contains the specific functions in the node conformance test suite.
+type ConformanceRemote struct{}
+
+func InitConformanceRemote() TestSuite {
+	return &ConformanceRemote{}
+}
+
+// getConformanceDirectory gets node conformance test build directory.
+func getConformanceDirectory() (string, error) {
+	k8sRoot, err := builder.GetK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(k8sRoot, "test", "e2e_node", "conformance", "build"), nil
+}
+
+// commandToString is a helper function which formats command to string.
+func commandToString(c *exec.Cmd) string {
+	return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ")
+}
+
+// Image path constants.
+const (
+	conformanceRegistry         = "gcr.io/google_containers"
+	conformanceArch             = runtime.GOARCH
+	conformanceTarfile          = "node_conformance.tar"
+	conformanceTestBinary       = "e2e_node.test"
+	conformanceImageLoadTimeout = time.Duration(30) * time.Second
+)
+
+// timestamp is used as an unique id of current test.
+var timestamp = getTimestamp()
+
+// getConformanceImageRepo returns conformance image full repo name.
+func getConformanceImageRepo() string {
+	return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp)
+}
+
+// buildConformanceTest builds node conformance test image tarball into binDir.
+func buildConformanceTest(binDir string) error {
+	// Get node conformance directory.
+	conformancePath, err := getConformanceDirectory()
+	if err != nil {
+		return fmt.Errorf("failed to get node conformance directory: %v", err)
+	}
+	// Build docker image.
+	cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir,
+		"REGISTRY="+conformanceRegistry,
+		"ARCH="+conformanceArch,
+		"VERSION="+timestamp)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q",
+			commandToString(cmd), err, output)
+	}
+	// Save docker image into tar file.
+	cmd = exec.Command("docker", "save", getConformanceImageRepo(), "-o", filepath.Join(binDir, conformanceTarfile))
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q",
+			commandToString(cmd), err, output)
+	}
+	return nil
+}
+
+// SetupTestPackage sets up the test package with binaries k8s required for node conformance test
+func (c *ConformanceRemote) SetupTestPackage(tardir string) error {
+	// Build the executables
+	if err := builder.BuildGo(); err != nil {
+		return fmt.Errorf("failed to build the depedencies: %v", err)
+	}
+
+	// Make sure we can find the newly built binaries
+	buildOutputDir, err := builder.GetK8sBuildOutputDir()
+	if err != nil {
+		return fmt.Errorf("failed to locate kubernetes build output directory %v", err)
+	}
+
+	// Build node conformance tarball.
+	if err := buildConformanceTest(buildOutputDir); err != nil {
+		return fmt.Errorf("failed to build node conformance test %v", err)
+	}
+
+	// Copy files
+	requiredFiles := []string{"kubelet", conformanceTestBinary, conformanceTarfile}
+	for _, file := range requiredFiles {
+		source := filepath.Join(buildOutputDir, file)
+		if _, err := os.Stat(source); err != nil {
+			return fmt.Errorf("failed to locate test file %s: %v", file, err)
+		}
+		output, err := exec.Command("cp", source, filepath.Join(tardir, file)).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to copy %q: error - %v output - %q", file, err, output)
+		}
+	}
+
+	return nil
+}
+
+// loadConformanceImage loads node conformance image from tar file.
+func loadConformanceImage(host, workspace string) error {
+	tarfile := filepath.Join(workspace, conformanceTarfile)
+	if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(),
+		"docker", "load", "-i", tarfile); err != nil {
+		return fmt.Errorf("failed to load node conformance image from tar file %q: error - %v output - %q",
+			tarfile, err, output)
+	}
+	return nil
+}
+
+// kubeletLauncherLog is the log of kubelet launcher.
+const kubeletLauncherLog = "kubelet-launcher.log"
+
+// kubeletPodManifestPath is a fixed known pod manifest path. We can not use the random pod
+// manifest directory generated in e2e_node.test because we need to mount the directory into
+// the conformance test container, it's easier if it's a known directory.
+// TODO(random-liu): Get rid of this once we switch to cluster e2e node bootstrap script.
+var kubeletPodManifestPath = "conformance-pod-manifest-" + timestamp
+
+// getPodManifestPath returns pod manifest full path.
+func getPodManifestPath(workspace string) string {
+	return filepath.Join(workspace, kubeletPodManifestPath)
+}
+
+// isSystemd returns whether the node is a systemd node.
+func isSystemd(host string) (bool, error) {
+	// Returns "systemd" if /run/systemd/system is found, empty string otherwise.
+	output, err := SSH(host, "test", "-e", "/run/systemd/system", "&&", "echo", "systemd", "||", "true")
+	if err != nil {
+		return false, fmt.Errorf("failed to check systemd: error - %v output - %q", err, output)
+	}
+	return strings.TrimSpace(output) != "", nil
+}
+
+// launchKubelet launches kubelet by running e2e_node.test binary in run-kubelet-mode.
+// This is a temporary solution, we should change node e2e to use the same node bootstrap
+// with cluster e2e and launch kubelet outside of the test for both regular node e2e and
+// node conformance test.
+// TODO(random-liu): Switch to use standard node bootstrap script.
+func launchKubelet(host, workspace, results, testArgs string) error {
+	podManifestPath := getPodManifestPath(workspace)
+	if output, err := SSH(host, "mkdir", podManifestPath); err != nil {
+		return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q",
+			podManifestPath, err, output)
+	}
+	startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --logtostderr --node-name=%s"+
+		" --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1",
+		conformanceTestBinary, host, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog))
+	var cmd []string
+	systemd, err := isSystemd(host)
+	if err != nil {
+		return fmt.Errorf("failed to check systemd: %v", err)
+	}
+	if systemd {
+		cmd = []string{
+			"systemd-run", "sh", "-c", getSSHCommand(" && ",
+				// Switch to workspace.
+				fmt.Sprintf("cd %s", workspace),
+				// Launch kubelet by running e2e_node.test in run-kubelet-mode.
+				startKubeletCmd,
+			),
+		}
+	} else {
+		cmd = []string{
+			"sh", "-c", getSSHCommand(" && ",
+				// Switch to workspace.
+				fmt.Sprintf("cd %s", workspace),
+				// Launch kubelet by running e2e_node.test in run-kubelet-mode with nohup.
+				fmt.Sprintf("(nohup %s &)", startKubeletCmd),
+			),
+		}
+	}
+	glog.Infof("Launch kubelet with command: %v", cmd)
+	output, err := SSH(host, cmd...)
+	if err != nil {
+		return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q",
+			cmd, err, output)
+	}
+	glog.Info("Successfully launch kubelet")
+	return nil
+}
+
+// kubeletStopGracePeriod is the grace period to wait before forcibly killing kubelet.
+const kubeletStopGracePeriod = 10 * time.Second
+
+// stopKubelet stops kubelet launcher and kubelet gracefully.
+func stopKubelet(host, workspace string) error {
+	glog.Info("Gracefully stop kubelet launcher")
+	if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil {
+		return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q",
+			err, output)
+	}
+	glog.Info("Wait for kubelet launcher to stop")
+	stopped := false
+	for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) {
+		// Check whehther the process is still running.
+		output, err := SSH(host, "pidof", conformanceTestBinary, "||", "true")
+		if err != nil {
+			return fmt.Errorf("failed to check kubelet stopping: error - %v output -%q",
+				err, output)
+		}
+		// Kubelet is stopped
+		if strings.TrimSpace(output) == "" {
+			stopped = true
+			break
+		}
+	}
+	if !stopped {
+		glog.Info("Forcibly stop kubelet")
+		if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil {
+			return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q",
+				err, output)
+		}
+	}
+	glog.Info("Successfully stop kubelet")
+	// Clean up the pod manifest path
+	podManifestPath := getPodManifestPath(workspace)
+	if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil {
+		return fmt.Errorf("failed to cleanup pod manifest directory %q: error - %v, output - %q",
+			podManifestPath, err, output)
+	}
+	return nil
+}
+
+// RunTest runs test on the node.
+func (c *ConformanceRemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) {
+	// Install the cni plugin.
+	if err := installCNI(host, workspace); err != nil {
+		return "", err
+	}
+
+	// Configure iptables firewall rules.
+	if err := configureFirewall(host); err != nil {
+		return "", err
+	}
+
+	// Kill any running node processes.
+	cleanupNodeProcesses(host)
+
+	// Load node conformance image.
+	if err := loadConformanceImage(host, workspace); err != nil {
+		return "", err
+	}
+
+	// Launch kubelet.
+	if err := launchKubelet(host, workspace, results, testArgs); err != nil {
+		return "", err
+	}
+	// Stop kubelet.
+	defer func() {
+		if err := stopKubelet(host, workspace); err != nil {
+			// Only log an error if failed to stop kubelet because it is not critical.
+			glog.Errorf("failed to stop kubelet: %v", err)
+		}
+	}()
+
+	// Run the tests
+	glog.Infof("Starting tests on %q", host)
+	podManifestPath := getPodManifestPath(workspace)
+	cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result %s'",
+		timeout.Seconds(), podManifestPath, podManifestPath, results, getConformanceImageRepo())
+	testOutput, err := SSH(host, "sh", "-c", cmd)
+	if err != nil {
+		return testOutput, err
+	}
+
+	return testOutput, nil
+}

test/e2e_node/remote/remote.go

@@ -98,8 +98,15 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, junit
 		return "", false, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
 	}
 
+	// Create the test result directory.
+	resultDir := filepath.Join(workspace, "results")
+	if output, err := SSHNoSudo(host, "mkdir", resultDir); err != nil {
+		// Exit failure with the error
+		return "", false, fmt.Errorf("failed to create test result directory %q on host %q: %v output: %q", resultDir, host, err, output)
+	}
+
 	glog.Infof("Running test on %q", host)
-	output, err := suite.RunTest(host, workspace, filepath.Join(workspace, "results"), junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds)
+	output, err := suite.RunTest(host, workspace, resultDir, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds)
 
 	aggErrs := []error{}
 	// Do not log the output here, let the caller deal with the test output.

test/e2e_node/runner/remote/run_remote.go

@@ -134,6 +134,8 @@ func parseFlags() {
 	// Parse subcommand.
 	subcommand := os.Args[1]
 	switch subcommand {
+	case "conformance":
+		suite = remote.InitConformanceRemote()
 	// TODO: Add subcommand for node soaking, node conformance, cri validation.
 	default:
 		// Use node e2e suite by default if no subcommand is specified.