Merge pull request #38153 from Random-Liu/node-conformance-ci

Automatic merge from submit-queue

Node Conformance: Node Conformance CI

For https://github.com/kubernetes/kubernetes/issues/37252.

The first 2 commits of this PR are from #38150 and #38152. Please review those 2 PRs first, they are both minor cleanup.

This PR:
* Add `TestSuite` interface in `test/e2e_node/remote` to separate test suite logic (packaging, deploy, run test) from VM lifecycle management logic, so that different test suites can share the same VM lifecycle management logic.
  * Different test suites such as node e2e, node conformance, node soaking, cri validation etc. should implement different `TestSuite`.
  * `test/e2e_node/runner/remote` will initialize and run different test suite based on the subcommand.
* Add `run-kubelet-mode` which only starts and monitors kubelet, similar with `run-services-mode`. The reason we need this:
  * Unlike node e2e, node conformance test doesn't start kubelet inside the test suite (in fact, in the future node e2e shouldn't do that either), it assumes kubelet is already running before the test.
  * In fact, node e2e should use similar node bootstrap script like cluster e2e, and the bootstrap script should initialize the node with all necessary node software including kubelet. However, it's not the case now.
  * The easiest way for now is to reuse the kubelet start logic in the test suite. So in this PR, we added `run-kubelet-mode`, and use the test binary as a kubelet launcher to start kubelet before running the test.
* Implement node e2e `TestSuite`.
* Implement node conformance `TestSuite`. Use `docker save` and `docker load` to create and deploy conformance docker image; Start kubelet by running test binary in `run-kubelet-mode`; Run conformance test with `docker run`.

This PR will make it easy to implement continuous integration node soaking test and cri validation test (https://github.com/kubernetes/kubernetes/pull/35266).

/cc @kubernetes/sig-node

hack/verify-flags/known-flags.txt

@@ -525,6 +525,7 @@ rkt-stage1-image
 root-ca-file
 root-dir
 route-reconciliation-period
+run-kubelet-mode
 run-proxy
 run-services-mode
 runtime-cgroups

test/e2e_node/e2e_node_suite_test.go

@@ -51,6 +51,7 @@ var e2es *services.E2EServices
 
 // TODO(random-liu): Change the following modes to sub-command.
 var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
+var runKubeletMode = flag.Bool("run-kubelet-mode", false, "If true, only start kubelet, and not run test.")
 var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.")
 
 func init() {
@@ -81,6 +82,11 @@ func TestE2eNode(t *testing.T) {
 		services.RunE2EServices()
 		return
 	}
+	if *runKubeletMode {
+		// If run-kubelet-mode is specified, only start kubelet.
+		services.RunKubelet()
+		return
+	}
 	if *systemValidateMode {
 		// If system-validate-mode is specified, only run system validation in current process.
 		if framework.TestContext.NodeConformance {

test/e2e_node/jenkins/conformance/conformance-jenkins.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script executed by jenkins to run node conformance test against gce
+# Usage: test/e2e_node/jenkins/conformance-node-jenkins.sh <path to properties>
+
+set -e
+set -x
+
+: "${1:?Usage test/e2e_node/jenkins/conformance-node-jenkins.sh <path to properties>}"
+
+. $1
+
+make generated_files
+
+WORKSPACE=${WORKSPACE:-"/tmp/"}
+ARTIFACTS=${WORKSPACE}/_artifacts
+TIMEOUT=${TIMEOUT:-"45m"}
+
+mkdir -p ${ARTIFACTS}
+
+go run test/e2e_node/runner/remote/run_remote.go  conformance \
+  --logtostderr --vmodule=*=4 --ssh-env="gce" \
+  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
+  --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
+  --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
+  --results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \
+  --test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \
+  --instance-metadata="$GCE_INSTANCE_METADATA"

test/e2e_node/jenkins/conformance/jenkins-conformance.properties

@@ -0,0 +1,6 @@
+GCE_HOSTS=
+GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
+GCE_ZONE=us-central1-f
+GCE_PROJECT=k8s-jkns-ci-node-e2e
+CLEANUP=true
+KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/'

test/e2e_node/remote/BUILD

@@ -10,8 +10,12 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "node_conformance.go",
+        "node_e2e.go",
         "remote.go",
         "ssh.go",
+        "types.go",
+        "utils.go",
     ],
     tags = ["automanaged"],
     deps = [

test/e2e_node/remote/node_conformance.go

@@ -0,0 +1,298 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/test/e2e_node/builder"
+)
+
+// ConformanceRemote contains the specific functions in the node conformance test suite.
+type ConformanceRemote struct{}
+
+func InitConformanceRemote() TestSuite {
+	return &ConformanceRemote{}
+}
+
+// getConformanceDirectory gets node conformance test build directory.
+func getConformanceDirectory() (string, error) {
+	k8sRoot, err := builder.GetK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(k8sRoot, "test", "e2e_node", "conformance", "build"), nil
+}
+
+// commandToString is a helper function which formats command to string.
+func commandToString(c *exec.Cmd) string {
+	return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ")
+}
+
+// Image path constants.
+const (
+	conformanceRegistry         = "gcr.io/google_containers"
+	conformanceArch             = runtime.GOARCH
+	conformanceTarfile          = "node_conformance.tar"
+	conformanceTestBinary       = "e2e_node.test"
+	conformanceImageLoadTimeout = time.Duration(30) * time.Second
+)
+
+// timestamp is used as an unique id of current test.
+var timestamp = getTimestamp()
+
+// getConformanceImageRepo returns conformance image full repo name.
+func getConformanceImageRepo() string {
+	return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp)
+}
+
+// buildConformanceTest builds node conformance test image tarball into binDir.
+func buildConformanceTest(binDir string) error {
+	// Get node conformance directory.
+	conformancePath, err := getConformanceDirectory()
+	if err != nil {
+		return fmt.Errorf("failed to get node conformance directory: %v", err)
+	}
+	// Build docker image.
+	cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir,
+		"REGISTRY="+conformanceRegistry,
+		"ARCH="+conformanceArch,
+		"VERSION="+timestamp)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q",
+			commandToString(cmd), err, output)
+	}
+	// Save docker image into tar file.
+	cmd = exec.Command("docker", "save", getConformanceImageRepo(), "-o", filepath.Join(binDir, conformanceTarfile))
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q",
+			commandToString(cmd), err, output)
+	}
+	return nil
+}
+
+// SetupTestPackage sets up the test package with binaries k8s required for node conformance test
+func (c *ConformanceRemote) SetupTestPackage(tardir string) error {
+	// Build the executables
+	if err := builder.BuildGo(); err != nil {
+		return fmt.Errorf("failed to build the depedencies: %v", err)
+	}
+
+	// Make sure we can find the newly built binaries
+	buildOutputDir, err := builder.GetK8sBuildOutputDir()
+	if err != nil {
+		return fmt.Errorf("failed to locate kubernetes build output directory %v", err)
+	}
+
+	// Build node conformance tarball.
+	if err := buildConformanceTest(buildOutputDir); err != nil {
+		return fmt.Errorf("failed to build node conformance test %v", err)
+	}
+
+	// Copy files
+	requiredFiles := []string{"kubelet", conformanceTestBinary, conformanceTarfile}
+	for _, file := range requiredFiles {
+		source := filepath.Join(buildOutputDir, file)
+		if _, err := os.Stat(source); err != nil {
+			return fmt.Errorf("failed to locate test file %s: %v", file, err)
+		}
+		output, err := exec.Command("cp", source, filepath.Join(tardir, file)).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to copy %q: error - %v output - %q", file, err, output)
+		}
+	}
+
+	return nil
+}
+
+// loadConformanceImage loads node conformance image from tar file.
+func loadConformanceImage(host, workspace string) error {
+	tarfile := filepath.Join(workspace, conformanceTarfile)
+	if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(),
+		"docker", "load", "-i", tarfile); err != nil {
+		return fmt.Errorf("failed to load node conformance image from tar file %q: error - %v output - %q",
+			tarfile, err, output)
+	}
+	return nil
+}
+
+// kubeletLauncherLog is the log of kubelet launcher.
+const kubeletLauncherLog = "kubelet-launcher.log"
+
+// kubeletPodManifestPath is a fixed known pod manifest path. We can not use the random pod
+// manifest directory generated in e2e_node.test because we need to mount the directory into
+// the conformance test container, it's easier if it's a known directory.
+// TODO(random-liu): Get rid of this once we switch to cluster e2e node bootstrap script.
+var kubeletPodManifestPath = "conformance-pod-manifest-" + timestamp
+
+// getPodManifestPath returns pod manifest full path.
+func getPodManifestPath(workspace string) string {
+	return filepath.Join(workspace, kubeletPodManifestPath)
+}
+
+// isSystemd returns whether the node is a systemd node.
+func isSystemd(host string) (bool, error) {
+	// Returns "systemd" if /run/systemd/system is found, empty string otherwise.
+	output, err := SSH(host, "test", "-e", "/run/systemd/system", "&&", "echo", "systemd", "||", "true")
+	if err != nil {
+		return false, fmt.Errorf("failed to check systemd: error - %v output - %q", err, output)
+	}
+	return strings.TrimSpace(output) != "", nil
+}
+
+// launchKubelet launches kubelet by running e2e_node.test binary in run-kubelet-mode.
+// This is a temporary solution, we should change node e2e to use the same node bootstrap
+// with cluster e2e and launch kubelet outside of the test for both regular node e2e and
+// node conformance test.
+// TODO(random-liu): Switch to use standard node bootstrap script.
+func launchKubelet(host, workspace, results, testArgs string) error {
+	podManifestPath := getPodManifestPath(workspace)
+	if output, err := SSH(host, "mkdir", podManifestPath); err != nil {
+		return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q",
+			podManifestPath, err, output)
+	}
+	startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --logtostderr --node-name=%s"+
+		" --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1",
+		conformanceTestBinary, host, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog))
+	var cmd []string
+	systemd, err := isSystemd(host)
+	if err != nil {
+		return fmt.Errorf("failed to check systemd: %v", err)
+	}
+	if systemd {
+		cmd = []string{
+			"systemd-run", "sh", "-c", getSSHCommand(" && ",
+				// Switch to workspace.
+				fmt.Sprintf("cd %s", workspace),
+				// Launch kubelet by running e2e_node.test in run-kubelet-mode.
+				startKubeletCmd,
+			),
+		}
+	} else {
+		cmd = []string{
+			"sh", "-c", getSSHCommand(" && ",
+				// Switch to workspace.
+				fmt.Sprintf("cd %s", workspace),
+				// Launch kubelet by running e2e_node.test in run-kubelet-mode with nohup.
+				fmt.Sprintf("(nohup %s &)", startKubeletCmd),
+			),
+		}
+	}
+	glog.V(2).Infof("Launch kubelet with command: %v", cmd)
+	output, err := SSH(host, cmd...)
+	if err != nil {
+		return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q",
+			cmd, err, output)
+	}
+	glog.Info("Successfully launch kubelet")
+	return nil
+}
+
+// kubeletStopGracePeriod is the grace period to wait before forcibly killing kubelet.
+const kubeletStopGracePeriod = 10 * time.Second
+
+// stopKubelet stops kubelet launcher and kubelet gracefully.
+func stopKubelet(host, workspace string) error {
+	glog.Info("Gracefully stop kubelet launcher")
+	if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil {
+		return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q",
+			err, output)
+	}
+	glog.Info("Wait for kubelet launcher to stop")
+	stopped := false
+	for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) {
+		// Check whehther the process is still running.
+		output, err := SSH(host, "pidof", conformanceTestBinary, "||", "true")
+		if err != nil {
+			return fmt.Errorf("failed to check kubelet stopping: error - %v output -%q",
+				err, output)
+		}
+		// Kubelet is stopped
+		if strings.TrimSpace(output) == "" {
+			stopped = true
+			break
+		}
+	}
+	if !stopped {
+		glog.Info("Forcibly stop kubelet")
+		if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil {
+			return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q",
+				err, output)
+		}
+	}
+	glog.Info("Successfully stop kubelet")
+	// Clean up the pod manifest path
+	podManifestPath := getPodManifestPath(workspace)
+	if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil {
+		return fmt.Errorf("failed to cleanup pod manifest directory %q: error - %v, output - %q",
+			podManifestPath, err, output)
+	}
+	return nil
+}
+
+// RunTest runs test on the node.
+func (c *ConformanceRemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) {
+	// Install the cni plugin.
+	if err := installCNI(host, workspace); err != nil {
+		return "", err
+	}
+
+	// Configure iptables firewall rules.
+	if err := configureFirewall(host); err != nil {
+		return "", err
+	}
+
+	// Kill any running node processes.
+	cleanupNodeProcesses(host)
+
+	// Load node conformance image.
+	if err := loadConformanceImage(host, workspace); err != nil {
+		return "", err
+	}
+
+	// Launch kubelet.
+	if err := launchKubelet(host, workspace, results, testArgs); err != nil {
+		return "", err
+	}
+	// Stop kubelet.
+	defer func() {
+		if err := stopKubelet(host, workspace); err != nil {
+			// Only log an error if failed to stop kubelet because it is not critical.
+			glog.Errorf("failed to stop kubelet: %v", err)
+		}
+	}()
+
+	// Run the tests
+	glog.V(2).Infof("Starting tests on %q", host)
+	podManifestPath := getPodManifestPath(workspace)
+	cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result %s'",
+		timeout.Seconds(), podManifestPath, podManifestPath, results, getConformanceImageRepo())
+	testOutput, err := SSH(host, "sh", "-c", cmd)
+	if err != nil {
+		return testOutput, err
+	}
+
+	return testOutput, nil
+}

test/e2e_node/remote/node_e2e.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/test/e2e_node/builder"
+)
+
+// NodeE2ERemote contains the specific functions in the node e2e test suite.
+type NodeE2ERemote struct{}
+
+func InitNodeE2ERemote() TestSuite {
+	// TODO: Register flags.
+	return &NodeE2ERemote{}
+}
+
+const localGCIMounterPath = "cluster/gce/gci/mounter/mounter"
+
+// SetupTestPackage sets up the test package with binaries k8s required for node e2e tests
+func (n *NodeE2ERemote) SetupTestPackage(tardir string) error {
+	// Build the executables
+	if err := builder.BuildGo(); err != nil {
+		return fmt.Errorf("failed to build the depedencies: %v", err)
+	}
+
+	// Make sure we can find the newly built binaries
+	buildOutputDir, err := builder.GetK8sBuildOutputDir()
+	if err != nil {
+		return fmt.Errorf("failed to locate kubernetes build output directory %v", err)
+	}
+
+	// Copy binaries
+	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo"}
+	for _, bin := range requiredBins {
+		source := filepath.Join(buildOutputDir, bin)
+		if _, err := os.Stat(source); err != nil {
+			return fmt.Errorf("failed to locate test binary %s: %v", bin, err)
+		}
+		out, err := exec.Command("cp", source, filepath.Join(tardir, bin)).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to copy %q: %v Output: %q", bin, err, out)
+		}
+	}
+
+	// Include the GCI mounter artifacts in the deployed tarball
+	k8sDir, err := builder.GetK8sRootDir()
+	if err != nil {
+		return fmt.Errorf("Could not find K8s root dir! Err: %v", err)
+	}
+	source := filepath.Join(k8sDir, localGCIMounterPath)
+
+	// Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves
+	if _, err := os.Stat(source); err != nil {
+		return fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err)
+	}
+
+	bindir := "cluster/gce/gci/mounter"
+	bin := "mounter"
+	destdir := filepath.Join(tardir, bindir)
+	dest := filepath.Join(destdir, bin)
+	out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out)
+	}
+	out, err = exec.Command("cp", source, dest).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out)
+	}
+	return nil
+}
+
+// updateGCIMounterPath updates kubelet flags to set gci mounter path. This will only take effect for
+// GCI image.
+func updateGCIMounterPath(args, host, workspace string) (string, error) {
+	// Determine if tests will run on a GCI node.
+	output, err := SSH(host, "cat", "/etc/os-release")
+	if err != nil {
+		return args, fmt.Errorf("issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
+	}
+	if !strings.Contains(output, "ID=gci") {
+		// This is not a GCI image
+		return args, nil
+	}
+
+	// If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args.
+	// We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run.
+
+	// Determine if the GCI mounter script exists locally.
+	k8sDir, err := builder.GetK8sRootDir()
+	if err != nil {
+		return args, fmt.Errorf("could not find K8s root dir! Err: %v", err)
+	}
+	source := filepath.Join(k8sDir, localGCIMounterPath)
+
+	// Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves
+	if _, err = os.Stat(source); err != nil {
+		return args, fmt.Errorf("could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err)
+	}
+
+	glog.V(2).Infof("GCI node and GCI mounter both detected, modifying --experimental-mounter-path accordingly")
+	// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
+	// here will tell us to update the remote test runner.
+	mounterPath := filepath.Join(workspace, localGCIMounterPath)
+	output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
+	if err != nil {
+		return args, fmt.Errorf("unabled to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output)
+	}
+	// Insert args at beginning of test args, so any values from command line take precedence
+	args = fmt.Sprintf("--kubelet-flags=--experimental-mounter-path=%s ", mounterPath) + args
+	return args, nil
+}
+
+// RunTest runs test on the node.
+func (n *NodeE2ERemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) {
+	// Install the cni plugin.
+	if err := installCNI(host, workspace); err != nil {
+		return "", err
+	}
+
+	// Configure iptables firewall rules
+	if err := configureFirewall(host); err != nil {
+		return "", err
+	}
+
+	// Kill any running node processes
+	cleanupNodeProcesses(host)
+
+	testArgs, err := updateGCIMounterPath(testArgs, host, workspace)
+	if err != nil {
+		return "", err
+	}
+
+	// Run the tests
+	glog.V(2).Infof("Starting tests on %q", host)
+	cmd := getSSHCommand(" && ",
+		fmt.Sprintf("cd %s", workspace),
+		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s %s",
+			timeout.Seconds(), ginkgoArgs, host, results, junitFilePrefix, testArgs),
+	)
+	return SSH(host, "sh", "-c", cmd)
+}

test/e2e_node/remote/remote.go

@@ -23,87 +23,33 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/golang/glog"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
-	"k8s.io/kubernetes/test/e2e_node/builder"
 )
 
 var testTimeoutSeconds = flag.Duration("test-timeout", 45*time.Minute, "How long (in golang duration format) to wait for ginkgo tests to complete.")
 var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.")
 
-const (
-	archiveName  = "e2e_node_test.tar.gz"
-	CNIRelease   = "07a8a28637e97b22eb8dfe710eeae1344f69d16e"
-	CNIDirectory = "cni"
-)
+const archiveName = "e2e_node_test.tar.gz"
 
-var CNIURL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-%s.tar.gz", CNIRelease)
-
-// CreateTestArchive builds the local source and creates a tar archive e2e_node_test.tar.gz containing
-// the binaries k8s required for node e2e tests
-func CreateTestArchive() (string, error) {
-	// Build the executables
-	if err := builder.BuildGo(); err != nil {
-		return "", fmt.Errorf("failed to build the depedencies: %v", err)
-	}
-
-	// Make sure we can find the newly built binaries
-	buildOutputDir, err := builder.GetK8sBuildOutputDir()
-	if err != nil {
-		return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err)
-	}
-
-	glog.Infof("Building archive...")
+func CreateTestArchive(suite TestSuite) (string, error) {
+	glog.V(2).Infof("Building archive...")
 	tardir, err := ioutil.TempDir("", "node-e2e-archive")
 	if err != nil {
 		return "", fmt.Errorf("failed to create temporary directory %v.", err)
 	}
 	defer os.RemoveAll(tardir)
 
-	// Copy binaries
-	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo"}
-	for _, bin := range requiredBins {
-		source := filepath.Join(buildOutputDir, bin)
-		if _, err := os.Stat(source); err != nil {
-			return "", fmt.Errorf("failed to locate test binary %s: %v", bin, err)
-		}
-		out, err := exec.Command("cp", source, filepath.Join(tardir, bin)).CombinedOutput()
-		if err != nil {
-			return "", fmt.Errorf("failed to copy %q: %v Output: %q", bin, err, out)
-		}
-	}
-
-	// Include the GCI mounter artifacts in the deployed tarball
-	k8sDir, err := builder.GetK8sRootDir()
+	// Call the suite function to setup the test package.
+	err = suite.SetupTestPackage(tardir)
 	if err != nil {
-		return "", fmt.Errorf("Could not find K8s root dir! Err: %v", err)
-	}
-	localSource := "cluster/gce/gci/mounter/mounter"
-	source := filepath.Join(k8sDir, localSource)
-
-	// Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves
-	if _, err := os.Stat(source); err != nil {
-		return "", fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err)
-	}
-
-	bindir := "cluster/gce/gci/mounter"
-	bin := "mounter"
-	destdir := filepath.Join(tardir, bindir)
-	dest := filepath.Join(destdir, bin)
-	out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out)
-	}
-	out, err = exec.Command("cp", source, dest).CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out)
+		return "", fmt.Errorf("failed to setup test package %q: %v", tardir, err)
 	}
 
 	// Build the tar
-	out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput()
+	out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput()
 	if err != nil {
 		return "", fmt.Errorf("failed to build tar %v.  Output:\n%s", err, out)
 	}
@@ -116,170 +62,60 @@ func CreateTestArchive() (string, error) {
 }
 
 // Returns the command output, whether the exit was ok, and any errors
-func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, testArgs string, ginkgoFlags string) (string, bool, error) {
+// TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name.
+func RunRemote(suite TestSuite, archive string, host string, cleanup bool, junitFilePrefix string, testArgs string, ginkgoArgs string) (string, bool, error) {
 	// Create the temp staging directory
-	glog.Infof("Staging test binaries on %s", host)
+	glog.V(2).Infof("Staging test binaries on %q", host)
 	workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp())
 	// Do not sudo here, so that we can use scp to copy test archive to the directdory.
 	if output, err := SSHNoSudo(host, "mkdir", workspace); err != nil {
 		// Exit failure with the error
-		return "", false, fmt.Errorf("failed to create workspace directory: %v output: %q", err, output)
+		return "", false, fmt.Errorf("failed to create workspace directory %q on host %q: %v output: %q", workspace, host, err, output)
 	}
 	if cleanup {
 		defer func() {
 			output, err := SSH(host, "rm", "-rf", workspace)
 			if err != nil {
-				glog.Errorf("failed to cleanup workspace %s on host %v.  Output:\n%s", workspace, err, output)
+				glog.Errorf("failed to cleanup workspace %q on host %q: %v.  Output:\n%s", workspace, host, err, output)
 			}
 		}()
 	}
 
-	// Install the cni plugin.
-	cniPath := filepath.Join(workspace, CNIDirectory)
-	cmd := getSSHCommand(" ; ",
-		fmt.Sprintf("mkdir -p %s", cniPath),
-		fmt.Sprintf("wget -O - %s | tar -xz -C %s", CNIURL, cniPath),
-	)
-	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
-		// Exit failure with the error
-		return "", false, fmt.Errorf("failed to install cni plugin: %v output: %q", err, output)
-	}
-
-	// Configure iptables firewall rules
-	// TODO: consider calling bootstrap script to configure host based on OS
-	output, err := SSH(host, "iptables", "-L", "INPUT")
-	if err != nil {
-		return "", false, fmt.Errorf("failed to get iptables INPUT: %v output: %q", err, output)
-	}
-	if strings.Contains(output, "Chain INPUT (policy DROP)") {
-		cmd = getSSHCommand("&&",
-			"(iptables -C INPUT -w -p TCP -j ACCEPT || iptables -A INPUT -w -p TCP -j ACCEPT)",
-			"(iptables -C INPUT -w -p UDP -j ACCEPT || iptables -A INPUT -w -p UDP -j ACCEPT)",
-			"(iptables -C INPUT -w -p ICMP -j ACCEPT || iptables -A INPUT -w -p ICMP -j ACCEPT)")
-		output, err := SSH(host, "sh", "-c", cmd)
-		if err != nil {
-			return "", false, fmt.Errorf("failed to configured firewall: %v output: %v", err, output)
-		}
-	}
-	output, err = SSH(host, "iptables", "-L", "FORWARD")
-	if err != nil {
-		return "", false, fmt.Errorf("failed to get iptables FORWARD: %v output: %q", err, output)
-	}
-	if strings.Contains(output, "Chain FORWARD (policy DROP)") {
-		cmd = getSSHCommand("&&",
-			"(iptables -C FORWARD -w -p TCP -j ACCEPT || iptables -A FORWARD -w -p TCP -j ACCEPT)",
-			"(iptables -C FORWARD -w -p UDP -j ACCEPT || iptables -A FORWARD -w -p UDP -j ACCEPT)",
-			"(iptables -C FORWARD -w -p ICMP -j ACCEPT || iptables -A FORWARD -w -p ICMP -j ACCEPT)")
-		output, err = SSH(host, "sh", "-c", cmd)
-		if err != nil {
-			return "", false, fmt.Errorf("failed to configured firewall: %v output: %v", err, output)
-		}
-	}
-
 	// Copy the archive to the staging directory
-	if output, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace)); err != nil {
+	if output, err := runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace)); err != nil {
 		// Exit failure with the error
 		return "", false, fmt.Errorf("failed to copy test archive: %v, output: %q", err, output)
 	}
 
-	// Kill any running node processes
-	cmd = getSSHCommand(" ; ",
-		"pkill kubelet",
-		"pkill kube-apiserver",
-		"pkill etcd",
-	)
-	// No need to log an error if pkill fails since pkill will fail if the commands are not running.
-	// If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd
-	// logs about failing to bind the required ports.
-	glog.Infof("Killing any existing node processes on %s", host)
-	SSH(host, "sh", "-c", cmd)
-
 	// Extract the archive
-	cmd = getSSHCommand(" && ",
+	cmd := getSSHCommand(" && ",
 		fmt.Sprintf("cd %s", workspace),
 		fmt.Sprintf("tar -xzvf ./%s", archiveName),
 	)
-	glog.Infof("Extracting tar on %s", host)
-	if output, err = SSH(host, "sh", "-c", cmd); err != nil {
+	glog.V(2).Infof("Extracting tar on %q", host)
+	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
 		// Exit failure with the error
 		return "", false, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
 	}
 
-	// If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args.
-	// We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run.
-
-	// Determine if the GCI mounter script exists locally.
-	k8sDir, err := builder.GetK8sRootDir()
-	if err != nil {
-		return "", false, fmt.Errorf("Could not find K8s root dir! Err: %v", err)
-	}
-	localSource := "cluster/gce/gci/mounter/mounter"
-	source := filepath.Join(k8sDir, localSource)
-
-	// Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves
-	if _, err = os.Stat(source); err != nil {
-		return "", false, fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err)
+	// Create the test result directory.
+	resultDir := filepath.Join(workspace, "results")
+	if output, err := SSHNoSudo(host, "mkdir", resultDir); err != nil {
+		// Exit failure with the error
+		return "", false, fmt.Errorf("failed to create test result directory %q on host %q: %v output: %q", resultDir, host, err, output)
 	}
 
-	// Determine if tests will run on a GCI node.
-	output, err = SSH(host, "sh", "-c", "'cat /etc/os-release'")
-	if err != nil {
-		glog.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
-		return "", false, fmt.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
-	}
-	if strings.Contains(output, "ID=gci") {
-		glog.Infof("GCI node and GCI mounter both detected, modifying --experimental-mounter-path accordingly")
-		// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
-		// here will tell us to update the remote test runner.
-		mounterPath := filepath.Join(workspace, "cluster/gce/gci/mounter/mounter")
-		output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
-		if err != nil {
-			glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output)
-			return "", false, err
-		}
-		// Insert args at beginning of testArgs, so any values from command line take precedence
-		testArgs = fmt.Sprintf("--kubelet-flags=--experimental-mounter-path=%s ", mounterPath) + testArgs
-	}
+	glog.V(2).Infof("Running test on %q", host)
+	output, err := suite.RunTest(host, workspace, resultDir, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds)
 
-	// Run the tests
-	cmd = getSSHCommand(" && ",
-		fmt.Sprintf("cd %s", workspace),
-		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
-			testTimeoutSeconds.Seconds(), ginkgoFlags, host, workspace, junitFilePrefix, testArgs),
-	)
 	aggErrs := []error{}
-
-	glog.Infof("Starting tests on %s", host)
-	output, err = SSH(host, "sh", "-c", cmd)
 	// Do not log the output here, let the caller deal with the test output.
 	if err != nil {
 		aggErrs = append(aggErrs, err)
-
-		// Encountered an unexpected error. The remote test harness may not
-		// have finished retrieved and stored all the logs in this case. Try
-		// to get some logs for debugging purposes.
-		// TODO: This is a best-effort, temporary hack that only works for
-		// journald nodes. We should have a more robust way to collect logs.
-		var (
-			logName  = "system.log"
-			logPath  = fmt.Sprintf("/tmp/%s-%s", getTimestamp(), logName)
-			destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName)
-		)
-		glog.Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)")
-		// Try getting the system logs from journald and store it to a file.
-		// Don't reuse the original test directory on the remote host because
-		// it could've be been removed if the node was rebooted.
-		if output, err := SSH(host, "sh", "-c", fmt.Sprintf("'journalctl --system --all > %s'", logPath)); err == nil {
-			glog.Infof("Got the system logs from journald; copying it back...")
-			if output, err := runSSHCommand("scp", fmt.Sprintf("%s:%s", GetHostnameOrIp(host), logPath), destPath); err != nil {
-				glog.Infof("Failed to copy the log: err: %v, output: %q", err, output)
-			}
-		} else {
-			glog.Infof("Failed to run journactl (normal if it doesn't exist on the node): %v, output: %q", err, output)
-		}
+		collectSystemLog(host, workspace)
 	}
 
-	glog.Infof("Copying test artifacts from %s", host)
+	glog.V(2).Infof("Copying test artifacts from %q", host)
 	scpErr := getTestArtifacts(host, workspace)
 	if scpErr != nil {
 		aggErrs = append(aggErrs, scpErr)
@@ -313,6 +149,33 @@ func getTestArtifacts(host, testDir string) error {
 	return nil
 }
 
+// collectSystemLog is a temporary hack to collect system log when encountered on
+// unexpected error.
+func collectSystemLog(host, workspace string) {
+	// Encountered an unexpected error. The remote test harness may not
+	// have finished retrieved and stored all the logs in this case. Try
+	// to get some logs for debugging purposes.
+	// TODO: This is a best-effort, temporary hack that only works for
+	// journald nodes. We should have a more robust way to collect logs.
+	var (
+		logName  = "system.log"
+		logPath  = fmt.Sprintf("/tmp/%s-%s", getTimestamp(), logName)
+		destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName)
+	)
+	glog.V(2).Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)")
+	// Try getting the system logs from journald and store it to a file.
+	// Don't reuse the original test directory on the remote host because
+	// it could've be been removed if the node was rebooted.
+	if output, err := SSH(host, "sh", "-c", fmt.Sprintf("'journalctl --system --all > %s'", logPath)); err == nil {
+		glog.V(2).Infof("Got the system logs from journald; copying it back...")
+		if output, err := runSSHCommand("scp", fmt.Sprintf("%s:%s", GetHostnameOrIp(host), logPath), destPath); err != nil {
+			glog.V(2).Infof("Failed to copy the log: err: %v, output: %q", err, output)
+		}
+	} else {
+		glog.V(2).Infof("Failed to run journactl (normal if it doesn't exist on the node): %v, output: %q", err, output)
+	}
+}
+
 // WriteLog is a temporary function to make it possible to write log
 // in the runner. This is used to collect serial console log.
 // TODO(random-liu): Use the log-dump script in cluster e2e.

test/e2e_node/remote/types.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+	"time"
+)
+
+// TestSuite is the interface of a test suite, such as node e2e, node conformance,
+// node soaking, cri validation etc.
+type TestSuite interface {
+	// SetupTestPackage setup the test package in the given directory. TestSuite
+	// should put all necessary binaries and dependencies into the path. The caller
+	// will:
+	// * create a tarball with the directory.
+	// * deploy the tarball to the testing host.
+	// * untar the tarball to the testing workspace on the testing host.
+	SetupTestPackage(path string) error
+	// RunTest runs test on the node in the given workspace and returns test output
+	// and test error if there is any.
+	// * host is the target node to run the test.
+	// * workspace is the directory on the testing host the test is running in. Note
+	// that the test package is unpacked in the workspace before running the test.
+	// * results is the directory the test should write result into. All logs should be
+	// saved as *.log, all junit file should start with junit*.
+	// * junitFilePrefix is the prefix of output junit file.
+	// * testArgs is the arguments passed to test.
+	// * ginkgoArgs is the arguments passed to ginkgo.
+	// * timeout is the test timeout.
+	RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error)
+}

test/e2e_node/remote/utils.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+// utils.go contains functions used accross test suites.
+
+const (
+	cniRelease   = "07a8a28637e97b22eb8dfe710eeae1344f69d16e"
+	cniDirectory = "cni"
+	cniURL       = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-" + cniRelease + ".tar.gz"
+)
+
+// Install the cni plugin.
+func installCNI(host, workspace string) error {
+	glog.V(2).Infof("Install CNI on %q", host)
+	cniPath := filepath.Join(workspace, cniDirectory)
+	cmd := getSSHCommand(" ; ",
+		fmt.Sprintf("mkdir -p %s", cniPath),
+		fmt.Sprintf("wget -O - %s | tar -xz -C %s", cniURL, cniPath),
+	)
+	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
+		return fmt.Errorf("failed to install cni plugin on %q: %v output: %q", host, err, output)
+	}
+	return nil
+}
+
+// configureFirewall configures iptable firewall rules.
+func configureFirewall(host string) error {
+	glog.V(2).Infof("Configure iptables firewall rules on %q", host)
+	// TODO: consider calling bootstrap script to configure host based on OS
+	output, err := SSH(host, "iptables", "-L", "INPUT")
+	if err != nil {
+		return fmt.Errorf("failed to get iptables INPUT on %q: %v output: %q", host, err, output)
+	}
+	if strings.Contains(output, "Chain INPUT (policy DROP)") {
+		cmd := getSSHCommand("&&",
+			"(iptables -C INPUT -w -p TCP -j ACCEPT || iptables -A INPUT -w -p TCP -j ACCEPT)",
+			"(iptables -C INPUT -w -p UDP -j ACCEPT || iptables -A INPUT -w -p UDP -j ACCEPT)",
+			"(iptables -C INPUT -w -p ICMP -j ACCEPT || iptables -A INPUT -w -p ICMP -j ACCEPT)")
+		output, err := SSH(host, "sh", "-c", cmd)
+		if err != nil {
+			return fmt.Errorf("failed to configured firewall on %q: %v output: %v", host, err, output)
+		}
+	}
+	output, err = SSH(host, "iptables", "-L", "FORWARD")
+	if err != nil {
+		return fmt.Errorf("failed to get iptables FORWARD on %q: %v output: %q", host, err, output)
+	}
+	if strings.Contains(output, "Chain FORWARD (policy DROP)") {
+		cmd := getSSHCommand("&&",
+			"(iptables -C FORWARD -w -p TCP -j ACCEPT || iptables -A FORWARD -w -p TCP -j ACCEPT)",
+			"(iptables -C FORWARD -w -p UDP -j ACCEPT || iptables -A FORWARD -w -p UDP -j ACCEPT)",
+			"(iptables -C FORWARD -w -p ICMP -j ACCEPT || iptables -A FORWARD -w -p ICMP -j ACCEPT)")
+		output, err = SSH(host, "sh", "-c", cmd)
+		if err != nil {
+			return fmt.Errorf("failed to configured firewall on %q: %v output: %v", host, err, output)
+		}
+	}
+	return nil
+}
+
+// cleanupNodeProcesses kills all running node processes may conflict with the test.
+func cleanupNodeProcesses(host string) {
+	glog.V(2).Infof("Killing any existing node processes on %q", host)
+	cmd := getSSHCommand(" ; ",
+		"pkill kubelet",
+		"pkill kube-apiserver",
+		"pkill etcd",
+		"pkill e2e_node.test",
+	)
+	// No need to log an error if pkill fails since pkill will fail if the commands are not running.
+	// If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd
+	// logs about failing to bind the required ports.
+	SSH(host, "sh", "-c", cmd)
+}

test/e2e_node/runner/remote/run_remote.go

@@ -66,6 +66,7 @@ const (
 var (
 	computeService *compute.Service
 	arc            Archive
+	suite          remote.TestSuite
 )
 
 type Archive struct {
@@ -125,12 +126,32 @@ type internalGCEImage struct {
 	tests    []string
 }
 
+// parseFlags parse subcommands and flags
+func parseFlags() {
+	if len(os.Args) <= 1 {
+		glog.Fatalf("Too few flags specified: %v", os.Args)
+	}
+	// Parse subcommand.
+	subcommand := os.Args[1]
+	switch subcommand {
+	case "conformance":
+		suite = remote.InitConformanceRemote()
+	// TODO: Add subcommand for node soaking, node conformance, cri validation.
+	default:
+		// Use node e2e suite by default if no subcommand is specified.
+		suite = remote.InitNodeE2ERemote()
+	}
+	// Parse test flags.
+	flag.CommandLine.Parse(os.Args[2:])
+}
+
 func main() {
-	flag.Parse()
+	parseFlags()
+
 	rand.Seed(time.Now().UTC().UnixNano())
 	if *buildOnly {
 		// Build the archive and exit
-		remote.CreateTestArchive()
+		remote.CreateTestArchive(suite)
 		return
 	}
 
@@ -301,7 +322,7 @@ func callGubernator(gubernator bool) {
 }
 
 func (a *Archive) getArchive() (string, error) {
-	a.Do(func() { a.path, a.err = remote.CreateTestArchive() })
+	a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite) })
 	return a.path, a.err
 }
 
@@ -363,7 +384,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, ginkgoFlags
 		}
 	}
 
-	output, exitOk, err := remote.RunRemote(path, host, deleteFiles, junitFilePrefix, *testArgs, ginkgoFlagsStr)
+	output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, junitFilePrefix, *testArgs, ginkgoFlagsStr)
 	return &TestResult{
 		output: output,
 		err:    err,

test/e2e_node/services/BUILD

@@ -17,6 +17,7 @@ go_library(
         "namespace_controller.go",
         "server.go",
         "services.go",
+        "util.go",
     ],
     tags = ["automanaged"],
     deps = [

test/e2e_node/services/internal_services.go

@@ -19,8 +19,6 @@ package services
 import (
 	"io/ioutil"
 	"os"
-	"os/signal"
-	"syscall"
 
 	"github.com/golang/glog"
 )
@@ -38,10 +36,6 @@ func newE2EServices() *e2eServices {
 	return &e2eServices{}
 }
 
-// terminationSignals are signals that cause the program to exit in the
-// supported platforms (linux, darwin, windows).
-var terminationSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
-
 // run starts all e2e services and wait for the termination signal. Once receives the
 // termination signal, it will stop the e2e services gracefully.
 func (es *e2eServices) run() error {
@@ -50,9 +44,7 @@ func (es *e2eServices) run() error {
 		return err
 	}
 	// Wait until receiving a termination signal.
-	sig := make(chan os.Signal, 1)
-	signal.Notify(sig, terminationSignals...)
-	<-sig
+	waitForTerminationSignal()
 	return nil
 }
 

test/e2e_node/services/kubelet.go

@@ -63,6 +63,22 @@ func init() {
 	flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.")
 }
 
+// RunKubelet starts kubelet and waits for termination signal. Once receives the
+// termination signal, it will stop the kubelet gracefully.
+func RunKubelet() {
+	var err error
+	// Enable monitorParent to make sure kubelet will receive termination signal
+	// when test process exits.
+	e := NewE2EServices(true /* monitorParent */)
+	defer e.Stop()
+	e.kubelet, err = e.startKubelet()
+	if err != nil {
+		glog.Fatalf("Failed to start kubelet: %v", err)
+	}
+	// Wait until receiving a termination signal.
+	waitForTerminationSignal()
+}
+
 const (
 	// Ports of different e2e services.
 	kubeletPort         = "10250"

test/e2e_node/services/util.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package services
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+// terminationSignals are signals that cause the program to exit in the
+// supported platforms (linux, darwin, windows).
+var terminationSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
+
+// waitForTerminationSignal waits for termination signal.
+func waitForTerminationSignal() {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, terminationSignals...)
+	<-sig
+}