Move utilities into different packages. Add local and remove runner.

hack/.linted_packages

@@ -191,6 +191,7 @@ plugin/pkg/client/auth/gcp
 test/e2e/cleanup
 test/e2e/generated
 test/e2e/perftype
+test/e2e_node/runner/local
 test/images/clusterapi-tester
 test/images/entrypoint-tester
 test/images/fakegitserver

hack/make-rules/test-e2e-node.sh

@@ -52,12 +52,6 @@ if  [[ $list_images == "true" ]]; then
   exit 0
 fi
 
-ginkgo=$(kube::util::find-binary "ginkgo")
-if [[ -z "${ginkgo}" ]]; then
-  echo "You do not appear to have ginkgo built. Try 'make WHAT=vendor/github.com/onsi/ginkgo/ginkgo'"
-  exit 1
-fi
-
 # Parse the flags to pass to ginkgo
 ginkgoflags=""
 if [[ $parallelism > 1 ]]; then
@@ -135,7 +129,7 @@ if [ $remote = true ] ; then
   echo "Ginkgo Flags: $ginkgoflags"
   echo "Instance Metadata: $metadata"
   # Invoke the runner
-  go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
+  go run test/e2e_node/runner/remote/run_remote.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
     --zone="$zone" --project="$project" --gubernator="$gubernator" \
     --hosts="$hosts" --images="$images" --cleanup="$cleanup" \
     --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \
@@ -171,8 +165,8 @@ else
 
   # Test using the host the script was run on
   # Provided for backwards compatibility
-  "${ginkgo}" $ginkgoflags "${KUBE_ROOT}/test/e2e_node/" --report-dir=${report} \
-    -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true \
-    --start-services=true --stop-services=true $test_args
+  go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \
+    --test-flags="--alsologtostderr --v 2 --report-dir=${report} --node-name $(hostname) \
+    --start-services=true --stop-services=true $test_args" --build-dependencies=true
   exit $?
 fi

hack/verify-flags/known-flags.txt

@@ -41,8 +41,8 @@ bind-address
 bind-pods-burst
 bind-pods-qps
 bounding-dirs
+build-dependencies
 build-only
-build-services
 build-tag
 cadvisor-port
 cert-dir
@@ -494,6 +494,7 @@ target-port
 target-ram-mb
 tcp-services
 terminated-pod-gc-threshold
+test-flags
 test-timeout
 tls-cert-file
 tls-private-key-file

test/e2e_node/build/build.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package build
 
 import (
 	"flag"
@@ -36,7 +36,7 @@ var buildTargets = []string{
 	"vendor/github.com/onsi/ginkgo/ginkgo",
 }
 
-func buildGo() error {
+func BuildGo() error {
 	glog.Infof("Building k8s binaries...")
 	k8sRoot, err := getK8sRootDir()
 	if err != nil {
@@ -74,7 +74,7 @@ func getK8sBin(bin string) (string, error) {
 		return filepath.Join(path, bin), nil
 	}
 
-	buildOutputDir, err := getK8sBuildOutputDir()
+	buildOutputDir, err := GetK8sBuildOutputDir()
 	if err != nil {
 		return "", err
 	}
@@ -101,7 +101,7 @@ func getK8sRootDir() (string, error) {
 	return "", fmt.Errorf("Could not find kubernetes source root directory.")
 }
 
-func getK8sBuildOutputDir() (string, error) {
+func GetK8sBuildOutputDir() (string, error) {
 	k8sRoot, err := getK8sRootDir()
 	if err != nil {
 		return "", err
@@ -113,7 +113,7 @@ func getK8sBuildOutputDir() (string, error) {
 	return buildOutputDir, nil
 }
 
-func getKubeletServerBin() string {
+func GetKubeletServerBin() string {
 	bin, err := getK8sBin("kubelet")
 	if err != nil {
 		glog.Fatalf("Could not locate kubelet binary %v.", err)

test/e2e_node/e2e_node_suite_test.go

@@ -35,6 +35,7 @@ import (
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	commontest "k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e_node/services"
 
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo"
@@ -44,7 +45,7 @@ import (
 	"github.com/spf13/pflag"
 )
 
-var e2es *E2EServices
+var e2es *services.E2EServices
 
 var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
 var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
@@ -61,7 +62,7 @@ func TestE2eNode(t *testing.T) {
 	pflag.Parse()
 	if *runServicesMode {
 		// If run-services-mode is specified, only run services in current process.
-		RunE2EServices()
+		services.RunE2EServices()
 		return
 	}
 	// If run-services-mode is not specified, run test.
@@ -85,10 +86,6 @@ func TestE2eNode(t *testing.T) {
 
 // Setup the kubelet on the node
 var _ = SynchronizedBeforeSuite(func() []byte {
-	if *buildServices {
-		Expect(buildGo()).To(Succeed())
-	}
-
 	// Initialize node name here, so that the following code can get right node name.
 	if framework.TestContext.NodeName == "" {
 		hostname, err := os.Hostname()
@@ -109,7 +106,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	maskLocksmithdOnCoreos()
 
 	if *startServices {
-		e2es = NewE2EServices()
+		e2es = services.NewE2EServices()
 		Expect(e2es.Start()).To(Succeed(), "should be able to start node services.")
 		glog.Infof("Node services started.  Running tests...")
 	} else {

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -42,7 +42,7 @@ TIMEOUT=${TIMEOUT:-"45m"}
 
 mkdir -p ${ARTIFACTS}
 
-go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
+go run test/e2e_node/runner/remote/run_remote.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
   --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
   --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
   --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \

test/e2e_node/remote/remote.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package remote
 
 import (
 	"flag"
@@ -31,6 +31,7 @@ import (
 
 	"github.com/golang/glog"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
+	"k8s.io/kubernetes/test/e2e_node/build"
 )
 
 var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
@@ -82,12 +83,12 @@ func GetHostnameOrIp(hostname string) string {
 // the binaries k8s required for node e2e tests
 func CreateTestArchive() (string, error) {
 	// Build the executables
-	if err := buildGo(); err != nil {
+	if err := build.BuildGo(); err != nil {
 		return "", fmt.Errorf("failed to build the depedencies: %v", err)
 	}
 
 	// Make sure we can find the newly built binaries
-	buildOutputDir, err := getK8sBuildOutputDir()
+	buildOutputDir, err := build.GetK8sBuildOutputDir()
 	if err != nil {
 		return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err)
 	}
@@ -194,7 +195,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
 	// Run the tests
 	cmd = getSshCommand(" && ",
 		fmt.Sprintf("cd %s", tmp),
-		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
+		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
 			testTimeoutSeconds.Seconds(), ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs),
 	)
 	aggErrs := []error{}

test/e2e_node/runner/local/run_local.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/kubernetes/test/e2e_node/build"
+
+	"github.com/golang/glog"
+)
+
+var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.")
+var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
+var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
+
+func main() {
+	flag.Parse()
+
+	// Build dependencies - ginkgo, kubelet and apiserver.
+	if *buildDependencies {
+		if err := build.BuildGo(); err != nil {
+			glog.Fatalf("Failed to build the dependencies: %v", err)
+		}
+	}
+
+	// Run node e2e test
+	outputDir, err := build.GetK8sBuildOutputDir()
+	if err != nil {
+		glog.Fatalf("Failed to get build output directory: %v", err)
+	}
+	ginkgo := filepath.Join(outputDir, "ginkgo")
+	test := filepath.Join(outputDir, "e2e_node.test")
+	runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags)
+	return
+}
+
+func runCommand(name string, args ...string) error {
+	cmd := exec.Command("sh", "-c", strings.Join(append([]string{name}, args...), " "))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

test/e2e_node/runner/remote/run_remote.go

@@ -14,10 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// To run the e2e tests against one or more hosts on gce:
-// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
-// To run the e2e tests against one or more images on gce and provision them:
-// $ go run run_e2e.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
+// To run the node e2e tests remotely against one or more hosts on gce:
+// $ go run run_remote.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
+// To run the node e2e tests remotely against one or more images on gce and provision them:
+// $ go run run_remote.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
 package main
 
 import (
@@ -34,7 +34,7 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/kubernetes/test/e2e_node"
+	"k8s.io/kubernetes/test/e2e_node/remote"
 
 	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
@@ -131,7 +131,7 @@ func main() {
 	rand.Seed(time.Now().UTC().UnixNano())
 	if *buildOnly {
 		// Build the archive and exit
-		e2e_node.CreateTestArchive()
+		remote.CreateTestArchive()
 		return
 	}
 
@@ -295,7 +295,7 @@ func callGubernator(gubernator bool) {
 }
 
 func (a *Archive) getArchive() (string, error) {
-	a.Do(func() { a.path, a.err = e2e_node.CreateTestArchive() })
+	a.Do(func() { a.path, a.err = remote.CreateTestArchive() })
 	return a.path, a.err
 }
 
@@ -346,7 +346,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b
 	}
 	externalIp := getExternalIp(instance)
 	if len(externalIp) > 0 {
-		e2e_node.AddHostnameIp(host, externalIp)
+		remote.AddHostnameIp(host, externalIp)
 	}
 
 	path, err := arc.getArchive()
@@ -357,7 +357,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b
 		}
 	}
 
-	output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr)
+	output, exitOk, err := remote.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr)
 	return &TestResult{
 		output: output,
 		err:    err,
@@ -497,10 +497,10 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 		}
 		externalIp := getExternalIp(instance)
 		if len(externalIp) > 0 {
-			e2e_node.AddHostnameIp(name, externalIp)
+			remote.AddHostnameIp(name, externalIp)
 		}
 		var output string
-		output, err = e2e_node.RunSshCommand("ssh", e2e_node.GetHostnameOrIp(name), "--", "sudo", "docker", "version")
+		output, err = remote.RunSshCommand("ssh", remote.GetHostnameOrIp(name), "--", "sudo", "docker", "version")
 		if err != nil {
 			err = fmt.Errorf("instance %s not running docker daemon - Command failed: %s", name, output)
 			continue

test/e2e_node/services/apiserver.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"fmt"

test/e2e_node/services/etcd.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"crypto/tls"

test/e2e_node/services/namespace_controller.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"time"

test/e2e_node/services/services.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"flag"
@@ -37,6 +37,7 @@ import (
 	"github.com/kardianos/osext"
 
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e_node/build"
 )
 
 // TODO(random-liu): Move this file to a separate package.
@@ -194,10 +195,6 @@ func (es *e2eService) run() error {
 }
 
 func (es *e2eService) start() error {
-	if _, err := getK8sBin("kubelet"); err != nil {
-		return err
-	}
-
 	err := es.startEtcd()
 	if err != nil {
 		return err
@@ -341,7 +338,7 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		// Since kubelet will typically be run as a service it also makes more
 		// sense to test it that way
 		unitName := fmt.Sprintf("kubelet-%d.service", rand.Int31())
-		cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--remain-after-exit", getKubeletServerBin())
+		cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--remain-after-exit", build.GetKubeletServerBin())
 		killCommand = exec.Command("sudo", "systemctl", "kill", unitName)
 		restartCommand = exec.Command("sudo", "systemctl", "restart", unitName)
 		es.logFiles["kubelet.log"] = logFileData{
@@ -349,7 +346,7 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		}
 		framework.TestContext.EvictionHard = adjustConfigForSystemd(framework.TestContext.EvictionHard)
 	} else {
-		cmdArgs = append(cmdArgs, getKubeletServerBin())
+		cmdArgs = append(cmdArgs, build.GetKubeletServerBin())
 		cmdArgs = append(cmdArgs,
 			"--runtime-cgroups=/docker-daemon",
 			"--kubelet-cgroups=/kubelet",
@@ -387,7 +384,8 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		}
 		cmdArgs = append(cmdArgs,
 			"--network-plugin=kubenet",
-			"--network-plugin-dir", filepath.Join(cwd, CNIDirectory, "bin")) // Enable kubenet
+			// TODO(random-liu): Make sure the cni directory name is the same with that in remote/remote.go
+			"--network-plugin-dir", filepath.Join(cwd, "cni", "bin")) // Enable kubenet
 	}
 
 	cmd := exec.Command("sudo", cmdArgs...)

test/e2e_node/util.go

@@ -20,8 +20,8 @@ import (
 	"flag"
 )
 
+// TODO(random-liu): Get this automatically from kubelet flag.
 var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
 
-var buildServices = flag.Bool("build-services", true, "If true, build local executables")
 var startServices = flag.Bool("start-services", true, "If true, start local node services")
 var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")