Merge pull request #30540 from Random-Liu/refactor-node-e2e-framework

Automatic merge from submit-queue

Node Conformance Test: Refactor node e2e framework

For #30122, #30174.
Based on #30348.

**Please only review the last 3 commits.**

This PR is part of our roadmap to package node conformance test.
The 1st commit is from #30348, it removed unnecessary dependencies in the node e2e test framework, because we've statically linked these dependencies.

The PR refactored the node e2e framework. Moving different utilities into different packages under `pkg/`.

We need to do this because:
1) Files like e2e_remote.go and e2e_build.go should only be used by runner, but they were compiled into the test suite because they were placed in the same package. The worst thing is that it will introduce some never used flags in the test suite binary.
2) Make the directory structure more clear. Only test should be placed in `test/e2e_node`, other utilities should be placed in different packages in `pkg/`.

@dchen1107 @vishh 
/cc @kubernetes/sig-node @kubernetes/sig-testing

hack/.linted_packages

@@ -193,6 +193,7 @@ plugin/pkg/client/auth/gcp
 test/e2e/cleanup
 test/e2e/generated
 test/e2e/perftype
+test/e2e_node/runner/local
 test/images/clusterapi-tester
 test/images/entrypoint-tester
 test/images/fakegitserver

hack/make-rules/test-e2e-node.sh

@@ -52,12 +52,6 @@ if  [[ $list_images == "true" ]]; then
   exit 0
 fi
 
-ginkgo=$(kube::util::find-binary "ginkgo")
-if [[ -z "${ginkgo}" ]]; then
-  echo "You do not appear to have ginkgo built. Try 'make WHAT=vendor/github.com/onsi/ginkgo/ginkgo'"
-  exit 1
-fi
-
 # Parse the flags to pass to ginkgo
 ginkgoflags=""
 if [[ $parallelism > 1 ]]; then
@@ -135,7 +129,7 @@ if [ $remote = true ] ; then
   echo "Ginkgo Flags: $ginkgoflags"
   echo "Instance Metadata: $metadata"
   # Invoke the runner
-  go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
+  go run test/e2e_node/runner/remote/run_remote.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
     --zone="$zone" --project="$project" --gubernator="$gubernator" \
     --hosts="$hosts" --images="$images" --cleanup="$cleanup" \
     --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \
@@ -171,8 +165,8 @@ else
 
   # Test using the host the script was run on
   # Provided for backwards compatibility
-  "${ginkgo}" $ginkgoflags "${KUBE_ROOT}/test/e2e_node/" --report-dir=${report} \
-    -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true \
-    --start-services=true --stop-services=true $test_args
+  go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \
+    --test-flags="--alsologtostderr --v 2 --report-dir=${report} --node-name $(hostname) \
+    --start-services=true --stop-services=true $test_args" --build-dependencies=true
   exit $?
 fi

hack/verify-flags/known-flags.txt

@@ -41,8 +41,8 @@ bind-address
 bind-pods-burst
 bind-pods-qps
 bounding-dirs
+build-dependencies
 build-only
-build-services
 build-tag
 cadvisor-port
 cert-dir
@@ -495,6 +495,7 @@ target-port
 target-ram-mb
 tcp-services
 terminated-pod-gc-threshold
+test-flags
 test-timeout
 tls-cert-file
 tls-private-key-file

test/e2e_node/build/build.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package build
 
 import (
 	"flag"
@@ -36,7 +36,7 @@ var buildTargets = []string{
 	"vendor/github.com/onsi/ginkgo/ginkgo",
 }
 
-func buildGo() error {
+func BuildGo() error {
 	glog.Infof("Building k8s binaries...")
 	k8sRoot, err := getK8sRootDir()
 	if err != nil {
@@ -74,7 +74,7 @@ func getK8sBin(bin string) (string, error) {
 		return filepath.Join(path, bin), nil
 	}
 
-	buildOutputDir, err := getK8sBuildOutputDir()
+	buildOutputDir, err := GetK8sBuildOutputDir()
 	if err != nil {
 		return "", err
 	}
@@ -101,7 +101,7 @@ func getK8sRootDir() (string, error) {
 	return "", fmt.Errorf("Could not find kubernetes source root directory.")
 }
 
-func getK8sBuildOutputDir() (string, error) {
+func GetK8sBuildOutputDir() (string, error) {
 	k8sRoot, err := getK8sRootDir()
 	if err != nil {
 		return "", err
@@ -113,7 +113,7 @@ func getK8sBuildOutputDir() (string, error) {
 	return buildOutputDir, nil
 }
 
-func getKubeletServerBin() string {
+func GetKubeletServerBin() string {
 	bin, err := getK8sBin("kubelet")
 	if err != nil {
 		glog.Fatalf("Could not locate kubelet binary %v.", err)

test/e2e_node/e2e_node_suite_test.go

@@ -35,6 +35,7 @@ import (
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	commontest "k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e_node/services"
 
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo"
@@ -44,7 +45,7 @@ import (
 	"github.com/spf13/pflag"
 )
 
-var e2es *E2EServices
+var e2es *services.E2EServices
 
 var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
 var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
@@ -61,7 +62,7 @@ func TestE2eNode(t *testing.T) {
 	pflag.Parse()
 	if *runServicesMode {
 		// If run-services-mode is specified, only run services in current process.
-		RunE2EServices()
+		services.RunE2EServices()
 		return
 	}
 	// If run-services-mode is not specified, run test.
@@ -85,10 +86,6 @@ func TestE2eNode(t *testing.T) {
 
 // Setup the kubelet on the node
 var _ = SynchronizedBeforeSuite(func() []byte {
-	if *buildServices {
-		Expect(buildGo()).To(Succeed())
-	}
-
 	// Initialize node name here, so that the following code can get right node name.
 	if framework.TestContext.NodeName == "" {
 		hostname, err := os.Hostname()
@@ -109,7 +106,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	maskLocksmithdOnCoreos()
 
 	if *startServices {
-		e2es = NewE2EServices()
+		e2es = services.NewE2EServices()
 		Expect(e2es.Start()).To(Succeed(), "should be able to start node services.")
 		glog.Infof("Node services started.  Running tests...")
 	} else {

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -42,7 +42,7 @@ TIMEOUT=${TIMEOUT:-"45m"}
 
 mkdir -p ${ARTIFACTS}
 
-go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
+go run test/e2e_node/runner/remote/run_remote.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
   --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
   --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
   --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \

test/e2e_node/remote/remote.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package remote
 
 import (
 	"flag"
@@ -31,6 +31,7 @@ import (
 
 	"github.com/golang/glog"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
+	"k8s.io/kubernetes/test/e2e_node/build"
 )
 
 var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
@@ -82,12 +83,12 @@ func GetHostnameOrIp(hostname string) string {
 // the binaries k8s required for node e2e tests
 func CreateTestArchive() (string, error) {
 	// Build the executables
-	if err := buildGo(); err != nil {
+	if err := build.BuildGo(); err != nil {
 		return "", fmt.Errorf("failed to build the depedencies: %v", err)
 	}
 
 	// Make sure we can find the newly built binaries
-	buildOutputDir, err := getK8sBuildOutputDir()
+	buildOutputDir, err := build.GetK8sBuildOutputDir()
 	if err != nil {
 		return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err)
 	}
@@ -194,7 +195,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
 	// Run the tests
 	cmd = getSshCommand(" && ",
 		fmt.Sprintf("cd %s", tmp),
-		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
+		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
 			testTimeoutSeconds.Seconds(), ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs),
 	)
 	aggErrs := []error{}

test/e2e_node/runner/local/run_local.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/kubernetes/test/e2e_node/build"
+
+	"github.com/golang/glog"
+)
+
+var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.")
+var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
+var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
+
+func main() {
+	flag.Parse()
+
+	// Build dependencies - ginkgo, kubelet and apiserver.
+	if *buildDependencies {
+		if err := build.BuildGo(); err != nil {
+			glog.Fatalf("Failed to build the dependencies: %v", err)
+		}
+	}
+
+	// Run node e2e test
+	outputDir, err := build.GetK8sBuildOutputDir()
+	if err != nil {
+		glog.Fatalf("Failed to get build output directory: %v", err)
+	}
+	ginkgo := filepath.Join(outputDir, "ginkgo")
+	test := filepath.Join(outputDir, "e2e_node.test")
+	runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags)
+	return
+}
+
+func runCommand(name string, args ...string) error {
+	cmd := exec.Command("sh", "-c", strings.Join(append([]string{name}, args...), " "))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

test/e2e_node/runner/remote/run_remote.go

@@ -14,10 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// To run the e2e tests against one or more hosts on gce:
-// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
-// To run the e2e tests against one or more images on gce and provision them:
-// $ go run run_e2e.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
+// To run the node e2e tests remotely against one or more hosts on gce:
+// $ go run run_remote.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
+// To run the node e2e tests remotely against one or more images on gce and provision them:
+// $ go run run_remote.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
 package main
 
 import (
@@ -34,7 +34,7 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/kubernetes/test/e2e_node"
+	"k8s.io/kubernetes/test/e2e_node/remote"
 
 	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
@@ -131,7 +131,7 @@ func main() {
 	rand.Seed(time.Now().UTC().UnixNano())
 	if *buildOnly {
 		// Build the archive and exit
-		e2e_node.CreateTestArchive()
+		remote.CreateTestArchive()
 		return
 	}
 
@@ -295,7 +295,7 @@ func callGubernator(gubernator bool) {
 }
 
 func (a *Archive) getArchive() (string, error) {
-	a.Do(func() { a.path, a.err = e2e_node.CreateTestArchive() })
+	a.Do(func() { a.path, a.err = remote.CreateTestArchive() })
 	return a.path, a.err
 }
 
@@ -346,7 +346,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b
 	}
 	externalIp := getExternalIp(instance)
 	if len(externalIp) > 0 {
-		e2e_node.AddHostnameIp(host, externalIp)
+		remote.AddHostnameIp(host, externalIp)
 	}
 
 	path, err := arc.getArchive()
@@ -357,7 +357,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b
 		}
 	}
 
-	output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr)
+	output, exitOk, err := remote.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr)
 	return &TestResult{
 		output: output,
 		err:    err,
@@ -497,10 +497,10 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 		}
 		externalIp := getExternalIp(instance)
 		if len(externalIp) > 0 {
-			e2e_node.AddHostnameIp(name, externalIp)
+			remote.AddHostnameIp(name, externalIp)
 		}
 		var output string
-		output, err = e2e_node.RunSshCommand("ssh", e2e_node.GetHostnameOrIp(name), "--", "sudo", "docker", "version")
+		output, err = remote.RunSshCommand("ssh", remote.GetHostnameOrIp(name), "--", "sudo", "docker", "version")
 		if err != nil {
 			err = fmt.Errorf("instance %s not running docker daemon - Command failed: %s", name, output)
 			continue

test/e2e_node/services/apiserver.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"fmt"

test/e2e_node/services/etcd.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"crypto/tls"

test/e2e_node/services/namespace_controller.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"time"

test/e2e_node/services/services.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_node
+package services
 
 import (
 	"flag"
@@ -37,6 +37,7 @@ import (
 	"github.com/kardianos/osext"
 
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e_node/build"
 )
 
 // TODO(random-liu): Move this file to a separate package.
@@ -194,10 +195,6 @@ func (es *e2eService) run() error {
 }
 
 func (es *e2eService) start() error {
-	if _, err := getK8sBin("kubelet"); err != nil {
-		return err
-	}
-
 	err := es.startEtcd()
 	if err != nil {
 		return err
@@ -341,7 +338,7 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		// Since kubelet will typically be run as a service it also makes more
 		// sense to test it that way
 		unitName := fmt.Sprintf("kubelet-%d.service", rand.Int31())
-		cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--remain-after-exit", getKubeletServerBin())
+		cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--remain-after-exit", build.GetKubeletServerBin())
 		killCommand = exec.Command("sudo", "systemctl", "kill", unitName)
 		restartCommand = exec.Command("sudo", "systemctl", "restart", unitName)
 		es.logFiles["kubelet.log"] = logFileData{
@@ -349,7 +346,7 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		}
 		framework.TestContext.EvictionHard = adjustConfigForSystemd(framework.TestContext.EvictionHard)
 	} else {
-		cmdArgs = append(cmdArgs, getKubeletServerBin())
+		cmdArgs = append(cmdArgs, build.GetKubeletServerBin())
 		cmdArgs = append(cmdArgs,
 			"--runtime-cgroups=/docker-daemon",
 			"--kubelet-cgroups=/kubelet",
@@ -387,7 +384,8 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		}
 		cmdArgs = append(cmdArgs,
 			"--network-plugin=kubenet",
-			"--network-plugin-dir", filepath.Join(cwd, CNIDirectory, "bin")) // Enable kubenet
+			// TODO(random-liu): Make sure the cni directory name is the same with that in remote/remote.go
+			"--network-plugin-dir", filepath.Join(cwd, "cni", "bin")) // Enable kubenet
 	}
 
 	cmd := exec.Command("sudo", cmdArgs...)

test/e2e_node/util.go

@@ -20,8 +20,8 @@ import (
 	"flag"
 )
 
+// TODO(random-liu): Get this automatically from kubelet flag.
 var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
 
-var buildServices = flag.Bool("build-services", true, "If true, build local executables")
 var startServices = flag.Bool("start-services", true, "If true, start local node services")
 var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")