From afb780d4ee1b6587ed107c4c74bf4723da7955f1 Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Thu, 11 Aug 2016 23:30:04 -0700 Subject: [PATCH] Move utilities into different packages. Add local and remove runner. --- hack/.linted_packages | 1 + hack/make-rules/test-e2e-node.sh | 14 ++--- hack/verify-flags/known-flags.txt | 3 +- .../e2e_node/{e2e_build.go => build/build.go} | 10 +-- test/e2e_node/e2e_node_suite_test.go | 11 ++-- test/e2e_node/jenkins/e2e-node-jenkins.sh | 2 +- .../{e2e_remote.go => remote/remote.go} | 9 +-- test/e2e_node/runner/local/run_local.go | 61 +++++++++++++++++++ .../{run_e2e.go => remote/run_remote.go} | 22 +++---- test/e2e_node/{ => services}/apiserver.go | 2 +- test/e2e_node/{ => services}/etcd.go | 2 +- .../{ => services}/namespace_controller.go | 2 +- .../{e2e_service.go => services/services.go} | 14 ++--- test/e2e_node/util.go | 2 +- 14 files changed, 104 insertions(+), 51 deletions(-) rename test/e2e_node/{e2e_build.go => build/build.go} (94%) rename test/e2e_node/{e2e_remote.go => remote/remote.go} (97%) create mode 100644 test/e2e_node/runner/local/run_local.go rename test/e2e_node/runner/{run_e2e.go => remote/run_remote.go} (95%) rename test/e2e_node/{ => services}/apiserver.go (99%) rename test/e2e_node/{ => services}/etcd.go (99%) rename test/e2e_node/{ => services}/namespace_controller.go (99%) rename test/e2e_node/{e2e_service.go => services/services.go} (98%) diff --git a/hack/.linted_packages b/hack/.linted_packages index 383f631ad23..b0517be3e45 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -191,6 +191,7 @@ plugin/pkg/client/auth/gcp test/e2e/cleanup test/e2e/generated test/e2e/perftype +test/e2e_node/runner/local test/images/clusterapi-tester test/images/entrypoint-tester test/images/fakegitserver diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index a21ff6e364f..eaaf2223f37 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -52,12 +52,6 @@ if [[ $list_images == "true" ]]; then exit 0 fi -ginkgo=$(kube::util::find-binary "ginkgo") -if [[ -z "${ginkgo}" ]]; then - echo "You do not appear to have ginkgo built. Try 'make WHAT=vendor/github.com/onsi/ginkgo/ginkgo'" - exit 1 -fi - # Parse the flags to pass to ginkgo ginkgoflags="" if [[ $parallelism > 1 ]]; then @@ -135,7 +129,7 @@ if [ $remote = true ] ; then echo "Ginkgo Flags: $ginkgoflags" echo "Instance Metadata: $metadata" # Invoke the runner - go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=2 --ssh-env="gce" \ + go run test/e2e_node/runner/remote/run_remote.go --logtostderr --vmodule=*=4 --ssh-env="gce" \ --zone="$zone" --project="$project" --gubernator="$gubernator" \ --hosts="$hosts" --images="$images" --cleanup="$cleanup" \ --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \ @@ -171,8 +165,8 @@ else # Test using the host the script was run on # Provided for backwards compatibility - "${ginkgo}" $ginkgoflags "${KUBE_ROOT}/test/e2e_node/" --report-dir=${report} \ - -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true \ - --start-services=true --stop-services=true $test_args + go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \ + --test-flags="--alsologtostderr --v 2 --report-dir=${report} --node-name $(hostname) \ + --start-services=true --stop-services=true $test_args" --build-dependencies=true exit $? fi diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 789bb6ca499..2465f74feef 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -41,8 +41,8 @@ bind-address bind-pods-burst bind-pods-qps bounding-dirs +build-dependencies build-only -build-services build-tag cadvisor-port cert-dir @@ -494,6 +494,7 @@ target-port target-ram-mb tcp-services terminated-pod-gc-threshold +test-flags test-timeout tls-cert-file tls-private-key-file diff --git a/test/e2e_node/e2e_build.go b/test/e2e_node/build/build.go similarity index 94% rename from test/e2e_node/e2e_build.go rename to test/e2e_node/build/build.go index b806dfb3ed1..b54d8b64119 100644 --- a/test/e2e_node/e2e_build.go +++ b/test/e2e_node/build/build.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_node +package build import ( "flag" @@ -36,7 +36,7 @@ var buildTargets = []string{ "vendor/github.com/onsi/ginkgo/ginkgo", } -func buildGo() error { +func BuildGo() error { glog.Infof("Building k8s binaries...") k8sRoot, err := getK8sRootDir() if err != nil { @@ -74,7 +74,7 @@ func getK8sBin(bin string) (string, error) { return filepath.Join(path, bin), nil } - buildOutputDir, err := getK8sBuildOutputDir() + buildOutputDir, err := GetK8sBuildOutputDir() if err != nil { return "", err } @@ -101,7 +101,7 @@ func getK8sRootDir() (string, error) { return "", fmt.Errorf("Could not find kubernetes source root directory.") } -func getK8sBuildOutputDir() (string, error) { +func GetK8sBuildOutputDir() (string, error) { k8sRoot, err := getK8sRootDir() if err != nil { return "", err @@ -113,7 +113,7 @@ func getK8sBuildOutputDir() (string, error) { return buildOutputDir, nil } -func getKubeletServerBin() string { +func GetKubeletServerBin() string { bin, err := getK8sBin("kubelet") if err != nil { glog.Fatalf("Could not locate kubelet binary %v.", err) diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 85378a5ae77..becc40812f3 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -35,6 +35,7 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e_node/services" "github.com/golang/glog" . "github.com/onsi/ginkgo" @@ -44,7 +45,7 @@ import ( "github.com/spf13/pflag" ) -var e2es *E2EServices +var e2es *services.E2EServices var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.") @@ -61,7 +62,7 @@ func TestE2eNode(t *testing.T) { pflag.Parse() if *runServicesMode { // If run-services-mode is specified, only run services in current process. - RunE2EServices() + services.RunE2EServices() return } // If run-services-mode is not specified, run test. @@ -85,10 +86,6 @@ func TestE2eNode(t *testing.T) { // Setup the kubelet on the node var _ = SynchronizedBeforeSuite(func() []byte { - if *buildServices { - Expect(buildGo()).To(Succeed()) - } - // Initialize node name here, so that the following code can get right node name. if framework.TestContext.NodeName == "" { hostname, err := os.Hostname() @@ -109,7 +106,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { maskLocksmithdOnCoreos() if *startServices { - e2es = NewE2EServices() + e2es = services.NewE2EServices() Expect(e2es.Start()).To(Succeed(), "should be able to start node services.") glog.Infof("Node services started. Running tests...") } else { diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh index d0f57f6078d..afe8dc74876 100755 --- a/test/e2e_node/jenkins/e2e-node-jenkins.sh +++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh @@ -42,7 +42,7 @@ TIMEOUT=${TIMEOUT:-"45m"} mkdir -p ${ARTIFACTS} -go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=4 --ssh-env="gce" \ +go run test/e2e_node/runner/remote/run_remote.go --logtostderr --vmodule=*=4 --ssh-env="gce" \ --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \ --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \ --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \ diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/remote/remote.go similarity index 97% rename from test/e2e_node/e2e_remote.go rename to test/e2e_node/remote/remote.go index cc1fc4e99c7..eb545e3005f 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/remote/remote.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_node +package remote import ( "flag" @@ -31,6 +31,7 @@ import ( "github.com/golang/glog" utilerrors "k8s.io/kubernetes/pkg/util/errors" + "k8s.io/kubernetes/test/e2e_node/build" ) var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") @@ -82,12 +83,12 @@ func GetHostnameOrIp(hostname string) string { // the binaries k8s required for node e2e tests func CreateTestArchive() (string, error) { // Build the executables - if err := buildGo(); err != nil { + if err := build.BuildGo(); err != nil { return "", fmt.Errorf("failed to build the depedencies: %v", err) } // Make sure we can find the newly built binaries - buildOutputDir, err := getK8sBuildOutputDir() + buildOutputDir, err := build.GetK8sBuildOutputDir() if err != nil { return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err) } @@ -194,7 +195,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), - fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", + fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --stop-services=%t --node-name=%s --report-dir=%s/results --report-prefix=%s %s", testTimeoutSeconds.Seconds(), ginkgoFlags, cleanup, host, tmp, junitFilePrefix, testArgs), ) aggErrs := []error{} diff --git a/test/e2e_node/runner/local/run_local.go b/test/e2e_node/runner/local/run_local.go new file mode 100644 index 00000000000..58466bdd667 --- /dev/null +++ b/test/e2e_node/runner/local/run_local.go @@ -0,0 +1,61 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/kubernetes/test/e2e_node/build" + + "github.com/golang/glog" +) + +var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.") +var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.") +var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.") + +func main() { + flag.Parse() + + // Build dependencies - ginkgo, kubelet and apiserver. + if *buildDependencies { + if err := build.BuildGo(); err != nil { + glog.Fatalf("Failed to build the dependencies: %v", err) + } + } + + // Run node e2e test + outputDir, err := build.GetK8sBuildOutputDir() + if err != nil { + glog.Fatalf("Failed to get build output directory: %v", err) + } + ginkgo := filepath.Join(outputDir, "ginkgo") + test := filepath.Join(outputDir, "e2e_node.test") + runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags) + return +} + +func runCommand(name string, args ...string) error { + cmd := exec.Command("sh", "-c", strings.Join(append([]string{name}, args...), " ")) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/remote/run_remote.go similarity index 95% rename from test/e2e_node/runner/run_e2e.go rename to test/e2e_node/runner/remote/run_remote.go index 85b24d15e36..6e6fa24031d 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// To run the e2e tests against one or more hosts on gce: -// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts -// To run the e2e tests against one or more images on gce and provision them: -// $ go run run_e2e.go --logtostderr --v 2 --project --zone --ssh-env gce --images +// To run the node e2e tests remotely against one or more hosts on gce: +// $ go run run_remote.go --logtostderr --v 2 --ssh-env gce --hosts +// To run the node e2e tests remotely against one or more images on gce and provision them: +// $ go run run_remote.go --logtostderr --v 2 --project --zone --ssh-env gce --images package main import ( @@ -34,7 +34,7 @@ import ( "sync" "time" - "k8s.io/kubernetes/test/e2e_node" + "k8s.io/kubernetes/test/e2e_node/remote" "github.com/ghodss/yaml" "github.com/golang/glog" @@ -131,7 +131,7 @@ func main() { rand.Seed(time.Now().UTC().UnixNano()) if *buildOnly { // Build the archive and exit - e2e_node.CreateTestArchive() + remote.CreateTestArchive() return } @@ -295,7 +295,7 @@ func callGubernator(gubernator bool) { } func (a *Archive) getArchive() (string, error) { - a.Do(func() { a.path, a.err = e2e_node.CreateTestArchive() }) + a.Do(func() { a.path, a.err = remote.CreateTestArchive() }) return a.path, a.err } @@ -346,7 +346,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b } externalIp := getExternalIp(instance) if len(externalIp) > 0 { - e2e_node.AddHostnameIp(host, externalIp) + remote.AddHostnameIp(host, externalIp) } path, err := arc.getArchive() @@ -357,7 +357,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode b } } - output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr) + output, exitOk, err := remote.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs, ginkgoFlagsStr) return &TestResult{ output: output, err: err, @@ -497,10 +497,10 @@ func createInstance(imageConfig *internalGCEImage) (string, error) { } externalIp := getExternalIp(instance) if len(externalIp) > 0 { - e2e_node.AddHostnameIp(name, externalIp) + remote.AddHostnameIp(name, externalIp) } var output string - output, err = e2e_node.RunSshCommand("ssh", e2e_node.GetHostnameOrIp(name), "--", "sudo", "docker", "version") + output, err = remote.RunSshCommand("ssh", remote.GetHostnameOrIp(name), "--", "sudo", "docker", "version") if err != nil { err = fmt.Errorf("instance %s not running docker daemon - Command failed: %s", name, output) continue diff --git a/test/e2e_node/apiserver.go b/test/e2e_node/services/apiserver.go similarity index 99% rename from test/e2e_node/apiserver.go rename to test/e2e_node/services/apiserver.go index 99c59e94c52..c3454412e96 100644 --- a/test/e2e_node/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_node +package services import ( "fmt" diff --git a/test/e2e_node/etcd.go b/test/e2e_node/services/etcd.go similarity index 99% rename from test/e2e_node/etcd.go rename to test/e2e_node/services/etcd.go index 7a587f4f93b..a1e3abf7383 100644 --- a/test/e2e_node/etcd.go +++ b/test/e2e_node/services/etcd.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_node +package services import ( "crypto/tls" diff --git a/test/e2e_node/namespace_controller.go b/test/e2e_node/services/namespace_controller.go similarity index 99% rename from test/e2e_node/namespace_controller.go rename to test/e2e_node/services/namespace_controller.go index a47cb9b8810..90cf20b3fd1 100644 --- a/test/e2e_node/namespace_controller.go +++ b/test/e2e_node/services/namespace_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_node +package services import ( "time" diff --git a/test/e2e_node/e2e_service.go b/test/e2e_node/services/services.go similarity index 98% rename from test/e2e_node/e2e_service.go rename to test/e2e_node/services/services.go index c41c9705fb8..22b87fb9bcd 100644 --- a/test/e2e_node/e2e_service.go +++ b/test/e2e_node/services/services.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_node +package services import ( "flag" @@ -37,6 +37,7 @@ import ( "github.com/kardianos/osext" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e_node/build" ) // TODO(random-liu): Move this file to a separate package. @@ -194,10 +195,6 @@ func (es *e2eService) run() error { } func (es *e2eService) start() error { - if _, err := getK8sBin("kubelet"); err != nil { - return err - } - err := es.startEtcd() if err != nil { return err @@ -341,7 +338,7 @@ func (es *e2eService) startKubeletServer() (*server, error) { // Since kubelet will typically be run as a service it also makes more // sense to test it that way unitName := fmt.Sprintf("kubelet-%d.service", rand.Int31()) - cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--remain-after-exit", getKubeletServerBin()) + cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--remain-after-exit", build.GetKubeletServerBin()) killCommand = exec.Command("sudo", "systemctl", "kill", unitName) restartCommand = exec.Command("sudo", "systemctl", "restart", unitName) es.logFiles["kubelet.log"] = logFileData{ @@ -349,7 +346,7 @@ func (es *e2eService) startKubeletServer() (*server, error) { } framework.TestContext.EvictionHard = adjustConfigForSystemd(framework.TestContext.EvictionHard) } else { - cmdArgs = append(cmdArgs, getKubeletServerBin()) + cmdArgs = append(cmdArgs, build.GetKubeletServerBin()) cmdArgs = append(cmdArgs, "--runtime-cgroups=/docker-daemon", "--kubelet-cgroups=/kubelet", @@ -387,7 +384,8 @@ func (es *e2eService) startKubeletServer() (*server, error) { } cmdArgs = append(cmdArgs, "--network-plugin=kubenet", - "--network-plugin-dir", filepath.Join(cwd, CNIDirectory, "bin")) // Enable kubenet + // TODO(random-liu): Make sure the cni directory name is the same with that in remote/remote.go + "--network-plugin-dir", filepath.Join(cwd, "cni", "bin")) // Enable kubenet } cmd := exec.Command("sudo", cmdArgs...) diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index ef4505074d0..ebb9f141917 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -20,8 +20,8 @@ import ( "flag" ) +// TODO(random-liu): Get this automatically from kubelet flag. var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet") -var buildServices = flag.Bool("build-services", true, "If true, build local executables") var startServices = flag.Bool("start-services", true, "If true, start local node services") var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")