diff --git a/hack/jenkins/job-configs/node-e2e.yaml b/hack/jenkins/job-configs/node-e2e.yaml index 0e9621f3963..e1748c36ce8 100644 --- a/hack/jenkins/job-configs/node-e2e.yaml +++ b/hack/jenkins/job-configs/node-e2e.yaml @@ -57,7 +57,7 @@ # owner: owner to be notified for job failures. test results are published to owner email # repoName: github repo to checkout e.g. kubernetes/kubernetes or google/cadvisor # gitbasedir: directory under $WORKSPACE/go/src to checkout source repo to - e.g. k8s.io/kubernetes or github.com/google/cadvisor -# shell: shell to execute from workspace +# shell: bash command to execute from gitbasedir. should be a single script such as {gitproject}-jenkins.sh - job-template: name: '{gitproject}-gce-e2e-ci' description: '{gitproject} continuous e2e tests.
Test Owner: {owner}.' @@ -66,7 +66,12 @@ numToKeep: 200 node: node builders: - - shell: '{shell}' + - shell: | + #!/bin/bash + set -e + set -x + cd go/src/{gitbasedir} + {shell} publishers: - claim-build - gcs-uploader @@ -125,11 +130,6 @@ gitbasedir: 'github.com/google/cadvisor' owner: 'vishnuk@google.com' shell: | - #!/bin/bash - set -e - set -x - cd go/src/github.com/google/cadvisor - go get -u github.com/tools/godep ./build/presubmit.sh @@ -145,28 +145,11 @@ repoName: 'kubernetes/heapster' gitbasedir: 'k8s.io/heapster' owner: 'pszczesniak@google.com' - shell: | - #!/bin/bash - set -e - set -x - cd go/src/k8s.io/heapster - - make test-unit test-integration + shell: 'make test-unit test-integration' - 'kubelet': repoName: 'kubernetes/kubernetes' gitbasedir: 'k8s.io/kubernetes' owner: 'pwittroc@google.com' - shell: | - #!/bin/bash - set -e - set -x - cd go/src/k8s.io/kubernetes - - go get -u github.com/tools/godep - go get -u github.com/onsi/ginkgo/ginkgo - go get -u github.com/onsi/gomega - - godep go build test/e2e_node/environment/conformance.go - godep go run test/e2e_node/runner/run_e2e.go --zone us-central1-f --hosts e2e-node-container-vm-v20151215,e2e-node-coreos-beta.c.kubernetes-jenkins.internal,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10 --logtostderr -v 2 + shell: 'test/e2e_node/jenkins/e2e-node-jenkins.sh test/e2e_node/jenkins/jenkins-ci.properties' jobs: - '{gitproject}-gce-e2e-ci' diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 9c2ed7ed91f..33d5e1ddfc7 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -151,6 +151,7 @@ input-dirs insecure-bind-address insecure-port insecure-skip-tls-verify +instance-name-prefix iptables-masquerade-bit iptables-sync-period ir-data-source diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 58ceb4398ba..69cd8c1c653 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors All rights reserved. +Copyright 2016 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -61,7 +61,7 @@ var _ = BeforeSuite(func() { } if *startServices { - e2es = newE2eService() + e2es = newE2eService(*nodeName) if err := e2es.start(); err != nil { Fail(fmt.Sprintf("Unable to start node services.\n%v", err)) } diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go index 1de8d0600a3..e8b65d108c0 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/e2e_remote.go @@ -121,19 +121,19 @@ func CreateTestArchive() string { func RunRemote(archive string, host string) (string, error) { // Create the temp staging directory tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) - _, err := runSshCommand("ssh", host, "--", "mkdir", tmp) + _, err := RunSshCommand("ssh", host, "--", "mkdir", tmp) if err != nil { return "", err } defer func() { - output, err := runSshCommand("ssh", host, "--", "rm", "-rf", tmp) + output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp) if err != nil { glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output) } }() // Copy the archive to the staging directory - _, err = runSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp)) + _, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp)) if err != nil { return "", err } @@ -142,18 +142,20 @@ func RunRemote(archive string, host string) (string, error) { cmd := getSshCommand(" ; ", "sudo pkill kubelet", "sudo pkill kube-apiserver", - "sudo pkill etcd") + "sudo pkill etcd", + ) // No need to log an error if pkill fails since pkill will fail if the commands are not running. // If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd // logs about failing to bind the required ports. - runSshCommand("ssh", host, "--", "sh", "-c", cmd) + RunSshCommand("ssh", host, "--", "sh", "-c", cmd) // Extract the archive and run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), fmt.Sprintf("tar -xzvf ./%s", archiveName), - "./e2e_node.test --logtostderr --v 2 --build-services=false --node-name `hostname`") - output, err := runSshCommand("ssh", host, "--", "sh", "-c", cmd) + fmt.Sprintf("./e2e_node.test --logtostderr --v 2 --build-services=false --node-name=%s", host), + ) + output, err := RunSshCommand("ssh", host, "--", "sh", "-c", cmd) if err != nil { return "", err } @@ -167,7 +169,7 @@ func getSshCommand(sep string, args ...string) string { } // runSshCommand executes the ssh or scp command, adding the flag provided --ssh-options -func runSshCommand(cmd string, args ...string) (string, error) { +func RunSshCommand(cmd string, args ...string) (string, error) { if env, found := sshOptionsMap[*sshEnv]; found { args = append(strings.Split(env, " "), args...) } @@ -176,7 +178,7 @@ func runSshCommand(cmd string, args ...string) (string, error) { } output, err := exec.Command(cmd, args...).CombinedOutput() if err != nil { - return fmt.Sprintf("%s", output), fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output) + return fmt.Sprintf("%s", output), fmt.Errorf("Command [%s %s] failed with error: %v and output:\n%s", cmd, strings.Join(args, " "), err, output) } return fmt.Sprintf("%s", output), nil } diff --git a/test/e2e_node/e2e_service.go b/test/e2e_node/e2e_service.go index 56f525fb0d1..9dddf141483 100644 --- a/test/e2e_node/e2e_service.go +++ b/test/e2e_node/e2e_service.go @@ -40,10 +40,11 @@ type e2eService struct { apiServerCombinedOut bytes.Buffer kubeletCmd *exec.Cmd kubeletCombinedOut bytes.Buffer + nodeName string } -func newE2eService() *e2eService { - return &e2eService{} +func newE2eService(nodeName string) *e2eService { + return &e2eService{nodeName: nodeName} } func (es *e2eService) start() error { @@ -141,7 +142,9 @@ func (es *e2eService) startKubeletServer() (*exec.Cmd, error) { "--v", "2", "--logtostderr", "--log_dir", "./", "--api-servers", "http://127.0.0.1:8080", "--address", "0.0.0.0", - "--port", "10250"}, + "--port", "10250", + "--hostname-override", es.nodeName, // Required because hostname is inconsistent across hosts + }, }) } diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh new file mode 100755 index 00000000000..d001a558e01 --- /dev/null +++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script executed by jenkins to run node e2e tests against gce +# Usage: test/e2e_node/jenkins/e2e-node-jenkins.sh +# Properties files: +# - test/e2e_node/jenkins/jenkins-ci.properties : for running jenkins ci +# - test/e2e_node/jenkins/jenkins-pull.properties : for running jenkins pull request builder +# - test/e2e_node/jenkins/template.properties : template for creating a properties file to run locally + +set -e +set -x + +: "${1:?Usage test/e2e_node/jenkins/e2e-node-jenkins.sh }" + +. $1 + +if [ "$INSTALL_GODEP" = true ] ; then + go get -u github.com/tools/godep + go get -u github.com/onsi/ginkgo/ginkgo + go get -u github.com/onsi/gomega +fi + +godep go build test/e2e_node/environment/conformance.go +godep go run test/e2e_node/runner/run_e2e.go --logtostderr --v="2" --ssh-env="gce" --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" diff --git a/test/e2e_node/jenkins/jenkins-ci.properties b/test/e2e_node/jenkins/jenkins-ci.properties new file mode 100644 index 00000000000..bfbf7442e02 --- /dev/null +++ b/test/e2e_node/jenkins/jenkins-ci.properties @@ -0,0 +1,5 @@ +GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-coreos-beta,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10 +GCE_IMAGES= +GCE_ZONE=us-central1-f +GCE_PROJECT=kubernetes-jenkins +INSTALL_GODEP=true diff --git a/test/e2e_node/jenkins/jenkins-pull.properties b/test/e2e_node/jenkins/jenkins-pull.properties new file mode 100644 index 00000000000..9bdd378d2bd --- /dev/null +++ b/test/e2e_node/jenkins/jenkins-pull.properties @@ -0,0 +1,5 @@ +GCE_HOSTS=e2e-node-ubuntu-trusty-docker10 +GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image +GCE_ZONE=us-central1-f +GCE_PROJECT=kubernetes-jenkins-pull +INSTALL_GODEP=true diff --git a/test/e2e_node/jenkins/template.properties b/test/e2e_node/jenkins/template.properties new file mode 100644 index 00000000000..aacdb9c3abd --- /dev/null +++ b/test/e2e_node/jenkins/template.properties @@ -0,0 +1,10 @@ +# Copy this file to your home directory and modify +# Names of gce hosts to test against (must be resolvable) or empty +GCE_HOSTS= +# Names of gce images to test or empty +GCE_IMAGES= +# Gce zone to use - required when using GCE_IMAGES +GCE_ZONE= +# Gce project to use - required when using GCE_IMAGES +GCE_PROJECT= +INSTALL_GODEP=false diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go index 69240dad1b9..70292efa042 100644 --- a/test/e2e_node/kubelet_test.go +++ b/test/e2e_node/kubelet_test.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors All rights reserved. +Copyright 2016 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index e2d5bb8d93c..7b84749ddea 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors All rights reserved. +Copyright 2016 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,22 +14,59 @@ See the License for the specific language governing permissions and limitations under the License. */ -// To run the e2e tests against one or more hosts on gce: $ go run run_e2e.go --hosts -// Requires gcloud compute ssh access to the hosts +// To run the e2e tests against one or more hosts on gce: +// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts +// To run the e2e tests against one or more images on gce and provision them: +// $ go run run_e2e.go --logtostderr --v 2 --project --zone --ssh-env gce --images package main import ( "flag" "fmt" + "net/http" "os" "strings" + "time" "k8s.io/kubernetes/test/e2e_node" + + "github.com/golang/glog" + "github.com/pborman/uuid" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/compute/v1" ) +var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names") +var zone = flag.String("zone", "", "gce zone the hosts live in") +var project = flag.String("project", "", "gce project the hosts live in") +var images = flag.String("images", "", "images to test") var hosts = flag.String("hosts", "", "hosts to test") +var computeService *compute.Service + +type TestResult struct { + output string + err error + host string +} + func main() { + flag.Parse() + + if *hosts == "" && *images == "" { + glog.Fatalf("Must specify one of --images or --hosts flag.") + } + if *images != "" && *zone == "" { + glog.Fatal("Must specify --zone flag") + } + if *images != "" && *project == "" { + glog.Fatal("Must specify --project flag") + } + if *instanceNamePrefix == "" { + *instanceNamePrefix = "tmp-node-e2e-" + uuid.NewUUID().String()[:8] + } + // Setup coloring stat, _ := os.Stdout.Stat() useColor := (stat.Mode() & os.ModeCharDevice) != 0 @@ -40,38 +77,58 @@ func main() { noColour = "\033[0m" } - flag.Parse() - if *hosts == "" { - fmt.Printf("Must specific --hosts flag") - } archive := e2e_node.CreateTestArchive() defer os.Remove(archive) results := make(chan *TestResult) - hs := strings.Split(*hosts, ",") - for _, h := range hs { - fmt.Printf("Starting tests on host %s.", h) - go func(host string) { - output, err := e2e_node.RunRemote(archive, host) - results <- &TestResult{ - output: output, - err: err, - host: host, + running := 0 + if *images != "" { + // Setup the gce client for provisioning instances + // Getting credentials on gce jenkins is flaky, so try a couple times + var err error + for i := 0; i < 10; i++ { + var client *http.Client + client, err = google.DefaultClient(oauth2.NoContext, compute.ComputeScope) + if err != nil { + continue } - }(h) + computeService, err = compute.New(client) + if err != nil { + continue + } + time.Sleep(time.Second * 6) + } + if err != nil { + glog.Fatalf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err) + } + + for _, image := range strings.Split(*images, ",") { + running++ + fmt.Printf("Initializing e2e tests using image %s.\n", image) + go func(image string) { results <- testImage(image, archive) }(image) + } + } + if *hosts != "" { + for _, host := range strings.Split(*hosts, ",") { + fmt.Printf("Initializing e2e tests using host %s.\n", host) + running++ + go func(host string) { + results <- testHost(host, archive) + }(host) + } } // Wait for all tests to complete and emit the results errCount := 0 - for i := 0; i < len(hs); i++ { + for i := 0; i < running; i++ { tr := <-results host := tr.host fmt.Printf("%s================================================================%s\n", blue, noColour) if tr.err != nil { errCount++ - fmt.Printf("Failure Finished Host %s Test Suite %s %v\n", host, tr.output, tr.err) + fmt.Printf("Failure Finished Host %s Test Suite\n%s\n%v\n", host, tr.output, tr.err) } else { - fmt.Printf("Success Finished Host %s Test Suite %s\n", host, tr.output) + fmt.Printf("Success Finished Host %s Test Suite\n%s\n", host, tr.output) } fmt.Printf("%s================================================================%s\n", blue, noColour) } @@ -83,8 +140,107 @@ func main() { } } -type TestResult struct { - output string - err error - host string +// Run tests in archive against host +func testHost(host, archive string) *TestResult { + output, err := e2e_node.RunRemote(archive, host) + return &TestResult{ + output: output, + err: err, + host: host, + } +} + +// Provision a gce instance using image and run the tests in archive against the instance. +// Delete the instance afterward. +func testImage(image, archive string) *TestResult { + host, err := createInstance(image) + defer deleteInstance(image) + if err != nil { + return &TestResult{ + err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s. %v", image, err), + } + } + return testHost(host, archive) +} + +// Provision a gce instance using image +func createInstance(image string) (string, error) { + name := imageToInstanceName(image) + i := &compute.Instance{ + Name: name, + MachineType: machineType(), + NetworkInterfaces: []*compute.NetworkInterface{ + { + AccessConfigs: []*compute.AccessConfig{ + { + Type: "ONE_TO_ONE_NAT", + Name: "External NAT", + }, + }}, + }, + Disks: []*compute.AttachedDisk{ + { + AutoDelete: true, + Boot: true, + Type: "PERSISTENT", + InitializeParams: &compute.AttachedDiskInitializeParams{ + SourceImage: sourceImage(image), + }, + }, + }, + } + op, err := computeService.Instances.Insert(*project, *zone, i).Do() + if err != nil { + return "", err + } + if op.Error != nil { + return "", fmt.Errorf("Could not create instance %s: %+v", name, op.Error) + } + + instanceRunning := false + for i := 0; i < 30 && !instanceRunning; i++ { + if i > 0 { + time.Sleep(time.Second * 20) + } + var instance *compute.Instance + instance, err = computeService.Instances.Get(*project, *zone, name).Do() + if err != nil { + continue + } + if strings.ToUpper(instance.Status) != "RUNNING" { + err = fmt.Errorf("Instance %s not in state RUNNING, was %s.", name, instance.Status) + continue + } + var output string + output, err = e2e_node.RunSshCommand("ssh", name, "--", "sudo", "docker", "version") + if err != nil { + err = fmt.Errorf("Instance %s not running docker daemon - Command failed: %s", name, output) + continue + } + if !strings.Contains(output, "Server") { + err = fmt.Errorf("Instance %s not running docker daemon - Server not found: %s", name, output) + continue + } + instanceRunning = true + } + return name, err +} + +func deleteInstance(image string) { + _, err := computeService.Instances.Delete(*project, *zone, imageToInstanceName(image)).Do() + if err != nil { + glog.Infof("Error deleting instance %s", imageToInstanceName(image)) + } +} + +func imageToInstanceName(image string) string { + return *instanceNamePrefix + "-" + image +} + +func sourceImage(image string) string { + return fmt.Sprintf("projects/%s/global/images/%s", *project, image) +} + +func machineType() string { + return fmt.Sprintf("zones/%s/machineTypes/n1-standard-1", *zone) }