From a11489e0ff1efe887bdf0be8d2d9401c369a7b72 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 25 Feb 2016 08:38:06 -0800 Subject: [PATCH] Node e2e documentations and minor features - Add README.md for node e2e tests - Add support for --cleanup=false to leave test files on remote hosts and temporary instances for debugging - Add ubuntu trusty instances for docker 1.8 and docker 1.9 to jenkins pr builder - Disable coreos-beta for jenkins ci since it is failing - need to investigate --- docs/devel/e2e-node-tests.md | 141 ++++++++++++++++++ hack/verify-flags/known-flags.txt | 2 + test/e2e_node/README.md | 37 +++++ test/e2e_node/e2e_node_suite_test.go | 5 +- test/e2e_node/e2e_remote.go | 16 +- test/e2e_node/e2e_service.go | 2 +- test/e2e_node/environment/setup_host.sh | 62 ++++++++ test/e2e_node/jenkins/e2e-node-jenkins.sh | 4 +- test/e2e_node/jenkins/jenkins-ci.properties | 3 +- test/e2e_node/jenkins/jenkins-pull.properties | 5 +- test/e2e_node/jenkins/template.properties | 7 +- test/e2e_node/runner/run_e2e.go | 17 ++- 12 files changed, 280 insertions(+), 21 deletions(-) create mode 100644 docs/devel/e2e-node-tests.md create mode 100644 test/e2e_node/README.md create mode 100755 test/e2e_node/environment/setup_host.sh diff --git a/docs/devel/e2e-node-tests.md b/docs/devel/e2e-node-tests.md new file mode 100644 index 00000000000..9df2d1db61f --- /dev/null +++ b/docs/devel/e2e-node-tests.md @@ -0,0 +1,141 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +# Node End-To-End tests + +Node e2e tests start kubelet and minimal supporting infrastructure to validate the kubelet on a host. +Tests can be run either locally, against a remote host or against a GCE image. + +*Note: Linux only. Mac and Windows unsupported.* + +## Running tests locally + +etcd must be installed and on the PATH to run the node e2e tests. To verify etcd is installed: `which etcd`. +You can find instructions for installing etcd [on the etcd releases page](https://github.com/coreos/etcd/releases). + +Run the tests locally: `make test_e2e_node` + +Running the node e2e tests locally will build the kubernetes go source files and then start the +kubelet, kube-apiserver, and etcd binaries on localhost before executing the ginkgo tests under +test/e2e_node against the local kubelet instance. + +## Running tests against a remote host + +The node e2e tests can be run against one or more remote hosts using one of +* [e2e-node-jenkins.sh](../../test/e2e_node/jenkins/e2e-node-jenkins.sh) (gce only) +* [run_e2e.go](../../test/e2e_node/runner/run_e2e.go) (requires passwordless ssh and remote passwordless sudo access over ssh) +* using [run_e2e.go](../../test/e2e_node/runner/run_e2e.go) to build a tar.gz and executing on host (requires host access w/ remote sudo) + +### Configuring a new remote host for testing + +The host must contain a environment capable of supporting a mini-kubernetes cluster. Includes: +* install etcd +* install docker +* install lxc and update grub commandline +* enable tty-less sudo access + +See [setup_host.sh](../../test/e2e_node/environment/setup_host.sh) + +### Running the tests + +1. If running against a host on gce + * Copy [template.properties](../../test/e2e_node/jenkins/template.properties) + * Fill in `GCE_HOSTS` + * Set `INSTALL_GODEP=true` to install `godep`, `gomega`, `ginkgo` + * Make sure host names are resolvable to ssh `ssh `. + * If needed, you can run `gcloud compute config-ssh` to add gce hostnames to your .ssh/config so they are resolvable by ssh. + * Run `test/e2e_node/jenkins/e2e-node-jenkins.sh ` + * **Must be run from kubernetes root** + +2. If running against a host anywhere else + * **Requires password-less ssh and sudo access** + * Make sure this works - e.g. `ssh -- sudo echo "ok"` + * If ssh flags are required (e.g. `-i`), they can be used and passed to the tests with `--ssh-options` + * `godep go run test/e2e_node/runner/run_e2e.go --logtostderr --hosts ` + * **Must be run from kubernetes root** + * requires (go get): `github.com/tools/godep`, `github.com/onsi/gomega`, `github.com/onsi/ginkgo/ginkgo` + +3. Alternatively, manually build and copy `e2e_node_test.tar.gz` to a remote host + * Build the tar.gz `godep go run test/e2e_node/runner/run_e2e.go --logtostderr --build-only` + * requires (go get): `github.com/tools/godep`, `github.com/onsi/gomega`, `github.com/onsi/ginkgo/ginkgo` + * Copy `e2e_node_test.tar.gz` to the remote host + * Extract the archive on the remote host `tar -xzvf e2e_node_test.tar.gz` + * Run the tests `./e2e_node.test --logtostderr --vmodule=*=2 --build-services=false --node-name=` + * Note: This must be run from the directory containing the kubelet and kube-apiserver binaries. + +## Running tests against a gce image + +* Build a gce image from a prepared gce host + * Create the host from a base image and configure it (see above) + * Run tests against this remote host to ensure that it is setup correctly before doing anything else + * Create a gce *snapshot* of the instance + * Create a gce *disk* from the snapshot + * Create a gce *image* from the disk +* Test that the necessary gcloud credentials are setup for the project + * `gcloud compute --project --zone images list` + * Verify that your image appears in the list +* Copy [template.properties](../../test/e2e_node/jenkins/template.properties) + * Fill in `GCE_PROJECT`, `GCE_ZONE`, `GCE_IMAGES` +* Run `test/e2e_node/jenkins/e2e-node-jenkins.sh ` + * **Must be run from kubernetes root** + +## Kubernetes Jenkins CI and PR builder + +Node e2e tests are run against a static list of host environments continuously or when manually triggered on a github.com +pull requests using the trigger phrase `@k8s-bot test node e2e experimental` - *results not yet publish, pending +evaluation of test stability.*. + + +### CI Host environments + +TBD + +### PR builder host environments + +| linux distro | distro version | docker version | etcd version | cloud provider | +|-----------------|----------------|----------------|--------------|----------------| +| containervm | | 1.8 | | gce | +| rhel | 7 | 1.10 | | gce | +| centos | 7 | 1.10 | | gce | +| coreos | stable | 1.8 | | gce | +| debian | jessie | 1.10 | | gce | +| ubuntu | trusty | 1.8 | | gce | +| ubuntu | trusty | 1.9 | | gce | +| ubuntu | trusty | 1.10 | | gce | +| ubuntu | wily | 1.10 | | gce | + + + + + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/devel/e2e-node-tests.md?pixel)]() + diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 33d5e1ddfc7..5781afc957e 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -28,6 +28,7 @@ bench-workers bind-address bind-pods-burst bind-pods-qps +build-only build-services cadvisor-port cert-dir @@ -35,6 +36,7 @@ certificate-authority cgroup-root chaos-chance clean-start +cleanup cleanup-iptables client-ca-file client-certificate diff --git a/test/e2e_node/README.md b/test/e2e_node/README.md new file mode 100644 index 00000000000..2d5f51d73cd --- /dev/null +++ b/test/e2e_node/README.md @@ -0,0 +1,37 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + + + +The latest release of this document can be found +[here](http://releases.k8s.io/release-1.1/docs/devel/collab.md). + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +See [e2e-node-tests](../../docs/devel/e2e-node-tests.md) + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/e2e_node/README.md?pixel)]() diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 69cd8c1c653..3ec0b3840db 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -15,8 +15,7 @@ limitations under the License. */ // To run tests in this suite -// NOTE: This test suite requires sudo capabilities to run the kubelet and kube-apiserver. -// $ sudo -v && ginkgo test/e2e_node/ -- --logtostderr --v 2 --node-name `hostname` --start-services +// NOTE: This test suite requires password-less sudo capabilities to run the kubelet and kube-apiserver. package e2e_node import ( @@ -37,7 +36,7 @@ var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080" var nodeName = flag.String("node-name", "", "Name of the node") var buildServices = flag.Bool("build-services", true, "If true, build local executables") var startServices = flag.Bool("start-services", true, "If true, start local node services") -var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tets") +var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests") var e2es *e2eService diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go index e8b65d108c0..73c84759353 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/e2e_remote.go @@ -118,19 +118,21 @@ func CreateTestArchive() string { } // RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test -func RunRemote(archive string, host string) (string, error) { +func RunRemote(archive string, host string, deleteFiles bool) (string, error) { // Create the temp staging directory tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) _, err := RunSshCommand("ssh", host, "--", "mkdir", tmp) if err != nil { return "", err } - defer func() { - output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp) - if err != nil { - glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output) - } - }() + if deleteFiles { + defer func() { + output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp) + if err != nil { + glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output) + } + }() + } // Copy the archive to the staging directory _, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp)) diff --git a/test/e2e_node/e2e_service.go b/test/e2e_node/e2e_service.go index 9dddf141483..0ed325e23ae 100644 --- a/test/e2e_node/e2e_service.go +++ b/test/e2e_node/e2e_service.go @@ -113,7 +113,7 @@ func (es *e2eService) startEtcd() (*exec.Cmd, error) { combinedOut: &es.etcdCombinedOut, healthCheckUrl: "http://127.0.0.1:4001/v2/keys", command: "etcd", - args: []string{"--data-dir", dataDir, "--name", "e2e-node"}, + args: []string{"--data-dir", dataDir}, }) } diff --git a/test/e2e_node/environment/setup_host.sh b/test/e2e_node/environment/setup_host.sh new file mode 100755 index 00000000000..662bc1b30e8 --- /dev/null +++ b/test/e2e_node/environment/setup_host.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script used to configure node e2e test hosts from gce base images. +# DISCLAIMER: This script is not actively tested or maintained. No guarantees that this will work +# on any host environment. Contributions encouraged! Send PRs to pwittrock (github.com). +# +# At some point has successfully configured the following distros: +# - ubuntu trusty +# - containervm (no-op) +# - rhel 7 +# - centos 7 +# - debian jessie + +set -e +set -x + +# Fixup sudoers require tty +sudo grep -q "# Defaults requiretty" /etc/sudoers +if [ $? -ne 0 ] ; then + sudo sed -i 's/Defaults requiretty/# Defaults requiretty/' /etc/sudoers +fi + +# Install etcd +hash etcd 2>/dev/null +if [ $? -ne 0 ]; then + curl -L https://github.com/coreos/etcd/releases/download/v2.2.5/etcd-v2.2.5-linux-amd64.tar.gz -o etcd-v2.2.5-linux-amd64.tar.gz + tar xzvf etcd-v2.2.5-linux-amd64.tar.gz + sudo mv etcd-v2.2.5-linux-amd64/etcd* /usr/local/bin/ + sudo chown root:root /usr/local/bin/etcd* + rm -r etcd-v2.2.5-linux-amd64* +fi + +# Install docker +hash docker 2>/dev/null +if [ $? -ne 0 ]; then + curl -fsSL https://get.docker.com/ | sh + sudo service docker start + sudo systemctl enable docker.service +fi + +# install lxc +cat /etc/*-release | grep "ID=debian" +if [ $? -ne 0 ]; then + sudo apt-get install lxc -y + lxc-checkconfig + sudo sed -i 's/GRUB_CMDLINE_LINUX="\(.*\)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory"/' /etc/default/grub + sudo update-grub +fi diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh index d001a558e01..426f483a2d7 100755 --- a/test/e2e_node/jenkins/e2e-node-jenkins.sh +++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh @@ -35,4 +35,6 @@ if [ "$INSTALL_GODEP" = true ] ; then fi godep go build test/e2e_node/environment/conformance.go -godep go run test/e2e_node/runner/run_e2e.go --logtostderr --v="2" --ssh-env="gce" --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" +godep go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=2 --ssh-env="gce" \ + --zone="$GCE_ZONE" --project="$GCE_PROJECT" \ + --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" diff --git a/test/e2e_node/jenkins/jenkins-ci.properties b/test/e2e_node/jenkins/jenkins-ci.properties index bfbf7442e02..88222a05dda 100644 --- a/test/e2e_node/jenkins/jenkins-ci.properties +++ b/test/e2e_node/jenkins/jenkins-ci.properties @@ -1,5 +1,6 @@ -GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-coreos-beta,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10 +GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10 GCE_IMAGES= GCE_ZONE=us-central1-f GCE_PROJECT=kubernetes-jenkins INSTALL_GODEP=true +CLEANUP=true diff --git a/test/e2e_node/jenkins/jenkins-pull.properties b/test/e2e_node/jenkins/jenkins-pull.properties index 9bdd378d2bd..7b286794a54 100644 --- a/test/e2e_node/jenkins/jenkins-pull.properties +++ b/test/e2e_node/jenkins/jenkins-pull.properties @@ -1,5 +1,6 @@ -GCE_HOSTS=e2e-node-ubuntu-trusty-docker10 -GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image +GCE_HOSTS= +GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image,e2e-node-ubuntu-trusty-docker9-image,e2e-node-ubuntu-trusty-docker8-image GCE_ZONE=us-central1-f GCE_PROJECT=kubernetes-jenkins-pull INSTALL_GODEP=true +CLEANUP=true diff --git a/test/e2e_node/jenkins/template.properties b/test/e2e_node/jenkins/template.properties index aacdb9c3abd..70d0a5506f2 100644 --- a/test/e2e_node/jenkins/template.properties +++ b/test/e2e_node/jenkins/template.properties @@ -1,10 +1,13 @@ # Copy this file to your home directory and modify -# Names of gce hosts to test against (must be resolvable) or empty +# Names of gce hosts to test against (must be resolvable) or empty (one or more of GCE_IMAGES, GCE_HOSTS is required) GCE_HOSTS= -# Names of gce images to test or empty +# Names of gce images to test or empty (one or more of GCE_IMAGES, GCE_HOSTS is required) GCE_IMAGES= # Gce zone to use - required when using GCE_IMAGES GCE_ZONE= # Gce project to use - required when using GCE_IMAGES GCE_PROJECT= +# If true, assume a pristine GOPATH and install necessary godeps INSTALL_GODEP=false +# If true, delete instances created from GCE_IMAGES and files copied to GCE_HOSTS +CLEANUP=true diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index 7b84749ddea..d9ea0c55f57 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -15,9 +15,9 @@ limitations under the License. */ // To run the e2e tests against one or more hosts on gce: -// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts +// $ godep go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts // To run the e2e tests against one or more images on gce and provision them: -// $ go run run_e2e.go --logtostderr --v 2 --project --zone --ssh-env gce --images +// $ godep go run run_e2e.go --logtostderr --v 2 --project --zone --ssh-env gce --images package main import ( @@ -42,6 +42,8 @@ var zone = flag.String("zone", "", "gce zone the hosts live in") var project = flag.String("project", "", "gce project the hosts live in") var images = flag.String("images", "", "images to test") var hosts = flag.String("hosts", "", "hosts to test") +var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances") +var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.") var computeService *compute.Service @@ -53,6 +55,11 @@ type TestResult struct { func main() { flag.Parse() + if *buildOnly { + // Build the archive and exit + e2e_node.CreateTestArchive() + return + } if *hosts == "" && *images == "" { glog.Fatalf("Must specify one of --images or --hosts flag.") @@ -142,7 +149,7 @@ func main() { // Run tests in archive against host func testHost(host, archive string) *TestResult { - output, err := e2e_node.RunRemote(archive, host) + output, err := e2e_node.RunRemote(archive, host, *cleanup) return &TestResult{ output: output, err: err, @@ -154,7 +161,9 @@ func testHost(host, archive string) *TestResult { // Delete the instance afterward. func testImage(image, archive string) *TestResult { host, err := createInstance(image) - defer deleteInstance(image) + if *cleanup { + defer deleteInstance(image) + } if err != nil { return &TestResult{ err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s. %v", image, err),