diff --git a/docs/devel/e2e-node-tests.md b/docs/devel/e2e-node-tests.md
new file mode 100644
index 00000000000..9df2d1db61f
--- /dev/null
+++ b/docs/devel/e2e-node-tests.md
@@ -0,0 +1,141 @@
+
+
+
+
+
+
+
+
+
+
+PLEASE NOTE: This document applies to the HEAD of the source tree
+
+If you are using a released version of Kubernetes, you should
+refer to the docs that go with that version.
+
+Documentation for other releases can be found at
+[releases.k8s.io](http://releases.k8s.io).
+
+--
+
+
+
+
+
+# Node End-To-End tests
+
+Node e2e tests start kubelet and minimal supporting infrastructure to validate the kubelet on a host.
+Tests can be run either locally, against a remote host or against a GCE image.
+
+*Note: Linux only. Mac and Windows unsupported.*
+
+## Running tests locally
+
+etcd must be installed and on the PATH to run the node e2e tests. To verify etcd is installed: `which etcd`.
+You can find instructions for installing etcd [on the etcd releases page](https://github.com/coreos/etcd/releases).
+
+Run the tests locally: `make test_e2e_node`
+
+Running the node e2e tests locally will build the kubernetes go source files and then start the
+kubelet, kube-apiserver, and etcd binaries on localhost before executing the ginkgo tests under
+test/e2e_node against the local kubelet instance.
+
+## Running tests against a remote host
+
+The node e2e tests can be run against one or more remote hosts using one of
+* [e2e-node-jenkins.sh](../../test/e2e_node/jenkins/e2e-node-jenkins.sh) (gce only)
+* [run_e2e.go](../../test/e2e_node/runner/run_e2e.go) (requires passwordless ssh and remote passwordless sudo access over ssh)
+* using [run_e2e.go](../../test/e2e_node/runner/run_e2e.go) to build a tar.gz and executing on host (requires host access w/ remote sudo)
+
+### Configuring a new remote host for testing
+
+The host must contain a environment capable of supporting a mini-kubernetes cluster. Includes:
+* install etcd
+* install docker
+* install lxc and update grub commandline
+* enable tty-less sudo access
+
+See [setup_host.sh](../../test/e2e_node/environment/setup_host.sh)
+
+### Running the tests
+
+1. If running against a host on gce
+ * Copy [template.properties](../../test/e2e_node/jenkins/template.properties)
+ * Fill in `GCE_HOSTS`
+ * Set `INSTALL_GODEP=true` to install `godep`, `gomega`, `ginkgo`
+ * Make sure host names are resolvable to ssh `ssh `.
+ * If needed, you can run `gcloud compute config-ssh` to add gce hostnames to your .ssh/config so they are resolvable by ssh.
+ * Run `test/e2e_node/jenkins/e2e-node-jenkins.sh `
+ * **Must be run from kubernetes root**
+
+2. If running against a host anywhere else
+ * **Requires password-less ssh and sudo access**
+ * Make sure this works - e.g. `ssh -- sudo echo "ok"`
+ * If ssh flags are required (e.g. `-i`), they can be used and passed to the tests with `--ssh-options`
+ * `godep go run test/e2e_node/runner/run_e2e.go --logtostderr --hosts `
+ * **Must be run from kubernetes root**
+ * requires (go get): `github.com/tools/godep`, `github.com/onsi/gomega`, `github.com/onsi/ginkgo/ginkgo`
+
+3. Alternatively, manually build and copy `e2e_node_test.tar.gz` to a remote host
+ * Build the tar.gz `godep go run test/e2e_node/runner/run_e2e.go --logtostderr --build-only`
+ * requires (go get): `github.com/tools/godep`, `github.com/onsi/gomega`, `github.com/onsi/ginkgo/ginkgo`
+ * Copy `e2e_node_test.tar.gz` to the remote host
+ * Extract the archive on the remote host `tar -xzvf e2e_node_test.tar.gz`
+ * Run the tests `./e2e_node.test --logtostderr --vmodule=*=2 --build-services=false --node-name=`
+ * Note: This must be run from the directory containing the kubelet and kube-apiserver binaries.
+
+## Running tests against a gce image
+
+* Build a gce image from a prepared gce host
+ * Create the host from a base image and configure it (see above)
+ * Run tests against this remote host to ensure that it is setup correctly before doing anything else
+ * Create a gce *snapshot* of the instance
+ * Create a gce *disk* from the snapshot
+ * Create a gce *image* from the disk
+* Test that the necessary gcloud credentials are setup for the project
+ * `gcloud compute --project --zone images list`
+ * Verify that your image appears in the list
+* Copy [template.properties](../../test/e2e_node/jenkins/template.properties)
+ * Fill in `GCE_PROJECT`, `GCE_ZONE`, `GCE_IMAGES`
+* Run `test/e2e_node/jenkins/e2e-node-jenkins.sh `
+ * **Must be run from kubernetes root**
+
+## Kubernetes Jenkins CI and PR builder
+
+Node e2e tests are run against a static list of host environments continuously or when manually triggered on a github.com
+pull requests using the trigger phrase `@k8s-bot test node e2e experimental` - *results not yet publish, pending
+evaluation of test stability.*.
+
+
+### CI Host environments
+
+TBD
+
+### PR builder host environments
+
+| linux distro | distro version | docker version | etcd version | cloud provider |
+|-----------------|----------------|----------------|--------------|----------------|
+| containervm | | 1.8 | | gce |
+| rhel | 7 | 1.10 | | gce |
+| centos | 7 | 1.10 | | gce |
+| coreos | stable | 1.8 | | gce |
+| debian | jessie | 1.10 | | gce |
+| ubuntu | trusty | 1.8 | | gce |
+| ubuntu | trusty | 1.9 | | gce |
+| ubuntu | trusty | 1.10 | | gce |
+| ubuntu | wily | 1.10 | | gce |
+
+
+
+
+
+
+
+
+[]()
+
diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt
index 33d5e1ddfc7..5781afc957e 100644
--- a/hack/verify-flags/known-flags.txt
+++ b/hack/verify-flags/known-flags.txt
@@ -28,6 +28,7 @@ bench-workers
bind-address
bind-pods-burst
bind-pods-qps
+build-only
build-services
cadvisor-port
cert-dir
@@ -35,6 +36,7 @@ certificate-authority
cgroup-root
chaos-chance
clean-start
+cleanup
cleanup-iptables
client-ca-file
client-certificate
diff --git a/test/e2e_node/README.md b/test/e2e_node/README.md
new file mode 100644
index 00000000000..2d5f51d73cd
--- /dev/null
+++ b/test/e2e_node/README.md
@@ -0,0 +1,37 @@
+
+
+
+
+
+
+
+
+
+
+PLEASE NOTE: This document applies to the HEAD of the source tree
+
+If you are using a released version of Kubernetes, you should
+refer to the docs that go with that version.
+
+
+
+The latest release of this document can be found
+[here](http://releases.k8s.io/release-1.1/docs/devel/collab.md).
+
+Documentation for other releases can be found at
+[releases.k8s.io](http://releases.k8s.io).
+
+--
+
+
+
+
+
+See [e2e-node-tests](../../docs/devel/e2e-node-tests.md)
+
+[]()
diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go
index 69cd8c1c653..3ec0b3840db 100644
--- a/test/e2e_node/e2e_node_suite_test.go
+++ b/test/e2e_node/e2e_node_suite_test.go
@@ -15,8 +15,7 @@ limitations under the License.
*/
// To run tests in this suite
-// NOTE: This test suite requires sudo capabilities to run the kubelet and kube-apiserver.
-// $ sudo -v && ginkgo test/e2e_node/ -- --logtostderr --v 2 --node-name `hostname` --start-services
+// NOTE: This test suite requires password-less sudo capabilities to run the kubelet and kube-apiserver.
package e2e_node
import (
@@ -37,7 +36,7 @@ var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080"
var nodeName = flag.String("node-name", "", "Name of the node")
var buildServices = flag.Bool("build-services", true, "If true, build local executables")
var startServices = flag.Bool("start-services", true, "If true, start local node services")
-var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tets")
+var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
var e2es *e2eService
diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go
index e8b65d108c0..73c84759353 100644
--- a/test/e2e_node/e2e_remote.go
+++ b/test/e2e_node/e2e_remote.go
@@ -118,19 +118,21 @@ func CreateTestArchive() string {
}
// RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test
-func RunRemote(archive string, host string) (string, error) {
+func RunRemote(archive string, host string, deleteFiles bool) (string, error) {
// Create the temp staging directory
tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
_, err := RunSshCommand("ssh", host, "--", "mkdir", tmp)
if err != nil {
return "", err
}
- defer func() {
- output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp)
- if err != nil {
- glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
- }
- }()
+ if deleteFiles {
+ defer func() {
+ output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp)
+ if err != nil {
+ glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
+ }
+ }()
+ }
// Copy the archive to the staging directory
_, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp))
diff --git a/test/e2e_node/e2e_service.go b/test/e2e_node/e2e_service.go
index 9dddf141483..0ed325e23ae 100644
--- a/test/e2e_node/e2e_service.go
+++ b/test/e2e_node/e2e_service.go
@@ -113,7 +113,7 @@ func (es *e2eService) startEtcd() (*exec.Cmd, error) {
combinedOut: &es.etcdCombinedOut,
healthCheckUrl: "http://127.0.0.1:4001/v2/keys",
command: "etcd",
- args: []string{"--data-dir", dataDir, "--name", "e2e-node"},
+ args: []string{"--data-dir", dataDir},
})
}
diff --git a/test/e2e_node/environment/setup_host.sh b/test/e2e_node/environment/setup_host.sh
new file mode 100755
index 00000000000..662bc1b30e8
--- /dev/null
+++ b/test/e2e_node/environment/setup_host.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script used to configure node e2e test hosts from gce base images.
+# DISCLAIMER: This script is not actively tested or maintained. No guarantees that this will work
+# on any host environment. Contributions encouraged! Send PRs to pwittrock (github.com).
+#
+# At some point has successfully configured the following distros:
+# - ubuntu trusty
+# - containervm (no-op)
+# - rhel 7
+# - centos 7
+# - debian jessie
+
+set -e
+set -x
+
+# Fixup sudoers require tty
+sudo grep -q "# Defaults requiretty" /etc/sudoers
+if [ $? -ne 0 ] ; then
+ sudo sed -i 's/Defaults requiretty/# Defaults requiretty/' /etc/sudoers
+fi
+
+# Install etcd
+hash etcd 2>/dev/null
+if [ $? -ne 0 ]; then
+ curl -L https://github.com/coreos/etcd/releases/download/v2.2.5/etcd-v2.2.5-linux-amd64.tar.gz -o etcd-v2.2.5-linux-amd64.tar.gz
+ tar xzvf etcd-v2.2.5-linux-amd64.tar.gz
+ sudo mv etcd-v2.2.5-linux-amd64/etcd* /usr/local/bin/
+ sudo chown root:root /usr/local/bin/etcd*
+ rm -r etcd-v2.2.5-linux-amd64*
+fi
+
+# Install docker
+hash docker 2>/dev/null
+if [ $? -ne 0 ]; then
+ curl -fsSL https://get.docker.com/ | sh
+ sudo service docker start
+ sudo systemctl enable docker.service
+fi
+
+# install lxc
+cat /etc/*-release | grep "ID=debian"
+if [ $? -ne 0 ]; then
+ sudo apt-get install lxc -y
+ lxc-checkconfig
+ sudo sed -i 's/GRUB_CMDLINE_LINUX="\(.*\)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory"/' /etc/default/grub
+ sudo update-grub
+fi
diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh
index d001a558e01..426f483a2d7 100755
--- a/test/e2e_node/jenkins/e2e-node-jenkins.sh
+++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh
@@ -35,4 +35,6 @@ if [ "$INSTALL_GODEP" = true ] ; then
fi
godep go build test/e2e_node/environment/conformance.go
-godep go run test/e2e_node/runner/run_e2e.go --logtostderr --v="2" --ssh-env="gce" --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" --images="$GCE_IMAGES"
+godep go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=2 --ssh-env="gce" \
+ --zone="$GCE_ZONE" --project="$GCE_PROJECT" \
+ --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP"
diff --git a/test/e2e_node/jenkins/jenkins-ci.properties b/test/e2e_node/jenkins/jenkins-ci.properties
index bfbf7442e02..88222a05dda 100644
--- a/test/e2e_node/jenkins/jenkins-ci.properties
+++ b/test/e2e_node/jenkins/jenkins-ci.properties
@@ -1,5 +1,6 @@
-GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-coreos-beta,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10
+GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10
GCE_IMAGES=
GCE_ZONE=us-central1-f
GCE_PROJECT=kubernetes-jenkins
INSTALL_GODEP=true
+CLEANUP=true
diff --git a/test/e2e_node/jenkins/jenkins-pull.properties b/test/e2e_node/jenkins/jenkins-pull.properties
index 9bdd378d2bd..7b286794a54 100644
--- a/test/e2e_node/jenkins/jenkins-pull.properties
+++ b/test/e2e_node/jenkins/jenkins-pull.properties
@@ -1,5 +1,6 @@
-GCE_HOSTS=e2e-node-ubuntu-trusty-docker10
-GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image
+GCE_HOSTS=
+GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image,e2e-node-ubuntu-trusty-docker9-image,e2e-node-ubuntu-trusty-docker8-image
GCE_ZONE=us-central1-f
GCE_PROJECT=kubernetes-jenkins-pull
INSTALL_GODEP=true
+CLEANUP=true
diff --git a/test/e2e_node/jenkins/template.properties b/test/e2e_node/jenkins/template.properties
index aacdb9c3abd..70d0a5506f2 100644
--- a/test/e2e_node/jenkins/template.properties
+++ b/test/e2e_node/jenkins/template.properties
@@ -1,10 +1,13 @@
# Copy this file to your home directory and modify
-# Names of gce hosts to test against (must be resolvable) or empty
+# Names of gce hosts to test against (must be resolvable) or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
GCE_HOSTS=
-# Names of gce images to test or empty
+# Names of gce images to test or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
GCE_IMAGES=
# Gce zone to use - required when using GCE_IMAGES
GCE_ZONE=
# Gce project to use - required when using GCE_IMAGES
GCE_PROJECT=
+# If true, assume a pristine GOPATH and install necessary godeps
INSTALL_GODEP=false
+# If true, delete instances created from GCE_IMAGES and files copied to GCE_HOSTS
+CLEANUP=true
diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go
index 7b84749ddea..d9ea0c55f57 100644
--- a/test/e2e_node/runner/run_e2e.go
+++ b/test/e2e_node/runner/run_e2e.go
@@ -15,9 +15,9 @@ limitations under the License.
*/
// To run the e2e tests against one or more hosts on gce:
-// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts
+// $ godep go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts
// To run the e2e tests against one or more images on gce and provision them:
-// $ go run run_e2e.go --logtostderr --v 2 --project --zone --ssh-env gce --images
+// $ godep go run run_e2e.go --logtostderr --v 2 --project --zone --ssh-env gce --images
package main
import (
@@ -42,6 +42,8 @@ var zone = flag.String("zone", "", "gce zone the hosts live in")
var project = flag.String("project", "", "gce project the hosts live in")
var images = flag.String("images", "", "images to test")
var hosts = flag.String("hosts", "", "hosts to test")
+var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances")
+var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
var computeService *compute.Service
@@ -53,6 +55,11 @@ type TestResult struct {
func main() {
flag.Parse()
+ if *buildOnly {
+ // Build the archive and exit
+ e2e_node.CreateTestArchive()
+ return
+ }
if *hosts == "" && *images == "" {
glog.Fatalf("Must specify one of --images or --hosts flag.")
@@ -142,7 +149,7 @@ func main() {
// Run tests in archive against host
func testHost(host, archive string) *TestResult {
- output, err := e2e_node.RunRemote(archive, host)
+ output, err := e2e_node.RunRemote(archive, host, *cleanup)
return &TestResult{
output: output,
err: err,
@@ -154,7 +161,9 @@ func testHost(host, archive string) *TestResult {
// Delete the instance afterward.
func testImage(image, archive string) *TestResult {
host, err := createInstance(image)
- defer deleteInstance(image)
+ if *cleanup {
+ defer deleteInstance(image)
+ }
if err != nil {
return &TestResult{
err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s. %v", image, err),