mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 04:33:26 +00:00
Merge pull request #21946 from pwittrock/node-e2e-images
Auto commit by PR queue bot
This commit is contained in:
k8s-merge-robot
commit
a53bf86f40
141
docs/devel/e2e-node-tests.md
Normal file
141
docs/devel/e2e-node-tests.md
Normal file
@ -0,0 +1,141 @@
|
||||
<!-- BEGIN MUNGE: UNVERSIONED_WARNING -->
|
||||
|
||||
<!-- BEGIN STRIP_FOR_RELEASE -->
|
||||
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
|
||||
<h2>PLEASE NOTE: This document applies to the HEAD of the source tree</h2>
|
||||
|
||||
If you are using a released version of Kubernetes, you should
|
||||
refer to the docs that go with that version.
|
||||
|
||||
Documentation for other releases can be found at
|
||||
[releases.k8s.io](http://releases.k8s.io).
|
||||
</strong>
|
||||
--
|
||||
|
||||
<!-- END STRIP_FOR_RELEASE -->
|
||||
|
||||
<!-- END MUNGE: UNVERSIONED_WARNING -->
|
||||
|
||||
# Node End-To-End tests
|
||||
|
||||
Node e2e tests start kubelet and minimal supporting infrastructure to validate the kubelet on a host.
|
||||
Tests can be run either locally, against a remote host or against a GCE image.
|
||||
|
||||
*Note: Linux only. Mac and Windows unsupported.*
|
||||
|
||||
## Running tests locally
|
||||
|
||||
etcd must be installed and on the PATH to run the node e2e tests. To verify etcd is installed: `which etcd`.
|
||||
You can find instructions for installing etcd [on the etcd releases page](https://github.com/coreos/etcd/releases).
|
||||
|
||||
Run the tests locally: `make test_e2e_node`
|
||||
|
||||
Running the node e2e tests locally will build the kubernetes go source files and then start the
|
||||
kubelet, kube-apiserver, and etcd binaries on localhost before executing the ginkgo tests under
|
||||
test/e2e_node against the local kubelet instance.
|
||||
|
||||
## Running tests against a remote host
|
||||
|
||||
The node e2e tests can be run against one or more remote hosts using one of
|
||||
* [e2e-node-jenkins.sh](../../test/e2e_node/jenkins/e2e-node-jenkins.sh) (gce only)
|
||||
* [run_e2e.go](../../test/e2e_node/runner/run_e2e.go) (requires passwordless ssh and remote passwordless sudo access over ssh)
|
||||
* using [run_e2e.go](../../test/e2e_node/runner/run_e2e.go) to build a tar.gz and executing on host (requires host access w/ remote sudo)
|
||||
|
||||
### Configuring a new remote host for testing
|
||||
|
||||
The host must contain a environment capable of supporting a mini-kubernetes cluster. Includes:
|
||||
* install etcd
|
||||
* install docker
|
||||
* install lxc and update grub commandline
|
||||
* enable tty-less sudo access
|
||||
|
||||
See [setup_host.sh](../../test/e2e_node/environment/setup_host.sh)
|
||||
|
||||
### Running the tests
|
||||
|
||||
1. If running against a host on gce
|
||||
* Copy [template.properties](../../test/e2e_node/jenkins/template.properties)
|
||||
* Fill in `GCE_HOSTS`
|
||||
* Set `INSTALL_GODEP=true` to install `godep`, `gomega`, `ginkgo`
|
||||
* Make sure host names are resolvable to ssh `ssh <host>`.
|
||||
* If needed, you can run `gcloud compute config-ssh` to add gce hostnames to your .ssh/config so they are resolvable by ssh.
|
||||
* Run `test/e2e_node/jenkins/e2e-node-jenkins.sh <path to properties file>`
|
||||
* **Must be run from kubernetes root**
|
||||
|
||||
2. If running against a host anywhere else
|
||||
* **Requires password-less ssh and sudo access**
|
||||
* Make sure this works - e.g. `ssh <hostname> -- sudo echo "ok"`
|
||||
* If ssh flags are required (e.g. `-i`), they can be used and passed to the tests with `--ssh-options`
|
||||
* `godep go run test/e2e_node/runner/run_e2e.go --logtostderr --hosts <comma separated hosts>`
|
||||
* **Must be run from kubernetes root**
|
||||
* requires (go get): `github.com/tools/godep`, `github.com/onsi/gomega`, `github.com/onsi/ginkgo/ginkgo`
|
||||
|
||||
3. Alternatively, manually build and copy `e2e_node_test.tar.gz` to a remote host
|
||||
* Build the tar.gz `godep go run test/e2e_node/runner/run_e2e.go --logtostderr --build-only`
|
||||
* requires (go get): `github.com/tools/godep`, `github.com/onsi/gomega`, `github.com/onsi/ginkgo/ginkgo`
|
||||
* Copy `e2e_node_test.tar.gz` to the remote host
|
||||
* Extract the archive on the remote host `tar -xzvf e2e_node_test.tar.gz`
|
||||
* Run the tests `./e2e_node.test --logtostderr --vmodule=*=2 --build-services=false --node-name=<hostname>`
|
||||
* Note: This must be run from the directory containing the kubelet and kube-apiserver binaries.
|
||||
|
||||
## Running tests against a gce image
|
||||
|
||||
* Build a gce image from a prepared gce host
|
||||
* Create the host from a base image and configure it (see above)
|
||||
* Run tests against this remote host to ensure that it is setup correctly before doing anything else
|
||||
* Create a gce *snapshot* of the instance
|
||||
* Create a gce *disk* from the snapshot
|
||||
* Create a gce *image* from the disk
|
||||
* Test that the necessary gcloud credentials are setup for the project
|
||||
* `gcloud compute --project <project> --zone <zone> images list`
|
||||
* Verify that your image appears in the list
|
||||
* Copy [template.properties](../../test/e2e_node/jenkins/template.properties)
|
||||
* Fill in `GCE_PROJECT`, `GCE_ZONE`, `GCE_IMAGES`
|
||||
* Run `test/e2e_node/jenkins/e2e-node-jenkins.sh <path to properties file>`
|
||||
* **Must be run from kubernetes root**
|
||||
|
||||
## Kubernetes Jenkins CI and PR builder
|
||||
|
||||
Node e2e tests are run against a static list of host environments continuously or when manually triggered on a github.com
|
||||
pull requests using the trigger phrase `@k8s-bot test node e2e experimental` - *results not yet publish, pending
|
||||
evaluation of test stability.*.
|
||||
|
||||
|
||||
### CI Host environments
|
||||
|
||||
TBD
|
||||
|
||||
### PR builder host environments
|
||||
|
||||
| linux distro | distro version | docker version | etcd version | cloud provider |
|
||||
|-----------------|----------------|----------------|--------------|----------------|
|
||||
| containervm | | 1.8 | | gce |
|
||||
| rhel | 7 | 1.10 | | gce |
|
||||
| centos | 7 | 1.10 | | gce |
|
||||
| coreos | stable | 1.8 | | gce |
|
||||
| debian | jessie | 1.10 | | gce |
|
||||
| ubuntu | trusty | 1.8 | | gce |
|
||||
| ubuntu | trusty | 1.9 | | gce |
|
||||
| ubuntu | trusty | 1.10 | | gce |
|
||||
| ubuntu | wily | 1.10 | | gce |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
|
||||
[]()
|
||||
<!-- END MUNGE: GENERATED_ANALYTICS -->
|
||||
@ -28,6 +28,7 @@ bench-workers
|
||||
bind-address
|
||||
bind-pods-burst
|
||||
bind-pods-qps
|
||||
build-only
|
||||
build-services
|
||||
cadvisor-port
|
||||
cert-dir
|
||||
@ -35,6 +36,7 @@ certificate-authority
|
||||
cgroup-root
|
||||
chaos-chance
|
||||
clean-start
|
||||
cleanup
|
||||
cleanup-iptables
|
||||
client-ca-file
|
||||
client-certificate
|
||||
|
||||
37
test/e2e_node/README.md
Normal file
37
test/e2e_node/README.md
Normal file
@ -0,0 +1,37 @@
|
||||
<!-- BEGIN MUNGE: UNVERSIONED_WARNING -->
|
||||
|
||||
<!-- BEGIN STRIP_FOR_RELEASE -->
|
||||
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
<img src="http://kubernetes.io/img/warning.png" alt="WARNING"
|
||||
width="25" height="25">
|
||||
|
||||
<h2>PLEASE NOTE: This document applies to the HEAD of the source tree</h2>
|
||||
|
||||
If you are using a released version of Kubernetes, you should
|
||||
refer to the docs that go with that version.
|
||||
|
||||
<!-- TAG RELEASE_LINK, added by the munger automatically -->
|
||||
<strong>
|
||||
The latest release of this document can be found
|
||||
[here](http://releases.k8s.io/release-1.1/docs/devel/collab.md).
|
||||
|
||||
Documentation for other releases can be found at
|
||||
[releases.k8s.io](http://releases.k8s.io).
|
||||
</strong>
|
||||
--
|
||||
|
||||
<!-- END STRIP_FOR_RELEASE -->
|
||||
|
||||
<!-- END MUNGE: UNVERSIONED_WARNING -->
|
||||
|
||||
See [e2e-node-tests](../../docs/devel/e2e-node-tests.md)
|
||||
|
||||
[]()
|
||||
@ -15,8 +15,7 @@ limitations under the License.
|
||||
*/
|
||||
|
||||
// To run tests in this suite
|
||||
// NOTE: This test suite requires sudo capabilities to run the kubelet and kube-apiserver.
|
||||
// $ sudo -v && ginkgo test/e2e_node/ -- --logtostderr --v 2 --node-name `hostname` --start-services
|
||||
// NOTE: This test suite requires password-less sudo capabilities to run the kubelet and kube-apiserver.
|
||||
package e2e_node
|
||||
|
||||
import (
|
||||
@ -37,7 +36,7 @@ var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080"
|
||||
var nodeName = flag.String("node-name", "", "Name of the node")
|
||||
var buildServices = flag.Bool("build-services", true, "If true, build local executables")
|
||||
var startServices = flag.Bool("start-services", true, "If true, start local node services")
|
||||
var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tets")
|
||||
var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
|
||||
|
||||
var e2es *e2eService
|
||||
|
||||
|
||||
@ -118,19 +118,21 @@ func CreateTestArchive() string {
|
||||
}
|
||||
|
||||
// RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test
|
||||
func RunRemote(archive string, host string) (string, error) {
|
||||
func RunRemote(archive string, host string, deleteFiles bool) (string, error) {
|
||||
// Create the temp staging directory
|
||||
tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
|
||||
_, err := RunSshCommand("ssh", host, "--", "mkdir", tmp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() {
|
||||
output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
|
||||
}
|
||||
}()
|
||||
if deleteFiles {
|
||||
defer func() {
|
||||
output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Copy the archive to the staging directory
|
||||
_, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp))
|
||||
|
||||
@ -113,7 +113,7 @@ func (es *e2eService) startEtcd() (*exec.Cmd, error) {
|
||||
combinedOut: &es.etcdCombinedOut,
|
||||
healthCheckUrl: "http://127.0.0.1:4001/v2/keys",
|
||||
command: "etcd",
|
||||
args: []string{"--data-dir", dataDir, "--name", "e2e-node"},
|
||||
args: []string{"--data-dir", dataDir},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
62
test/e2e_node/environment/setup_host.sh
Executable file
62
test/e2e_node/environment/setup_host.sh
Executable file
@ -0,0 +1,62 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Script used to configure node e2e test hosts from gce base images.
|
||||
# DISCLAIMER: This script is not actively tested or maintained. No guarantees that this will work
|
||||
# on any host environment. Contributions encouraged! Send PRs to pwittrock (github.com).
|
||||
#
|
||||
# At some point has successfully configured the following distros:
|
||||
# - ubuntu trusty
|
||||
# - containervm (no-op)
|
||||
# - rhel 7
|
||||
# - centos 7
|
||||
# - debian jessie
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# Fixup sudoers require tty
|
||||
sudo grep -q "# Defaults requiretty" /etc/sudoers
|
||||
if [ $? -ne 0 ] ; then
|
||||
sudo sed -i 's/Defaults requiretty/# Defaults requiretty/' /etc/sudoers
|
||||
fi
|
||||
|
||||
# Install etcd
|
||||
hash etcd 2>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
curl -L https://github.com/coreos/etcd/releases/download/v2.2.5/etcd-v2.2.5-linux-amd64.tar.gz -o etcd-v2.2.5-linux-amd64.tar.gz
|
||||
tar xzvf etcd-v2.2.5-linux-amd64.tar.gz
|
||||
sudo mv etcd-v2.2.5-linux-amd64/etcd* /usr/local/bin/
|
||||
sudo chown root:root /usr/local/bin/etcd*
|
||||
rm -r etcd-v2.2.5-linux-amd64*
|
||||
fi
|
||||
|
||||
# Install docker
|
||||
hash docker 2>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
curl -fsSL https://get.docker.com/ | sh
|
||||
sudo service docker start
|
||||
sudo systemctl enable docker.service
|
||||
fi
|
||||
|
||||
# install lxc
|
||||
cat /etc/*-release | grep "ID=debian"
|
||||
if [ $? -ne 0 ]; then
|
||||
sudo apt-get install lxc -y
|
||||
lxc-checkconfig
|
||||
sudo sed -i 's/GRUB_CMDLINE_LINUX="\(.*\)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory"/' /etc/default/grub
|
||||
sudo update-grub
|
||||
fi
|
||||
@ -35,4 +35,6 @@ if [ "$INSTALL_GODEP" = true ] ; then
|
||||
fi
|
||||
|
||||
godep go build test/e2e_node/environment/conformance.go
|
||||
godep go run test/e2e_node/runner/run_e2e.go --logtostderr --v="2" --ssh-env="gce" --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" --images="$GCE_IMAGES"
|
||||
godep go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=2 --ssh-env="gce" \
|
||||
--zone="$GCE_ZONE" --project="$GCE_PROJECT" \
|
||||
--hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP"
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-coreos-beta,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10
|
||||
GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10
|
||||
GCE_IMAGES=
|
||||
GCE_ZONE=us-central1-f
|
||||
GCE_PROJECT=kubernetes-jenkins
|
||||
INSTALL_GODEP=true
|
||||
CLEANUP=true
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
GCE_HOSTS=e2e-node-ubuntu-trusty-docker10
|
||||
GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image
|
||||
GCE_HOSTS=
|
||||
GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image,e2e-node-ubuntu-trusty-docker9-image,e2e-node-ubuntu-trusty-docker8-image
|
||||
GCE_ZONE=us-central1-f
|
||||
GCE_PROJECT=kubernetes-jenkins-pull
|
||||
INSTALL_GODEP=true
|
||||
CLEANUP=true
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
# Copy this file to your home directory and modify
|
||||
# Names of gce hosts to test against (must be resolvable) or empty
|
||||
# Names of gce hosts to test against (must be resolvable) or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
|
||||
GCE_HOSTS=
|
||||
# Names of gce images to test or empty
|
||||
# Names of gce images to test or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
|
||||
GCE_IMAGES=
|
||||
# Gce zone to use - required when using GCE_IMAGES
|
||||
GCE_ZONE=
|
||||
# Gce project to use - required when using GCE_IMAGES
|
||||
GCE_PROJECT=
|
||||
# If true, assume a pristine GOPATH and install necessary godeps
|
||||
INSTALL_GODEP=false
|
||||
# If true, delete instances created from GCE_IMAGES and files copied to GCE_HOSTS
|
||||
CLEANUP=true
|
||||
|
||||
@ -15,9 +15,9 @@ limitations under the License.
|
||||
*/
|
||||
|
||||
// To run the e2e tests against one or more hosts on gce:
|
||||
// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
|
||||
// $ godep go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
|
||||
// To run the e2e tests against one or more images on gce and provision them:
|
||||
// $ go run run_e2e.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
|
||||
// $ godep go run run_e2e.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
|
||||
package main
|
||||
|
||||
import (
|
||||
@ -42,6 +42,8 @@ var zone = flag.String("zone", "", "gce zone the hosts live in")
|
||||
var project = flag.String("project", "", "gce project the hosts live in")
|
||||
var images = flag.String("images", "", "images to test")
|
||||
var hosts = flag.String("hosts", "", "hosts to test")
|
||||
var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances")
|
||||
var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
|
||||
|
||||
var computeService *compute.Service
|
||||
|
||||
@ -53,6 +55,11 @@ type TestResult struct {
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if *buildOnly {
|
||||
// Build the archive and exit
|
||||
e2e_node.CreateTestArchive()
|
||||
return
|
||||
}
|
||||
|
||||
if *hosts == "" && *images == "" {
|
||||
glog.Fatalf("Must specify one of --images or --hosts flag.")
|
||||
@ -142,7 +149,7 @@ func main() {
|
||||
|
||||
// Run tests in archive against host
|
||||
func testHost(host, archive string) *TestResult {
|
||||
output, err := e2e_node.RunRemote(archive, host)
|
||||
output, err := e2e_node.RunRemote(archive, host, *cleanup)
|
||||
return &TestResult{
|
||||
output: output,
|
||||
err: err,
|
||||
@ -154,7 +161,9 @@ func testHost(host, archive string) *TestResult {
|
||||
// Delete the instance afterward.
|
||||
func testImage(image, archive string) *TestResult {
|
||||
host, err := createInstance(image)
|
||||
defer deleteInstance(image)
|
||||
if *cleanup {
|
||||
defer deleteInstance(image)
|
||||
}
|
||||
if err != nil {
|
||||
return &TestResult{
|
||||
err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s. %v", image, err),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user