diff --git a/test/e2e_node/conformance/build/Dockerfile b/test/e2e_node/conformance/build/Dockerfile new file mode 100644 index 00000000000..8e490331f54 --- /dev/null +++ b/test/e2e_node/conformance/build/Dockerfile @@ -0,0 +1,44 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +COPY ginkgo /usr/local/bin/ +COPY e2e_node.test /usr/local/bin + +# The following environment variables can be override when starting the container. +# FOCUS is regex matching test to run. By default run all conformance test. +# SKIP is regex matching test to skip. By default empty. +# PARALLELISM is the number of processes the test will run in parallel. +# REPORT_PATH is the path in the container to save test result and logs. +# MANIFEST_PATH is the kubelet manifest path in the container. +# FLAKE_ATTEMPTS is the time to retry when there is a test failure. By default 2. +# TEST_ARGS is the test arguments passed into the test. +ENV FOCUS="\[Conformance\]" \ + SKIP="\[Flaky\]|\[Serial\]" \ + PARALLELISM=8 \ + REPORT_PATH="/var/result" \ + MANIFEST_PATH="/etc/manifest" \ + FLAKE_ATTEMPTS=2 \ + TEST_ARGS="" + +ENTRYPOINT ginkgo --focus="$FOCUS" \ + --skip="$SKIP" \ + --nodes=$PARALLELISM \ + --flakeAttempts=$FLAKE_ATTEMPTS \ + /usr/local/bin/e2e_node.test \ + -- --conformance=true \ + --prepull-images=false \ + --manifest-path="$MANIFEST_PATH"\ + --report-dir="$REPORT_PATH $TEST_ARGS" diff --git a/test/e2e_node/conformance/build/Makefile b/test/e2e_node/conformance/build/Makefile new file mode 100644 index 00000000000..2ac003f747b --- /dev/null +++ b/test/e2e_node/conformance/build/Makefile @@ -0,0 +1,60 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the node-test image. +# +# Usage: +# [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] [BIN_DIR="../../../../_output/bin"] make (build|push) VERSION={some_version_number e.g. 0.1} + +# TODO(random-liu): Add this into release progress. +REGISTRY?=gcr.io/google_containers +ARCH?=amd64 +# BIN_DIR is the directory to find binaries, overwrite with ../../../../_output/bin +# for local development. +BIN_DIR?=../../../../_output/dockerized/bin/linux/${ARCH} +TEMP_DIR:=$(shell mktemp -d) + +BASEIMAGE_amd64=debian:jessie +BASEIMAGE_arm=armel/debian:jessie +BASEIMAGE_arm64=aarch64/debian:jessie +BASEIMAGE_ppc64le=ppc64le/debian:jessie + +BASEIMAGE?=${BASEIMAGE_${ARCH}} + +all: build + +build: + +ifndef VERSION + $(error VERSION is undefined) +endif + cp -r ./* ${TEMP_DIR} + + cp ${BIN_DIR}/ginkgo ${TEMP_DIR} + cp ${BIN_DIR}/e2e_node.test ${TEMP_DIR} + + cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile + + # Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer + # they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers. + cd ${TEMP_DIR} && chmod a+rx \ + e2e_node.test \ + ginkgo + + docker build -t ${REGISTRY}/node-test-${ARCH}:${VERSION} ${TEMP_DIR} + +push: build + gcloud docker push ${REGISTRY}/node-test-${ARCH}:${VERSION} + +.PHONY: all diff --git a/test/e2e_node/conformance/run_test.sh b/test/e2e_node/conformance/run_test.sh new file mode 100755 index 00000000000..a1ad7af0f5c --- /dev/null +++ b/test/e2e_node/conformance/run_test.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is only for demonstrating how to use the node test container. In +# production environment, kubelet bootstrap will be more complicated, user +# should configure the node test container accordingly. +# In addition, this script will also be used in the node e2e test to let it use +# the containerized test suite. + +# TODO(random-liu): Use standard installer to install kubelet. +# TODO(random-liu): Use standard tool to start kubelet in production way (such +# as systemd, supervisord etc.) +# TODO(random-liu): Initialize kubelet with standard configmap after dynamic +# configuration landing, so that all test could get the current kubelet +# configuration and react accordingly. + +# Refresh sudo credentials if not running on GCE. +if ! ping -c 1 -q metadata.google.internal &> /dev/null; then + sudo -v || exit 1 +fi + +# FOCUS is ginkgo focus to select which tests to run. By default, FOCUS is +# initialized as "\[Conformance\]" in the test container to run all conformance +# test. +FOCUS=${FOCUS:-""} + +# SKIP is ginkgo skip to select which tests to skip. By default, SKIP is +# initialized as "\[Flaky\]|\[Serial\]" in the test container skipping all +# flaky and serial test. +SKIP=${SKIP:-""} + +# REGISTRY is the image registry for node test image. +REGISTRY=${REGISTRY:-"gcr.io/google_containers"} + +# ARCH is the architecture of current machine, the script will use this to +# select corresponding test container image. +ARCH=${ARCH:-"amd64"} + +# VERSION is the version of the test container image. +VERSION=${VERSION:-"0.1"} + +# KUBELET_BIN is the kubelet binary name. If it is not specified, use the +# default binary name "kubelet". +KUBELET_BIN=${KUBELET_BIN:-"kubelet"} + +# KUBELET is the kubelet binary path. If it is not specified, assume kubelet is +# in PATH. +KUBELET=${KUBELET:-"`which $KUBELET_BIN`"} + +# LOG_DIR is the absolute path of the directory where the test will collect all +# logs to. By default, use the current directory. +LOG_DIR=${LOG_DIR:-`pwd`} +mkdir -p $LOG_DIR + +# NETWORK_PLUGIN is the network plugin used by kubelet. Do not use network +# plugin by default. +NETWORK_PLUGIN=${NETWORK_PLUGIN:-""} + +# NETWORK_PLUGIN_PATH is the path to network plugin binary. +NETWORK_PLUGIN_PATH=${NETWORK_PLUGIN_PATH:-""} + +# start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log. +kubelet_log=kubelet.log +start_kubelet() { + echo "Starting kubelet..." + sudo -b $KUBELET $@ &>$LOG_DIR/$kubelet_log + if [ $? -ne 0 ]; then + echo "Failed to start kubelet" + exit 1 + fi +} + +# wait_kubelet retris for 10 times for kubelet to be ready by checking http://127.0.0.1:10255/healthz. +wait_kubelet() { + echo "Health checking kubelet..." + healthCheckURL=http://127.0.0.1:10255/healthz + local maxRetry=10 + local cur=1 + while [ $cur -le $maxRetry ]; do + curl -s $healthCheckURL > /dev/null + if [ $? -eq 0 ]; then + echo "Kubelet is ready" + break + fi + if [ $cur -eq $maxRetry]; then + echo "Health check exceeds max retry" + exit 1 + fi + echo "Kubelet is not ready" + sleep 1 + ((cur++)) + done +} + +# kill_kubelet kills kubelet. +kill_kubelet() { + echo "Stopping kubelet..." + sudo pkill $KUBELET_BIN + if [ $? -ne 0 ]; then + echo "Failed to stop kubelet." + exit 1 + fi +} + +# run_test runs the node test container. +run_test() { + env="" + if [ ! -z "$FOCUS" ]; then + env="$env -e FOCUS=$FOCUS" + fi + if [ ! -z "$SKIP" ]; then + env="$env -e SKIP=$SKIP" + fi + # The test assumes that inside the container: + # * kubelet manifest path is mounted to /etc/manifest; + # * log collect directory is mounted to /var/result; + # * root file system is mounted to /rootfs. + sudo docker run -it --rm --privileged=true --net=host -v /:/rootfs \ + -v $config_dir:/etc/manifest -v $LOG_DIR:/var/result $env $REGISTRY/node-test-$ARCH:$VERSION +} + +# Check whether kubelet is running. If kubelet is running, tell the user to stop +# it before running the test. +pid=`pidof $KUBELET_BIN` +if [ ! -z $pid ]; then + echo "Kubelet is running (pid=$pid), please stop it before running the test." + exit 1 +fi + +apiserver=http://localhost:8080 +volume_stats_agg_period=10s +allow_privileged=true +serialize_image_pulls=false +config_dir=`mktemp -d` +file_check_frequency=10s +pod_cidr=10.180.0.0/24 +log_level=4 +start_kubelet --api-servers $apiserver \ + --volume-stats-agg-period $volume_stats_agg_period \ + --allow-privileged=$allow_privileged \ + --serialize-image-pulls=$serialize_image_pulls \ + --config $config_dir \ + --file-check-frequency $file_check_frequency \ + --pod-cidr=$pod_cidr \ + --runtime-cgroups=/docker-daemon \ + --kubelet-cgroups=/kubelet \ + --system-cgroups=/system \ + --cgroup-root=/ \ + --network-plugin=$NETWORK_PLUGIN \ + --network-plugin-dir=$NETWORK_PLUGIN_PATH \ + --v=$log_level \ + --logtostderr + +wait_kubelet + +run_test + +kill_kubelet + +# Clean up the kubelet config directory +sudo rm -rf $config_dir diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index b5d275034ee..b0a87f70059 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -28,6 +28,7 @@ import ( "os" "os/exec" "path" + "syscall" "testing" "time" @@ -71,6 +72,10 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +// When running the containerized conformance test, we'll mount the +// host root filesystem as readonly to /rootfs. +const rootfs = "/rootfs" + func TestE2eNode(t *testing.T) { if *runServicesMode { // If run-services-mode is specified, only run services in current process. @@ -79,6 +84,15 @@ func TestE2eNode(t *testing.T) { } if *systemValidateMode { // If system-validate-mode is specified, only run system validation in current process. + if framework.TestContext.NodeConformance { + // Chroot to /rootfs to make system validation can check system + // as in the root filesystem. + // TODO(random-liu): Consider to chroot the whole test process to make writing + // test easier. + if err := syscall.Chroot(rootfs); err != nil { + glog.Exitf("chroot %q failed: %v", rootfs, err) + } + } if err := system.Validate(); err != nil { glog.Exitf("system validation failed: %v", err) } @@ -172,6 +186,7 @@ func validateSystem() error { if err != nil { return fmt.Errorf("can't get current binary: %v", err) } + // Pass all flags into the child process, so that it will see the same flag set. output, err := exec.Command(testBin, append([]string{"--system-validate-mode"}, os.Args[1:]...)...).CombinedOutput() // The output of system validation should have been formatted, directly print here. fmt.Print(string(output)) diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 040b506a6f7..80370070c4f 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -148,6 +148,7 @@ func (e *E2EServices) startInternalServices() (*server, error) { if err != nil { return nil, fmt.Errorf("can't get current binary: %v", err) } + // Pass all flags into the child process, so that it will see the same flag set. startCmd := exec.Command(testBin, append([]string{"--run-services-mode"}, os.Args[1:]...)...) server := newServer("services", startCmd, nil, nil, getServicesHealthCheckURLs(), servicesLogFile, e.monitorParent, false) return server, server.start()