Add Dockerfile and Makefile to containerize node conformance test.

test/e2e_node/conformance/build/Dockerfile

@@ -0,0 +1,44 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+COPY ginkgo /usr/local/bin/
+COPY e2e_node.test /usr/local/bin
+
+# The following environment variables can be override when starting the container.
+# FOCUS is regex matching test to run. By default run all conformance test.
+# SKIP is regex matching test to skip. By default empty.
+# PARALLELISM is the number of processes the test will run in parallel.
+# REPORT_PATH is the path in the container to save test result and logs.
+# MANIFEST_PATH is the kubelet manifest path in the container.
+# FLAKE_ATTEMPTS is the time to retry when there is a test failure. By default 2.
+# TEST_ARGS is the test arguments passed into the test.
+ENV FOCUS="\[Conformance\]" \
+	   SKIP="\[Flaky\]|\[Serial\]" \
+	   PARALLELISM=8 \
+	   REPORT_PATH="/var/result" \
+	   MANIFEST_PATH="/etc/manifest" \
+	   FLAKE_ATTEMPTS=2 \
+	   TEST_ARGS=""
+
+ENTRYPOINT ginkgo --focus="$FOCUS" \
+	--skip="$SKIP" \
+	--nodes=$PARALLELISM \
+	--flakeAttempts=$FLAKE_ATTEMPTS \
+	/usr/local/bin/e2e_node.test \
+	-- --conformance=true \
+	--prepull-images=false \
+	--manifest-path="$MANIFEST_PATH"\
+	--report-dir="$REPORT_PATH $TEST_ARGS"

test/e2e_node/conformance/build/Makefile

@@ -0,0 +1,60 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the node-test image.
+#
+# Usage:
+#   [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] [BIN_DIR="../../../../_output/bin"] make (build|push) VERSION={some_version_number e.g. 0.1}
+
+# TODO(random-liu): Add this into release progress.
+REGISTRY?=gcr.io/google_containers
+ARCH?=amd64
+# BIN_DIR is the directory to find binaries, overwrite with ../../../../_output/bin
+# for local development.
+BIN_DIR?=../../../../_output/dockerized/bin/linux/${ARCH}
+TEMP_DIR:=$(shell mktemp -d)
+
+BASEIMAGE_amd64=debian:jessie
+BASEIMAGE_arm=armel/debian:jessie
+BASEIMAGE_arm64=aarch64/debian:jessie
+BASEIMAGE_ppc64le=ppc64le/debian:jessie
+
+BASEIMAGE?=${BASEIMAGE_${ARCH}}
+
+all: build
+
+build:
+
+ifndef VERSION
+    $(error VERSION is undefined)
+endif
+	cp -r ./* ${TEMP_DIR}
+
+	cp ${BIN_DIR}/ginkgo ${TEMP_DIR}
+	cp ${BIN_DIR}/e2e_node.test ${TEMP_DIR}
+
+	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
+
+	# Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer
+	# they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers.
+	cd ${TEMP_DIR} && chmod a+rx \
+		e2e_node.test \
+		ginkgo
+
+	docker build -t ${REGISTRY}/node-test-${ARCH}:${VERSION} ${TEMP_DIR}
+
+push: build
+	gcloud docker push ${REGISTRY}/node-test-${ARCH}:${VERSION}
+
+.PHONY: all

test/e2e_node/conformance/run_test.sh

@@ -0,0 +1,174 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is only for demonstrating how to use the node test container. In
+# production environment, kubelet bootstrap will be more complicated, user
+# should configure the node test container accordingly.
+# In addition, this script will also be used in the node e2e test to let it use
+# the containerized test suite.
+
+# TODO(random-liu): Use standard installer to install kubelet.
+# TODO(random-liu): Use standard tool to start kubelet in production way (such
+# as systemd, supervisord etc.)
+# TODO(random-liu): Initialize kubelet with standard configmap after dynamic
+# configuration landing, so that all test could get the current kubelet
+# configuration and react accordingly.
+
+# Refresh sudo credentials if not running on GCE.
+if ! ping -c 1 -q metadata.google.internal &> /dev/null; then
+  sudo -v || exit 1
+fi
+
+# FOCUS is ginkgo focus to select which tests to run. By default, FOCUS is
+# initialized as "\[Conformance\]" in the test container to run all conformance
+# test.
+FOCUS=${FOCUS:-""}
+
+# SKIP is ginkgo skip to select which tests to skip. By default, SKIP is
+# initialized as "\[Flaky\]|\[Serial\]" in the test container skipping all
+# flaky and serial test.
+SKIP=${SKIP:-""}
+
+# REGISTRY is the image registry for node test image.
+REGISTRY=${REGISTRY:-"gcr.io/google_containers"}
+
+# ARCH is the architecture of current machine, the script will use this to
+# select corresponding test container image.
+ARCH=${ARCH:-"amd64"}
+
+# VERSION is the version of the test container image.
+VERSION=${VERSION:-"0.1"}
+
+# KUBELET_BIN is the kubelet binary name. If it is not specified, use the
+# default binary name "kubelet".
+KUBELET_BIN=${KUBELET_BIN:-"kubelet"}
+
+# KUBELET is the kubelet binary path. If it is not specified, assume kubelet is
+# in PATH.
+KUBELET=${KUBELET:-"`which $KUBELET_BIN`"}
+
+# LOG_DIR is the absolute path of the directory where the test will collect all
+# logs to. By default, use the current directory.
+LOG_DIR=${LOG_DIR:-`pwd`}
+mkdir -p $LOG_DIR
+
+# NETWORK_PLUGIN is the network plugin used by kubelet. Do not use network
+# plugin by default.
+NETWORK_PLUGIN=${NETWORK_PLUGIN:-""}
+
+# NETWORK_PLUGIN_PATH is the path to network plugin binary.
+NETWORK_PLUGIN_PATH=${NETWORK_PLUGIN_PATH:-""}
+
+# start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log.
+kubelet_log=kubelet.log
+start_kubelet() {
+  echo "Starting kubelet..."
+  sudo -b $KUBELET $@ &>$LOG_DIR/$kubelet_log
+  if [ $? -ne 0 ]; then
+    echo "Failed to start kubelet"
+    exit 1
+  fi
+}
+
+# wait_kubelet retris for 10 times for kubelet to be ready by checking http://127.0.0.1:10255/healthz.
+wait_kubelet() {
+  echo "Health checking kubelet..."
+  healthCheckURL=http://127.0.0.1:10255/healthz
+  local maxRetry=10
+  local cur=1
+  while [ $cur -le $maxRetry ]; do
+    curl -s $healthCheckURL > /dev/null
+    if [ $? -eq 0 ]; then
+      echo "Kubelet is ready"
+      break
+    fi
+    if [ $cur -eq $maxRetry]; then
+      echo "Health check exceeds max retry"
+      exit 1
+    fi
+    echo "Kubelet is not ready"
+    sleep 1
+    ((cur++))
+  done
+}
+
+# kill_kubelet kills kubelet.
+kill_kubelet() {
+  echo "Stopping kubelet..."
+  sudo pkill $KUBELET_BIN
+  if [ $? -ne 0 ]; then
+    echo "Failed to stop kubelet."
+    exit 1
+  fi
+}
+
+# run_test runs the node test container.
+run_test() {
+  env=""
+  if [ ! -z "$FOCUS" ]; then
+    env="$env -e FOCUS=$FOCUS"
+  fi
+  if [ ! -z "$SKIP" ]; then
+    env="$env -e SKIP=$SKIP"
+  fi
+  # The test assumes that inside the container:
+  # * kubelet manifest path is mounted to /etc/manifest;
+  # * log collect directory is mounted to /var/result;
+  # * root file system is mounted to /rootfs.
+  sudo docker run -it --rm --privileged=true --net=host  -v /:/rootfs \
+    -v $config_dir:/etc/manifest -v $LOG_DIR:/var/result $env $REGISTRY/node-test-$ARCH:$VERSION
+}
+
+# Check whether kubelet is running. If kubelet is running, tell the user to stop
+# it before running the test.
+pid=`pidof $KUBELET_BIN`
+if [ ! -z $pid ]; then
+  echo "Kubelet is running (pid=$pid), please stop it before running the test."
+  exit 1
+fi
+
+apiserver=http://localhost:8080
+volume_stats_agg_period=10s
+allow_privileged=true
+serialize_image_pulls=false
+config_dir=`mktemp -d`
+file_check_frequency=10s
+pod_cidr=10.180.0.0/24
+log_level=4
+start_kubelet --api-servers $apiserver \
+  --volume-stats-agg-period $volume_stats_agg_period \
+  --allow-privileged=$allow_privileged \
+  --serialize-image-pulls=$serialize_image_pulls \
+  --config $config_dir \
+  --file-check-frequency $file_check_frequency \
+  --pod-cidr=$pod_cidr \
+  --runtime-cgroups=/docker-daemon \
+  --kubelet-cgroups=/kubelet \
+  --system-cgroups=/system \
+  --cgroup-root=/ \
+  --network-plugin=$NETWORK_PLUGIN \
+  --network-plugin-dir=$NETWORK_PLUGIN_PATH \
+  --v=$log_level \
+  --logtostderr
+
+wait_kubelet
+
+run_test
+
+kill_kubelet
+
+# Clean up the kubelet config directory
+sudo rm -rf $config_dir

test/e2e_node/e2e_node_suite_test.go

@@ -28,6 +28,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"syscall"
 	"testing"
 	"time"
 
@@ -71,6 +72,10 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+// When running the containerized conformance test, we'll mount the
+// host root filesystem as readonly to /rootfs.
+const rootfs = "/rootfs"
+
 func TestE2eNode(t *testing.T) {
 	if *runServicesMode {
 		// If run-services-mode is specified, only run services in current process.
@@ -79,6 +84,15 @@ func TestE2eNode(t *testing.T) {
 	}
 	if *systemValidateMode {
 		// If system-validate-mode is specified, only run system validation in current process.
+		if framework.TestContext.NodeConformance {
+			// Chroot to /rootfs to make system validation can check system
+			// as in the root filesystem.
+			// TODO(random-liu): Consider to chroot the whole test process to make writing
+			// test easier.
+			if err := syscall.Chroot(rootfs); err != nil {
+				glog.Exitf("chroot %q failed: %v", rootfs, err)
+			}
+		}
 		if err := system.Validate(); err != nil {
 			glog.Exitf("system validation failed: %v", err)
 		}
@@ -172,6 +186,7 @@ func validateSystem() error {
 	if err != nil {
 		return fmt.Errorf("can't get current binary: %v", err)
 	}
+	// Pass all flags into the child process, so that it will see the same flag set.
 	output, err := exec.Command(testBin, append([]string{"--system-validate-mode"}, os.Args[1:]...)...).CombinedOutput()
 	// The output of system validation should have been formatted, directly print here.
 	fmt.Print(string(output))

test/e2e_node/services/services.go

@@ -148,6 +148,7 @@ func (e *E2EServices) startInternalServices() (*server, error) {
 	if err != nil {
 		return nil, fmt.Errorf("can't get current binary: %v", err)
 	}
+	// Pass all flags into the child process, so that it will see the same flag set.
 	startCmd := exec.Command(testBin, append([]string{"--run-services-mode"}, os.Args[1:]...)...)
 	server := newServer("services", startCmd, nil, nil, getServicesHealthCheckURLs(), servicesLogFile, e.monitorParent, false)
 	return server, server.start()