diff --git a/hack/jenkins/dockerized-e2e-runner.sh b/hack/jenkins/dockerized-e2e-runner.sh
new file mode 100755
index 00000000000..91bdf284659
--- /dev/null
+++ b/hack/jenkins/dockerized-e2e-runner.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Save environment variables in $WORKSPACE/env.list and then run the Jenkins e2e
+# test runner inside the kubekins-test Docker image.
+
+env -u HOME -u PATH -u PWD -u WORKSPACE >${WORKSPACE}/env.list
+docker run --rm=true -i \
+ -v "${WORKSPACE}/_artifacts":/workspace/_artifacts \
+ -v /etc/localtime:/etc/localtime:ro \
+ -v /var/lib/jenkins/gce_keys:/workspace/.ssh:ro \
+ --env-file "${WORKSPACE}/env.list" \
+ -e "HOME=/workspace" \
+ -e "WORKSPACE=/workspace" \
+ gcr.io/google_containers/kubekins-test:0.9 \
+ bash -c "bash <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh')"
diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh
index e2c1c09c468..ba8c3d781a7 100755
--- a/hack/jenkins/e2e-runner.sh
+++ b/hack/jenkins/e2e-runner.sh
@@ -21,6 +21,10 @@ set -o nounset
set -o pipefail
set -o xtrace
+function running_in_docker() {
+ grep -q docker /proc/self/cgroup
+}
+
function check_dirty_workspace() {
if [[ "${JENKINS_TOLERATE_DIRTY_WORKSPACE:-}" =~ ^[yY]$ ]]; then
echo "Tolerating dirty workspace (because JENKINS_TOLERATE_DIRTY_WORKSPACE)."
@@ -143,6 +147,43 @@ function get_latest_trusty_image() {
rm -rf .gsutil &> /dev/null
}
+function install_google_cloud_sdk_tarball() {
+ local -r tarball=$1
+ local -r install_dir=$2
+ mkdir -p "${install_dir}"
+ tar xzf "${tarball}" -C "${install_dir}"
+
+ export CLOUDSDK_CORE_DISABLE_PROMPTS=1
+ "${install_dir}/google-cloud-sdk/install.sh" --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
+ export PATH=${install_dir}/google-cloud-sdk/bin:${PATH}
+}
+
+### Pre Set Up ###
+if running_in_docker; then
+ curl -fsSL --retry 3 -o "${WORKSPACE}/google-cloud-sdk.tar.gz" 'https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz'
+ install_google_cloud_sdk_tarball "${WORKSPACE}/google-cloud-sdk.tar.gz" /
+fi
+
+# Install gcloud from a custom path if provided. Used to test GKE with gcloud
+# at HEAD, release candidate.
+# TODO: figure out how to avoid installing the cloud sdk twice if run inside Docker.
+if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then
+ # Retry the download a few times to mitigate transient server errors and
+ # race conditions where the bucket contents change under us as we download.
+ for n in $(seq 3); do
+ gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1
+ # Delete any temporary files from the download so that we start from
+ # scratch when we retry.
+ rm -rf ~/.gsutil
+ done
+ rm -rf ~/repo ~/cloudsdk
+ mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
+ export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json
+ install_google_cloud_sdk_tarball ~/repo/google-cloud-sdk.tar.gz ~/cloudsdk
+ # TODO: is this necessary? this won't work inside Docker currently.
+ export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud
+fi
+
# We get the image project and name for Trusty dynamically.
if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then
trusty_image_project="$(get_trusty_image_project)"
@@ -212,13 +253,23 @@ fi
# # Move the permissions for the keys to Jenkins.
# $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/
# $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/
-if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
- echo "Skipping SSH key copying for AWS"
-else
- mkdir -p ${WORKSPACE}/.ssh/
- cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/
- cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/
-fi
+case "${KUBERNETES_PROVIDER}" in
+ gce|gke|kubemark)
+ if ! running_in_docker; then
+ mkdir -p ${WORKSPACE}/.ssh/
+ cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/
+ cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/
+ fi
+ if [[ ! -f ${WORKSPACE}/.ssh/google_compute_engine ]]; then
+ echo "google_compute_engine ssh key missing!"
+ exit 1
+ fi
+ ;;
+
+ default)
+ echo "Not copying ssh keys for ${KUBERNETES_PROVIDER}"
+ ;;
+esac
cd kubernetes
@@ -231,6 +282,9 @@ fi
# Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
ARTIFACTS=${WORKSPACE}/_artifacts
mkdir -p ${ARTIFACTS}
+# When run inside Docker, we need to make sure all files are world-readable
+# (since they will be owned by root on the host).
+trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
export E2E_REPORT_DIR=${ARTIFACTS}
declare -r gcp_list_resources_script="./cluster/gce/list-resources.sh"
declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt"
@@ -244,29 +298,6 @@ else
gcp_list_resources="false"
fi
-### Pre Set Up ###
-# Install gcloud from a custom path if provided. Used to test GKE with gcloud
-# at HEAD, release candidate.
-if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then
- # Retry the download a few times to mitigate transient server errors and
- # race conditions where the bucket contents change under us as we download.
- for n in $(seq 3); do
- gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1
- # Delete any temporary files from the download so that we start from
- # scratch when we retry.
- rm -rf ~/.gsutil
- done
- rm -rf ~/repo ~/cloudsdk
- mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
- mkdir ~/cloudsdk
- tar zxf ~/repo/google-cloud-sdk.tar.gz -C ~/cloudsdk
- export CLOUDSDK_CORE_DISABLE_PROMPTS=1
- export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json
- ~/cloudsdk/google-cloud-sdk/install.sh --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
- export PATH=${HOME}/cloudsdk/google-cloud-sdk/bin:${PATH}
- export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud
-fi
-
### Set up ###
if [[ "${E2E_UP,,}" == "true" ]]; then
go run ./hack/e2e.go ${E2E_OPT:-} -v --down
diff --git a/hack/jenkins/job-configs/global.yaml b/hack/jenkins/job-configs/global.yaml
index 741ac3b08fc..7fc2ce14de8 100644
--- a/hack/jenkins/job-configs/global.yaml
+++ b/hack/jenkins/job-configs/global.yaml
@@ -138,6 +138,7 @@
branch: 'master'
job-env: ''
runner: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh")
+ dockerized-runner: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/dockerized-e2e-runner.sh")
old-runner-1-1: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.1/hack/jenkins/e2e.sh")
old-runner-1-0: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.0/hack/jenkins/e2e.sh")
provider-env: ''
diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
index d4c8ad09efe..1d583d599b5 100644
--- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
+++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
@@ -15,6 +15,8 @@
description: '{description} Test owner: {test-owner}.'
logrotate:
daysToKeep: 7
+ node: '{jenkins_node}'
+ jenkins_node: 'master'
disabled: '{obj:disable_job}'
builders:
- shell: |
@@ -105,6 +107,8 @@
export KUBE_ADMISSION_CONTROL="NamespaceLifecycle,InitialResources,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
- 'gce-flaky':
description: 'Run the flaky tests on GCE, sequentially.'
+ jenkins_node: 'e2e'
+ runner: '{dockerized-runner}'
timeout: 180
job-env: |
export GINKGO_TEST_ARGS="--ginkgo.focus=\[Flaky\] \
@@ -213,6 +217,8 @@
- client (kubectl): ci/latest.txt
- cluster (k8s): ci/latest.txt
- tests: ci/latest.txt
+ jenkins_node: 'e2e'
+ runner: '{dockerized-runner}'
timeout: 300
job-env: |
export PROJECT="k8s-jkns-e2e-gke-ci-flaky"