From 253711016e586b69bdcd40eba275eda0c8f851c4 Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 30 Mar 2016 11:33:12 -0700 Subject: [PATCH 1/2] Make e2e-runner.sh work inside Docker and add wrapper script --- hack/jenkins/dockerized-e2e-runner.sh | 29 +++++++++ hack/jenkins/e2e-runner.sh | 91 ++++++++++++++++++--------- 2 files changed, 90 insertions(+), 30 deletions(-) create mode 100755 hack/jenkins/dockerized-e2e-runner.sh diff --git a/hack/jenkins/dockerized-e2e-runner.sh b/hack/jenkins/dockerized-e2e-runner.sh new file mode 100755 index 00000000000..91bdf284659 --- /dev/null +++ b/hack/jenkins/dockerized-e2e-runner.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Save environment variables in $WORKSPACE/env.list and then run the Jenkins e2e +# test runner inside the kubekins-test Docker image. + +env -u HOME -u PATH -u PWD -u WORKSPACE >${WORKSPACE}/env.list +docker run --rm=true -i \ + -v "${WORKSPACE}/_artifacts":/workspace/_artifacts \ + -v /etc/localtime:/etc/localtime:ro \ + -v /var/lib/jenkins/gce_keys:/workspace/.ssh:ro \ + --env-file "${WORKSPACE}/env.list" \ + -e "HOME=/workspace" \ + -e "WORKSPACE=/workspace" \ + gcr.io/google_containers/kubekins-test:0.9 \ + bash -c "bash <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh')" diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh index e2c1c09c468..ba8c3d781a7 100755 --- a/hack/jenkins/e2e-runner.sh +++ b/hack/jenkins/e2e-runner.sh @@ -21,6 +21,10 @@ set -o nounset set -o pipefail set -o xtrace +function running_in_docker() { + grep -q docker /proc/self/cgroup +} + function check_dirty_workspace() { if [[ "${JENKINS_TOLERATE_DIRTY_WORKSPACE:-}" =~ ^[yY]$ ]]; then echo "Tolerating dirty workspace (because JENKINS_TOLERATE_DIRTY_WORKSPACE)." @@ -143,6 +147,43 @@ function get_latest_trusty_image() { rm -rf .gsutil &> /dev/null } +function install_google_cloud_sdk_tarball() { + local -r tarball=$1 + local -r install_dir=$2 + mkdir -p "${install_dir}" + tar xzf "${tarball}" -C "${install_dir}" + + export CLOUDSDK_CORE_DISABLE_PROMPTS=1 + "${install_dir}/google-cloud-sdk/install.sh" --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false + export PATH=${install_dir}/google-cloud-sdk/bin:${PATH} +} + +### Pre Set Up ### +if running_in_docker; then + curl -fsSL --retry 3 -o "${WORKSPACE}/google-cloud-sdk.tar.gz" 'https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz' + install_google_cloud_sdk_tarball "${WORKSPACE}/google-cloud-sdk.tar.gz" / +fi + +# Install gcloud from a custom path if provided. Used to test GKE with gcloud +# at HEAD, release candidate. +# TODO: figure out how to avoid installing the cloud sdk twice if run inside Docker. +if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then + # Retry the download a few times to mitigate transient server errors and + # race conditions where the bucket contents change under us as we download. + for n in $(seq 3); do + gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1 + # Delete any temporary files from the download so that we start from + # scratch when we retry. + rm -rf ~/.gsutil + done + rm -rf ~/repo ~/cloudsdk + mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo + export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json + install_google_cloud_sdk_tarball ~/repo/google-cloud-sdk.tar.gz ~/cloudsdk + # TODO: is this necessary? this won't work inside Docker currently. + export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud +fi + # We get the image project and name for Trusty dynamically. if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then trusty_image_project="$(get_trusty_image_project)" @@ -212,13 +253,23 @@ fi # # Move the permissions for the keys to Jenkins. # $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/ # $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/ -if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then - echo "Skipping SSH key copying for AWS" -else - mkdir -p ${WORKSPACE}/.ssh/ - cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/ - cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/ -fi +case "${KUBERNETES_PROVIDER}" in + gce|gke|kubemark) + if ! running_in_docker; then + mkdir -p ${WORKSPACE}/.ssh/ + cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/ + cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/ + fi + if [[ ! -f ${WORKSPACE}/.ssh/google_compute_engine ]]; then + echo "google_compute_engine ssh key missing!" + exit 1 + fi + ;; + + default) + echo "Not copying ssh keys for ${KUBERNETES_PROVIDER}" + ;; +esac cd kubernetes @@ -231,6 +282,9 @@ fi # Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml ARTIFACTS=${WORKSPACE}/_artifacts mkdir -p ${ARTIFACTS} +# When run inside Docker, we need to make sure all files are world-readable +# (since they will be owned by root on the host). +trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM export E2E_REPORT_DIR=${ARTIFACTS} declare -r gcp_list_resources_script="./cluster/gce/list-resources.sh" declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt" @@ -244,29 +298,6 @@ else gcp_list_resources="false" fi -### Pre Set Up ### -# Install gcloud from a custom path if provided. Used to test GKE with gcloud -# at HEAD, release candidate. -if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then - # Retry the download a few times to mitigate transient server errors and - # race conditions where the bucket contents change under us as we download. - for n in $(seq 3); do - gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1 - # Delete any temporary files from the download so that we start from - # scratch when we retry. - rm -rf ~/.gsutil - done - rm -rf ~/repo ~/cloudsdk - mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo - mkdir ~/cloudsdk - tar zxf ~/repo/google-cloud-sdk.tar.gz -C ~/cloudsdk - export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json - ~/cloudsdk/google-cloud-sdk/install.sh --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false - export PATH=${HOME}/cloudsdk/google-cloud-sdk/bin:${PATH} - export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud -fi - ### Set up ### if [[ "${E2E_UP,,}" == "true" ]]; then go run ./hack/e2e.go ${E2E_OPT:-} -v --down From fde3545351596733801e42049c15e59d3556c08b Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Thu, 7 Apr 2016 16:55:51 -0700 Subject: [PATCH 2/2] Use dockerized e2e for g[ck]e-flaky jobs, and run on the Jenkins slave Force all other e2e jobs to run on master. --- hack/jenkins/job-configs/global.yaml | 1 + .../job-configs/kubernetes-jenkins/kubernetes-e2e.yaml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/hack/jenkins/job-configs/global.yaml b/hack/jenkins/job-configs/global.yaml index 741ac3b08fc..7fc2ce14de8 100644 --- a/hack/jenkins/job-configs/global.yaml +++ b/hack/jenkins/job-configs/global.yaml @@ -138,6 +138,7 @@ branch: 'master' job-env: '' runner: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh") + dockerized-runner: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/dockerized-e2e-runner.sh") old-runner-1-1: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.1/hack/jenkins/e2e.sh") old-runner-1-0: bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.0/hack/jenkins/e2e.sh") provider-env: '' diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml index 4712def74fc..47e734ea370 100644 --- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml +++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml @@ -15,6 +15,8 @@ description: '{description} Test owner: {test-owner}.' logrotate: daysToKeep: 7 + node: '{jenkins_node}' + jenkins_node: 'master' disabled: '{obj:disable_job}' builders: - shell: | @@ -105,6 +107,8 @@ export KUBE_ADMISSION_CONTROL="NamespaceLifecycle,InitialResources,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" - 'gce-flaky': description: 'Run the flaky tests on GCE, sequentially.' + jenkins_node: 'e2e' + runner: '{dockerized-runner}' timeout: 180 job-env: | export GINKGO_TEST_ARGS="--ginkgo.focus=\[Flaky\] \ @@ -213,6 +217,8 @@ - client (kubectl): ci/latest.txt
- cluster (k8s): ci/latest.txt
- tests: ci/latest.txt + jenkins_node: 'e2e' + runner: '{dockerized-runner}' timeout: 300 job-env: | export PROJECT="k8s-jkns-e2e-gke-ci-flaky"