diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh
index 93c3ed81301..5ea2a3a6709 100755
--- a/hack/jenkins/e2e-runner.sh
+++ b/hack/jenkins/e2e-runner.sh
@@ -94,11 +94,69 @@ function unpack_binaries() {
tar -xzf kubernetes-test.tar.gz
}
+# GCP Project to fetch Trusty images.
+function get_trusty_image_project() {
+ local project=""
+ # Retry the gsutil command a couple times to mitigate the effect of
+ # transient server errors.
+ for n in $(seq 3); do
+ project="$(gsutil cat "gs://trusty-images/image-project.txt")" && break || sleep 1
+ done
+ if [[ -z "${project}" ]]; then
+ echo "Failed to find the image project for Trusty images."
+ exit 1
+ fi
+ echo "${project}"
+ # Clean up gsutil artifacts otherwise the later test stage will complain.
+ rm -rf .config &> /dev/null
+ rm -rf .gsutil &> /dev/null
+}
+
+# Get the latest Trusty image for a Jenkins job.
+function get_latest_trusty_image() {
+ local image_project="$1"
+ local image_type="$2"
+ local image_index=""
+ if [[ "${image_type}" == head ]]; then
+ image_index="trusty-head"
+ elif [[ "${image_type}" == dev ]]; then
+ image_index="trusty-dev"
+ elif [[ "${image_type}" == beta ]]; then
+ image_index="trusty-beta"
+ elif [[ "${image_type}" == stable ]]; then
+ image_index="trusty-stable"
+ fi
+
+ local image=""
+ # Retry the gsutil command a couple times to mitigate the effect of
+ # transient server errors.
+ for n in $(seq 3); do
+ image="$(gsutil cat "gs://${image_project}/image-indices/latest-test-image-${image_index}")" && break || sleep 1
+ done
+ if [[ -z "${image}" ]]; then
+ echo "Failed to find Trusty image for ${image_type}"
+ exit 1
+ fi
+ echo "${image}"
+ # Clean up gsutil artifacts otherwise the later test stage will complain.
+ rm -rf .config &> /dev/null
+ rm -rf .gsutil &> /dev/null
+}
+
echo "--------------------------------------------------------------------------------"
echo "Test Environment:"
printenv | sort
echo "--------------------------------------------------------------------------------"
+# We get the image project and name for Trusty dynamically.
+if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then
+ trusty_image_project="$(get_trusty_image_project)"
+ trusty_image="$(get_latest_trusty_image "${trusty_image_project}" "head")"
+ export KUBE_MASTER_IMAGE_PROJECT="${trusty_image_project}"
+ export KUBE_MASTER_IMAGE="${trusty_image}"
+ export KUBE_OS_DISTRIBUTION="trusty"
+fi
+
# We get the Kubernetes tarballs unless we are going to use old ones
if [[ "${JENKINS_USE_EXISTING_BINARIES:-}" =~ ^[yY]$ ]]; then
echo "Using existing binaries; not cleaning, fetching, or unpacking new ones."
diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
index 4f1e132fc93..d72e964ba38 100644
--- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
+++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
@@ -40,25 +40,7 @@
- timestamps
- workspace-cleanup
-# Template for e2e test jobs that run on GCE with Trusty images.
-- job-template:
- name: 'kubernetes-e2e-gce-trusty-{suffix}'
- <<: *e2e_job_defaults
- triggers:
- - timed: 'H H/8 * * *'
- publishers:
- - e2e-publishers:
- recipients: '{emails}'
- - description-setter:
- regexp: KUBE_GCE_MINION_IMAGE=(.*)
- - groovy-postbuild:
- script: |
- def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MINION_IMAGE=(.*)")
- if(trustyImageMatcher?.matches()) manager.addShortText("Trusty Image: " + trustyImageMatcher.group(1) + "", "grey", "white", "0px", "white")
- def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
- if(k8sVersionMatcher?.matches()) manager.addShortText("
Kubernetes version: " + k8sVersionMatcher.group(1) + "", "grey", "white", "0px", "white")
-
-# Template for the rest of e2e test jobs.
+# Template for most e2e test jobs.
- job-template:
name: 'kubernetes-e2e-{suffix}'
<<: *e2e_job_defaults
@@ -285,7 +267,7 @@
export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
jobs:
- 'kubernetes-e2e-{suffix}'
-
+
- project:
name: kubernetes-e2e-gke-1-2
trigger-job: 'kubernetes-build-1.2'
@@ -390,47 +372,6 @@
jobs:
- 'kubernetes-e2e-{suffix}'
-- project:
- name: kubernetes-e2e-gke-trusty
- trigger-job: 'kubernetes-build-1.1'
- test-owner: 'wonderfly@google.com'
- branch: 'release-1.1'
- runner: '{old-runner-1-1}'
- post-env: ''
- emails: 'wonderfly@google.com,qzheng@google.com'
- suffix:
- - 'gke-trusty-prod':
- timeout: 180
- description: |
- Run e2e tests with Trusty as node image using the following config:
- - provider: GKE
- - api proxy: prod
- - borg job: prod
- - client (kubectl): release/stable.txt
- - cluster (k8s): release/stable.txt
- - tests: release/stable.txt
- - 'gke-trusty-staging':
- timeout: 300
- description: |
- Run e2e tests with Trusty as node image using the following config:
- - provider: GKE
- - api proxy: staging
- - borg job: staging
- - client (kubectl): release/stable.txt
- - cluster (k8s): release/stable.txt
- - tests: release/stable.txt
- - 'gke-trusty-test':
- timeout: 300
- description: |
- Run e2e tests with Trusty as node image using the following config:
- - provider: GKE
- - api proxy: staging
- - borg job: test
- - client (kubectl): release/stable.txt
- - cluster (k8s): release/stable.txt
- - tests: release/stable.txt
- jobs:
- - 'kubernetes-e2e-{suffix}'
- project:
name: kubernetes-e2e-gce-1.1
@@ -543,44 +484,6 @@
jobs:
- 'kubernetes-e2e-{suffix}'
-- project:
- name: kubernetes-e2e-gce-trusty
- test-owner: 'wonderfly@google.com'
- branch: 'release-1.1'
- emails: 'wonderfly@google.com,qzheng@google.com'
- runner: '{old-runner-1-1}'
- post-env: ''
- suffix:
- - 'head-release':
- description: 'Continuously test Trusty build against latest k8s release.'
- timeout: 150
- - 'dev-release':
- description: 'Continuously test Trusty dev build against latest k8s release.'
- timeout: 150
- - 'beta-release':
- description: 'Continuously test Trusty beta build against latest k8s release.'
- timeout: 150
- - 'stable-release':
- description: 'Continuously test Trusty stable build against latest k8s release.'
- timeout: 150
- - 'head-slow':
- description: 'Run slow E2E tests on latest Trusty build.'
- timeout: 270
- - 'dev-slow':
- # Constantly failing due to a known issue of the image. Disabled
- # until the issue is resolved.
- disable_job: true
- description: 'Run slow E2E tests on latest Trusty dev build.'
- timeout: 270
- - 'beta-slow':
- description: 'Run slow E2E tests on latest Trusty beta build.'
- timeout: 270
- - 'stable-slow':
- description: 'Run slow E2E tests on latest Trusty stable build.'
- timeout: 270
- jobs:
- - 'kubernetes-e2e-gce-trusty-{suffix}'
-
- project:
name: kubernetes-e2e-gce-enormous-cluster
test-owner: 'gmarek'
@@ -620,3 +523,194 @@
jobs:
- 'kubernetes-e2e-{suffix}'
+# ==============================================================================
+# NOTE: From here on all jobs use Trusty as the image for master and/or nodes.
+# Please add templates/groups/projects/jobs that use ContainerVm above/below
+# this section (search "End of Trusty jobs" for the ending separator).
+
+# Template for e2e test jobs that run on GCE with Trusty images against
+# kubernetes HEAD.
+- job-template:
+ name: 'kubernetes-e2e-gce-trusty-ci-{suffix}'
+ <<: *e2e_job_defaults
+ triggers:
+ - reverse:
+ jobs: '{trigger-job}'
+ result: success
+ - timed: '{cron-string}'
+ publishers:
+ - e2e-publishers:
+ recipients: '{emails}'
+ - description-setter:
+ regexp: KUBE_GCE_MASTER_IMAGE=(.*)
+ - groovy-postbuild:
+ script: |
+ def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MASTER_IMAGE=(.*)")
+ if(trustyImageMatcher?.matches()) manager.addShortText("Trusty Image: " + trustyImageMatcher.group(1) + "", "grey", "white", "0px", "white")
+ def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
+ if(k8sVersionMatcher?.matches()) manager.addShortText("
Kubernetes version: " + k8sVersionMatcher.group(1) + "", "grey", "white", "0px", "white")
+
+# CI jobs that run on GCE with both master and nodes running Trusty images. We
+# use these to guard k8s+Trusty compatibility.
+- project:
+ name: kubernetes-e2e-gce-trusty-ci-master
+ trigger-job: 'kubernetes-build'
+ test-owner: 'wonderfly@google.com'
+ emails: 'wonderfly@google.com,qzheng@google.com'
+ provider-env: |
+ {gce-provider-env}
+ export JENKINS_USE_TRUSTY_IMAGES="y"
+ suffix:
+ # TODO(wonderfly): For Trusty, we currently only run CI and slow tests.
+ # More test coverage under way.
+ - 'master':
+ description: 'Runs all non-slow, non-serial, non-flaky, tests on GCE with Trusty images in parallel on the master branch.'
+ timeout: 30
+ job-env: |
+ export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+ export GINKGO_PARALLEL="y"
+ export PROJECT="e2e-gce-trusty-ci-master"
+ - 'slow-master':
+ description: 'Runs slow tests on GCE with Trusty images, sequentially on the master branch.'
+ timeout: 60
+ job-env: |
+ export GINKGO_TEST_ARGS="--ginkgo.focus=\[Slow\] \
+ --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+ export GINKGO_PARALLEL="y"
+ export PROJECT="e2e-gce-trusty-ci-master-slow"
+ jobs:
+ - 'kubernetes-e2e-gce-trusty-ci-{suffix}'
+
+- project:
+ name: kubernetes-e2e-gce-trusty-ci-1-2
+ trigger-job: 'kubernetes-build-1.2'
+ test-owner: 'wonderfly@google.com'
+ emails: 'wonderfly@google.com,qzheng@google.com'
+ provider-env: |
+ {gce-provider-env}
+ export JENKINS_PUBLISHED_VERSION="ci/latest-1.2"
+ export JENKINS_USE_TRUSTY_IMAGES="y"
+ suffix:
+ # TODO(wonderfly): For Trusty, we currently only run CI and slow tests.
+ # More test coverage under way.
+ - 'release-1.2':
+ description: 'Runs all non-slow, non-serial, non-flaky, tests on GCE with Trusty images in parallel on the release-1.2 branch.'
+ timeout: 30
+ job-env: |
+ export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+ export GINKGO_PARALLEL="y"
+ export PROJECT="e2e-gce-trusty-ci-1-2"
+ - 'slow-release-1.2':
+ description: 'Runs slow tests on GCE with Trusty images, sequentially on the release-1.2 branch.'
+ timeout: 60
+ job-env: |
+ export GINKGO_TEST_ARGS="--ginkgo.focus=\[Slow\] \
+ --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+ export GINKGO_PARALLEL="y"
+ export PROJECT="e2e-gce-trusty-ci-slow-1-2"
+ jobs:
+ - 'kubernetes-e2e-gce-trusty-ci-{suffix}'
+
+# Template for e2e test jobs that run on GCE with Trusty images against last
+# released k8s version. We use these jobs to test Trusty's continuous build.
+- job-template:
+ name: 'kubernetes-e2e-gce-trusty-{suffix}'
+ <<: *e2e_job_defaults
+ triggers:
+ - timed: 'H H/8 * * *'
+ publishers:
+ - e2e-publishers:
+ recipients: '{emails}'
+ - description-setter:
+ regexp: KUBE_GCE_MINION_IMAGE=(.*)
+ - groovy-postbuild:
+ script: |
+ def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MINION_IMAGE=(.*)")
+ if(trustyImageMatcher?.matches()) manager.addShortText("Trusty Image: " + trustyImageMatcher.group(1) + "", "grey", "white", "0px", "white")
+ def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
+ if(k8sVersionMatcher?.matches()) manager.addShortText("
Kubernetes version: " + k8sVersionMatcher.group(1) + "", "grey", "white", "0px", "white")
+
+# Jobs that run e2e tests on GCE with Trusty images against the latest k8s
+# release.
+- project:
+ name: kubernetes-e2e-gce-trusty
+ test-owner: 'wonderfly@google.com'
+ branch: 'release-1.1'
+ emails: 'wonderfly@google.com,qzheng@google.com'
+ runner: '{old-runner-1-1}'
+ post-env: ''
+ suffix:
+ - 'head-release':
+ description: 'Continuously test Trusty build against latest k8s release.'
+ timeout: 150
+ - 'dev-release':
+ description: 'Continuously test Trusty dev build against latest k8s release.'
+ timeout: 150
+ - 'beta-release':
+ description: 'Continuously test Trusty beta build against latest k8s release.'
+ timeout: 150
+ - 'stable-release':
+ description: 'Continuously test Trusty stable build against latest k8s release.'
+ timeout: 150
+ - 'head-slow':
+ description: 'Run slow E2E tests on latest Trusty build.'
+ timeout: 270
+ - 'dev-slow':
+ # Constantly failing due to a known issue of the image. Disabled
+ # until the issue is resolved.
+ # TODO(wonderfly): Re-enable it once the internal issue is fixed.
+ disable_job: true
+ description: 'Run slow E2E tests on latest Trusty dev build.'
+ timeout: 270
+ - 'beta-slow':
+ description: 'Run slow E2E tests on latest Trusty beta build.'
+ timeout: 270
+ - 'stable-slow':
+ description: 'Run slow E2E tests on latest Trusty stable build.'
+ timeout: 270
+ jobs:
+ - 'kubernetes-e2e-gce-trusty-{suffix}'
+
+# Jobs that run e2e tests on GKE with Trusty as node image on the release-1.1 branch.
+- project:
+ name: kubernetes-e2e-gke-trusty
+ trigger-job: 'kubernetes-build-1.1'
+ test-owner: 'wonderfly@google.com'
+ branch: 'release-1.1'
+ runner: '{old-runner-1-1}'
+ post-env: ''
+ emails: 'wonderfly@google.com,qzheng@google.com'
+ suffix:
+ - 'gke-trusty-prod':
+ timeout: 180
+ description: |
+ Run e2e tests with Trusty as node image using the following config:
+ - provider: GKE
+ - api proxy: prod
+ - borg job: prod
+ - client (kubectl): release/stable.txt
+ - cluster (k8s): release/stable.txt
+ - tests: release/stable.txt
+ - 'gke-trusty-staging':
+ timeout: 300
+ description: |
+ Run e2e tests with Trusty as node image using the following config:
+ - provider: GKE
+ - api proxy: staging
+ - borg job: staging
+ - client (kubectl): release/stable.txt
+ - cluster (k8s): release/stable.txt
+ - tests: release/stable.txt
+ - 'gke-trusty-test':
+ timeout: 300
+ description: |
+ Run e2e tests with Trusty as node image using the following config:
+ - provider: GKE
+ - api proxy: staging
+ - borg job: test
+ - client (kubectl): release/stable.txt
+ - cluster (k8s): release/stable.txt
+ - tests: release/stable.txt
+ jobs:
+ - 'kubernetes-e2e-{suffix}'
+#============================== End of Trusty jobs =============================