Merge pull request #24636 from ixdy/dockerized-e2e

Automatic merge from submit-queue

Run Kubemark builds inside Docker

Since Docker-in-Docker is tricky to get right (esp. wrt volume mounts), I'm only enabling it when necessary for a build (e.g. for kubemark).

cc @spxtr @fejta @wojtek-t

hack/jenkins/dockerized-e2e-runner.sh

@@ -21,6 +21,10 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+export REPO_DIR=${REPO_DIR:-$(pwd)}
+export HOST_ARTIFACTS_DIR=${WORKSPACE}/_artifacts
+mkdir -p "${HOST_ARTIFACTS_DIR}"
+
 env -u HOME -u PATH -u PWD -u WORKSPACE >${WORKSPACE}/env.list
 
 # Add all uncommented lines for metadata.google.internal in /etc/hosts to the
@@ -30,6 +34,17 @@ readonly METADATA_SERVER_ADD_HOST_ARGS=($(
   cut -f1 -d' ' |\
   xargs -r printf -- '--add-host="metadata.google.internal:%s"\n'))
 
+docker_extra_args=()
+if [[ "${JENKINS_ENABLE_DOCKER_IN_DOCKER:-}" =~ ^[yY]$ ]]; then
+    docker_extra_args+=(\
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      -v "$(which docker)":/bin/docker:ro \
+      -v "${REPO_DIR}":/go/src/k8s.io/kubernetes \
+      -e "REPO_DIR=${REPO_DIR}" \
+      -e "HOST_ARTIFACTS_DIR=${HOST_ARTIFACTS_DIR}" \
+    )
+fi
+
 docker run --rm=true -i \
   -v "${WORKSPACE}/_artifacts":/workspace/_artifacts \
   -v /etc/localtime:/etc/localtime:ro \
@@ -37,6 +52,7 @@ docker run --rm=true -i \
   --env-file "${WORKSPACE}/env.list" \
   -e "HOME=/workspace" \
   -e "WORKSPACE=/workspace" \
+  "${docker_extra_args[@]:+${docker_extra_args[@]}}" \
   "${METADATA_SERVER_ADD_HOST_ARGS[@]:+${METADATA_SERVER_ADD_HOST_ARGS[@]}}" \
   gcr.io/google_containers/kubekins-test:0.11 \
   bash -c "bash <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh')"

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-kubemark.yaml

@@ -3,13 +3,13 @@
     description: '{description} Test owner: gmarek'
     logrotate:
         daysToKeep: 7
-    node: 'master'
+    node: 'e2e'
     builders:
         - shell: |
             {provider-env}
             {job-env}
             {post-env}
-            timeout -k {kill-timeout}m {timeout}m {legacy-runner} && rc=$? || rc=$?
+            timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$?
             {report-rc}
     properties:
         - mail-watcher
@@ -56,6 +56,8 @@
                 export KUBE_GCE_ZONE="us-central1-f"
                 export KUBEMARK_MASTER_SIZE="n1-standard-1"
                 export KUBEMARK_NUM_NODES="5"
+                # The kubemark scripts build a Docker image
+                export JENKINS_ENABLE_DOCKER_IN_DOCKER="y"
         - '100-gce':
             description: 'Run small-ish kubemark cluster to continuously run performance experiments'
             timeout: 240
@@ -74,6 +76,8 @@
                 export KUBE_GCE_ZONE="us-central1-f"
                 export KUBEMARK_MASTER_SIZE="n1-standard-4"
                 export KUBEMARK_NUM_NODES="100"
+                # The kubemark scripts build a Docker image
+                export JENKINS_ENABLE_DOCKER_IN_DOCKER="y"
         - 'high-density-100-gce':
             description: 'Run Kubemark high-density (100 pods/node) test on a fake 100 node cluster.'
             timeout: 160
@@ -92,6 +96,8 @@
                 export KUBE_GCE_ZONE="us-east1-d"
                 export KUBEMARK_MASTER_SIZE="n1-standard-4"
                 export KUBEMARK_NUM_NODES="100"
+                # The kubemark scripts build a Docker image
+                export JENKINS_ENABLE_DOCKER_IN_DOCKER="y"
         - '500-gce':
             description: 'Run Kubemark test on a fake 500 node cluster to test for regressions on bigger clusters'
             timeout: 300
@@ -112,6 +118,8 @@
                 export KUBE_GCE_ZONE="us-central1-f"
                 export KUBEMARK_MASTER_SIZE="n1-standard-16"
                 export KUBEMARK_NUM_NODES="500"
+                # The kubemark scripts build a Docker image
+                export JENKINS_ENABLE_DOCKER_IN_DOCKER="y"
         - 'gce-scale':
             description: 'Run Density test on Kubemark in very large cluster. Currently only scheduled to run every 12 hours so as not to waste too many resources.'
             # 12h - load tests take really, really, really long time.
@@ -136,5 +144,7 @@
                 export KUBEMARK_MASTER_SIZE="n1-standard-16"
                 export KUBEMARK_NUM_NODES="1000"
                 export KUBE_GCE_ZONE="us-central1-f"
+                # The kubemark scripts build a Docker image
+                export JENKINS_ENABLE_DOCKER_IN_DOCKER="y"
     jobs:
         - 'kubernetes-kubemark-{suffix}'