From 72d66be439b123ac974886a40f50b4fc473c08e9 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 16 Oct 2019 07:50:02 -0700 Subject: [PATCH] test images: Adds E2E test image automated build In order for the E2E test images to be automatically built and published to the staging registry (from which they will be promoted to the regular E2E test registry), the cloudbuild.yaml file has been added. The file was added in conformance with [1]. Adds the ability to build all test images: make -C test/images WHAT=all-images [1] https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md --- test/images/cloudbuild.yaml | 29 +++++++++++++++++++ test/images/image-util.sh | 55 ++++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 test/images/cloudbuild.yaml diff --git a/test/images/cloudbuild.yaml b/test/images/cloudbuild.yaml new file mode 100644 index 00000000000..fe7c3bb970a --- /dev/null +++ b/test/images/cloudbuild.yaml @@ -0,0 +1,29 @@ +# See https://cloud.google.com/cloud-build/docs/build-config + +# this must be specified in seconds. If omitted, defaults to 600s (10 mins) +# the timeout was increased because some images might take longer to build (e.g.: sample-apiserver, pets/redis-installer) +timeout: 3000s +# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE +steps: + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4' + entrypoint: make + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - WHAT=$_WHAT + - REGISTRY=gcr.io/k8s-staging-e2e-test-images + args: + - all-push +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and + # can be used as a substitution + _GIT_TAG: '12345' + # _PULL_BASE_REF will contain the ref that was pushed to to trigger this build - + # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. + _PULL_BASE_REF: 'master' + # _WHAT will contain the image name to be built and published to the staging registry. + _WHAT: 'all-conformance' diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 726c50817fe..2e7791f8394 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -19,7 +19,7 @@ set -o nounset set -o pipefail TASK=$1 -IMAGE=$2 +WHAT=$2 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" source "${KUBE_ROOT}/hack/lib/util.sh" @@ -29,7 +29,8 @@ declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["pp # Returns list of all supported architectures from BASEIMAGE file listArchs() { - cut -d "=" -f 1 "${IMAGE}"/BASEIMAGE + image=$1 + cut -d "=" -f 1 "${image}"/BASEIMAGE } # Returns baseimage need to used in Dockerfile for any given architecture @@ -43,8 +44,9 @@ getBaseImage() { # it will build for all the supported arch list - amd64, arm, # arm64, ppc64le, s390x build() { - if [[ -f ${IMAGE}/BASEIMAGE ]]; then - archs=$(listArchs) + image=$1 + if [[ -f ${image}/BASEIMAGE ]]; then + archs=$(listArchs "$image") else archs=${!QEMUARCHS[*]} fi @@ -52,7 +54,7 @@ build() { kube::util::ensure-gnu-sed for arch in ${archs}; do - echo "Building image for ${IMAGE} ARCH: ${arch}..." + echo "Building image for ${image} ARCH: ${arch}..." # Create a temporary directory for every architecture and copy the image content # and build the image from temporary directory @@ -60,11 +62,11 @@ build() { temp_dir=$(mktemp -d "${KUBE_ROOT}"/_tmp/test-images-build.XXXXXX) kube::util::trap_add "rm -rf ${temp_dir}" EXIT - cp -r "${IMAGE}"/* "${temp_dir}" - if [[ -f ${IMAGE}/Makefile ]]; then + cp -r "${image}"/* "${temp_dir}" + if [[ -f ${image}/Makefile ]]; then # make bin will take care of all the prerequisites needed # for building the docker image - make -C "${IMAGE}" bin ARCH="${arch}" TARGET="${temp_dir}" + make -C "${image}" bin ARCH="${arch}" TARGET="${temp_dir}" fi pushd "${temp_dir}" # image tag @@ -96,7 +98,7 @@ build() { fi fi - docker build --pull -t "${REGISTRY}/${IMAGE}-${arch}:${TAG}" . + docker build --pull -t "${REGISTRY}/${image}-${arch}:${TAG}" . popd done @@ -114,28 +116,29 @@ docker_version_check() { # This function will push the docker images push() { + image=$1 docker_version_check - TAG=$(<"${IMAGE}"/VERSION) - if [[ -f ${IMAGE}/BASEIMAGE ]]; then - archs=$(listArchs) + TAG=$(<"${image}"/VERSION) + if [[ -f ${image}/BASEIMAGE ]]; then + archs=$(listArchs "$image") else archs=${!QEMUARCHS[*]} fi for arch in ${archs}; do - docker push "${REGISTRY}/${IMAGE}-${arch}:${TAG}" + docker push "${REGISTRY}/${image}-${arch}:${TAG}" done kube::util::ensure-gnu-sed # The manifest command is still experimental as of Docker 18.09.2 export DOCKER_CLI_EXPERIMENTAL="enabled" - # Make archs list into image manifest. Eg: 'amd64 ppc64le' to '${REGISTRY}/${IMAGE}-amd64:${TAG} ${REGISTRY}/${IMAGE}-ppc64le:${TAG}' - while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$archs" | ${SED} -e "s~[^ ]*~$REGISTRY\/$IMAGE\-&:$TAG~g") - docker manifest create --amend "${REGISTRY}/${IMAGE}:${TAG}" "${manifest[@]}" + # Make archs list into image manifest. Eg: 'amd64 ppc64le' to '${REGISTRY}/${image}-amd64:${TAG} ${REGISTRY}/${image}-ppc64le:${TAG}' + while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$archs" | ${SED} -e "s~[^ ]*~$REGISTRY\/$image\-&:$TAG~g") + docker manifest create --amend "${REGISTRY}/${image}:${TAG}" "${manifest[@]}" for arch in ${archs}; do - docker manifest annotate --arch "${arch}" "${REGISTRY}/${IMAGE}:${TAG}" "${REGISTRY}/${IMAGE}-${arch}:${TAG}" + docker manifest annotate --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}-${arch}:${TAG}" done - docker manifest push --purge "${REGISTRY}/${IMAGE}:${TAG}" + docker manifest push --purge "${REGISTRY}/${image}:${TAG}" } # This function is for building the go code @@ -156,4 +159,18 @@ bin() { shift -eval "${TASK}" "$@" +if [[ "${WHAT}" == "all-conformance" ]]; then + # NOTE(claudiub): Building *ALL* the images under the kubernetes/test/images folder takes an extremely + # long time (especially some images), and some images are rarely used and rarely updated, so there's + # no point in rebuilding all of them every time. This will only build the Conformance-related images. + # Discussed during Conformance Office Hours Meeting (2019.12.17): + # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh + # echoserver image not included: https://github.com/kubernetes/kubernetes/issues/84158 + conformance_images=("agnhost" "dnsutils" "jessie-dnsutils" "kitten" "mounttest" "mounttest-user"\ + "nautilus" "nonewprivs" "resource-consumer" "resource-consumer-controller" "sample-apiserver" "test-webserver") + for image in "${conformance_images[@]}"; do + eval "${TASK}" "${image}" + done +else + eval "${TASK}" "$@" +fi