From 94ffe582e98614fb601497092e2134a749c236a8 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Fri, 13 Mar 2020 03:55:54 -0700 Subject: [PATCH] Image promoter: Reenable Windows test image building The build times are a bit high for the image builder (~50 minutes), and it will a bit more when Windows support will be added to the other test images. This commit changes the machineType to N1_HIGHCPU_8. Reenables Windows test image building. Added DOCKER_CERT_BASE_PATH (default value: $HOME), which will contain the path where the certificates needed for Remote Docker Connection can be found. If a REMOTE_DOCKER_URL was not set for a particular OS version, exclude that image from the manifest list. This fixes an issue where, if REMOTE_DOCKER_URL was not set for Windows Server 1909, the Windows were completely excluded from the manifest list, including for Windows Server 1809 and 1903 which could have been built and pushed. Sets "test-webserver" as the default CMD for kitten and nautilus. Since they are now based on agnhost, they should be set to run test-webserver to maintain previous behaviour. Bumps the agnhost version to 2.13, as 2.12 has already been promoted. 2.13 will contain Windows support. Adds Windows support for the kitten and nautilus images, so they can promoted together with agnhost (they were not previously promoted). Adds OWNERS files to: agnhost, busybox, kitten, nautilus. --- test/images/Makefile | 1 + test/images/agnhost/OWNERS | 2 ++ test/images/agnhost/README.md | 14 +++++++------- test/images/agnhost/VERSION | 2 +- test/images/agnhost/agnhost.go | 2 +- test/images/busybox/OWNERS | 2 ++ test/images/cloudbuild.yaml | 5 +++++ test/images/image-util.sh | 22 +++++++++++----------- test/images/kitten/BASEIMAGE | 13 ++++++++----- test/images/kitten/Dockerfile | 1 + test/images/kitten/OWNERS | 3 +++ test/images/nautilus/BASEIMAGE | 13 ++++++++----- test/images/nautilus/Dockerfile | 1 + test/images/nautilus/OWNERS | 3 +++ 14 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 test/images/agnhost/OWNERS create mode 100644 test/images/busybox/OWNERS create mode 100644 test/images/kitten/OWNERS create mode 100644 test/images/nautilus/OWNERS diff --git a/test/images/Makefile b/test/images/Makefile index deca1cd2b67..c653f87f8c0 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -14,6 +14,7 @@ REGISTRY ?= gcr.io/kubernetes-e2e-test-images GOARM ?= 7 +DOCKER_CERT_BASE_PATH ?= QEMUVERSION=v2.9.1 GOLANG_VERSION=1.13.9 export diff --git a/test/images/agnhost/OWNERS b/test/images/agnhost/OWNERS new file mode 100644 index 00000000000..83a3a4d90cc --- /dev/null +++ b/test/images/agnhost/OWNERS @@ -0,0 +1,2 @@ +approvers: + - claudiubelu diff --git a/test/images/agnhost/README.md b/test/images/agnhost/README.md index d929fd3022a..bc630551f71 100644 --- a/test/images/agnhost/README.md +++ b/test/images/agnhost/README.md @@ -40,7 +40,7 @@ For example, let's consider the following `pod.yaml` file: containers: - args: - dns-suffix - image: us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 + image: us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14 name: agnhost dnsConfig: nameservers: @@ -207,7 +207,7 @@ Usage: ```console guestbook="test/e2e/testing-manifests/guestbook" -sed_expr="s|{{.AgnhostImage}}|us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11|" +sed_expr="s|{{.AgnhostImage}}|us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14|" # create the services. kubectl create -f ${guestbook}/frontend-service.yaml @@ -290,14 +290,14 @@ Examples: ```console docker run -i \ - us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ + us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14 \ logs-generator --log-lines-total 10 --run-duration 1s ``` ```console kubectl run logs-generator \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14 \ --restart=Never \ -- logs-generator -t 10 -d 1s ``` @@ -464,7 +464,7 @@ Usage: ```console kubectl run test-agnhost \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14 \ --restart=Never \ --env "POD_IP=" \ --env "NODE_IP=" \ @@ -519,7 +519,7 @@ Usage: ```console kubectl run test-agnhost \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14 \ --restart=Never \ --env "BIND_ADDRESS=localhost" \ --env "BIND_PORT=8080" \ @@ -647,6 +647,6 @@ The Windows `agnhost` image includes a `nc` binary that is 100% compliant with i ## Image -The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11` for both Linux and +The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.14` for both Linux and Windows containers (based on `mcr.microsoft.com/windows/servercore:ltsc2019`, `mcr.microsoft.com/windows/servercore:1903`, and `mcr.microsoft.com/windows/servercore:1909`). diff --git a/test/images/agnhost/VERSION b/test/images/agnhost/VERSION index ae656d47364..123a39a8e91 100644 --- a/test/images/agnhost/VERSION +++ b/test/images/agnhost/VERSION @@ -1 +1 @@ -2.13 +2.14 diff --git a/test/images/agnhost/agnhost.go b/test/images/agnhost/agnhost.go index c8436aacd4a..ade9f23c1e8 100644 --- a/test/images/agnhost/agnhost.go +++ b/test/images/agnhost/agnhost.go @@ -49,7 +49,7 @@ import ( ) func main() { - rootCmd := &cobra.Command{Use: "app", Version: "2.13"} + rootCmd := &cobra.Command{Use: "app", Version: "2.14"} rootCmd.AddCommand(auditproxy.CmdAuditProxy) rootCmd.AddCommand(connect.CmdConnect) diff --git a/test/images/busybox/OWNERS b/test/images/busybox/OWNERS new file mode 100644 index 00000000000..83a3a4d90cc --- /dev/null +++ b/test/images/busybox/OWNERS @@ -0,0 +1,2 @@ +approvers: + - claudiubelu diff --git a/test/images/cloudbuild.yaml b/test/images/cloudbuild.yaml index 2912b934601..0408b7e891b 100644 --- a/test/images/cloudbuild.yaml +++ b/test/images/cloudbuild.yaml @@ -7,6 +7,7 @@ timeout: 5400s # or any new substitutions added in the future. options: substitution_option: ALLOW_LOOSE + machineType: 'N1_HIGHCPU_8' steps: - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4' entrypoint: make @@ -17,6 +18,10 @@ steps: - BASE_REF=$_PULL_BASE_REF - WHAT=$_WHAT - REGISTRY=gcr.io/k8s-staging-e2e-test-images + - DOCKER_CERT_BASE_PATH=/root + - REMOTE_DOCKER_URL_1809=tcp://img-promoter-1809.eastus.cloudapp.azure.com:2376 + - REMOTE_DOCKER_URL_1903=tcp://img-promoter-1903.eastus.cloudapp.azure.com:2376 + - REMOTE_DOCKER_URL_1909=tcp://img-promoter-1909.eastus.cloudapp.azure.com:2376 # TODO(claudiub): Readd the REMOTE_DOCKER_URL_${os_version} to reenable the Windows test image building process. args: - all-build-and-push diff --git a/test/images/image-util.sh b/test/images/image-util.sh index cd4b929692d..a027cec8c75 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -21,6 +21,11 @@ set -o pipefail TASK=$1 WHAT=$2 +# Connecting to a Remote Docker requires certificates for authentication, which can be found +# at this path. By default, they can be found in the ${HOME} folder. We're expecting to find +# here ".docker-${os_version}" folders which contains the necessary certificates. +DOCKER_CERT_BASE_PATH="${DOCKER_CERT_BASE_PATH:-${HOME}}" + KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" source "${KUBE_ROOT}/hack/lib/logging.sh" source "${KUBE_ROOT}/hack/lib/util.sh" @@ -135,8 +140,8 @@ build() { # The node requires TLS authentication, and thus it is expected that the # ca.pem, cert.pem, key.pem files can be found in the ${HOME}/.docker-${os_version} folder. # TODO(claudiub): add "build --isolation=hyperv" once GCE introduces Hyper-V support. - docker --tlsverify --tlscacert "${HOME}/.docker-${os_version}/ca.pem" \ - --tlscert "${HOME}/.docker-${os_version}/cert.pem" --tlskey "${HOME}/.docker-${os_version}/key.pem" \ + docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \ + --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \ -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" \ --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name . else @@ -192,21 +197,16 @@ push() { docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then # NOTE(claudiub): We're pushing the image we built on the remote Windows node. - docker --tlsverify --tlscacert "${HOME}/.docker-${os_version}/ca.pem" \ - --tlscert "${HOME}/.docker-${os_version}/cert.pem" --tlskey "${HOME}/.docker-${os_version}/key.pem" \ + docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \ + --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \ -H "${REMOTE_DOCKER_URL}" push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" else echo "Cannot push the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL_${os_version} should be set, containing the URL to a Windows docker daemon." + # we should exclude this image from the manifest list as well, we couldn't build / push it. + os_archs=$(printf "%s\n" "$os_archs" | grep -v "$os_arch" || true) fi done - # NOTE(claudiub): if the REMOTE_DOCKER_URL var is not set, or it is an empty string, we mustn't include - # Windows images into the manifest list. - if test -z "${REMOTE_DOCKER_URL:-}" && printf "%s\n" "$os_archs" | grep -q '^windows'; then - echo "Skipping pushing the image '${image}' for Windows. REMOTE_DOCKER_URL_\${os_version} should be set, containing the URL to a Windows docker daemon." - os_archs=$(printf "%s\n" "$os_archs" | grep -v "^windows" || true) - fi - if test -z "${os_archs}"; then # this can happen for Windows-only images if they have been skipped entirely. echo "No image for the manifest list. Skipping ${image}." diff --git a/test/images/kitten/BASEIMAGE b/test/images/kitten/BASEIMAGE index 4a0492bdfce..ca4cc5610bf 100644 --- a/test/images/kitten/BASEIMAGE +++ b/test/images/kitten/BASEIMAGE @@ -1,5 +1,8 @@ -linux/amd64=REGISTRY/agnhost:2.13-linux-amd64 -linux/arm=REGISTRY/agnhost:2.13-linux-arm -linux/arm64=REGISTRY/agnhost:2.13-linux-arm64 -linux/ppc64le=REGISTRY/agnhost:2.13-linux-ppc64le -linux/s390x=REGISTRY/agnhost:2.13-linux-s390x +linux/amd64=REGISTRY/agnhost:2.14-linux-amd64 +linux/arm=REGISTRY/agnhost:2.14-linux-arm +linux/arm64=REGISTRY/agnhost:2.14-linux-arm64 +linux/ppc64le=REGISTRY/agnhost:2.14-linux-ppc64le +linux/s390x=REGISTRY/agnhost:2.14-linux-s390x +windows/amd64/1809=REGISTRY/agnhost:2.14-windows-amd64-1809 +windows/amd64/1903=REGISTRY/agnhost:2.14-windows-amd64-1903 +windows/amd64/1909=REGISTRY/agnhost:2.14-windows-amd64-1909 diff --git a/test/images/kitten/Dockerfile b/test/images/kitten/Dockerfile index 48d6f860d88..f92dd7a89d4 100644 --- a/test/images/kitten/Dockerfile +++ b/test/images/kitten/Dockerfile @@ -16,3 +16,4 @@ ARG BASEIMAGE FROM $BASEIMAGE COPY html/kitten.jpg kitten.jpg COPY html/data.json data.json +CMD ["test-webserver"] diff --git a/test/images/kitten/OWNERS b/test/images/kitten/OWNERS new file mode 100644 index 00000000000..a31515e32b5 --- /dev/null +++ b/test/images/kitten/OWNERS @@ -0,0 +1,3 @@ +approvers: + - claudiubelu + - mkumatag diff --git a/test/images/nautilus/BASEIMAGE b/test/images/nautilus/BASEIMAGE index 4a0492bdfce..ca4cc5610bf 100644 --- a/test/images/nautilus/BASEIMAGE +++ b/test/images/nautilus/BASEIMAGE @@ -1,5 +1,8 @@ -linux/amd64=REGISTRY/agnhost:2.13-linux-amd64 -linux/arm=REGISTRY/agnhost:2.13-linux-arm -linux/arm64=REGISTRY/agnhost:2.13-linux-arm64 -linux/ppc64le=REGISTRY/agnhost:2.13-linux-ppc64le -linux/s390x=REGISTRY/agnhost:2.13-linux-s390x +linux/amd64=REGISTRY/agnhost:2.14-linux-amd64 +linux/arm=REGISTRY/agnhost:2.14-linux-arm +linux/arm64=REGISTRY/agnhost:2.14-linux-arm64 +linux/ppc64le=REGISTRY/agnhost:2.14-linux-ppc64le +linux/s390x=REGISTRY/agnhost:2.14-linux-s390x +windows/amd64/1809=REGISTRY/agnhost:2.14-windows-amd64-1809 +windows/amd64/1903=REGISTRY/agnhost:2.14-windows-amd64-1903 +windows/amd64/1909=REGISTRY/agnhost:2.14-windows-amd64-1909 diff --git a/test/images/nautilus/Dockerfile b/test/images/nautilus/Dockerfile index c344116bf8c..2cb4784a8fa 100644 --- a/test/images/nautilus/Dockerfile +++ b/test/images/nautilus/Dockerfile @@ -16,3 +16,4 @@ ARG BASEIMAGE FROM $BASEIMAGE COPY html/nautilus.jpg nautilus.jpg COPY html/data.json data.json +CMD ["test-webserver"] diff --git a/test/images/nautilus/OWNERS b/test/images/nautilus/OWNERS new file mode 100644 index 00000000000..a31515e32b5 --- /dev/null +++ b/test/images/nautilus/OWNERS @@ -0,0 +1,3 @@ +approvers: + - claudiubelu + - mkumatag