From 22a5fbcfa98a750a0b9b9f0bceabde2493ebd721 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 1 May 2019 21:43:49 -0700 Subject: [PATCH 1/9] test images: Bumps image versions The way the images are built is going to be changed, and in order to avoid overwritting and breaking the current images, the image versions are bumped. --- test/images/agnhost/VERSION | 2 +- test/images/agnhost/agnhost.go | 2 +- test/images/apparmor-loader/VERSION | 2 +- test/images/cuda-vector-add/VERSION | 2 +- test/images/echoserver/VERSION | 2 +- test/images/ipc-utils/VERSION | 2 +- test/images/jessie-dnsutils/VERSION | 2 +- test/images/kitten/BASEIMAGE | 10 +++++----- test/images/metadata-concealment/VERSION | 2 +- test/images/nautilus/BASEIMAGE | 10 +++++----- test/images/node-perf/npb-ep/VERSION | 2 +- test/images/node-perf/npb-is/VERSION | 2 +- test/images/node-perf/tf-wide-deep/VERSION | 2 +- test/images/nonewprivs/VERSION | 2 +- test/images/nonroot/VERSION | 2 +- test/images/pets/peer-finder/VERSION | 2 +- test/images/pets/redis-installer/VERSION | 2 +- test/images/pets/zookeeper-installer/VERSION | 2 +- test/images/redis/VERSION | 2 +- test/images/regression-issue-74839/VERSION | 2 +- test/images/resource-consumer/README.md | 4 ++-- test/images/resource-consumer/VERSION | 2 +- test/images/sample-apiserver/VERSION | 2 +- test/images/sample-device-plugin/VERSION | 2 +- test/images/volume/gluster/VERSION | 2 +- test/images/volume/iscsi/VERSION | 2 +- test/images/volume/nfs/VERSION | 2 +- test/images/volume/rbd/VERSION | 2 +- 28 files changed, 37 insertions(+), 37 deletions(-) diff --git a/test/images/agnhost/VERSION b/test/images/agnhost/VERSION index 37989bd16b4..6a5fe6e8977 100644 --- a/test/images/agnhost/VERSION +++ b/test/images/agnhost/VERSION @@ -1 +1 @@ -2.10 +2.11 diff --git a/test/images/agnhost/agnhost.go b/test/images/agnhost/agnhost.go index 6d6510d8c71..ca82e93c2b8 100644 --- a/test/images/agnhost/agnhost.go +++ b/test/images/agnhost/agnhost.go @@ -48,7 +48,7 @@ import ( ) func main() { - rootCmd := &cobra.Command{Use: "app", Version: "2.10"} + rootCmd := &cobra.Command{Use: "app", Version: "2.11"} rootCmd.AddCommand(auditproxy.CmdAuditProxy) rootCmd.AddCommand(connect.CmdConnect) diff --git a/test/images/apparmor-loader/VERSION b/test/images/apparmor-loader/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/apparmor-loader/VERSION +++ b/test/images/apparmor-loader/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/cuda-vector-add/VERSION b/test/images/cuda-vector-add/VERSION index cd5ac039d67..879b416e609 100644 --- a/test/images/cuda-vector-add/VERSION +++ b/test/images/cuda-vector-add/VERSION @@ -1 +1 @@ -2.0 +2.1 diff --git a/test/images/echoserver/VERSION b/test/images/echoserver/VERSION index 8bbe6cf74a1..bb576dbde10 100644 --- a/test/images/echoserver/VERSION +++ b/test/images/echoserver/VERSION @@ -1 +1 @@ -2.2 +2.3 diff --git a/test/images/ipc-utils/VERSION b/test/images/ipc-utils/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/ipc-utils/VERSION +++ b/test/images/ipc-utils/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/jessie-dnsutils/VERSION b/test/images/jessie-dnsutils/VERSION index 5625e59da88..7e32cd56983 100644 --- a/test/images/jessie-dnsutils/VERSION +++ b/test/images/jessie-dnsutils/VERSION @@ -1 +1 @@ -1.2 +1.3 diff --git a/test/images/kitten/BASEIMAGE b/test/images/kitten/BASEIMAGE index 20b2972e552..a6c10a149fe 100644 --- a/test/images/kitten/BASEIMAGE +++ b/test/images/kitten/BASEIMAGE @@ -1,5 +1,5 @@ -amd64=gcr.io/kubernetes-e2e-test-images/agnhost-amd64:2.7 -arm=gcr.io/kubernetes-e2e-test-images/agnhost-arm:2.7 -arm64=gcr.io/kubernetes-e2e-test-images/agnhost-arm64:2.7 -ppc64le=gcr.io/kubernetes-e2e-test-images/agnhost-ppc64le:2.7 -s390x=gcr.io/kubernetes-e2e-test-images/agnhost-s390x:2.7 +amd64=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-amd64:2.11 +arm=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-arm:2.11 +arm64=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-arm64:2.11 +ppc64le=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-ppc64le:2.11 +s390x=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-s390x:2.11 diff --git a/test/images/metadata-concealment/VERSION b/test/images/metadata-concealment/VERSION index 5625e59da88..7e32cd56983 100644 --- a/test/images/metadata-concealment/VERSION +++ b/test/images/metadata-concealment/VERSION @@ -1 +1 @@ -1.2 +1.3 diff --git a/test/images/nautilus/BASEIMAGE b/test/images/nautilus/BASEIMAGE index 20b2972e552..a6c10a149fe 100644 --- a/test/images/nautilus/BASEIMAGE +++ b/test/images/nautilus/BASEIMAGE @@ -1,5 +1,5 @@ -amd64=gcr.io/kubernetes-e2e-test-images/agnhost-amd64:2.7 -arm=gcr.io/kubernetes-e2e-test-images/agnhost-arm:2.7 -arm64=gcr.io/kubernetes-e2e-test-images/agnhost-arm64:2.7 -ppc64le=gcr.io/kubernetes-e2e-test-images/agnhost-ppc64le:2.7 -s390x=gcr.io/kubernetes-e2e-test-images/agnhost-s390x:2.7 +amd64=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-amd64:2.11 +arm=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-arm:2.11 +arm64=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-arm64:2.11 +ppc64le=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-ppc64le:2.11 +s390x=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost-s390x:2.11 diff --git a/test/images/node-perf/npb-ep/VERSION b/test/images/node-perf/npb-ep/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/node-perf/npb-ep/VERSION +++ b/test/images/node-perf/npb-ep/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/node-perf/npb-is/VERSION b/test/images/node-perf/npb-is/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/node-perf/npb-is/VERSION +++ b/test/images/node-perf/npb-is/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/node-perf/tf-wide-deep/VERSION b/test/images/node-perf/tf-wide-deep/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/node-perf/tf-wide-deep/VERSION +++ b/test/images/node-perf/tf-wide-deep/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/nonewprivs/VERSION b/test/images/nonewprivs/VERSION index 9459d4ba2a0..5625e59da88 100644 --- a/test/images/nonewprivs/VERSION +++ b/test/images/nonewprivs/VERSION @@ -1 +1 @@ -1.1 +1.2 diff --git a/test/images/nonroot/VERSION b/test/images/nonroot/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/nonroot/VERSION +++ b/test/images/nonroot/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/pets/peer-finder/VERSION b/test/images/pets/peer-finder/VERSION index 7e32cd56983..c068b2447cc 100644 --- a/test/images/pets/peer-finder/VERSION +++ b/test/images/pets/peer-finder/VERSION @@ -1 +1 @@ -1.3 +1.4 diff --git a/test/images/pets/redis-installer/VERSION b/test/images/pets/redis-installer/VERSION index 7e32cd56983..c068b2447cc 100644 --- a/test/images/pets/redis-installer/VERSION +++ b/test/images/pets/redis-installer/VERSION @@ -1 +1 @@ -1.3 +1.4 diff --git a/test/images/pets/zookeeper-installer/VERSION b/test/images/pets/zookeeper-installer/VERSION index 7e32cd56983..c068b2447cc 100644 --- a/test/images/pets/zookeeper-installer/VERSION +++ b/test/images/pets/zookeeper-installer/VERSION @@ -1 +1 @@ -1.3 +1.4 diff --git a/test/images/redis/VERSION b/test/images/redis/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/redis/VERSION +++ b/test/images/redis/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/regression-issue-74839/VERSION b/test/images/regression-issue-74839/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/regression-issue-74839/VERSION +++ b/test/images/regression-issue-74839/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/resource-consumer/README.md b/test/images/resource-consumer/README.md index e6a168c716d..a88dda3de53 100644 --- a/test/images/resource-consumer/README.md +++ b/test/images/resource-consumer/README.md @@ -48,7 +48,7 @@ Custom metrics in Prometheus format are exposed on "/metrics" endpoint. ### CURL example ```console -kubectl run resource-consumer --image=gcr.io/kubernetes-e2e-test-images/resource-consumer:1.4 --expose --service-overrides='{ "spec": { "type": "LoadBalancer" } }' --port 8080 --requests='cpu=500m,memory=256Mi' +kubectl run resource-consumer --image=gcr.io/kubernetes-e2e-test-images/resource-consumer:1.7 --expose --service-overrides='{ "spec": { "type": "LoadBalancer" } }' --port 8080 --requests='cpu=500m,memory=256Mi' kubectl get services resource-consumer ``` @@ -62,7 +62,7 @@ curl --data "millicores=300&durationSec=600" http://:8080/ConsumeCP ## Image -Docker image of Resource Consumer can be found in Google Container Registry as gcr.io/kubernetes-e2e-test-images/resource-consumer:1.4 +Docker image of Resource Consumer can be found in Google Container Registry as gcr.io/kubernetes-e2e-test-images/resource-consumer:1.7 ## Use cases diff --git a/test/images/resource-consumer/VERSION b/test/images/resource-consumer/VERSION index 810ee4e91e2..d3bdbdf1fda 100644 --- a/test/images/resource-consumer/VERSION +++ b/test/images/resource-consumer/VERSION @@ -1 +1 @@ -1.6 +1.7 diff --git a/test/images/sample-apiserver/VERSION b/test/images/sample-apiserver/VERSION index 06fb41b6322..b9a05a6dc1d 100644 --- a/test/images/sample-apiserver/VERSION +++ b/test/images/sample-apiserver/VERSION @@ -1 +1 @@ -1.17.2 +1.17.3 diff --git a/test/images/sample-device-plugin/VERSION b/test/images/sample-device-plugin/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/sample-device-plugin/VERSION +++ b/test/images/sample-device-plugin/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/volume/gluster/VERSION b/test/images/volume/gluster/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/volume/gluster/VERSION +++ b/test/images/volume/gluster/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/volume/iscsi/VERSION b/test/images/volume/iscsi/VERSION index cd5ac039d67..879b416e609 100644 --- a/test/images/volume/iscsi/VERSION +++ b/test/images/volume/iscsi/VERSION @@ -1 +1 @@ -2.0 +2.1 diff --git a/test/images/volume/nfs/VERSION b/test/images/volume/nfs/VERSION index d3827e75a5c..9459d4ba2a0 100644 --- a/test/images/volume/nfs/VERSION +++ b/test/images/volume/nfs/VERSION @@ -1 +1 @@ -1.0 +1.1 diff --git a/test/images/volume/rbd/VERSION b/test/images/volume/rbd/VERSION index 7dea76edb3d..6d7de6e6abe 100644 --- a/test/images/volume/rbd/VERSION +++ b/test/images/volume/rbd/VERSION @@ -1 +1 @@ -1.0.1 +1.0.2 From efcdb929dec18c201342c601008babc8005b54f8 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 17 Apr 2019 13:08:24 +0000 Subject: [PATCH 2/9] images: Adds linux/ prefix to BASEIMAGE entries Windows images will require other base images, and thus, we will need to explicitly specify the OS type a base image is for in order to avoid confusion or errors. --- test/images/agnhost/BASEIMAGE | 10 ++-- test/images/apparmor-loader/BASEIMAGE | 8 +-- test/images/cuda-vector-add/BASEIMAGE | 4 +- test/images/echoserver/BASEIMAGE | 10 ++-- test/images/image-util.sh | 54 ++++++++++++++----- test/images/ipc-utils/BASEIMAGE | 8 +-- test/images/jessie-dnsutils/BASEIMAGE | 10 ++-- test/images/kitten/BASEIMAGE | 10 ++-- test/images/nautilus/BASEIMAGE | 10 ++-- test/images/node-perf/npb-ep/BASEIMAGE | 6 +-- test/images/node-perf/npb-is/BASEIMAGE | 6 +-- test/images/node-perf/tf-wide-deep/BASEIMAGE | 4 +- test/images/nonewprivs/BASEIMAGE | 8 +-- test/images/nonroot/BASEIMAGE | 10 ++-- test/images/pets/redis-installer/BASEIMAGE | 8 +-- .../images/pets/zookeeper-installer/BASEIMAGE | 8 +-- test/images/redis/BASEIMAGE | 10 ++-- test/images/resource-consumer/BASEIMAGE | 10 ++-- test/images/sample-apiserver/BASEIMAGE | 10 ++-- test/images/sample-device-plugin/BASEIMAGE | 10 ++-- test/images/volume/gluster/BASEIMAGE | 6 +-- test/images/volume/iscsi/BASEIMAGE | 6 +-- test/images/volume/nfs/BASEIMAGE | 6 +-- test/images/volume/rbd/BASEIMAGE | 6 +-- 24 files changed, 132 insertions(+), 106 deletions(-) diff --git a/test/images/agnhost/BASEIMAGE b/test/images/agnhost/BASEIMAGE index 7bad7a6d3a2..ad69ccfb016 100644 --- a/test/images/agnhost/BASEIMAGE +++ b/test/images/agnhost/BASEIMAGE @@ -1,5 +1,5 @@ -amd64=alpine:3.6 -arm=arm32v6/alpine:3.6 -arm64=arm64v8/alpine:3.6 -ppc64le=ppc64le/alpine:3.6 -s390x=s390x/alpine:3.6 +linux/amd64=alpine:3.6 +linux/arm=arm32v6/alpine:3.6 +linux/arm64=arm64v8/alpine:3.6 +linux/ppc64le=ppc64le/alpine:3.6 +linux/s390x=s390x/alpine:3.6 diff --git a/test/images/apparmor-loader/BASEIMAGE b/test/images/apparmor-loader/BASEIMAGE index 960ad547f94..8d99f110a96 100644 --- a/test/images/apparmor-loader/BASEIMAGE +++ b/test/images/apparmor-loader/BASEIMAGE @@ -1,4 +1,4 @@ -amd64=alpine:3.8 -arm=arm32v6/alpine:3.8 -arm64=arm64v8/alpine:3.8 -ppc64le=ppc64le/alpine:3.8 +linux/amd64=alpine:3.8 +linux/arm=arm32v6/alpine:3.8 +linux/arm64=arm64v8/alpine:3.8 +linux/ppc64le=ppc64le/alpine:3.8 diff --git a/test/images/cuda-vector-add/BASEIMAGE b/test/images/cuda-vector-add/BASEIMAGE index 89b44efb366..5e5cd4dac28 100644 --- a/test/images/cuda-vector-add/BASEIMAGE +++ b/test/images/cuda-vector-add/BASEIMAGE @@ -1,2 +1,2 @@ -amd64=nvidia/cuda:10.0-devel-ubuntu18.04 -ppc64le=nvidia/cuda-ppc64le:10.0-devel-ubuntu18.04 +linux/amd64=nvidia/cuda:10.0-devel-ubuntu18.04 +linux/ppc64le=nvidia/cuda-ppc64le:10.0-devel-ubuntu18.04 diff --git a/test/images/echoserver/BASEIMAGE b/test/images/echoserver/BASEIMAGE index d27954b62e8..269c92ac3ad 100644 --- a/test/images/echoserver/BASEIMAGE +++ b/test/images/echoserver/BASEIMAGE @@ -1,5 +1,5 @@ -amd64=nginx:1.15-alpine -arm=arm32v6/nginx:1.15-alpine -arm64=arm64v8/nginx:1.15-alpine -ppc64le=ppc64le/nginx:1.15-alpine -s390x=s390x/nginx:1.15-alpine +linux/amd64=nginx:1.15-alpine +linux/arm=arm32v6/nginx:1.15-alpine +linux/arm64=arm64v8/nginx:1.15-alpine +linux/ppc64le=ppc64le/nginx:1.15-alpine +linux/s390x=s390x/nginx:1.15-alpine diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 99e91f4d803..2fabb902095 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -29,15 +29,16 @@ source "${KUBE_ROOT}/hack/lib/util.sh" declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["ppc64le"]="ppc64le" ["s390x"]="s390x" ) # Returns list of all supported architectures from BASEIMAGE file -listArchs() { +listOsArchs() { image=$1 cut -d "=" -f 1 "${image}"/BASEIMAGE } # Returns baseimage need to used in Dockerfile for any given architecture getBaseImage() { - arch=$1 - grep "${arch}=" BASEIMAGE | cut -d= -f2 + os_name=$1 + arch=$2 + grep "${os_name}/${arch}=" BASEIMAGE | cut -d= -f2 } # This function will build test image for all the architectures @@ -47,15 +48,24 @@ getBaseImage() { build() { image=$1 if [[ -f ${image}/BASEIMAGE ]]; then - archs=$(listArchs "$image") + os_archs=$(listOsArchs "$image") else - archs=${!QEMUARCHS[*]} + # prepend linux/ to the QEMUARCHS items. + os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[*]}") fi kube::util::ensure-gnu-sed - for arch in ${archs}; do - echo "Building image for ${image} ARCH: ${arch}..." + for os_arch in ${os_archs}; do + if [[ $os_arch =~ .*/.* ]]; then + os_name=$(echo "$os_arch" | cut -d "/" -f 1) + arch=$(echo "$os_arch" | cut -d "/" -f 2) + else + echo "The BASEIMAGE file for the ${image} image is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead." + exit 1 + fi + + echo "Building image for ${image} OS/ARCH: ${os_arch}..." # Create a temporary directory for every architecture and copy the image content # and build the image from temporary directory @@ -74,7 +84,7 @@ build() { TAG=$( Date: Wed, 17 Apr 2019 15:29:49 +0000 Subject: [PATCH 3/9] images: Configurable BASEIMAGE hierarchy When building images, their REGISTRY can be set to a custom one, instead of the default "gcr.io/kubernetes-e2e-test-images" or "us.gcr.io/k8s-artifacts-prod/e2e-test-images". Some images are based on other images we're already building (e.g.: kitten, nautilus), but their base images are set in the default registry name, which can be undesirable. This commit addresses this issue. --- test/images/image-util.sh | 2 +- test/images/kitten/BASEIMAGE | 10 +++++----- test/images/nautilus/BASEIMAGE | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 2fabb902095..75bf7968a71 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -84,7 +84,7 @@ build() { TAG=$( Date: Wed, 17 Apr 2019 15:13:51 +0000 Subject: [PATCH 4/9] images: Changes the image naming template Changes the image naming template from: $REGISTRY/$image-$arch:$TAG to $REGISTRY/$image:$TAG-$os_name-$arch The previous naming template would generate a plethora of images (Ai * N images, where Ai is the number of OS/architectures for the image i and N is the number of images), while the new naming template will reduce the number of images to N. The new template also includes the OS name, as we plan to integrate Windows images into the manifest lists as well. --- test/images/image-util.sh | 10 +++++----- test/images/kitten/BASEIMAGE | 10 +++++----- test/images/nautilus/BASEIMAGE | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 75bf7968a71..9ab51bd54fe 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -109,7 +109,7 @@ build() { fi fi - docker build --pull -t "${REGISTRY}/${image}-${arch}:${TAG}" . + docker build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" . popd done @@ -145,7 +145,7 @@ push() { exit 1 fi - docker push "${REGISTRY}/${image}-${arch}:${TAG}" + docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" done kube::util::ensure-gnu-sed @@ -154,8 +154,8 @@ push() { export DOCKER_CLI_EXPERIMENTAL="enabled" # reset manifest list; needed in case multiple images are being built / pushed. manifest=() - # Make os_archs list into image manifest. Eg: 'linux/amd64 linux/ppc64le' to '${REGISTRY}/${image}-amd64:${TAG} ${REGISTRY}/${image}-ppc64le:${TAG}' - while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$os_archs" | ${SED} "s~linux\/~~" | ${SED} -e "s~[^ ]*~$REGISTRY\/$image\-&:$TAG~g") + # Make os_archs list into image manifest. Eg: 'linux/amd64 linux/ppc64le' to '${REGISTRY}/${image}:${TAG}-linux-amd64 ${REGISTRY}/${image}:${TAG}-linux-ppc64le' + while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$os_archs" | ${SED} "s~\/~-~" | ${SED} -e "s~[^ ]*~$REGISTRY\/$image:$TAG\-&~g") docker manifest create --amend "${REGISTRY}/${image}:${TAG}" "${manifest[@]}" for os_arch in ${os_archs}; do if [[ $os_arch =~ .*/.* ]]; then @@ -165,7 +165,7 @@ push() { echo "The BASEIMAGE file for the ${image} image is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead." exit 1 fi - docker manifest annotate --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}-${arch}:${TAG}" + docker manifest annotate --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" done docker manifest push --purge "${REGISTRY}/${image}:${TAG}" } diff --git a/test/images/kitten/BASEIMAGE b/test/images/kitten/BASEIMAGE index fa30cda5081..6ccbb5ba8ad 100644 --- a/test/images/kitten/BASEIMAGE +++ b/test/images/kitten/BASEIMAGE @@ -1,5 +1,5 @@ -linux/amd64=REGISTRY/agnhost-amd64:2.11 -linux/arm=REGISTRY/agnhost-arm:2.11 -linux/arm64=REGISTRY/agnhost-arm64:2.11 -linux/ppc64le=REGISTRY/agnhost-ppc64le:2.11 -linux/s390x=REGISTRY/agnhost-s390x:2.11 +linux/amd64=REGISTRY/agnhost:2.11-linux-amd64 +linux/arm=REGISTRY/agnhost:2.11-linux-arm +linux/arm64=REGISTRY/agnhost:2.11-linux-arm64 +linux/ppc64le=REGISTRY/agnhost:2.11-linux-ppc64le +linux/s390x=REGISTRY/agnhost:2.11-linux-s390x diff --git a/test/images/nautilus/BASEIMAGE b/test/images/nautilus/BASEIMAGE index fa30cda5081..6ccbb5ba8ad 100644 --- a/test/images/nautilus/BASEIMAGE +++ b/test/images/nautilus/BASEIMAGE @@ -1,5 +1,5 @@ -linux/amd64=REGISTRY/agnhost-amd64:2.11 -linux/arm=REGISTRY/agnhost-arm:2.11 -linux/arm64=REGISTRY/agnhost-arm64:2.11 -linux/ppc64le=REGISTRY/agnhost-ppc64le:2.11 -linux/s390x=REGISTRY/agnhost-s390x:2.11 +linux/amd64=REGISTRY/agnhost:2.11-linux-amd64 +linux/arm=REGISTRY/agnhost:2.11-linux-arm +linux/arm64=REGISTRY/agnhost:2.11-linux-arm64 +linux/ppc64le=REGISTRY/agnhost:2.11-linux-ppc64le +linux/s390x=REGISTRY/agnhost:2.11-linux-s390x From 296464d968bf1122164d5f7b3e29b7690bb312a1 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 17 Apr 2019 13:56:50 +0000 Subject: [PATCH 5/9] test images: Adds Windows support (part 1) Adds Windows support to the test/images/image-util.sh script. A Windows node with Docker installed is required to build Windows images. The connection URL to it must be set in the REMOTE_DOCKER_URL env variable. Additionally, the authentication to the remote docker node is done through certificates, which must be found in ~/.docker. By default, the REMOTE_DOCKER_URL env variable is set to "" in the Makefile, and because of it, the image-util.sh script will skip building and pushing Windows images. Added GOOS argument to the go build process in order to be able to build Windows binaries. Additionally, the OS env variable was added to the images Makefiles (default value is "linux") in order to maintain default behaviour. Some images require a different Dockerfile for Windows images, since they have different ways of installing dependencies. Because of this, if a image needs to be built for Windows, it will first check for a Dockerfile_windows file instead of the default one. If there isn't one, it means that the same Dockerfile can be used for both Windows and Linux. All Windows images will be based on the image "mcr.microsoft.com/windows/servercore:ltsc2019". There are a couple of features that are needed from this image, especially powershell. Added busybox image for Windows. Most Windows images will be based on it, which will help reduce the command line differences between Linux and Windows, but not entirely. Added Windows support for agnhost image. --- test/images/BUILD | 1 + test/images/Makefile | 1 + test/images/agnhost/BASEIMAGE | 1 + test/images/agnhost/Dockerfile | 3 +- test/images/agnhost/Dockerfile_windows | 64 +++++++++++++++++++ test/images/agnhost/Makefile | 1 + test/images/agnhost/README.md | 26 +++++--- test/images/apparmor-loader/Dockerfile | 3 +- test/images/apparmor-loader/Makefile | 1 + test/images/busybox/BASEIMAGE | 1 + test/images/busybox/BUILD | 31 +++++++++ test/images/busybox/Dockerfile_windows | 34 ++++++++++ test/images/busybox/Makefile | 27 ++++++++ test/images/busybox/VERSION | 1 + test/images/busybox/hostname.go | 50 +++++++++++++++ test/images/cuda-vector-add/Dockerfile | 3 +- test/images/echoserver/Dockerfile | 3 +- test/images/image-util.sh | 45 ++++++++++--- test/images/ipc-utils/Dockerfile | 3 +- test/images/jessie-dnsutils/Dockerfile | 3 +- test/images/kitten/Dockerfile | 3 +- test/images/metadata-concealment/Makefile | 1 + test/images/nautilus/Dockerfile | 3 +- test/images/node-perf/npb-ep/Dockerfile | 6 +- test/images/node-perf/npb-is/Dockerfile | 6 +- test/images/node-perf/tf-wide-deep/Dockerfile | 3 +- test/images/nonewprivs/Dockerfile | 3 +- test/images/nonewprivs/Makefile | 1 + test/images/nonroot/Dockerfile | 3 +- test/images/pets/peer-finder/Dockerfile | 3 +- test/images/pets/peer-finder/Makefile | 1 + test/images/pets/redis-installer/Dockerfile | 3 +- test/images/pets/redis-installer/Makefile | 1 + .../pets/zookeeper-installer/Dockerfile | 3 +- test/images/pets/zookeeper-installer/Makefile | 1 + test/images/redis/Dockerfile | 3 +- test/images/regression-issue-74839/Makefile | 1 + test/images/resource-consumer/Dockerfile | 3 +- test/images/resource-consumer/Makefile | 1 + test/images/sample-apiserver/Dockerfile | 3 +- test/images/sample-device-plugin/Dockerfile | 3 +- test/images/sample-device-plugin/Makefile | 1 + test/images/volume/gluster/Dockerfile | 3 +- test/images/volume/iscsi/Dockerfile | 3 +- test/images/volume/nfs/Dockerfile | 3 +- test/images/volume/rbd/Dockerfile | 3 +- 46 files changed, 325 insertions(+), 45 deletions(-) create mode 100644 test/images/agnhost/Dockerfile_windows create mode 100644 test/images/busybox/BASEIMAGE create mode 100644 test/images/busybox/BUILD create mode 100644 test/images/busybox/Dockerfile_windows create mode 100644 test/images/busybox/Makefile create mode 100644 test/images/busybox/VERSION create mode 100644 test/images/busybox/hostname.go diff --git a/test/images/BUILD b/test/images/BUILD index 2f887f0b644..a068ae0951d 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -13,6 +13,7 @@ filegroup( ":package-srcs", "//test/images/agnhost:all-srcs", "//test/images/apparmor-loader:all-srcs", + "//test/images/busybox:all-srcs", "//test/images/echoserver:all-srcs", "//test/images/metadata-concealment:all-srcs", "//test/images/nonewprivs:all-srcs", diff --git a/test/images/Makefile b/test/images/Makefile index 92c48f5283b..e1efc8f2de0 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -14,6 +14,7 @@ REGISTRY ?= gcr.io/kubernetes-e2e-test-images GOARM ?= 7 +REMOTE_DOCKER_URL ?= QEMUVERSION=v2.9.1 GOLANG_VERSION=1.13.6 export diff --git a/test/images/agnhost/BASEIMAGE b/test/images/agnhost/BASEIMAGE index ad69ccfb016..5fdb1fa08ea 100644 --- a/test/images/agnhost/BASEIMAGE +++ b/test/images/agnhost/BASEIMAGE @@ -3,3 +3,4 @@ linux/arm=arm32v6/alpine:3.6 linux/arm64=arm64v8/alpine:3.6 linux/ppc64le=ppc64le/alpine:3.6 linux/s390x=s390x/alpine:3.6 +windows/amd64=REGISTRY/busybox:1.29-windows-amd64 diff --git a/test/images/agnhost/Dockerfile b/test/images/agnhost/Dockerfile index c551802c792..3cfe56c9cb2 100644 --- a/test/images/agnhost/Dockerfile +++ b/test/images/agnhost/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/agnhost/Dockerfile_windows b/test/images/agnhost/Dockerfile_windows new file mode 100644 index 00000000000..76edbdbb510 --- /dev/null +++ b/test/images/agnhost/Dockerfile_windows @@ -0,0 +1,64 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASEIMAGE +FROM $BASEIMAGE + +# from dnsutils image +# install necessary packages: +# - bind-tools: contains dig, which can used in DNS tests. +# - CoreDNS: used in some DNS tests. +# from hostexec image +# installed necessary packages: +# - curl, nc: used by a lot of e2e tests (inherited from BASEIMAGE) +# from iperf image +# install necessary packages: iperf +ENV chocolateyUseWindowsCompression false +RUN powershell -Command "\ + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ + choco feature disable --name showDownloadProgress; \ + choco install bind-toolsonly --version 9.10.3 -y +RUN powershell -Command "\ + wget -uri 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -OutFile C:\coredns.tgz;\ + tar -xzvf C:\coredns.tgz;\ + Remove-Item C:\coredns.tgz" + +RUN powershell -Command "\ + wget -uri 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -OutFile C:\iperf.zip;\ + Expand-Archive -Path C:\iperf.zip -DestinationPath C:\ -Force;\ + Rename-Item C:\iperf-2.0.9-win64 C:\iperf;\ + Remove-Item C:\iperf.zip" + +# PORT 80 needed by: test-webserver +# PORT 8080 needed by: netexec, nettest +# PORT 8081 needed by: netexec +# PORT 9376 needed by: serve-hostname +EXPOSE 80 8080 8081 9376 + +# from netexec +RUN mkdir C:\uploads + +# from porter +ADD porter/localhost.crt localhost.crt +ADD porter/localhost.key localhost.key + +ADD agnhost agnhost + +# needed for the entrypoint-tester related tests. Some of the entrypoint-tester related tests +# overrides this image's entrypoint with agnhost-2 binary, and will verify that the correct +# entrypoint is used by the containers. +RUN mklink agnhost-2 agnhost + +ENTRYPOINT ["/agnhost"] +CMD ["pause"] diff --git a/test/images/agnhost/Makefile b/test/images/agnhost/Makefile index c21639c38c2..c5eeeea4162 100644 --- a/test/images/agnhost/Makefile +++ b/test/images/agnhost/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=agnhost +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/agnhost/README.md b/test/images/agnhost/README.md index b797ee47f57..ce241dcd770 100644 --- a/test/images/agnhost/README.md +++ b/test/images/agnhost/README.md @@ -40,7 +40,7 @@ For example, let's consider the following `pod.yaml` file: containers: - args: - dns-suffix - image: us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 + image: us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 name: agnhost dnsConfig: nameservers: @@ -207,7 +207,7 @@ Usage: ```console guestbook="test/e2e/testing-manifests/guestbook" -sed_expr="s|{{.AgnhostImage}}|us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10|" +sed_expr="s|{{.AgnhostImage}}|us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11|" # create the services. kubectl create -f ${guestbook}/frontend-service.yaml @@ -290,14 +290,14 @@ Examples: ```console docker run -i \ - us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ logs-generator --log-lines-total 10 --run-duration 1s ``` ```console kubectl run logs-generator \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ --restart=Never \ -- logs-generator -t 10 -d 1s ``` @@ -455,7 +455,7 @@ Usage: ```console kubectl run test-agnhost \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ --restart=Never \ --env "POD_IP=" \ --env "NODE_IP=" \ @@ -510,7 +510,7 @@ Usage: ```console kubectl run test-agnhost \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ --restart=Never \ --env "BIND_ADDRESS=localhost" \ --env "BIND_PORT=8080" \ @@ -626,11 +626,17 @@ Usage: ## Other tools -The image contains `iperf`, `curl`, `dns-tools` (including `dig`), CoreDNS. +The image contains `iperf`, `curl`, `dns-tools` (including `dig`), CoreDNS, for both Windows and Linux. + +For Windows, the image is based on `busybox`, meaning that most of the Linux common tools are also +available on it, making it possible to run most Linux commands in the `agnhost` Windows container +as well. Keep in mind that there might still be some differences though (e.g.: `wget` does not +have the `-T` argument on Windows). + +The Windows `agnhost` image includes a `nc` binary that is 100% compliant with its Linux equivalent. ## Image -The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10` for Linux -containers, and `e2eteam/agnhost:2.10` for Windows containers. In the future, the same -repository can be used for both OSes. +The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11` for both Linux and +Windows containers (based on `mcr.microsoft.com/windows/servercore:ltsc2019`). diff --git a/test/images/apparmor-loader/Dockerfile b/test/images/apparmor-loader/Dockerfile index 0cad6e26eef..c6e38ae7e75 100644 --- a/test/images/apparmor-loader/Dockerfile +++ b/test/images/apparmor-loader/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/apparmor-loader/Makefile b/test/images/apparmor-loader/Makefile index 807d3a2a489..c0b6bd8e481 100644 --- a/test/images/apparmor-loader/Makefile +++ b/test/images/apparmor-loader/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=loader +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/busybox/BASEIMAGE b/test/images/busybox/BASEIMAGE new file mode 100644 index 00000000000..932c698c514 --- /dev/null +++ b/test/images/busybox/BASEIMAGE @@ -0,0 +1 @@ +windows/amd64=mcr.microsoft.com/windows/servercore:ltsc2019 diff --git a/test/images/busybox/BUILD b/test/images/busybox/BUILD new file mode 100644 index 00000000000..f1d2087ff43 --- /dev/null +++ b/test/images/busybox/BUILD @@ -0,0 +1,31 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "hostname", + embed = [":go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = ["hostname.go"], + importpath = "k8s.io/kubernetes/test/images/busybox", +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/images/busybox/Dockerfile_windows b/test/images/busybox/Dockerfile_windows new file mode 100644 index 00000000000..c4a508fa96a --- /dev/null +++ b/test/images/busybox/Dockerfile_windows @@ -0,0 +1,34 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASEIMAGE +from $BASEIMAGE as prep + +ENV CURL_VERSION 7.57.0 +WORKDIR /curl +ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab +RUN expand /R curl.cab /F:* . + +FROM $BASEIMAGE + +COPY --from=prep /curl/AMD64 /curl +COPY --from=prep /curl/CURL.LIC /curl +ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe +ADD https://github.com/diegocr/netcat/raw/master/nc.exe /bin/nc.exe +ADD hostname /bin/hostname.exe +USER ContainerAdministrator +RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe +RUN setx /M PATH "C:\bin;C:\curl\;%PATH%" &\ + mkdir C:\tmp +ENTRYPOINT ["cmd.exe", "/s", "/c"] diff --git a/test/images/busybox/Makefile b/test/images/busybox/Makefile new file mode 100644 index 00000000000..ca3bd80c5ae --- /dev/null +++ b/test/images/busybox/Makefile @@ -0,0 +1,27 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS=hostname +OS ?= linux +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOARM = 7 +GOLANG_VERSION ?= latest +SRC_DIR = $(notdir $(shell pwd)) +export + +bin: + ../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/busybox/VERSION b/test/images/busybox/VERSION new file mode 100644 index 00000000000..9de53f1932a --- /dev/null +++ b/test/images/busybox/VERSION @@ -0,0 +1 @@ +1.29 diff --git a/test/images/busybox/hostname.go b/test/images/busybox/hostname.go new file mode 100644 index 00000000000..77694dad16a --- /dev/null +++ b/test/images/busybox/hostname.go @@ -0,0 +1,50 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" +) + +func getOutboundIP() net.IP { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + log.Fatal(err) + } + defer conn.Close() + localAddr := conn.LocalAddr().(*net.UDPAddr) + return localAddr.IP +} + +func main() { + flagIP := flag.Bool("i", false, "a string") + flag.Parse() + if *flagIP { + ip := getOutboundIP() + fmt.Print(ip.String()) + } else { + hostname, err := os.Hostname() + if err != nil { + log.Fatal(err) + } + fmt.Print(hostname) + } +} diff --git a/test/images/cuda-vector-add/Dockerfile b/test/images/cuda-vector-add/Dockerfile index 968626bf424..af82e6a5e9b 100644 --- a/test/images/cuda-vector-add/Dockerfile +++ b/test/images/cuda-vector-add/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/echoserver/Dockerfile b/test/images/echoserver/Dockerfile index 919f39dba8e..3329cdf458c 100644 --- a/test/images/echoserver/Dockerfile +++ b/test/images/echoserver/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 9ab51bd54fe..3917aa67a8e 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -77,7 +77,7 @@ build() { if [[ -f ${image}/Makefile ]]; then # make bin will take care of all the prerequisites needed # for building the docker image - make -C "${image}" bin ARCH="${arch}" TARGET="${temp_dir}" + make -C "${image}" bin OS="${os_name}" ARCH="${arch}" TARGET="${temp_dir}" fi pushd "${temp_dir}" # image tag @@ -85,8 +85,16 @@ build() { if [[ -f BASEIMAGE ]]; then BASEIMAGE=$(getBaseImage "${os_name}" "${arch}" | ${SED} "s|REGISTRY|${REGISTRY}|g") - ${SED} -i "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile - ${SED} -i "s|BASEARCH|${arch}|g" Dockerfile + + # NOTE(claudiub): Some Windows images might require their own Dockerfile + # while simpler ones will not. If we're building for Windows, check if + # "Dockerfile_windows" exists or not. + dockerfile_name="Dockerfile" + if [[ "$os_name" = "windows" && -f "Dockerfile_windows" ]]; then + dockerfile_name="Dockerfile_windows" + fi + + ${SED} -i "s|BASEARCH|${arch}|g" $dockerfile_name fi # copy the qemu-*-static binary to docker image to build the multi architecture image on x86 platform @@ -109,8 +117,16 @@ build() { fi fi - docker build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" . - + if [[ "$os_name" = "linux" ]]; then + docker build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" --build-arg BASEIMAGE="${BASEIMAGE}" . + elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then + # NOTE(claudiub): We're using a remote Windows node to build the Windows Docker images. + # The node requires TLS authentication, and thus it is expected that the + # ca.pem, cert.pem, key.pem files can be found in the ~/.docker folder. + docker --tlsverify -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name . + else + echo "Cannot build the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL should be set, containing the URL to a Windows docker daemon." + fi popd done } @@ -132,6 +148,12 @@ push() { TAG=$(<"${image}"/VERSION) if [[ -f ${image}/BASEIMAGE ]]; then os_archs=$(listOsArchs "$image") + # NOTE(claudiub): if the REMOTE_DOCKER_URL var is not set, or it is an empty string, we must skip + # pushing the Windows image and including it into the manifest list. + if test -z "${REMOTE_DOCKER_URL:-}" && printf "%s\n" "$os_archs" | grep -q '^windows'; then + echo "Skipping pushing the image '${image}' for Windows. REMOTE_DOCKER_URL should be set, containing the URL to a Windows docker daemon." + os_archs=$(printf "%s\n" "$os_archs" | grep -v "^windows") + fi else # prepend linux/ to the QEMUARCHS items. os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[*]}") @@ -145,7 +167,12 @@ push() { exit 1 fi - docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + if [[ "$os_name" = "linux" ]]; then + docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + else + # NOTE(claudiub): We're pushing the image we built on the remote Windows node. + docker --tlsverify -H "${REMOTE_DOCKER_URL}" push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + fi done kube::util::ensure-gnu-sed @@ -165,7 +192,7 @@ push() { echo "The BASEIMAGE file for the ${image} image is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead." exit 1 fi - docker manifest annotate --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + docker manifest annotate --os "${os_name}" --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" done docker manifest push --purge "${REGISTRY}/${image}:${TAG}" } @@ -182,7 +209,7 @@ bin() { golang:"${GOLANG_VERSION}" \ /bin/bash -c "\ cd /go/src/k8s.io/kubernetes/test/images/${SRC_DIR} && \ - CGO_ENABLED=0 ${arch_prefix} GOARCH=${ARCH} go build -a -installsuffix cgo --ldflags '-w' -o ${TARGET}/${SRC} ./$(dirname "${SRC}")" + CGO_ENABLED=0 ${arch_prefix} GOOS=${OS} GOARCH=${ARCH} go build -a -installsuffix cgo --ldflags '-w' -o ${TARGET}/${SRC} ./$(dirname "${SRC}")" done } @@ -195,7 +222,7 @@ if [[ "${WHAT}" == "all-conformance" ]]; then # Discussed during Conformance Office Hours Meeting (2019.12.17): # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh # echoserver image not included: https://github.com/kubernetes/kubernetes/issues/84158 - conformance_images=("agnhost" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver") + conformance_images=("busybox" "agnhost" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver") for image in "${conformance_images[@]}"; do eval "${TASK}" "${image}" done diff --git a/test/images/ipc-utils/Dockerfile b/test/images/ipc-utils/Dockerfile index 3edaa272083..994618d2d04 100644 --- a/test/images/ipc-utils/Dockerfile +++ b/test/images/ipc-utils/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/jessie-dnsutils/Dockerfile b/test/images/jessie-dnsutils/Dockerfile index 40e867fc279..eedc1ec5d73 100644 --- a/test/images/jessie-dnsutils/Dockerfile +++ b/test/images/jessie-dnsutils/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/kitten/Dockerfile b/test/images/kitten/Dockerfile index 29834a005df..48d6f860d88 100644 --- a/test/images/kitten/Dockerfile +++ b/test/images/kitten/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY html/kitten.jpg kitten.jpg COPY html/data.json data.json diff --git a/test/images/metadata-concealment/Makefile b/test/images/metadata-concealment/Makefile index 970ccd3babe..48708b14e71 100644 --- a/test/images/metadata-concealment/Makefile +++ b/test/images/metadata-concealment/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=check_metadata_concealment +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/nautilus/Dockerfile b/test/images/nautilus/Dockerfile index 92ba02a4ddb..c344116bf8c 100644 --- a/test/images/nautilus/Dockerfile +++ b/test/images/nautilus/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY html/nautilus.jpg nautilus.jpg COPY html/data.json data.json diff --git a/test/images/node-perf/npb-ep/Dockerfile b/test/images/node-perf/npb-ep/Dockerfile index d2b6b9f68ff..a87ad660845 100644 --- a/test/images/node-perf/npb-ep/Dockerfile +++ b/test/images/node-perf/npb-ep/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE as build_node_perf_npb_ep +ARG BASEIMAGE +FROM $BASEIMAGE as build_node_perf_npb_ep CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ @@ -33,7 +34,8 @@ RUN make EP CLASS=D # main container in the second build stage. RUN mkdir -p /lib-copy && find /usr/lib -name "*.so.*" -exec cp {} /lib-copy \; -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY --from=build_node_perf_npb_ep /NPB3.3.1/NPB3.3-OMP/bin/ep.D.x / COPY --from=build_node_perf_npb_ep /lib-copy /lib-copy diff --git a/test/images/node-perf/npb-is/Dockerfile b/test/images/node-perf/npb-is/Dockerfile index c956658481e..cc43af6f797 100644 --- a/test/images/node-perf/npb-is/Dockerfile +++ b/test/images/node-perf/npb-is/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE as build_node_perf_npb_is +ARG BASEIMAGE +FROM $BASEIMAGE as build_node_perf_npb_is CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ @@ -35,7 +36,8 @@ RUN make IS CLASS=D # main container in the second build stage. RUN mkdir -p /lib-copy && find /usr/lib -name "*.so.*" -exec cp {} /lib-copy \; -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY --from=build_node_perf_npb_is /NPB3.3.1/NPB3.3-OMP/bin/is.D.x / COPY --from=build_node_perf_npb_is /lib-copy /lib-copy diff --git a/test/images/node-perf/tf-wide-deep/Dockerfile b/test/images/node-perf/tf-wide-deep/Dockerfile index 45e3e2496eb..9526f2974f7 100644 --- a/test/images/node-perf/tf-wide-deep/Dockerfile +++ b/test/images/node-perf/tf-wide-deep/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/nonewprivs/Dockerfile b/test/images/nonewprivs/Dockerfile index 47e49f8d530..a2a5f5f25fb 100644 --- a/test/images/nonewprivs/Dockerfile +++ b/test/images/nonewprivs/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/nonewprivs/Makefile b/test/images/nonewprivs/Makefile index 660b2f2f7ec..255eadd5d6c 100644 --- a/test/images/nonewprivs/Makefile +++ b/test/images/nonewprivs/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = nnp +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/nonroot/Dockerfile b/test/images/nonroot/Dockerfile index 6a8c6ea84f8..d71f5d5dcc8 100644 --- a/test/images/nonroot/Dockerfile +++ b/test/images/nonroot/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE USER 1234 diff --git a/test/images/pets/peer-finder/Dockerfile b/test/images/pets/peer-finder/Dockerfile index b233e2cf21f..707b3d4e8b4 100644 --- a/test/images/pets/peer-finder/Dockerfile +++ b/test/images/pets/peer-finder/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/pets/peer-finder/Makefile b/test/images/pets/peer-finder/Makefile index b76395dcd54..773b4167cc7 100644 --- a/test/images/pets/peer-finder/Makefile +++ b/test/images/pets/peer-finder/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = peer-finder +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/pets/redis-installer/Dockerfile b/test/images/pets/redis-installer/Dockerfile index 6e3b681ea2d..9b8c1483986 100644 --- a/test/images/pets/redis-installer/Dockerfile +++ b/test/images/pets/redis-installer/Dockerfile @@ -14,7 +14,8 @@ # TODO: get rid of bash dependency and switch to plain busybox. # The tar in busybox also doesn't seem to understand compression. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/pets/redis-installer/Makefile b/test/images/pets/redis-installer/Makefile index b76395dcd54..773b4167cc7 100644 --- a/test/images/pets/redis-installer/Makefile +++ b/test/images/pets/redis-installer/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = peer-finder +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/pets/zookeeper-installer/Dockerfile b/test/images/pets/zookeeper-installer/Dockerfile index e29a8def314..64b3b8f9ceb 100644 --- a/test/images/pets/zookeeper-installer/Dockerfile +++ b/test/images/pets/zookeeper-installer/Dockerfile @@ -14,7 +14,8 @@ # TODO: get rid of bash dependency and switch to plain busybox. # The tar in busybox also doesn't seem to understand compression. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/pets/zookeeper-installer/Makefile b/test/images/pets/zookeeper-installer/Makefile index b76395dcd54..773b4167cc7 100644 --- a/test/images/pets/zookeeper-installer/Makefile +++ b/test/images/pets/zookeeper-installer/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = peer-finder +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/redis/Dockerfile b/test/images/redis/Dockerfile index 0add11c3d44..1c37c14c444 100644 --- a/test/images/redis/Dockerfile +++ b/test/images/redis/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/regression-issue-74839/Makefile b/test/images/regression-issue-74839/Makefile index bace6da6235..1fd10ab9f24 100644 --- a/test/images/regression-issue-74839/Makefile +++ b/test/images/regression-issue-74839/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=regression-issue-74839 +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOARM ?= 7 diff --git a/test/images/resource-consumer/Dockerfile b/test/images/resource-consumer/Dockerfile index ea9e3acd048..8bbe3a0c000 100644 --- a/test/images/resource-consumer/Dockerfile +++ b/test/images/resource-consumer/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/resource-consumer/Makefile b/test/images/resource-consumer/Makefile index 44b90a03920..b5102ba1679 100644 --- a/test/images/resource-consumer/Makefile +++ b/test/images/resource-consumer/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = consumer consume-cpu/consume-cpu +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/sample-apiserver/Dockerfile b/test/images/sample-apiserver/Dockerfile index fcdadf58d51..23c1d1299ed 100644 --- a/test/images/sample-apiserver/Dockerfile +++ b/test/images/sample-apiserver/Dockerfile @@ -32,7 +32,8 @@ RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=BASEARCH go get k8s.io/sample # we can copy it out from this throw away container image from a standard location RUN find /go/bin -name sample-apiserver -exec cp {} / \; -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY --from=build_k8s_1_17_sample_apiserver /sample-apiserver /sample-apiserver ENTRYPOINT ["/sample-apiserver"] diff --git a/test/images/sample-device-plugin/Dockerfile b/test/images/sample-device-plugin/Dockerfile index 206e0ba645c..0e05f8e56da 100644 --- a/test/images/sample-device-plugin/Dockerfile +++ b/test/images/sample-device-plugin/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE ADD sampledeviceplugin /sampledeviceplugin ENTRYPOINT ["/sampledeviceplugin", "-alsologtostderr"] diff --git a/test/images/sample-device-plugin/Makefile b/test/images/sample-device-plugin/Makefile index a7921439c67..c3fa3cb23e4 100644 --- a/test/images/sample-device-plugin/Makefile +++ b/test/images/sample-device-plugin/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=sampledeviceplugin +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/volume/gluster/Dockerfile b/test/images/volume/gluster/Dockerfile index 3cfa358eb01..cbbcb71efb4 100644 --- a/test/images/volume/gluster/Dockerfile +++ b/test/images/volume/gluster/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/volume/iscsi/Dockerfile b/test/images/volume/iscsi/Dockerfile index bff98b575be..02106d54835 100644 --- a/test/images/volume/iscsi/Dockerfile +++ b/test/images/volume/iscsi/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/volume/nfs/Dockerfile b/test/images/volume/nfs/Dockerfile index 2af5f1c4b4c..f6b184c6dc6 100644 --- a/test/images/volume/nfs/Dockerfile +++ b/test/images/volume/nfs/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ RUN yum -y install /usr/bin/ps nfs-utils && yum clean all diff --git a/test/images/volume/rbd/Dockerfile b/test/images/volume/rbd/Dockerfile index ad6bb314ef5..40742f60dec 100644 --- a/test/images/volume/rbd/Dockerfile +++ b/test/images/volume/rbd/Dockerfile @@ -15,7 +15,8 @@ # CEPH all in one # Based on image by Ricardo Rocha, ricardo@catalyst.net.nz -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ From 3cdb7a89cbb6266ef14a5607ad006e1452de9154 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Thu, 1 Aug 2019 16:08:51 -0700 Subject: [PATCH 6/9] test images: Adds multiple Windows channels support This commit adds support for building test images for multiple Windows versions, as we have to support both LTS and SAC channels. With this, the format for Windows images in the BASEIMAGE files is: OS/ARCH/OS_VERSION Also adds --isolation-hyperv to the Windows docker build command, making sure that container images for multiple OS versions can be built using the same Windows node. --- test/images/agnhost/BASEIMAGE | 4 +++- test/images/agnhost/README.md | 3 ++- test/images/busybox/BASEIMAGE | 4 +++- test/images/image-util.sh | 41 +++++++++++++++++++++++++---------- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/test/images/agnhost/BASEIMAGE b/test/images/agnhost/BASEIMAGE index 5fdb1fa08ea..cc7c45a7657 100644 --- a/test/images/agnhost/BASEIMAGE +++ b/test/images/agnhost/BASEIMAGE @@ -3,4 +3,6 @@ linux/arm=arm32v6/alpine:3.6 linux/arm64=arm64v8/alpine:3.6 linux/ppc64le=ppc64le/alpine:3.6 linux/s390x=s390x/alpine:3.6 -windows/amd64=REGISTRY/busybox:1.29-windows-amd64 +windows/amd64/1809=REGISTRY/busybox:1.29-windows-amd64-1809 +windows/amd64/1903=REGISTRY/busybox:1.29-windows-amd64-1903 +windows/amd64/1909=REGISTRY/busybox:1.29-windows-amd64-1909 diff --git a/test/images/agnhost/README.md b/test/images/agnhost/README.md index ce241dcd770..0b44449c231 100644 --- a/test/images/agnhost/README.md +++ b/test/images/agnhost/README.md @@ -639,4 +639,5 @@ The Windows `agnhost` image includes a `nc` binary that is 100% compliant with i ## Image The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11` for both Linux and -Windows containers (based on `mcr.microsoft.com/windows/servercore:ltsc2019`). +Windows containers (based on `mcr.microsoft.com/windows/servercore:ltsc2019`, +`mcr.microsoft.com/windows/servercore:1903`, and `mcr.microsoft.com/windows/servercore:1909`). diff --git a/test/images/busybox/BASEIMAGE b/test/images/busybox/BASEIMAGE index 932c698c514..ce71d103b81 100644 --- a/test/images/busybox/BASEIMAGE +++ b/test/images/busybox/BASEIMAGE @@ -1 +1,3 @@ -windows/amd64=mcr.microsoft.com/windows/servercore:ltsc2019 +windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019 +windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903 +windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909 diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 3917aa67a8e..20278983696 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -36,9 +36,8 @@ listOsArchs() { # Returns baseimage need to used in Dockerfile for any given architecture getBaseImage() { - os_name=$1 - arch=$2 - grep "${os_name}/${arch}=" BASEIMAGE | cut -d= -f2 + os_arch=$1 + grep "${os_arch}=" BASEIMAGE | cut -d= -f2 } # This function will build test image for all the architectures @@ -57,7 +56,13 @@ build() { kube::util::ensure-gnu-sed for os_arch in ${os_archs}; do - if [[ $os_arch =~ .*/.* ]]; then + if [[ $os_arch =~ .*/.*/.* ]]; then + # for Windows, we have to support both LTS and SAC channels, so we're building multiple Windows images. + # the format for this case is: OS/ARCH/OS_VERSION. + os_name=$(echo "$os_arch" | cut -d "/" -f 1) + arch=$(echo "$os_arch" | cut -d "/" -f 2) + os_version=$(echo "$os_arch" | cut -d "/" -f 3) + elif [[ $os_arch =~ .*/.* ]]; then os_name=$(echo "$os_arch" | cut -d "/" -f 1) arch=$(echo "$os_arch" | cut -d "/" -f 2) else @@ -84,7 +89,7 @@ build() { TAG=$( Date: Mon, 30 Dec 2019 07:29:21 -0800 Subject: [PATCH 7/9] test images: Use multiple Windows nodes to build images In order to build Windows container images for multiple OS versions, --isolation=hyperv is required. However, not all clouds / nodes supports or have it enabled by default, which is why we're going to rely on having multiple nodes to build the Windows images, until this issue is addressed. --- test/images/image-util.sh | 43 +++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 20278983696..3c62ecd736e 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -62,6 +62,12 @@ build() { os_name=$(echo "$os_arch" | cut -d "/" -f 1) arch=$(echo "$os_arch" | cut -d "/" -f 2) os_version=$(echo "$os_arch" | cut -d "/" -f 3) + + # currently, GCE does not have Hyper-V support, which means that the same node cannot be used to build + # multiple versions of Windows images. Which is why we have $REMOTE_DOCKER_URL_$os_version URLs configured. + # TODO(claudiub): once Hyper-V support has been added to GCE, revert this to just $REMOTE_DOCKER_URL. + remote_docker_url_name="REMOTE_DOCKER_URL_$os_version" + REMOTE_DOCKER_URL=$(eval echo "\${${remote_docker_url_name}:-}") elif [[ $os_arch =~ .*/.* ]]; then os_name=$(echo "$os_arch" | cut -d "/" -f 1) arch=$(echo "$os_arch" | cut -d "/" -f 2) @@ -127,10 +133,14 @@ build() { elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then # NOTE(claudiub): We're using a remote Windows node to build the Windows Docker images. # The node requires TLS authentication, and thus it is expected that the - # ca.pem, cert.pem, key.pem files can be found in the ~/.docker folder. - docker --tlsverify -H "${REMOTE_DOCKER_URL}" build --isolation=hyperv --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name . + # ca.pem, cert.pem, key.pem files can be found in the ${HOME}/.docker-${os_version} folder. + # TODO(claudiub): add "build --isolation=hyperv" once GCE introduces Hyper-V support. + docker --tlsverify --tlscacert "${HOME}/.docker-${os_version}/ca.pem" \ + --tlscert "${HOME}/.docker-${os_version}/cert.pem" --tlskey "${HOME}/.docker-${os_version}/key.pem" \ + -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" \ + --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name . else - echo "Cannot build the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL should be set, containing the URL to a Windows docker daemon." + echo "Cannot build the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL_$os_version should be set, containing the URL to a Windows docker daemon." fi popd done @@ -153,12 +163,6 @@ push() { TAG=$(<"${image}"/VERSION) if [[ -f ${image}/BASEIMAGE ]]; then os_archs=$(listOsArchs "$image") - # NOTE(claudiub): if the REMOTE_DOCKER_URL var is not set, or it is an empty string, we must skip - # pushing the Windows image and including it into the manifest list. - if test -z "${REMOTE_DOCKER_URL:-}" && printf "%s\n" "$os_archs" | grep -q '^windows'; then - echo "Skipping pushing the image '${image}' for Windows. REMOTE_DOCKER_URL should be set, containing the URL to a Windows docker daemon." - os_archs=$(printf "%s\n" "$os_archs" | grep -v "^windows") - fi else # prepend linux/ to the QEMUARCHS items. os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[*]}") @@ -170,6 +174,12 @@ push() { os_name=$(echo "$os_arch" | cut -d "/" -f 1) arch=$(echo "$os_arch" | cut -d "/" -f 2) os_version=$(echo "$os_arch" | cut -d "/" -f 3) + + # currently, GCE does not have Hyper-V support, which means that the same node cannot be used to build + # multiple versions of Windows images. Which is why we have $REMOTE_DOCKER_URL_$os_version URLs configured. + # TODO(claudiub): once Hyper-V support has been added to GCE, revert this to just $REMOTE_DOCKER_URL. + remote_docker_url_name="REMOTE_DOCKER_URL_$os_version" + REMOTE_DOCKER_URL=$(eval echo "\${${remote_docker_url_name}:-}") elif [[ $os_arch =~ .*/.* ]]; then os_name=$(echo "$os_arch" | cut -d "/" -f 1) arch=$(echo "$os_arch" | cut -d "/" -f 2) @@ -180,12 +190,23 @@ push() { if [[ "$os_name" = "linux" ]]; then docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" - else + elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then # NOTE(claudiub): We're pushing the image we built on the remote Windows node. - docker --tlsverify -H "${REMOTE_DOCKER_URL}" push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" + docker --tlsverify --tlscacert "${HOME}/.docker-${os_version}/ca.pem" \ + --tlscert "${HOME}/.docker-${os_version}/cert.pem" --tlskey "${HOME}/.docker-${os_version}/key.pem" \ + -H "${REMOTE_DOCKER_URL}" push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" + else + echo "Cannot push the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL_${os_version} should be set, containing the URL to a Windows docker daemon." fi done + # NOTE(claudiub): if the REMOTE_DOCKER_URL var is not set, or it is an empty string, we mustn't include + # Windows images into the manifest list. + if test -z "${REMOTE_DOCKER_URL:-}" && printf "%s\n" "$os_archs" | grep -q '^windows'; then + echo "Skipping pushing the image '${image}' for Windows. REMOTE_DOCKER_URL_\${os_version} should be set, containing the URL to a Windows docker daemon." + os_archs=$(printf "%s\n" "$os_archs" | grep -v "^windows") + fi + kube::util::ensure-gnu-sed # The manifest command is still experimental as of Docker 18.09.2 From b71fbdb364390c40e2cd417192dcb9de9b7aab87 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Thu, 9 Jan 2020 04:04:04 -0800 Subject: [PATCH 8/9] Image Promoter: Adds Windows build nodes for Windows test images We have added and enabled the Image Promoter on the k/k test images, which will build the conformance images after a PR that affects kubernetes/test/images merges. We have added support for image-util.sh to handle external Windows Docker connections in order to build Windows images. This PR enables the Image Promoter to use some Windows nodes to build the necessary Windows images. --- test/images/cloudbuild.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/images/cloudbuild.yaml b/test/images/cloudbuild.yaml index 29be75e48bc..099d4615930 100644 --- a/test/images/cloudbuild.yaml +++ b/test/images/cloudbuild.yaml @@ -17,6 +17,9 @@ steps: - BASE_REF=$_PULL_BASE_REF - WHAT=$_WHAT - REGISTRY=gcr.io/k8s-staging-e2e-test-images + - REMOTE_DOCKER_URL_1809=tcp://img-promoter-1809.eastus.cloudapp.azure.com:2376 + - REMOTE_DOCKER_URL_1903=tcp://img-promoter-1903.eastus.cloudapp.azure.com:2376 + - REMOTE_DOCKER_URL_1909=tcp://img-promoter-1909.eastus.cloudapp.azure.com:2376 args: - all-push substitutions: From 91dc590cde756de650a35645c47857dd3d5b4a9b Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 1 May 2019 21:20:25 -0700 Subject: [PATCH 9/9] test images: Adds building README Adds a README explaining the image building process, including the Windows Container image building process. --- test/images/Makefile | 1 - test/images/README.md | 92 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/test/images/Makefile b/test/images/Makefile index e1efc8f2de0..92c48f5283b 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -14,7 +14,6 @@ REGISTRY ?= gcr.io/kubernetes-e2e-test-images GOARM ?= 7 -REMOTE_DOCKER_URL ?= QEMUVERSION=v2.9.1 GOLANG_VERSION=1.13.6 export diff --git a/test/images/README.md b/test/images/README.md index 4561aa1e33c..e4ed1ddf585 100644 --- a/test/images/README.md +++ b/test/images/README.md @@ -18,6 +18,75 @@ is recommended in order to avoid certain issues. The node must be able to push the images to the desired container registry, make sure you are authenticated with the registry you're pushing to. +Windows Container images are not built by default, since they cannot be built on Linux. For +that, a Windows node with Docker installed and configured for remote management is required. + + +### Windows node(s) setup + +In order to build the Windows container images, a node with Windows 10 or Windows Server 2019 +with the latest updates installed is required. The node will have to have Docker installed, +preferably version 18.06.0 or newer. + +Keep in mind that the Windows node might not be able to build container images for newer OS versions +than itself (even with `--isolation=hyperv`), so keeping the node up to date and / or upgrading it +to the latest Windows Server edition is ideal. + +Windows test images must be built for Windows Server 2019 (1809) and Windows Server 1903, thus, +if the node does not have Hyper-V enabled, or it is not supported, multiple Windows nodes are required, +one per OS version. + +Additionally, remote management must be configured for the node's Docker daemon. Exposing the +Docker daemon without requiring any authentication is not recommended, and thus, it must be +configured with TLS to ensure that only authorised people can interact with it. For this, the +following `powershell` script can be executed: + +```powershell +mkdir .docker +docker run --isolation=hyperv --user=ContainerAdministrator --rm ` + -e SERVER_NAME=$(hostname) ` + -e IP_ADDRESSES=127.0.0.1,YOUR_WINDOWS_BUILD_NODE_IP ` + -v "c:\programdata\docker:c:\programdata\docker" ` + -v "$env:USERPROFILE\.docker:c:\users\containeradministrator\.docker" stefanscherer/dockertls-windows:2.5.5 +# restart the Docker daemon. +Restart-Service docker +``` + +For more information about the above commands, you can check [here](https://hub.docker.com/r/stefanscherer/dockertls-windows/). + +A firewall rule to allow connections to the Docker daemon is necessary: + +```powershell +New-NetFirewallRule -DisplayName 'Docker SSL Inbound' -Profile @('Domain', 'Public', 'Private') -Direction Inbound -Action Allow -Protocol TCP -LocalPort 2376 +``` + +If your Windows build node is hosted by a cloud provider, make sure the port `2376` is open for the node. +For example, in Azure, this is done by running the following command: + +```console +az vm open-port -g GROUP-NAME -n NODE-NAME --port 2376 +``` + +The `ca.pem`, `cert.pem`, and `key.pem` files that can be found in `$env:USERPROFILE\.docker` +will have to copied to the `~/.docker-${os_version)/` on the Linux build node, where `${os_version}` +is `1809` or `1903`. + +```powershell +scp.exe -r $env:USERPROFILE\.docker ubuntu@YOUR_LINUX_BUILD_NODE:/home/ubuntu/.docker-$os_version +``` + +After all this, the Linux build node should be able to connect to the Windows build node: + +```bash +docker --tlsverify --tlscacert ~/.docker-${os_version}/ca.pem --tlscert ~/.docker-${os_version}/cert.pem --tlskey ~/.docker-${os_version}/key.pem -H "$REMOTE_DOCKER_URL" version +``` + +For more information and troubleshooting about enabling Docker remote management, see +[here](https://docs.microsoft.com/en-us/virtualization/windowscontainers/management/manage_remotehost) + +Finally, the node must be able to push the images to the desired container registry, make sure you are +authenticated with the registry you're pushing to. + ## Making changes to images @@ -63,6 +132,9 @@ For this, you will need the image manifest list's digest, which can be obtained manifest-tool inspect --raw gcr.io/k8s-staging-e2e-test-images/${IMAGE_NAME}:${VERSION} | jq '.[0].Digest' ``` +The images are built through `make`. Since some images (e.g.: `busybox`) are used as a base for +other images, it is recommended to build them first, if needed. + ## Building images @@ -88,6 +160,14 @@ registry. That can changed by running this command instead: REGISTRY=foo_registry make all-push WHAT=agnhost ``` +In order to also include Windows Container images into the final manifest lists, the `REMOTE_DOCKER_URL` argument +in the form `tcp://[host]:[port][path]` (for more details, see [here]([https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-socket-option]/)) +will also have to be specified: + +```bash +REMOTE_DOCKER_URL_1909=remote_docker_url_1909 REMOTE_DOCKER_URL_1903=remote_docker_url_1903 REMOTE_DOCKER_URL_1809=remote_docker_url_1809 REGISTRY=foo_registry make all-push WHAT=test-webserver +``` + *NOTE* (for test `gcr.io` image publishers): Some tests (e.g.: `should serve a basic image on each replica with a private image`) require the `agnhost` image to be published in an authenticated repo as well: @@ -135,3 +215,15 @@ After all the above has been done, run the desired tests. ```bash sudo chmod o+x /etc/docker ``` + +`nc` is being used by some E2E tests, which is why we are including a Linux-like `nc.exe` into the Windows `busybox` image. The image could fail to build during that step with an error that looks like this: + +```console +re-exec error: exit status 1: output: time="..." level=error msg="hcsshim::ImportLayer failed in Win32: The system cannot find the path specified. (0x3) path=\\\\?\\C:\\ProgramData\\... +``` + +The issue is caused by the Windows Defender which is removing the `nc.exe` binary from the filesystem. For more details on this issue, see [here](https://github.com/diegocr/netcat/issues/6). To fix this, you can simply run the following powershell command to temporarily disable Windows Defender: + +```powershell +Set-MpPreference -DisableRealtimeMonitoring $true +```