From 296464d968bf1122164d5f7b3e29b7690bb312a1 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 17 Apr 2019 13:56:50 +0000 Subject: [PATCH] test images: Adds Windows support (part 1) Adds Windows support to the test/images/image-util.sh script. A Windows node with Docker installed is required to build Windows images. The connection URL to it must be set in the REMOTE_DOCKER_URL env variable. Additionally, the authentication to the remote docker node is done through certificates, which must be found in ~/.docker. By default, the REMOTE_DOCKER_URL env variable is set to "" in the Makefile, and because of it, the image-util.sh script will skip building and pushing Windows images. Added GOOS argument to the go build process in order to be able to build Windows binaries. Additionally, the OS env variable was added to the images Makefiles (default value is "linux") in order to maintain default behaviour. Some images require a different Dockerfile for Windows images, since they have different ways of installing dependencies. Because of this, if a image needs to be built for Windows, it will first check for a Dockerfile_windows file instead of the default one. If there isn't one, it means that the same Dockerfile can be used for both Windows and Linux. All Windows images will be based on the image "mcr.microsoft.com/windows/servercore:ltsc2019". There are a couple of features that are needed from this image, especially powershell. Added busybox image for Windows. Most Windows images will be based on it, which will help reduce the command line differences between Linux and Windows, but not entirely. Added Windows support for agnhost image. --- test/images/BUILD | 1 + test/images/Makefile | 1 + test/images/agnhost/BASEIMAGE | 1 + test/images/agnhost/Dockerfile | 3 +- test/images/agnhost/Dockerfile_windows | 64 +++++++++++++++++++ test/images/agnhost/Makefile | 1 + test/images/agnhost/README.md | 26 +++++--- test/images/apparmor-loader/Dockerfile | 3 +- test/images/apparmor-loader/Makefile | 1 + test/images/busybox/BASEIMAGE | 1 + test/images/busybox/BUILD | 31 +++++++++ test/images/busybox/Dockerfile_windows | 34 ++++++++++ test/images/busybox/Makefile | 27 ++++++++ test/images/busybox/VERSION | 1 + test/images/busybox/hostname.go | 50 +++++++++++++++ test/images/cuda-vector-add/Dockerfile | 3 +- test/images/echoserver/Dockerfile | 3 +- test/images/image-util.sh | 45 ++++++++++--- test/images/ipc-utils/Dockerfile | 3 +- test/images/jessie-dnsutils/Dockerfile | 3 +- test/images/kitten/Dockerfile | 3 +- test/images/metadata-concealment/Makefile | 1 + test/images/nautilus/Dockerfile | 3 +- test/images/node-perf/npb-ep/Dockerfile | 6 +- test/images/node-perf/npb-is/Dockerfile | 6 +- test/images/node-perf/tf-wide-deep/Dockerfile | 3 +- test/images/nonewprivs/Dockerfile | 3 +- test/images/nonewprivs/Makefile | 1 + test/images/nonroot/Dockerfile | 3 +- test/images/pets/peer-finder/Dockerfile | 3 +- test/images/pets/peer-finder/Makefile | 1 + test/images/pets/redis-installer/Dockerfile | 3 +- test/images/pets/redis-installer/Makefile | 1 + .../pets/zookeeper-installer/Dockerfile | 3 +- test/images/pets/zookeeper-installer/Makefile | 1 + test/images/redis/Dockerfile | 3 +- test/images/regression-issue-74839/Makefile | 1 + test/images/resource-consumer/Dockerfile | 3 +- test/images/resource-consumer/Makefile | 1 + test/images/sample-apiserver/Dockerfile | 3 +- test/images/sample-device-plugin/Dockerfile | 3 +- test/images/sample-device-plugin/Makefile | 1 + test/images/volume/gluster/Dockerfile | 3 +- test/images/volume/iscsi/Dockerfile | 3 +- test/images/volume/nfs/Dockerfile | 3 +- test/images/volume/rbd/Dockerfile | 3 +- 46 files changed, 325 insertions(+), 45 deletions(-) create mode 100644 test/images/agnhost/Dockerfile_windows create mode 100644 test/images/busybox/BASEIMAGE create mode 100644 test/images/busybox/BUILD create mode 100644 test/images/busybox/Dockerfile_windows create mode 100644 test/images/busybox/Makefile create mode 100644 test/images/busybox/VERSION create mode 100644 test/images/busybox/hostname.go diff --git a/test/images/BUILD b/test/images/BUILD index 2f887f0b644..a068ae0951d 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -13,6 +13,7 @@ filegroup( ":package-srcs", "//test/images/agnhost:all-srcs", "//test/images/apparmor-loader:all-srcs", + "//test/images/busybox:all-srcs", "//test/images/echoserver:all-srcs", "//test/images/metadata-concealment:all-srcs", "//test/images/nonewprivs:all-srcs", diff --git a/test/images/Makefile b/test/images/Makefile index 92c48f5283b..e1efc8f2de0 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -14,6 +14,7 @@ REGISTRY ?= gcr.io/kubernetes-e2e-test-images GOARM ?= 7 +REMOTE_DOCKER_URL ?= QEMUVERSION=v2.9.1 GOLANG_VERSION=1.13.6 export diff --git a/test/images/agnhost/BASEIMAGE b/test/images/agnhost/BASEIMAGE index ad69ccfb016..5fdb1fa08ea 100644 --- a/test/images/agnhost/BASEIMAGE +++ b/test/images/agnhost/BASEIMAGE @@ -3,3 +3,4 @@ linux/arm=arm32v6/alpine:3.6 linux/arm64=arm64v8/alpine:3.6 linux/ppc64le=ppc64le/alpine:3.6 linux/s390x=s390x/alpine:3.6 +windows/amd64=REGISTRY/busybox:1.29-windows-amd64 diff --git a/test/images/agnhost/Dockerfile b/test/images/agnhost/Dockerfile index c551802c792..3cfe56c9cb2 100644 --- a/test/images/agnhost/Dockerfile +++ b/test/images/agnhost/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/agnhost/Dockerfile_windows b/test/images/agnhost/Dockerfile_windows new file mode 100644 index 00000000000..76edbdbb510 --- /dev/null +++ b/test/images/agnhost/Dockerfile_windows @@ -0,0 +1,64 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASEIMAGE +FROM $BASEIMAGE + +# from dnsutils image +# install necessary packages: +# - bind-tools: contains dig, which can used in DNS tests. +# - CoreDNS: used in some DNS tests. +# from hostexec image +# installed necessary packages: +# - curl, nc: used by a lot of e2e tests (inherited from BASEIMAGE) +# from iperf image +# install necessary packages: iperf +ENV chocolateyUseWindowsCompression false +RUN powershell -Command "\ + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ + choco feature disable --name showDownloadProgress; \ + choco install bind-toolsonly --version 9.10.3 -y +RUN powershell -Command "\ + wget -uri 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -OutFile C:\coredns.tgz;\ + tar -xzvf C:\coredns.tgz;\ + Remove-Item C:\coredns.tgz" + +RUN powershell -Command "\ + wget -uri 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -OutFile C:\iperf.zip;\ + Expand-Archive -Path C:\iperf.zip -DestinationPath C:\ -Force;\ + Rename-Item C:\iperf-2.0.9-win64 C:\iperf;\ + Remove-Item C:\iperf.zip" + +# PORT 80 needed by: test-webserver +# PORT 8080 needed by: netexec, nettest +# PORT 8081 needed by: netexec +# PORT 9376 needed by: serve-hostname +EXPOSE 80 8080 8081 9376 + +# from netexec +RUN mkdir C:\uploads + +# from porter +ADD porter/localhost.crt localhost.crt +ADD porter/localhost.key localhost.key + +ADD agnhost agnhost + +# needed for the entrypoint-tester related tests. Some of the entrypoint-tester related tests +# overrides this image's entrypoint with agnhost-2 binary, and will verify that the correct +# entrypoint is used by the containers. +RUN mklink agnhost-2 agnhost + +ENTRYPOINT ["/agnhost"] +CMD ["pause"] diff --git a/test/images/agnhost/Makefile b/test/images/agnhost/Makefile index c21639c38c2..c5eeeea4162 100644 --- a/test/images/agnhost/Makefile +++ b/test/images/agnhost/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=agnhost +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/agnhost/README.md b/test/images/agnhost/README.md index b797ee47f57..ce241dcd770 100644 --- a/test/images/agnhost/README.md +++ b/test/images/agnhost/README.md @@ -40,7 +40,7 @@ For example, let's consider the following `pod.yaml` file: containers: - args: - dns-suffix - image: us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 + image: us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 name: agnhost dnsConfig: nameservers: @@ -207,7 +207,7 @@ Usage: ```console guestbook="test/e2e/testing-manifests/guestbook" -sed_expr="s|{{.AgnhostImage}}|us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10|" +sed_expr="s|{{.AgnhostImage}}|us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11|" # create the services. kubectl create -f ${guestbook}/frontend-service.yaml @@ -290,14 +290,14 @@ Examples: ```console docker run -i \ - us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ logs-generator --log-lines-total 10 --run-duration 1s ``` ```console kubectl run logs-generator \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ --restart=Never \ -- logs-generator -t 10 -d 1s ``` @@ -455,7 +455,7 @@ Usage: ```console kubectl run test-agnhost \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ --restart=Never \ --env "POD_IP=" \ --env "NODE_IP=" \ @@ -510,7 +510,7 @@ Usage: ```console kubectl run test-agnhost \ --generator=run-pod/v1 \ - --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10 \ + --image=us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11 \ --restart=Never \ --env "BIND_ADDRESS=localhost" \ --env "BIND_PORT=8080" \ @@ -626,11 +626,17 @@ Usage: ## Other tools -The image contains `iperf`, `curl`, `dns-tools` (including `dig`), CoreDNS. +The image contains `iperf`, `curl`, `dns-tools` (including `dig`), CoreDNS, for both Windows and Linux. + +For Windows, the image is based on `busybox`, meaning that most of the Linux common tools are also +available on it, making it possible to run most Linux commands in the `agnhost` Windows container +as well. Keep in mind that there might still be some differences though (e.g.: `wget` does not +have the `-T` argument on Windows). + +The Windows `agnhost` image includes a `nc` binary that is 100% compliant with its Linux equivalent. ## Image -The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.10` for Linux -containers, and `e2eteam/agnhost:2.10` for Windows containers. In the future, the same -repository can be used for both OSes. +The image can be found at `us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.11` for both Linux and +Windows containers (based on `mcr.microsoft.com/windows/servercore:ltsc2019`). diff --git a/test/images/apparmor-loader/Dockerfile b/test/images/apparmor-loader/Dockerfile index 0cad6e26eef..c6e38ae7e75 100644 --- a/test/images/apparmor-loader/Dockerfile +++ b/test/images/apparmor-loader/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/apparmor-loader/Makefile b/test/images/apparmor-loader/Makefile index 807d3a2a489..c0b6bd8e481 100644 --- a/test/images/apparmor-loader/Makefile +++ b/test/images/apparmor-loader/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=loader +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/busybox/BASEIMAGE b/test/images/busybox/BASEIMAGE new file mode 100644 index 00000000000..932c698c514 --- /dev/null +++ b/test/images/busybox/BASEIMAGE @@ -0,0 +1 @@ +windows/amd64=mcr.microsoft.com/windows/servercore:ltsc2019 diff --git a/test/images/busybox/BUILD b/test/images/busybox/BUILD new file mode 100644 index 00000000000..f1d2087ff43 --- /dev/null +++ b/test/images/busybox/BUILD @@ -0,0 +1,31 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "hostname", + embed = [":go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = ["hostname.go"], + importpath = "k8s.io/kubernetes/test/images/busybox", +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/images/busybox/Dockerfile_windows b/test/images/busybox/Dockerfile_windows new file mode 100644 index 00000000000..c4a508fa96a --- /dev/null +++ b/test/images/busybox/Dockerfile_windows @@ -0,0 +1,34 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASEIMAGE +from $BASEIMAGE as prep + +ENV CURL_VERSION 7.57.0 +WORKDIR /curl +ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab +RUN expand /R curl.cab /F:* . + +FROM $BASEIMAGE + +COPY --from=prep /curl/AMD64 /curl +COPY --from=prep /curl/CURL.LIC /curl +ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe +ADD https://github.com/diegocr/netcat/raw/master/nc.exe /bin/nc.exe +ADD hostname /bin/hostname.exe +USER ContainerAdministrator +RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe +RUN setx /M PATH "C:\bin;C:\curl\;%PATH%" &\ + mkdir C:\tmp +ENTRYPOINT ["cmd.exe", "/s", "/c"] diff --git a/test/images/busybox/Makefile b/test/images/busybox/Makefile new file mode 100644 index 00000000000..ca3bd80c5ae --- /dev/null +++ b/test/images/busybox/Makefile @@ -0,0 +1,27 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS=hostname +OS ?= linux +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOARM = 7 +GOLANG_VERSION ?= latest +SRC_DIR = $(notdir $(shell pwd)) +export + +bin: + ../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/busybox/VERSION b/test/images/busybox/VERSION new file mode 100644 index 00000000000..9de53f1932a --- /dev/null +++ b/test/images/busybox/VERSION @@ -0,0 +1 @@ +1.29 diff --git a/test/images/busybox/hostname.go b/test/images/busybox/hostname.go new file mode 100644 index 00000000000..77694dad16a --- /dev/null +++ b/test/images/busybox/hostname.go @@ -0,0 +1,50 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" +) + +func getOutboundIP() net.IP { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + log.Fatal(err) + } + defer conn.Close() + localAddr := conn.LocalAddr().(*net.UDPAddr) + return localAddr.IP +} + +func main() { + flagIP := flag.Bool("i", false, "a string") + flag.Parse() + if *flagIP { + ip := getOutboundIP() + fmt.Print(ip.String()) + } else { + hostname, err := os.Hostname() + if err != nil { + log.Fatal(err) + } + fmt.Print(hostname) + } +} diff --git a/test/images/cuda-vector-add/Dockerfile b/test/images/cuda-vector-add/Dockerfile index 968626bf424..af82e6a5e9b 100644 --- a/test/images/cuda-vector-add/Dockerfile +++ b/test/images/cuda-vector-add/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/echoserver/Dockerfile b/test/images/echoserver/Dockerfile index 919f39dba8e..3329cdf458c 100644 --- a/test/images/echoserver/Dockerfile +++ b/test/images/echoserver/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 9ab51bd54fe..3917aa67a8e 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -77,7 +77,7 @@ build() { if [[ -f ${image}/Makefile ]]; then # make bin will take care of all the prerequisites needed # for building the docker image - make -C "${image}" bin ARCH="${arch}" TARGET="${temp_dir}" + make -C "${image}" bin OS="${os_name}" ARCH="${arch}" TARGET="${temp_dir}" fi pushd "${temp_dir}" # image tag @@ -85,8 +85,16 @@ build() { if [[ -f BASEIMAGE ]]; then BASEIMAGE=$(getBaseImage "${os_name}" "${arch}" | ${SED} "s|REGISTRY|${REGISTRY}|g") - ${SED} -i "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile - ${SED} -i "s|BASEARCH|${arch}|g" Dockerfile + + # NOTE(claudiub): Some Windows images might require their own Dockerfile + # while simpler ones will not. If we're building for Windows, check if + # "Dockerfile_windows" exists or not. + dockerfile_name="Dockerfile" + if [[ "$os_name" = "windows" && -f "Dockerfile_windows" ]]; then + dockerfile_name="Dockerfile_windows" + fi + + ${SED} -i "s|BASEARCH|${arch}|g" $dockerfile_name fi # copy the qemu-*-static binary to docker image to build the multi architecture image on x86 platform @@ -109,8 +117,16 @@ build() { fi fi - docker build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" . - + if [[ "$os_name" = "linux" ]]; then + docker build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" --build-arg BASEIMAGE="${BASEIMAGE}" . + elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then + # NOTE(claudiub): We're using a remote Windows node to build the Windows Docker images. + # The node requires TLS authentication, and thus it is expected that the + # ca.pem, cert.pem, key.pem files can be found in the ~/.docker folder. + docker --tlsverify -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name . + else + echo "Cannot build the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL should be set, containing the URL to a Windows docker daemon." + fi popd done } @@ -132,6 +148,12 @@ push() { TAG=$(<"${image}"/VERSION) if [[ -f ${image}/BASEIMAGE ]]; then os_archs=$(listOsArchs "$image") + # NOTE(claudiub): if the REMOTE_DOCKER_URL var is not set, or it is an empty string, we must skip + # pushing the Windows image and including it into the manifest list. + if test -z "${REMOTE_DOCKER_URL:-}" && printf "%s\n" "$os_archs" | grep -q '^windows'; then + echo "Skipping pushing the image '${image}' for Windows. REMOTE_DOCKER_URL should be set, containing the URL to a Windows docker daemon." + os_archs=$(printf "%s\n" "$os_archs" | grep -v "^windows") + fi else # prepend linux/ to the QEMUARCHS items. os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[*]}") @@ -145,7 +167,12 @@ push() { exit 1 fi - docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + if [[ "$os_name" = "linux" ]]; then + docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + else + # NOTE(claudiub): We're pushing the image we built on the remote Windows node. + docker --tlsverify -H "${REMOTE_DOCKER_URL}" push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + fi done kube::util::ensure-gnu-sed @@ -165,7 +192,7 @@ push() { echo "The BASEIMAGE file for the ${image} image is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead." exit 1 fi - docker manifest annotate --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" + docker manifest annotate --os "${os_name}" --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" done docker manifest push --purge "${REGISTRY}/${image}:${TAG}" } @@ -182,7 +209,7 @@ bin() { golang:"${GOLANG_VERSION}" \ /bin/bash -c "\ cd /go/src/k8s.io/kubernetes/test/images/${SRC_DIR} && \ - CGO_ENABLED=0 ${arch_prefix} GOARCH=${ARCH} go build -a -installsuffix cgo --ldflags '-w' -o ${TARGET}/${SRC} ./$(dirname "${SRC}")" + CGO_ENABLED=0 ${arch_prefix} GOOS=${OS} GOARCH=${ARCH} go build -a -installsuffix cgo --ldflags '-w' -o ${TARGET}/${SRC} ./$(dirname "${SRC}")" done } @@ -195,7 +222,7 @@ if [[ "${WHAT}" == "all-conformance" ]]; then # Discussed during Conformance Office Hours Meeting (2019.12.17): # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh # echoserver image not included: https://github.com/kubernetes/kubernetes/issues/84158 - conformance_images=("agnhost" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver") + conformance_images=("busybox" "agnhost" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver") for image in "${conformance_images[@]}"; do eval "${TASK}" "${image}" done diff --git a/test/images/ipc-utils/Dockerfile b/test/images/ipc-utils/Dockerfile index 3edaa272083..994618d2d04 100644 --- a/test/images/ipc-utils/Dockerfile +++ b/test/images/ipc-utils/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/jessie-dnsutils/Dockerfile b/test/images/jessie-dnsutils/Dockerfile index 40e867fc279..eedc1ec5d73 100644 --- a/test/images/jessie-dnsutils/Dockerfile +++ b/test/images/jessie-dnsutils/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/kitten/Dockerfile b/test/images/kitten/Dockerfile index 29834a005df..48d6f860d88 100644 --- a/test/images/kitten/Dockerfile +++ b/test/images/kitten/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY html/kitten.jpg kitten.jpg COPY html/data.json data.json diff --git a/test/images/metadata-concealment/Makefile b/test/images/metadata-concealment/Makefile index 970ccd3babe..48708b14e71 100644 --- a/test/images/metadata-concealment/Makefile +++ b/test/images/metadata-concealment/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=check_metadata_concealment +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/nautilus/Dockerfile b/test/images/nautilus/Dockerfile index 92ba02a4ddb..c344116bf8c 100644 --- a/test/images/nautilus/Dockerfile +++ b/test/images/nautilus/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY html/nautilus.jpg nautilus.jpg COPY html/data.json data.json diff --git a/test/images/node-perf/npb-ep/Dockerfile b/test/images/node-perf/npb-ep/Dockerfile index d2b6b9f68ff..a87ad660845 100644 --- a/test/images/node-perf/npb-ep/Dockerfile +++ b/test/images/node-perf/npb-ep/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE as build_node_perf_npb_ep +ARG BASEIMAGE +FROM $BASEIMAGE as build_node_perf_npb_ep CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ @@ -33,7 +34,8 @@ RUN make EP CLASS=D # main container in the second build stage. RUN mkdir -p /lib-copy && find /usr/lib -name "*.so.*" -exec cp {} /lib-copy \; -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY --from=build_node_perf_npb_ep /NPB3.3.1/NPB3.3-OMP/bin/ep.D.x / COPY --from=build_node_perf_npb_ep /lib-copy /lib-copy diff --git a/test/images/node-perf/npb-is/Dockerfile b/test/images/node-perf/npb-is/Dockerfile index c956658481e..cc43af6f797 100644 --- a/test/images/node-perf/npb-is/Dockerfile +++ b/test/images/node-perf/npb-is/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE as build_node_perf_npb_is +ARG BASEIMAGE +FROM $BASEIMAGE as build_node_perf_npb_is CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ @@ -35,7 +36,8 @@ RUN make IS CLASS=D # main container in the second build stage. RUN mkdir -p /lib-copy && find /usr/lib -name "*.so.*" -exec cp {} /lib-copy \; -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY --from=build_node_perf_npb_is /NPB3.3.1/NPB3.3-OMP/bin/is.D.x / COPY --from=build_node_perf_npb_is /lib-copy /lib-copy diff --git a/test/images/node-perf/tf-wide-deep/Dockerfile b/test/images/node-perf/tf-wide-deep/Dockerfile index 45e3e2496eb..9526f2974f7 100644 --- a/test/images/node-perf/tf-wide-deep/Dockerfile +++ b/test/images/node-perf/tf-wide-deep/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/nonewprivs/Dockerfile b/test/images/nonewprivs/Dockerfile index 47e49f8d530..a2a5f5f25fb 100644 --- a/test/images/nonewprivs/Dockerfile +++ b/test/images/nonewprivs/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/nonewprivs/Makefile b/test/images/nonewprivs/Makefile index 660b2f2f7ec..255eadd5d6c 100644 --- a/test/images/nonewprivs/Makefile +++ b/test/images/nonewprivs/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = nnp +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/nonroot/Dockerfile b/test/images/nonroot/Dockerfile index 6a8c6ea84f8..d71f5d5dcc8 100644 --- a/test/images/nonroot/Dockerfile +++ b/test/images/nonroot/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE USER 1234 diff --git a/test/images/pets/peer-finder/Dockerfile b/test/images/pets/peer-finder/Dockerfile index b233e2cf21f..707b3d4e8b4 100644 --- a/test/images/pets/peer-finder/Dockerfile +++ b/test/images/pets/peer-finder/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/pets/peer-finder/Makefile b/test/images/pets/peer-finder/Makefile index b76395dcd54..773b4167cc7 100644 --- a/test/images/pets/peer-finder/Makefile +++ b/test/images/pets/peer-finder/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = peer-finder +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/pets/redis-installer/Dockerfile b/test/images/pets/redis-installer/Dockerfile index 6e3b681ea2d..9b8c1483986 100644 --- a/test/images/pets/redis-installer/Dockerfile +++ b/test/images/pets/redis-installer/Dockerfile @@ -14,7 +14,8 @@ # TODO: get rid of bash dependency and switch to plain busybox. # The tar in busybox also doesn't seem to understand compression. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/pets/redis-installer/Makefile b/test/images/pets/redis-installer/Makefile index b76395dcd54..773b4167cc7 100644 --- a/test/images/pets/redis-installer/Makefile +++ b/test/images/pets/redis-installer/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = peer-finder +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/pets/zookeeper-installer/Dockerfile b/test/images/pets/zookeeper-installer/Dockerfile index e29a8def314..64b3b8f9ceb 100644 --- a/test/images/pets/zookeeper-installer/Dockerfile +++ b/test/images/pets/zookeeper-installer/Dockerfile @@ -14,7 +14,8 @@ # TODO: get rid of bash dependency and switch to plain busybox. # The tar in busybox also doesn't seem to understand compression. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/pets/zookeeper-installer/Makefile b/test/images/pets/zookeeper-installer/Makefile index b76395dcd54..773b4167cc7 100644 --- a/test/images/pets/zookeeper-installer/Makefile +++ b/test/images/pets/zookeeper-installer/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = peer-finder +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/redis/Dockerfile b/test/images/redis/Dockerfile index 0add11c3d44..1c37c14c444 100644 --- a/test/images/redis/Dockerfile +++ b/test/images/redis/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/regression-issue-74839/Makefile b/test/images/regression-issue-74839/Makefile index bace6da6235..1fd10ab9f24 100644 --- a/test/images/regression-issue-74839/Makefile +++ b/test/images/regression-issue-74839/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=regression-issue-74839 +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOARM ?= 7 diff --git a/test/images/resource-consumer/Dockerfile b/test/images/resource-consumer/Dockerfile index ea9e3acd048..8bbe3a0c000 100644 --- a/test/images/resource-consumer/Dockerfile +++ b/test/images/resource-consumer/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/resource-consumer/Makefile b/test/images/resource-consumer/Makefile index 44b90a03920..b5102ba1679 100644 --- a/test/images/resource-consumer/Makefile +++ b/test/images/resource-consumer/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS = consumer consume-cpu/consume-cpu +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/sample-apiserver/Dockerfile b/test/images/sample-apiserver/Dockerfile index fcdadf58d51..23c1d1299ed 100644 --- a/test/images/sample-apiserver/Dockerfile +++ b/test/images/sample-apiserver/Dockerfile @@ -32,7 +32,8 @@ RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=BASEARCH go get k8s.io/sample # we can copy it out from this throw away container image from a standard location RUN find /go/bin -name sample-apiserver -exec cp {} / \; -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE COPY --from=build_k8s_1_17_sample_apiserver /sample-apiserver /sample-apiserver ENTRYPOINT ["/sample-apiserver"] diff --git a/test/images/sample-device-plugin/Dockerfile b/test/images/sample-device-plugin/Dockerfile index 206e0ba645c..0e05f8e56da 100644 --- a/test/images/sample-device-plugin/Dockerfile +++ b/test/images/sample-device-plugin/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE ADD sampledeviceplugin /sampledeviceplugin ENTRYPOINT ["/sampledeviceplugin", "-alsologtostderr"] diff --git a/test/images/sample-device-plugin/Makefile b/test/images/sample-device-plugin/Makefile index a7921439c67..c3fa3cb23e4 100644 --- a/test/images/sample-device-plugin/Makefile +++ b/test/images/sample-device-plugin/Makefile @@ -13,6 +13,7 @@ # limitations under the License. SRCS=sampledeviceplugin +OS ?= linux ARCH ?= amd64 TARGET ?= $(CURDIR) GOLANG_VERSION ?= latest diff --git a/test/images/volume/gluster/Dockerfile b/test/images/volume/gluster/Dockerfile index 3cfa358eb01..cbbcb71efb4 100644 --- a/test/images/volume/gluster/Dockerfile +++ b/test/images/volume/gluster/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/volume/iscsi/Dockerfile b/test/images/volume/iscsi/Dockerfile index bff98b575be..02106d54835 100644 --- a/test/images/volume/iscsi/Dockerfile +++ b/test/images/volume/iscsi/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ diff --git a/test/images/volume/nfs/Dockerfile b/test/images/volume/nfs/Dockerfile index 2af5f1c4b4c..f6b184c6dc6 100644 --- a/test/images/volume/nfs/Dockerfile +++ b/test/images/volume/nfs/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ RUN yum -y install /usr/bin/ps nfs-utils && yum clean all diff --git a/test/images/volume/rbd/Dockerfile b/test/images/volume/rbd/Dockerfile index ad6bb314ef5..40742f60dec 100644 --- a/test/images/volume/rbd/Dockerfile +++ b/test/images/volume/rbd/Dockerfile @@ -15,7 +15,8 @@ # CEPH all in one # Based on image by Ricardo Rocha, ricardo@catalyst.net.nz -FROM BASEIMAGE +ARG BASEIMAGE +FROM $BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/