From 82ac28cc529c287e0f7e62d44a24ac714e6f42ee Mon Sep 17 00:00:00 2001 From: Ben Moss Date: Fri, 22 Mar 2019 17:53:40 -0400 Subject: [PATCH] Adds support for building Windows pause image We can use docker buildx in order to build and push Windows images from the same Linux node, as long as the Dockerfile does not have any RUN commands in the Windows step. We also need to create a non-default builder instance in order to be able to build and push Windows images. The Windows images have to be built and pushed directly to the registry. Because of this, the make target "push" has been removed (the target "all" will build and push the images). We need wincat for a few kubectl proxy scenarios. For Windows containers without Hyper-V isolation, the host OS Version and the Container OS Version need to match, which is why we added multiple Windows OS Versions to the building process. Adds support for Windows OS Versions: 1809, 1903, 1909, 2004. Bumps pause image version to 3.4. Co-Authored-By: Claudiu Belu Co-Authored-By: Ben Moss Signed-off-by: Leah Hanson --- build/BUILD | 1 + build/dependencies.yaml | 2 +- build/pause/Dockerfile | 5 +- build/pause/Dockerfile_windows | 25 +++++ build/pause/Makefile | 139 +++++++++++++++++---------- build/pause/cloudbuild.yaml | 1 + build/pause/{ => linux}/orphan.c | 0 build/pause/{ => linux}/pause.c | 0 build/pause/windows/pause.c | 49 ++++++++++ build/pause/windows/wincat/BUILD | 28 ++++++ build/pause/windows/wincat/wincat.go | 78 +++++++++++++++ 11 files changed, 276 insertions(+), 52 deletions(-) create mode 100644 build/pause/Dockerfile_windows rename build/pause/{ => linux}/orphan.c (100%) rename build/pause/{ => linux}/pause.c (100%) create mode 100644 build/pause/windows/pause.c create mode 100644 build/pause/windows/wincat/BUILD create mode 100644 build/pause/windows/wincat/wincat.go diff --git a/build/BUILD b/build/BUILD index 20793e1f7b9..a5318ffea88 100644 --- a/build/BUILD +++ b/build/BUILD @@ -20,6 +20,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//build/pause/windows/wincat:all-srcs", "//build/release-tars:all-srcs", "//build/visible_to:all-srcs", ], diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 868ba8d01b4..64624d294ee 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -152,7 +152,7 @@ dependencies: match: tag = - name: "k8s.gcr.io/pause" - version: 3.3 + version: 3.4 refPaths: - path: build/pause/Makefile match: TAG = diff --git a/build/pause/Dockerfile b/build/pause/Dockerfile index 09f713292b3..250284b2aa4 100644 --- a/build/pause/Dockerfile +++ b/build/pause/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM scratch +ARG BASE +FROM ${BASE} ARG ARCH -ADD bin/pause-${ARCH} /pause +ADD bin/pause-linux-${ARCH} /pause ENTRYPOINT ["/pause"] diff --git a/build/pause/Dockerfile_windows b/build/pause/Dockerfile_windows new file mode 100644 index 00000000000..daa7558ec00 --- /dev/null +++ b/build/pause/Dockerfile_windows @@ -0,0 +1,25 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE +FROM ${BASE} +ARG ARCH +ADD bin/pause-windows-${ARCH}.exe /pause.exe +ADD windows/wincat.exe /Windows/System32/wincat.exe + +# NOTE(claudiub): docker buildx sets the PATH env variable to a Linux-like PATH, +# which is not desirable. See: https://github.com/moby/buildkit/issues/1560 +# TODO(claudiub): remove this once the issue has been resolved. +ENV PATH="C:\Windows\system32;C:\Windows;" +ENTRYPOINT ["/pause.exe"] diff --git a/build/pause/Makefile b/build/pause/Makefile index 9fa573d7d70..4791ac91751 100644 --- a/build/pause/Makefile +++ b/build/pause/Makefile @@ -12,91 +12,132 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: all push container clean orphan all-push push-manifest +.PHONY: all container clean orphan all-push push-manifest REGISTRY ?= staging-k8s.gcr.io IMAGE = $(REGISTRY)/pause -IMAGE_WITH_ARCH = $(IMAGE)-$(ARCH) +IMAGE_WITH_OS_ARCH = $(IMAGE)-$(OS)-$(ARCH) -TAG = 3.3 +TAG = 3.4 REV = $(shell git describe --contains --always --match='v*') # Architectures supported: amd64, arm, arm64, ppc64le and s390x ARCH ?= amd64 +# Operating systems supported: linux, windows +OS ?= linux +# OS Version for the Windows images: 1809, 1903, 1909 2004 +OSVERSION ?= 1809 1903 1909 2004 -ALL_ARCH = amd64 arm arm64 ppc64le s390x +# The output type could either be docker (local), or registry. +# If it is registry, it will also allow us to push the Windows images. +OUTPUT_TYPE ?= docker + +ALL_OS = linux windows +ALL_ARCH.linux = amd64 arm arm64 ppc64le s390x +ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch)) +ALL_ARCH.windows = amd64 +ALL_OSVERSIONS.windows := 1809 1903 1909 2004 +ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-$(arch)-${osversion})) +ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}}) CFLAGS = -Os -Wall -Werror -static -DVERSION=v$(TAG)-$(REV) -KUBE_CROSS_IMAGE ?= k8s.gcr.io/build-image/kube-cross -KUBE_CROSS_VERSION ?= $(shell cat ../build-image/cross/VERSION) +KUBE_CROSS_IMAGE.linux ?= k8s.gcr.io/build-image/kube-cross +KUBE_CROSS_VERSION.linux ?= $(shell cat ../build-image/cross/VERSION) +KUBE_CROSS_IMAGE.windows ?= dockcross/windows-static-x64 +KUBE_CROSS_VERSION.windows ?= latest +KUBE_CROSS_IMAGE := ${KUBE_CROSS_IMAGE.${OS}} +KUBE_CROSS_VERSION := ${KUBE_CROSS_VERSION.${OS}} -BIN = pause -SRCS = pause.c +# NOTE(claudiub): The Windows pause image also requires the wincat binary we're compiling for the +# port-forwarding scenarios. If it's no longer necessary, it can be removed. +# For more information, see: https://github.com/kubernetes/kubernetes/pull/91452 +BIN.linux = pause +BIN.windows = pause wincat +BIN := ${BIN.${OS}} +SRCS.linux = linux/pause.c +SRCS.windows = windows/pause.c +SRCS := ${SRCS.${OS}} -ifeq ($(ARCH),amd64) - TRIPLE ?= x86_64-linux-gnu -endif +EXTENSION.linux = +EXTENSION.windows = .exe +EXTENSION := ${EXTENSION.${OS}} -ifeq ($(ARCH),arm) - TRIPLE ?= arm-linux-gnueabihf -endif +# The manifest command is still experimental as of Docker 18.09.3 +export DOCKER_CLI_EXPERIMENTAL=enabled -ifeq ($(ARCH),arm64) - TRIPLE ?= aarch64-linux-gnu -endif - -ifeq ($(ARCH),ppc64le) - TRIPLE ?= powerpc64le-linux-gnu -endif - -ifeq ($(ARCH),s390x) - TRIPLE ?= s390x-linux-gnu -endif +TRIPLE.windows-amd64 := x86_64-w64-mingw32.static +TRIPLE.linux-amd64 := x86_64-linux-gnu +TRIPLE.linux-arm := arm-linux-gnueabihf +TRIPLE.linux-arm64 := aarch64-linux-gnu +TRIPLE.linux-ppc64le := powerpc64le-linux-gnu +TRIPLE.linux-s390x := s390x-linux-gnu +TRIPLE := ${TRIPLE.${OS}-${ARCH}} +BASE.linux := scratch +BASE.windows := mcr.microsoft.com/windows/nanoserver +BASE := ${BASE.${OS}} # If you want to build AND push all containers, see the 'all-push' rule. -all: all-container +all: all-container-docker -all-push: all-push-images push-manifest +# NOTE(claudiub): A non-default builder instance is needed in order to build Windows images. +all-push: all-container-registry push-manifest push-manifest: - docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g") - set -x; for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done + docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g") + set -x; for arch in $(ALL_ARCH.linux); do docker manifest annotate --os linux --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-linux-$${arch}:${TAG}; done + # For Windows images, we also need to include the "os.version" in the manifest list, so the Windows node can pull the proper image it needs. + # At the moment, docker manifest annotate doesn't allow us to set the os.version, so we'll have to it ourselves. The manifest list can be found locally as JSONs. + # See: https://github.com/moby/moby/issues/41417 + # TODO(claudiub): Clean this up once the above issue has been fixed. + set -x; \ + registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq "[a-z]*") && echo "docker.io/" || echo ""); \ + manifest_image_folder=`echo "$${registry_prefix}${IMAGE}" | sed "s|/|_|g" | sed "s/:/-/"`; \ + for arch in $(ALL_ARCH.windows); do \ + for osversion in ${ALL_OSVERSIONS.windows}; do \ + docker manifest annotate --os windows --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-windows-$${arch}-$${osversion}:${TAG}; \ + BASEIMAGE=${BASE.windows}:$${osversion}; \ + full_version=`docker manifest inspect ${BASE.windows}:$${osversion} | grep "os.version" | head -n 1 | awk '{print $$2}'` || true; \ + sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":$${full_version}/" "${HOME}/.docker/manifests/$${manifest_image_folder}-${TAG}/$${manifest_image_folder}-windows-$${arch}-$${osversion}-${TAG}"; \ + done; \ + done docker manifest push --purge ${IMAGE}:${TAG} +all-container-docker: $(addprefix sub-container-docker-,$(ALL_OS_ARCH.linux)) +all-container-registry: $(addprefix sub-container-registry-,$(ALL_OS_ARCH)) + +# split words on hyphen, access by 1-index +word-hyphen = $(word $2,$(subst -, ,$1)) sub-container-%: - $(MAKE) ARCH=$* container + $(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) container -sub-push-%: - $(MAKE) ARCH=$* push +build: $(foreach binary, ${BIN}, bin/${binary}-${OS}-${ARCH}) -all-container: $(addprefix sub-container-,$(ALL_ARCH)) - -all-push-images: $(addprefix sub-push-,$(ALL_ARCH)) - -build: bin/$(BIN)-$(ARCH) - -bin/$(BIN)-$(ARCH): $(SRCS) +bin/${BIN.linux}-$(OS)-$(ARCH): $(SRCS) mkdir -p bin docker run --rm -u $$(id -u):$$(id -g) -v $$(pwd):/build \ $(KUBE_CROSS_IMAGE):$(KUBE_CROSS_VERSION) \ /bin/bash -c "\ cd /build && \ $(TRIPLE)-gcc $(CFLAGS) -o $@ $^ && \ - $(TRIPLE)-strip $@" + $(TRIPLE)-strip $(foreach binary, $@, ${binary}${EXTENSION})" -container: .container-$(ARCH) -.container-$(ARCH): bin/$(BIN)-$(ARCH) - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --load --pull --platform linux/$(ARCH) -t $(IMAGE_WITH_ARCH):$(TAG) --build-arg ARCH=$(ARCH) . +bin/wincat-windows-${ARCH}: + CGO_ENABLED=0 GOOS=windows GOARCH=${ARCH} go build -o windows/wincat.exe windows/wincat/wincat.go + +container: .container-${OS}-$(ARCH) +.container-linux-$(ARCH): bin/$(BIN)-$(OS)-$(ARCH) + docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/$(ARCH) \ + -t $(IMAGE_WITH_OS_ARCH):$(TAG) --build-arg BASE=${BASE} --build-arg ARCH=$(ARCH) . touch $@ -push: .push-$(ARCH) -.push-$(ARCH): .container-$(ARCH) - docker push $(IMAGE_WITH_ARCH):$(TAG) +.container-windows-$(ARCH): $(foreach binary, ${BIN}, bin/${binary}-${OS}-${ARCH}) + docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/$(ARCH) \ + -t $(IMAGE_WITH_OS_ARCH)-${OSVERSION}:$(TAG) --build-arg BASE=${BASE}:${OSVERSION} --build-arg ARCH=$(ARCH) -f Dockerfile_windows . touch $@ # Useful for testing, not automatically included in container image -orphan: bin/orphan-$(ARCH) -bin/orphan-$(ARCH): orphan.c +orphan: bin/orphan-linux-$(ARCH) +bin/orphan-linux-$(ARCH): linux/orphan.c mkdir -p bin docker run -u $$(id -u):$$(id -g) -v $$(pwd):/build \ $(KUBE_CROSS_IMAGE):$(KUBE_CROSS_VERSION) \ @@ -106,4 +147,4 @@ bin/orphan-$(ARCH): orphan.c $(TRIPLE)-strip $@" clean: - rm -rf .container-* .push-* bin/ + rm -rf .*-container-* .push-* bin/ diff --git a/build/pause/cloudbuild.yaml b/build/pause/cloudbuild.yaml index 6c83082b4f2..2b354129886 100644 --- a/build/pause/cloudbuild.yaml +++ b/build/pause/cloudbuild.yaml @@ -16,4 +16,5 @@ steps: - '-c' - | gcloud auth configure-docker \ + && docker buildx create --name img-builder --use \ && make all-push diff --git a/build/pause/orphan.c b/build/pause/linux/orphan.c similarity index 100% rename from build/pause/orphan.c rename to build/pause/linux/orphan.c diff --git a/build/pause/pause.c b/build/pause/linux/pause.c similarity index 100% rename from build/pause/pause.c rename to build/pause/linux/pause.c diff --git a/build/pause/windows/pause.c b/build/pause/windows/pause.c new file mode 100644 index 00000000000..3af570a25b9 --- /dev/null +++ b/build/pause/windows/pause.c @@ -0,0 +1,49 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include + +BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) +{ + switch (fdwCtrlType) + { + case CTRL_C_EVENT: + fprintf(stderr, "Shutting down, got signal\n"); + exit(0); + + case CTRL_BREAK_EVENT: + fprintf(stderr, "Shutting down, got signal\n"); + exit(0); + + default: + return FALSE; + } +} + +int main(void) +{ + if (SetConsoleCtrlHandler(CtrlHandler, TRUE)) + { + Sleep(INFINITE); + } + else + { + printf("\nERROR: Could not set control handler\n"); + return 1; + } + return 0; +} diff --git a/build/pause/windows/wincat/BUILD b/build/pause/windows/wincat/BUILD new file mode 100644 index 00000000000..beb1fbc5d28 --- /dev/null +++ b/build/pause/windows/wincat/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["wincat.go"], + importpath = "k8s.io/kubernetes/build/pause/windows/wincat", + visibility = ["//visibility:private"], +) + +go_binary( + name = "windows", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/build/pause/windows/wincat/wincat.go b/build/pause/windows/wincat/wincat.go new file mode 100644 index 00000000000..1bd57a3f067 --- /dev/null +++ b/build/pause/windows/wincat/wincat.go @@ -0,0 +1,78 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// package wincat connects to the given host and port and redirects its stdin to the connection and +// the connection's output to stdout. This is currently being used for port-forwarding for Windows Pods. +package wincat + +import ( + "fmt" + "io" + "log" + "net" + "os" + "sync" +) + +func main() { + if len(os.Args) != 3 { + log.Fatalln("usage: wincat ") + } + host := os.Args[1] + port := os.Args[2] + + addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%s", host, port)) + if err != nil { + log.Fatalf("Failed to resolve TCP addr %v %v", host, port) + } + + conn, err := net.DialTCP("tcp", nil, addr) + if err != nil { + log.Fatalf("Failed to connect to %s:%s because %s", host, port, err) + } + defer conn.Close() + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer func() { + os.Stdout.Close() + os.Stdin.Close() + conn.CloseRead() + wg.Done() + }() + + _, err := io.Copy(os.Stdout, conn) + if err != nil { + log.Printf("error while copying stream to stdout: %v", err) + } + }() + + go func() { + defer func() { + conn.CloseWrite() + wg.Done() + }() + + _, err := io.Copy(conn, os.Stdin) + if err != nil { + log.Printf("error while copying stream from stdin: %v", err) + } + }() + + wg.Wait() +}