Merge pull request #59664 from mkumatag/manifest_infra

Manifest kubernetes infra images

build/debian-base/Makefile

@@ -12,19 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-all: build
+all: all-build
 
 REGISTRY ?= staging-k8s.gcr.io
-IMAGE ?= debian-base
+IMAGE ?= $(REGISTRY)/debian-base
 BUILD_IMAGE ?= debian-build
 
 TAG ?= 0.3.2
 
 TAR_FILE ?= rootfs.tar
 ARCH?=amd64
+ALL_ARCH = amd64 arm arm64 ppc64le s390x
+
 TEMP_DIR:=$(shell mktemp -d)
 QEMUVERSION=v2.9.1
 
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
+
 ifeq ($(ARCH),amd64)
 	BASEIMAGE?=debian:stretch
 endif
@@ -45,6 +50,23 @@ ifeq ($(ARCH),s390x)
 	QEMUARCH=s390x
 endif
 
+sub-build-%:
+	$(MAKE) ARCH=$* build
+
+all-build: $(addprefix sub-build-,$(ALL_ARCH))
+
+sub-push-image-%:
+	$(MAKE) ARCH=$* push
+
+all-push-images: $(addprefix sub-push-image-,$(ALL_ARCH))
+
+all-push: all-push-images push-manifest
+
+push-manifest:
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
+	@for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done
+	docker manifest push ${IMAGE}:${TAG}
+
 build: clean
 	cp ./* $(TEMP_DIR)
 	cat Dockerfile.build \
@@ -69,13 +91,13 @@ endif
 	docker build --pull -t $(BUILD_IMAGE) -f $(TEMP_DIR)/Dockerfile.build $(TEMP_DIR)
 	docker create --name $(BUILD_IMAGE) $(BUILD_IMAGE)
 	docker export $(BUILD_IMAGE) > $(TEMP_DIR)/$(TAR_FILE)
-	docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
+	docker build -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
 	rm -rf $(TEMP_DIR)
 
 push: build
-	docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG)
+	docker push $(IMAGE)-$(ARCH):$(TAG)
 
 clean:
-	docker rmi -f $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) || true
+	docker rmi -f $(IMAGE)-$(ARCH):$(TAG) || true
 	docker rmi -f $(BUILD_IMAGE)   || true
 	docker rm  -f $(BUILD_IMAGE)   || true

build/debian-hyperkube-base/Makefile

@@ -18,9 +18,10 @@
 #   [ARCH=amd64] [REGISTRY="staging-k8s.gcr.io"] make (build|push)
 
 REGISTRY?=staging-k8s.gcr.io
-IMAGE?=debian-hyperkube-base
+IMAGE?=$(REGISTRY)/debian-hyperkube-base
 TAG=0.10.2
 ARCH?=amd64
+ALL_ARCH = amd64 arm arm64 ppc64le s390x
 CACHEBUST?=1
 
 BASEIMAGE=k8s.gcr.io/debian-base-$(ARCH):0.3.2
@@ -29,9 +30,29 @@ CNI_VERSION=v0.6.0
 TEMP_DIR:=$(shell mktemp -d)
 CNI_TARBALL=cni-plugins-$(ARCH)-$(CNI_VERSION).tgz
 
-.PHONY: all build push clean
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
 
-all: push
+.PHONY: all build push clean all-build all-push-images all-push push-manifest
+
+all: all-push
+
+sub-build-%:
+	$(MAKE) ARCH=$* build
+
+all-build: $(addprefix sub-build-,$(ALL_ARCH))
+
+sub-push-image-%:
+	$(MAKE) ARCH=$* push
+
+all-push-images: $(addprefix sub-push-image-,$(ALL_ARCH))
+
+all-push: all-push-images push-manifest
+
+push-manifest:
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
+	@for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done
+	docker manifest push ${IMAGE}:${TAG}
 
 cni-tars/$(CNI_TARBALL):
 	mkdir -p cni-tars/
@@ -55,8 +76,8 @@ ifneq ($(ARCH),amd64)
 	# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
 	docker run --rm --privileged multiarch/qemu-user-static:register --reset
 endif
-	docker build --pull -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
+	docker build --pull -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
 	rm -rf $(TEMP_DIR)
 
 push: build
-	docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG)
+	docker push $(IMAGE)-$(ARCH):$(TAG)

build/debian-hyperkube-base/README.md

@@ -10,24 +10,16 @@ This image is compiled for multiple architectures.
 If you're editing the Dockerfile or some other thing, please bump the `TAG` in the Makefile.
 
 ```console
-# Build for linux/amd64 (default)
+# Build and  push images for all the architectures
-$ make push ARCH=amd64
+$ make all-push
 # ---> staging-k8s.gcr.io/debian-hyperkube-base-amd64:TAG
-
-$ make push ARCH=arm
 # ---> staging-k8s.gcr.io/debian-hyperkube-base-arm:TAG
-
-$ make push ARCH=arm64
 # ---> staging-k8s.gcr.io/debian-hyperkube-base-arm64:TAG
-
-$ make push ARCH=ppc64le
 # ---> staging-k8s.gcr.io/debian-hyperkube-base-ppc64le:TAG
-
-$ make push ARCH=s390x
 # ---> staging-k8s.gcr.io/debian-hyperkube-base-s390x:TAG
 ```
 
-If you don't want to push the images, run `make build` instead
+If you don't want to push the images, run `make all-build` instead
 
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/build/debian-hyperkube-base/README.md?pixel)]()

build/debian-iptables/Makefile

@@ -12,16 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY:	build push
+.PHONY:	build push all all-build all-push-images all-push push-manifest
 
 REGISTRY?="staging-k8s.gcr.io"
-IMAGE=debian-iptables
+IMAGE=$(REGISTRY)/debian-iptables
 TAG?=v10.2
 ARCH?=amd64
+ALL_ARCH = amd64 arm arm64 ppc64le s390x
 TEMP_DIR:=$(shell mktemp -d)
 
 BASEIMAGE?=k8s.gcr.io/debian-base-$(ARCH):0.3.2
 
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
+
 build:
 	cp ./* $(TEMP_DIR)
 	cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
@@ -31,9 +35,26 @@ ifneq ($(ARCH),amd64)
 	docker run --rm --privileged multiarch/qemu-user-static:register --reset
 endif
 
-	docker build --pull -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
+	docker build --pull -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
 
 push: build
-	docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG)
+	docker push $(IMAGE)-$(ARCH):$(TAG)
 
-all: push
+sub-build-%:
+	$(MAKE) ARCH=$* build
+
+all-build: $(addprefix sub-build-,$(ALL_ARCH))
+
+sub-push-image-%:
+	$(MAKE) ARCH=$* push
+
+all-push-images: $(addprefix sub-push-image-,$(ALL_ARCH))
+
+all-push: all-push-images push-manifest
+
+push-manifest:
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
+	@for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done
+	docker manifest push ${IMAGE}:${TAG}
+
+all: all-push

build/debian-iptables/README.md

@@ -9,24 +9,16 @@ This image is compiled for multiple architectures.
 If you're editing the Dockerfile or some other thing, please bump the `TAG` in the Makefile.
 
 ```console
-# Build for linux/amd64 (default)
+Build and  push images for all the architectures
-$ make push ARCH=amd64
+$ make all-push
 # ---> staging-k8s.gcr.io/debian-iptables-amd64:TAG
-
-$ make push ARCH=arm
 # ---> staging-k8s.gcr.io/debian-iptables-arm:TAG
-
-$ make push ARCH=arm64
 # ---> staging-k8s.gcr.io/debian-iptables-arm64:TAG
-
-$ make push ARCH=ppc64le
 # ---> staging-k8s.gcr.io/debian-iptables-ppc64le:TAG
-
-$ make push ARCH=s390x
 # ---> staging-k8s.gcr.io/debian-iptables-s390x:TAG
 ```
 
-If you don't want to push the images, run `make` or `make build` instead
+If you don't want to push the images, run `make build ARCH={target_arch}` or `make all-build` instead
 
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/build/debian-iptables/README.md?pixel)]()

cluster/images/etcd/Makefile

@@ -34,17 +34,23 @@ LATEST_ETCD_VERSION?=3.2.24
 # REVISION provides a version number fo this image and all it's bundled
 # artifacts. It should start at zero for each LATEST_ETCD_VERSION and increment
 # for each revision of this image at that etcd version.
-REVISION?=0
+REVISION?=1
 
 # IMAGE_TAG Uniquely identifies k8s.gcr.io/etcd docker image with a tag of the form "<etcd-version>-<revision>".
 IMAGE_TAG=$(LATEST_ETCD_VERSION)-$(REVISION)
 
 ARCH?=amd64
+ALL_ARCH = amd64 arm arm64 ppc64le s390x
 # Image should be pulled from k8s.gcr.io, which will auto-detect
 # region (us, eu, asia, ...) and pull from the closest.
 REGISTRY?=k8s.gcr.io
 # Images should be pushed to staging-k8s.gcr.io.
 PUSH_REGISTRY?=staging-k8s.gcr.io
+
+MANIFEST_IMAGE := $(PUSH_REGISTRY)/etcd
+
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
 # golang version should match the golang version from https://github.com/coreos/etcd/releases for the current ETCD_VERSION.
 GOLANG_VERSION?=1.8.7
 GOARM=7
@@ -118,14 +124,25 @@ endif
 	docker build --pull -t $(REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG) $(TEMP_DIR)
 
 push: build
-	docker tag $(REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG) $(PUSH_REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG)
+	docker tag $(REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG) $(MANIFEST_IMAGE)-$(ARCH):$(IMAGE_TAG)
-	docker push $(PUSH_REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG)
+	docker push $(MANIFEST_IMAGE)-$(ARCH):$(IMAGE_TAG)
 
-ifeq ($(ARCH),amd64)
+sub-build-%:
-	# Backward compatibility. TODO: deprecate this image tag
+	$(MAKE) ARCH=$* build
-	docker tag $(REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG) $(PUSH_REGISTRY)/etcd:$(IMAGE_TAG)
+
-	docker push $(PUSH_REGISTRY)/etcd:$(IMAGE_TAG)
+all-build: $(addprefix sub-build-,$(ALL_ARCH))
-endif
+
+sub-push-image-%:
+	$(MAKE) ARCH=$* push
+
+all-push-images: $(addprefix sub-push-image-,$(ALL_ARCH))
+
+all-push: all-push-images push-manifest
+
+push-manifest:
+	docker manifest create --amend $(MANIFEST_IMAGE):$(IMAGE_TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(MANIFEST_IMAGE)\-&:$(IMAGE_TAG)~g")
+	@for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${MANIFEST_IMAGE}:${IMAGE_TAG} ${MANIFEST_IMAGE}-$${arch}:${IMAGE_TAG}; done
+	docker manifest push ${MANIFEST_IMAGE}:${IMAGE_TAG}
 
 unit-test:
 	docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
@@ -146,5 +163,5 @@ integration-test:
 
 integration-build-test: build-integration-test-image integration-test
 test: unit-test integration-build-test
-all: build test
+all: all-build test
-.PHONY:	build push unit-test build-integration-test-image integration-test integration-build-test test
+.PHONY:	build push push-manifest all-push all-push-images all-build unit-test build-integration-test-image integration-test integration-build-test test

cluster/images/etcd/README.md

@@ -62,22 +62,14 @@ $ make build test
 Last, build and push the docker images for all supported architectures.
 
 ```console
-# Build for linux/amd64 (default)
+# Build images for all the architecture and push the manifest image as well
-$ make push ARCH=amd64
+$ make all-push
-# ---> staging-k8s.gcr.io/etcd-amd64:TAG
-# ---> staging-k8s.gcr.io/etcd:TAG
 
-$ make push ARCH=arm
+# Build images for all the architecture 
-# ---> staging-k8s.gcr.io/etcd-arm:TAG
+$ make all-build
 
-$ make push ARCH=arm64
+# Build image for target architecture(default=amd64)
-# ---> staging-k8s.gcr.io/etcd-arm64:TAG
+$ make build ARCH=ppc64le
-
-$ make push ARCH=ppc64le
-# ---> staging-k8s.gcr.io/etcd-ppc64le:TAG
-
-$ make push ARCH=s390x
-# ---> staging-k8s.gcr.io/etcd-s390x:TAG
 ```
 
 If you don't want to push the images, run `make` or `make build` instead