Build Kubernetes, etcd and flannel for arm64 and ppc64le

build/build-image/cross/Dockerfile

@@ -19,11 +19,14 @@ FROM golang:1.6.0
 
 ENV GOARM 6
 ENV KUBE_DYNAMIC_CROSSPLATFORMS \
-  armel
+  armel \
+  arm64 \
+  ppc64el
 
 ENV KUBE_CROSSPLATFORMS \
   linux/386 \
-  linux/arm \
+  linux/arm linux/arm64 \
+  linux/ppc64le \
   darwin/amd64 darwin/386 \
   windows/amd64 windows/386
 

build/build-image/cross/Makefile

@@ -20,7 +20,7 @@ TAG = $(shell cat VERSION)
 build:
 	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .
 
-push:	build
+push: build
 	gcloud docker --server=gcr.io push gcr.io/google_containers/$(IMAGE):$(TAG)
 
-all:	push
+all: push

build/build-image/cross/VERSION

@@ -1 +1 @@
-v1.6.0-1
+v1.6.0-2

build/common.sh

@@ -48,7 +48,7 @@ readonly KUBE_GCS_DELETE_EXISTING="${KUBE_GCS_DELETE_EXISTING:-n}"
 
 # Constants
 readonly KUBE_BUILD_IMAGE_REPO=kube-build
-readonly KUBE_BUILD_IMAGE_CROSS_TAG="v1.6.0-1"
+readonly KUBE_BUILD_IMAGE_CROSS_TAG="v1.6.0-2"
 # KUBE_BUILD_DATA_CONTAINER_NAME=kube-build-data-<hash>"
 
 # Here we map the output directories across both the local and remote _output
@@ -672,9 +672,8 @@ function kube::release::clean_cruft() {
 function kube::release::package_hyperkube() {
   # If we have these variables set then we want to build all docker images.
   if [[ -n "${KUBE_DOCKER_IMAGE_TAG-}" && -n "${KUBE_DOCKER_REGISTRY-}" ]]; then
-    for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
+    for arch in "${KUBE_SERVER_PLATFORMS[@]##*/}"; do
 
-      local arch=${platform##*/}
       kube::log::status "Building hyperkube image for arch: ${arch}"
       REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" ARCH="${arch}" make -C cluster/images/hyperkube/ build
     done
@@ -1515,6 +1514,7 @@ function kube::release::gcs::publish() {
 # Globals:
 #   KUBE_DOCKER_REGISTRY
 #   KUBE_DOCKER_IMAGE_TAG
+#   KUBE_SERVER_PLATFORMS
 # Returns:
 #   If new pushing docker images was successful.
 function kube::release::docker::release() {
@@ -1526,11 +1526,6 @@ function kube::release::docker::release() {
     "hyperkube"
   )
 
-  local archs=(
-    "amd64"
-    "arm"
-  )
-
   local docker_push_cmd=("${DOCKER[@]}")
   if [[ "${KUBE_DOCKER_REGISTRY}" == "gcr.io/"* ]]; then
     docker_push_cmd=("gcloud" "docker")
@@ -1540,7 +1535,8 @@ function kube::release::docker::release() {
     # Activate credentials for the k8s.production.user@gmail.com
     gcloud config set account k8s.production.user@gmail.com
   fi
-  for arch in "${archs[@]}"; do
+      
+  for arch in "${KUBE_SERVER_PLATFORMS[@]##*/}"; do
     for binary in "${binaries[@]}"; do
 
       local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}"

cluster/images/etcd/Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 FROM BASEIMAGE
-MAINTAINER  Dawn Chen <dawnchen@google.com>
+MAINTAINER Dawn Chen <dawnchen@google.com>
 
-COPY ./etcd /usr/local/bin/etcd
+COPY etcd /usr/local/bin/
-COPY ./etcdctl /usr/local/bin/etcdctl
+COPY etcdctl /usr/local/bin/

cluster/images/etcd/Makefile

@@ -17,45 +17,57 @@
 # Usage:
 # 	[TAG=2.2.1] [REGISTRY=gcr.io/google_containers] [ARCH=amd64] [BASEIMAGE=busybox] make (build|push)
 
-TAG?=2.2.1
+TAG?=2.2.5
 ARCH?=amd64
 REGISTRY?=gcr.io/google_containers
+GOLANG_VERSION?=1.6.0
+GOARM=6
 TEMP_DIR:=$(shell mktemp -d)
 
-# Default base images for different architectures
-# '/' in images has to be written as '\\/'
 ifeq ($(ARCH),amd64)
 	BASEIMAGE?=busybox
 endif
 ifeq ($(ARCH),arm)
-	BASEIMAGE?=hypriot\\/armhf-busybox
+	BASEIMAGE?=armel/busybox
+endif
+ifeq ($(ARCH),arm64)
+	BASEIMAGE?=aarch64/busybox
+endif
+ifeq ($(ARCH),ppc64le)
+	BASEIMAGE?=ppc64le/busybox
 endif
 
 build:
-	# Download etcd source in a golang container and cross-compile it statically
-	# or download official binaries if the ARCH is amd64
-	./build-etcd.sh ${TAG} ${ARCH} ${TEMP_DIR}
-
 	# Copy the content in this dir to the temp dir
-	cp ./* ${TEMP_DIR}
+	cp ./* $(TEMP_DIR)
-
-	# Replace BASEIMAGE with the real base image
-	cd ${TEMP_DIR} && sed -i "s/BASEIMAGE/${BASEIMAGE}/g" Dockerfile
-
-	# And build the image
-	docker build -t ${REGISTRY}/etcd-${ARCH}:${TAG} ${TEMP_DIR}
 
 ifeq ($(ARCH),amd64)
-	# Backward compatibility. TODO: deprecate this image tag
+
-	docker tag -f ${REGISTRY}/etcd-${ARCH}:${TAG} ${REGISTRY}/etcd:${TAG}
+	# Do not compile if we should make an image for amd64, use the official etcd binaries instead
+	curl -sSL --retry 5 https://github.com/coreos/etcd/releases/download/v$(TAG)/etcd-v$(TAG)-linux-amd64.tar.gz | tar -xz -C $(TEMP_DIR) --strip-components=1
+else
+	
+	# Download etcd in a golang container and cross-compile it statically
+	docker run -it -v $(TEMP_DIR):/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \
+		"git clone https://github.com/coreos/etcd \
+		&& cd etcd && git checkout v$(TAG) \
+		&& GOARM=$(GOARM) GOARCH=$(ARCH) ./build \
+		&& cp bin/* /etcdbin"
 endif
 
-push: build
+	# Replace BASEIMAGE with the real base image
-	gcloud docker push ${REGISTRY}/etcd-${ARCH}:${TAG}
+	cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
+
+	# And build the image
+	docker build -t $(REGISTRY)/etcd-$(ARCH):$(TAG) $(TEMP_DIR)
+
+push: build
+	gcloud docker push $(REGISTRY)/etcd-$(ARCH):$(TAG)
 
-	# Backward compatibility. TODO: deprecate this image tag
 ifeq ($(ARCH),amd64)
-	gcloud docker push ${REGISTRY}/etcd:${TAG}
+	# Backward compatibility. TODO: deprecate this image tag
+	docker tag -f $(REGISTRY)/etcd-$(ARCH):$(TAG) $(REGISTRY)/etcd:$(TAG)
+	gcloud docker push $(REGISTRY)/etcd:$(TAG)
 endif
 
 all: build

cluster/images/etcd/README.md

@@ -0,0 +1,29 @@
+### etcd
+
+This is a small etcd image used in Kubernetes setups where `etcd` is deployed as a docker image.
+
+For `amd64`, official `etcd` and `etcdctl` binaries are downloaded from Github to maintain official support.
+For other architectures, `etcd` is cross-compiled from source. Arch-specific `busybox` images serve as base images.
+
+#### How to release
+
+```console
+# Build for linux/amd64 (default)
+$ make push ARCH=amd64
+# ---> gcr.io/google_containers/etcd-amd64:TAG
+# ---> gcr.io/google_containers/etcd:TAG
+
+$ make push ARCH=arm
+# ---> gcr.io/google_containers/etcd-arm:TAG
+
+$ make push ARCH=arm64
+# ---> gcr.io/google_containers/etcd-arm64:TAG
+
+$ make push ARCH=ppc64le
+# ---> gcr.io/google_containers/etcd-ppc64le:TAG
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/etcd/README.md?pixel)]()

cluster/images/etcd/build-etcd.sh

@@ -1,57 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Build etcd from source in a docker container
-
-# Require three arguments
-if [[ $# != 3 ]]; then
-	echo "Usage: ./build-etcd.sh TAG ARCH TARGET_DIR"
-	exit
-fi
-
-# Example tag should be 2.2.1, not v2.2.1
-TAG=$1
-ARCH=$2
-TARGET_DIR=$3
-
-GOLANG_VERSION=${GOLANG_VERSION:-1.6.0}
-GOARM=6
-
-# Create the ${TARGET_DIR} directory, if it doesn't exist
-mkdir -p ${TARGET_DIR}
-
-# Do not compile if we should make an image for amd64, use the "official" etcd binaries instead
-if [[ ${ARCH} == "amd64" ]]; then
-
-	# Just download the binaries to ${TARGET_DIR}
-	curl -sSL https://github.com/coreos/etcd/releases/download/v${TAG}/etcd-v${TAG}-linux-amd64.tar.gz | tar -xz -C ${TARGET_DIR} --strip-components=1
-else
-	# Download etcd in a golang container and cross-compile it statically
-	CID=$(docker run -d golang:${GOLANG_VERSION} /bin/sh -c \
-		"mkdir /etcd \
-		&& curl -sSL https://github.com/coreos/etcd/archive/v${TAG}.tar.gz | tar -C /etcd -xz --strip-components=1 \
-		&& cd /etcd \
-		&& GOARM=${GOARM} GOARCH=${ARCH} ./build")
-	
-	# Wait until etcd has compiled
-	docker wait ${CID}
-
-	# Copy out the bin folder to ${TARGET_DIR}
-	docker cp ${CID}:/etcd/bin ${TARGET_DIR}
-
-	# Move the contents in bin to the target directory
-	mv ${TARGET_DIR}/bin/* ${TARGET_DIR}
-fi

cluster/images/flannel/Makefile

@@ -20,7 +20,7 @@
 TAG?=0.5.5
 ARCH?=amd64
 REGISTRY?=gcr.io/google_containers
-KUBE_CROSS_TAG=v1.4.2-1
+KUBE_CROSS_TAG=v1.6.0-2
 GOARM=6
 TEMP_DIR:=$(shell mktemp -d)
 BASEIMAGE?=gcr.io/google_containers/debian-iptables-${ARCH}:v2
@@ -28,10 +28,16 @@ BASEIMAGE?=gcr.io/google_containers/debian-iptables-${ARCH}:v2
 ifeq ($(ARCH),arm)
 	CC=arm-linux-gnueabi-gcc
 endif
+ifeq ($(ARCH),arm64)
+	CC=aarch64-linux-gnu-gcc
+endif
+ifeq ($(ARCH),ppc64le)
+	CC=powerpc64le-linux-gnu-gcc
+endif
 
 build:
 ifeq ($(ARCH),amd64)
-	# If we should build an amd64 flannel, go with the official one
+	# If we should build an amd64 flannel image, go with the official one
 	docker pull quay.io/coreos/flannel:$(TAG)
 
 	docker tag -f quay.io/coreos/flannel:$(TAG) $(REGISTRY)/flannel-$(ARCH):$(TAG)
@@ -40,8 +46,8 @@ else
 	cp ./* $(TEMP_DIR)
 
 	docker run -it -v $(TEMP_DIR):/flannel/bin gcr.io/google_containers/kube-cross:$(KUBE_CROSS_TAG) /bin/bash -c \
-    	"curl -sSL https://github.com/coreos/flannel/archive/v${TAG}.tar.gz | tar -C /flannel -xz --strip-components=1 \
+		"curl -sSL https://github.com/coreos/flannel/archive/v${TAG}.tar.gz | tar -C /flannel -xz --strip-components=1 \
-    	&& cd /flannel && GOARM=$(GOARM) GOARCH=$(ARCH) CC=$(CC) CGO_ENABLED=1 ./build"
+		&& cd /flannel && GOARM=$(GOARM) GOARCH=$(ARCH) CC=$(CC) CGO_ENABLED=1 ./build"
 
 	# Replace BASEIMAGE with the real base image
 	cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile

cluster/images/flannel/README.md

@@ -14,6 +14,12 @@ $ make push ARCH=amd64
 
 $ make push ARCH=arm
 # ---> gcr.io/google_containers/flannel-arm:TAG
+
+$ make push ARCH=arm64
+# ---> gcr.io/google_containers/flannel-arm64:TAG
+
+$ make push ARCH=ppc64le
+# ---> gcr.io/google_containers/flannel-ppc64le:TAG
 ```
 
 If you don't want to push the images, run `make` or `make build` instead

cluster/images/hyperkube/Dockerfile

@@ -19,8 +19,7 @@ FROM BASEIMAGE
 CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get -yy -q \
+    && DEBIAN_FRONTEND=noninteractive apt-get -yy -q install \
-    install \
     iptables \
     ethtool \
     ca-certificates \
@@ -34,21 +33,26 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \
 
 RUN cp /usr/bin/nsenter /nsenter
 
+# Copy the hyperkube binary
 COPY hyperkube /hyperkube
-RUN chmod a+rx /hyperkube
-
-COPY master-multi.json /etc/kubernetes/manifests-multi/master.json
-COPY kube-proxy.json /etc/kubernetes/manifests-multi/kube-proxy.json
 
+# Manifests for the docker guide
 COPY master.json /etc/kubernetes/manifests/master.json
 COPY etcd.json /etc/kubernetes/manifests/etcd.json
 COPY kube-proxy.json /etc/kubernetes/manifests/kube-proxy.json
 
+# Manifests for the docker-multinode guide
+COPY master-multi.json /etc/kubernetes/manifests-multi/master.json
+COPY kube-proxy.json /etc/kubernetes/manifests-multi/kube-proxy.json
+
+# Other required scripts for the setup
 COPY safe_format_and_mount /usr/share/google/safe_format_and_mount
-RUN chmod a+rx /usr/share/google/safe_format_and_mount
-
 COPY setup-files.sh /setup-files.sh
-RUN chmod a+rx /setup-files.sh
-
 COPY make-ca-cert.sh /make-ca-cert.sh
-RUN chmod a+x /make-ca-cert.sh
+
+# Make scripts executable
+RUN chmod a+rx \
+        /hyperkube \
+        /usr/share/google/safe_format_and_mount \
+        /setup-files.sh \
+        /make-ca-cert.sh

cluster/images/hyperkube/Makefile

@@ -15,7 +15,7 @@
 # Build the hyperkube image.
 #
 # Usage:
-#   VERSION=v1.2.0 [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make build
+#   [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make (build|push) VERSION={some_version_number e.g. v1.2.0}
 
 REGISTRY?="gcr.io/google_containers"
 ARCH?=amd64
@@ -27,11 +27,21 @@ ifeq ($(ARCH),amd64)
 endif
 ifeq ($(ARCH),arm)
 	BASEIMAGE?=armel/debian:jessie
+	QEMUARCH=arm
+endif
+ifeq ($(ARCH),arm64)
+	BASEIMAGE?=aarch64/debian:jessie
+	QEMUARCH=aarch64
+endif
+ifeq ($(ARCH),ppc64le)
+	BASEIMAGE?=ppc64le/debian:jessie
+	QEMUARCH=ppc64le
 endif
 
 all: build
 
 build:
+
 ifndef VERSION
     $(error VERSION is undefined)
 endif
@@ -40,7 +50,8 @@ endif
 	cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR}
 	cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR}
 	cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json
-	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json Dockerfile
+	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json
+	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${QEMUARCH}|g" Dockerfile
 	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
 	rm ${TEMP_DIR}/*.back
 
@@ -51,7 +62,7 @@ else
 	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
 	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
 	docker run --rm --privileged multiarch/qemu-user-static:register --reset
-	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-${ARCH}-static.tar.xz | tar -xJ -C ${TEMP_DIR}
+	curl -sSL --retry 5 https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-${QEMUARCH}-static.tar.xz | tar -xJ -C ${TEMP_DIR}
 	cd ${TEMP_DIR} && sed -i "s/CROSS_BUILD_//g" Dockerfile
 endif
 

cluster/images/hyperkube/README.md

@@ -4,12 +4,12 @@
 Also, it's very easy to run this `hyperkube` setup dockerized.
 See http://kubernetes.io/docs/getting-started-guides/docker/ for up-to-date commands.
 
-`hyperkube` is built for multiple architectures and pushed on every release.
+`hyperkube` is built for multiple architectures and _pushed automatically on every release._
 
 #### How to release by hand
 
 ```console
-# First, build the 
+# First, build the binaries
 $ build/run.sh hack/build-cross.sh
 
 # Build for linux/amd64 (default)
@@ -19,6 +19,12 @@ $ make push VERSION={target_version} ARCH=amd64
 
 $ make push VERSION={target_version} ARCH=arm
 # ---> gcr.io/google_containers/hyperkube-arm:VERSION
+
+$ make push VERSION={target_version} ARCH=arm64
+# ---> gcr.io/google_containers/hyperkube-arm64:VERSION
+
+$ make push VERSION={target_version} ARCH=ppc64le
+# ---> gcr.io/google_containers/hyperkube-ppc64le:VERSION
 ```
 
 If you don't want to push the images, run `make` or `make build` instead

cluster/images/hyperkube/etcd.json

@@ -7,7 +7,7 @@
     "containers": [
       {
         "name": "etcd",
-        "image": "gcr.io/google_containers/etcd-ARCH:2.2.1",
+        "image": "gcr.io/google_containers/etcd-ARCH:2.2.5",
         "command": [
                 "/usr/local/bin/etcd",
                 "--listen-client-urls=http://127.0.0.1:4001",

hack/lib/golang.sh

@@ -64,6 +64,8 @@ else
   readonly KUBE_SERVER_PLATFORMS=(
     linux/amd64
     linux/arm
+    linux/arm64
+    linux/ppc64le
   )
 
   # If we update this we should also update the set of golang compilers we build
@@ -72,6 +74,8 @@ else
     linux/amd64
     linux/386
     linux/arm
+    linux/arm64
+    linux/ppc64le
     darwin/amd64
     darwin/386
     windows/amd64