Script based hyperkube

Update bazel based build and make the containers match as well.

build/lib/release.sh

@@ -28,6 +28,7 @@ readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage"
 readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars"
 readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images"
 
+KUBE_BUILD_HYPERKUBE=${KUBE_BUILD_HYPERKUBE:-y}
 KUBE_BUILD_CONFORMANCE=${KUBE_BUILD_CONFORMANCE:-y}
 KUBE_BUILD_PULL_LATEST_IMAGES=${KUBE_BUILD_PULL_LATEST_IMAGES:-y}
 
@@ -278,6 +279,23 @@ function kube::release::sha1() {
   fi
 }
 
+function kube::release::build_hyperkube_image() {
+  local -r arch="$1"
+  local -r registry="$2"
+  local -r version="$3"
+  local -r save_dir="${4-}"
+  kube::log::status "Building hyperkube image for arch: ${arch}"
+  ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \
+    make -C cluster/images/hyperkube/ build >/dev/null
+
+  local hyperkube_tag="${registry}/hyperkube-${arch}:${version}"
+  if [[ -n "${save_dir}" ]]; then
+    "${DOCKER[@]}" save "${hyperkube_tag}" > "${save_dir}/hyperkube-${arch}.tar"
+  fi
+  kube::log::status "Deleting hyperkube image ${hyperkube_tag}"
+  "${DOCKER[@]}" rmi "${hyperkube_tag}" &>/dev/null || true
+}
+
 function kube::release::build_conformance_image() {
   local -r arch="$1"
   local -r registry="$2"
@@ -378,6 +396,10 @@ EOF
       ) &
     done
 
+    if [[ "${KUBE_BUILD_HYPERKUBE}" =~ [yY] ]]; then
+      kube::release::build_hyperkube_image "${arch}" "${docker_registry}" \
+        "${docker_tag}" "${images_dir}" &
+    fi
     if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
       kube::release::build_conformance_image "${arch}" "${docker_registry}" \
         "${docker_tag}" "${images_dir}" &

cluster/images/hyperkube/BUILD

@@ -1,16 +1,42 @@
+load("@io_bazel_rules_docker//container:container.bzl", "container_layer")
 load("//build:container.bzl", "multi_arch_container")
 load("//build:platforms.bzl", "SERVER_PLATFORMS")
 
+container_layer(
+    name = "scripts",
+    directory = "/",
+    files = [
+        "hyperkube",
+    ],
+)
+
+container_layer(
+    name = "bins",
+    directory = "/usr/local/bin",
+    files = [
+        "//cmd/kube-apiserver",
+        "//cmd/kube-controller-manager",
+        "//cmd/kube-proxy",
+        "//cmd/kube-scheduler",
+        "//cmd/kubectl",
+        "//cmd/kubelet",
+    ],
+)
+
 multi_arch_container(
-    name = "hyperkube",
+    name = "image",
     architectures = SERVER_PLATFORMS["linux"],
     base = "@debian-hyperkube-base-{ARCH}//image",
+    cmd = [
+        "/hyperkube",
+    ],
     # {ARCH} is replaced by the macro, but STABLE_ vars are replaced by the
     # build stamping, so we need to escape them
     docker_push_tags = ["{{STABLE_DOCKER_PUSH_REGISTRY}}/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"],
     docker_tags = ["{{STABLE_DOCKER_REGISTRY}}/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"],
-    files = [
+    layers = [
-        "//cmd/hyperkube",
+        ":bins",
+        ":scripts",
     ],
     stamp = True,
     tags = ["manual"],

cluster/images/hyperkube/Dockerfile

@@ -14,5 +14,10 @@
 
 FROM BASEIMAGE
 
-# Copy the hyperkube binary
+# Cleanup all the soft links
+ADD binaries.tgz /usr/local/bin
+
+# Copy the shell script
 COPY hyperkube /hyperkube
+
+ENTRYPOINT ["/hyperkube"]

cluster/images/hyperkube/Makefile

@@ -20,9 +20,13 @@
 REGISTRY?=staging-k8s.gcr.io
 ARCH?=amd64
 OUT_DIR?=_output
-HYPERKUBE_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/hyperkube
 
 BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.12.1
+
+LOCAL_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/local/bin/linux/$(ARCH)
+DOCKERIZED_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)
+OUTPUT_PATH?=$(shell test -d $(LOCAL_OUTPUT_PATH) && echo $(LOCAL_OUTPUT_PATH) || echo $(DOCKERIZED_OUTPUT_PATH))
+
 TEMP_DIR:=$(shell mktemp -d -t hyperkubeXXXXXX)
 
 all: build
@@ -33,7 +37,9 @@ ifndef VERSION
     $(error VERSION is undefined)
 endif
 	cp -r ./* ${TEMP_DIR}
-	cp ${HYPERKUBE_BIN} ${TEMP_DIR}
+
+	tar -cvzf ${TEMP_DIR}/binaries.tgz  -C ${OUTPUT_PATH} kube-apiserver kube-controller-manager \
+		kube-proxy kube-scheduler kubectl kubelet
 
 	chmod a+rx ${TEMP_DIR}/hyperkube
 

cluster/images/hyperkube/hyperkube

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+BINS=(
+  kube-apiserver
+  kube-controller-manager
+  kube-proxy
+  kube-scheduler
+  kubectl
+  kubelet
+)
+
+function array_contains() {
+  local search="$1"
+  local element
+  shift
+  for element; do
+    if [[ "${element}" == "${search}" ]]; then
+      return 0
+     fi
+  done
+  return 1
+}
+
+function print_usage() {
+  cat <<EOF
+Usage:
+  $(basename "$0") [command]
+
+Available Commands:
+  help                     Help about any command
+  kube-apiserver
+  kube-controller-manager
+  kube-proxy
+  kube-scheduler
+  kubectl                  kubectl controls the Kubernetes cluster manager
+  kubelet
+EOF
+  exit 0
+}
+
+function main() {
+  if [[ "$#" -lt 1 || "${1:-}" == "--help" || "${1:-}" == "help" ]]; then
+    print_usage
+  fi
+  if ! array_contains "$1" "${BINS[@]}"; then
+    echo "$1: command not supported"
+    print_usage
+  fi
+  command=${1}
+  shift
+  if ! command -v "${command}" &>/dev/null; then
+    echo "${command}: command not found"
+    exit 1
+  fi
+  exec "${command}" "${@}"
+}
+
+main "${@}"