From 0e99cd912bb6f18ce26b9eb4519de3b028f0d8d5 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Fri, 1 Nov 2019 16:46:47 -0400 Subject: [PATCH 1/2] Script based hyperkube Update bazel based build and make the containers match as well. --- build/lib/release.sh | 22 +++++++++ cluster/images/hyperkube/BUILD | 32 ++++++++++-- cluster/images/hyperkube/Dockerfile | 7 ++- cluster/images/hyperkube/Makefile | 10 +++- cluster/images/hyperkube/hyperkube | 76 +++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 6 deletions(-) create mode 100755 cluster/images/hyperkube/hyperkube diff --git a/build/lib/release.sh b/build/lib/release.sh index 5c2fc49579e..b538c1e4880 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -28,6 +28,7 @@ readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage" readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars" readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images" +KUBE_BUILD_HYPERKUBE=${KUBE_BUILD_HYPERKUBE:-y} KUBE_BUILD_CONFORMANCE=${KUBE_BUILD_CONFORMANCE:-y} KUBE_BUILD_PULL_LATEST_IMAGES=${KUBE_BUILD_PULL_LATEST_IMAGES:-y} @@ -278,6 +279,23 @@ function kube::release::sha1() { fi } +function kube::release::build_hyperkube_image() { + local -r arch="$1" + local -r registry="$2" + local -r version="$3" + local -r save_dir="${4-}" + kube::log::status "Building hyperkube image for arch: ${arch}" + ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \ + make -C cluster/images/hyperkube/ build >/dev/null + + local hyperkube_tag="${registry}/hyperkube-${arch}:${version}" + if [[ -n "${save_dir}" ]]; then + "${DOCKER[@]}" save "${hyperkube_tag}" > "${save_dir}/hyperkube-${arch}.tar" + fi + kube::log::status "Deleting hyperkube image ${hyperkube_tag}" + "${DOCKER[@]}" rmi "${hyperkube_tag}" &>/dev/null || true +} + function kube::release::build_conformance_image() { local -r arch="$1" local -r registry="$2" @@ -378,6 +396,10 @@ EOF ) & done + if [[ "${KUBE_BUILD_HYPERKUBE}" =~ [yY] ]]; then + kube::release::build_hyperkube_image "${arch}" "${docker_registry}" \ + "${docker_tag}" "${images_dir}" & + fi if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then kube::release::build_conformance_image "${arch}" "${docker_registry}" \ "${docker_tag}" "${images_dir}" & diff --git a/cluster/images/hyperkube/BUILD b/cluster/images/hyperkube/BUILD index cc2653bf9dd..59d3b7a4ed7 100644 --- a/cluster/images/hyperkube/BUILD +++ b/cluster/images/hyperkube/BUILD @@ -1,16 +1,42 @@ +load("@io_bazel_rules_docker//container:container.bzl", "container_layer") load("//build:container.bzl", "multi_arch_container") load("//build:platforms.bzl", "SERVER_PLATFORMS") +container_layer( + name = "scripts", + directory = "/", + files = [ + "hyperkube", + ], +) + +container_layer( + name = "bins", + directory = "/usr/local/bin", + files = [ + "//cmd/kube-apiserver", + "//cmd/kube-controller-manager", + "//cmd/kube-proxy", + "//cmd/kube-scheduler", + "//cmd/kubectl", + "//cmd/kubelet", + ], +) + multi_arch_container( - name = "hyperkube", + name = "image", architectures = SERVER_PLATFORMS["linux"], base = "@debian-hyperkube-base-{ARCH}//image", + cmd = [ + "/hyperkube", + ], # {ARCH} is replaced by the macro, but STABLE_ vars are replaced by the # build stamping, so we need to escape them docker_push_tags = ["{{STABLE_DOCKER_PUSH_REGISTRY}}/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"], docker_tags = ["{{STABLE_DOCKER_REGISTRY}}/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"], - files = [ - "//cmd/hyperkube", + layers = [ + ":bins", + ":scripts", ], stamp = True, tags = ["manual"], diff --git a/cluster/images/hyperkube/Dockerfile b/cluster/images/hyperkube/Dockerfile index 61fa949e364..3347b9b4c26 100644 --- a/cluster/images/hyperkube/Dockerfile +++ b/cluster/images/hyperkube/Dockerfile @@ -14,5 +14,10 @@ FROM BASEIMAGE -# Copy the hyperkube binary +# Cleanup all the soft links +ADD binaries.tgz /usr/local/bin + +# Copy the shell script COPY hyperkube /hyperkube + +ENTRYPOINT ["/hyperkube"] \ No newline at end of file diff --git a/cluster/images/hyperkube/Makefile b/cluster/images/hyperkube/Makefile index 340f85e82e9..06f3c95fdee 100644 --- a/cluster/images/hyperkube/Makefile +++ b/cluster/images/hyperkube/Makefile @@ -20,9 +20,13 @@ REGISTRY?=staging-k8s.gcr.io ARCH?=amd64 OUT_DIR?=_output -HYPERKUBE_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/hyperkube BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.12.1 + +LOCAL_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/local/bin/linux/$(ARCH) +DOCKERIZED_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH) +OUTPUT_PATH?=$(shell test -d $(LOCAL_OUTPUT_PATH) && echo $(LOCAL_OUTPUT_PATH) || echo $(DOCKERIZED_OUTPUT_PATH)) + TEMP_DIR:=$(shell mktemp -d -t hyperkubeXXXXXX) all: build @@ -33,7 +37,9 @@ ifndef VERSION $(error VERSION is undefined) endif cp -r ./* ${TEMP_DIR} - cp ${HYPERKUBE_BIN} ${TEMP_DIR} + + tar -cvzf ${TEMP_DIR}/binaries.tgz -C ${OUTPUT_PATH} kube-apiserver kube-controller-manager \ + kube-proxy kube-scheduler kubectl kubelet chmod a+rx ${TEMP_DIR}/hyperkube diff --git a/cluster/images/hyperkube/hyperkube b/cluster/images/hyperkube/hyperkube new file mode 100755 index 00000000000..c353797c2c6 --- /dev/null +++ b/cluster/images/hyperkube/hyperkube @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +BINS=( + kube-apiserver + kube-controller-manager + kube-proxy + kube-scheduler + kubectl + kubelet +) + +function array_contains() { + local search="$1" + local element + shift + for element; do + if [[ "${element}" == "${search}" ]]; then + return 0 + fi + done + return 1 +} + +function print_usage() { + cat </dev/null; then + echo "${command}: command not found" + exit 1 + fi + exec "${command}" "${@}" +} + +main "${@}" From 30965676f3fa8e261c90a32ed399bab04840b21b Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sun, 3 Nov 2019 07:56:24 -0500 Subject: [PATCH 2/2] Drop cmd/hyperkube --- cmd/BUILD | 1 - cmd/hyperkube/BUILD | 48 ---------------- cmd/hyperkube/OWNERS | 4 -- cmd/hyperkube/main.go | 125 ------------------------------------------ 4 files changed, 178 deletions(-) delete mode 100644 cmd/hyperkube/BUILD delete mode 100644 cmd/hyperkube/OWNERS delete mode 100644 cmd/hyperkube/main.go diff --git a/cmd/BUILD b/cmd/BUILD index 28f5bfaf120..5f243e13d75 100644 --- a/cmd/BUILD +++ b/cmd/BUILD @@ -20,7 +20,6 @@ filegroup( "//cmd/genswaggertypedocs:all-srcs", "//cmd/genutils:all-srcs", "//cmd/genyaml:all-srcs", - "//cmd/hyperkube:all-srcs", "//cmd/importverifier:all-srcs", "//cmd/kube-apiserver:all-srcs", "//cmd/kube-controller-manager:all-srcs", diff --git a/cmd/hyperkube/BUILD b/cmd/hyperkube/BUILD deleted file mode 100644 index a3f4dd22b2b..00000000000 --- a/cmd/hyperkube/BUILD +++ /dev/null @@ -1,48 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "hyperkube", - embed = [":go_default_library"], - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["main.go"], - importpath = "k8s.io/kubernetes/cmd/hyperkube", - deps = [ - "//cmd/cloud-controller-manager/app:go_default_library", - "//cmd/kube-apiserver/app:go_default_library", - "//cmd/kube-controller-manager/app:go_default_library", - "//cmd/kube-proxy/app:go_default_library", - "//cmd/kube-scheduler/app:go_default_library", - "//cmd/kubelet/app:go_default_library", - "//pkg/kubectl/cmd:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/restclient:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/hyperkube/OWNERS b/cmd/hyperkube/OWNERS deleted file mode 100644 index c28de235565..00000000000 --- a/cmd/hyperkube/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -labels: -- sig/release diff --git a/cmd/hyperkube/main.go b/cmd/hyperkube/main.go deleted file mode 100644 index 9b606bd139e..00000000000 --- a/cmd/hyperkube/main.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// A binary that can morph into all of the other kubernetes binaries. You can -// also soft-link to it busybox style. -// -package main - -import ( - goflag "flag" - "math/rand" - "os" - "path/filepath" - "time" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - cloudcontrollermanager "k8s.io/kubernetes/cmd/cloud-controller-manager/app" - kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" - kubecontrollermanager "k8s.io/kubernetes/cmd/kube-controller-manager/app" - kubeproxy "k8s.io/kubernetes/cmd/kube-proxy/app" - kubescheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" - kubelet "k8s.io/kubernetes/cmd/kubelet/app" - kubectl "k8s.io/kubernetes/pkg/kubectl/cmd" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - hyperkubeCommand, allCommandFns := NewHyperKubeCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - // cliflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - basename := filepath.Base(os.Args[0]) - if err := commandFor(basename, hyperkubeCommand, allCommandFns).Execute(); err != nil { - os.Exit(1) - } -} - -func commandFor(basename string, defaultCommand *cobra.Command, commands []func() *cobra.Command) *cobra.Command { - for _, commandFn := range commands { - command := commandFn() - if command.Name() == basename { - return command - } - for _, alias := range command.Aliases { - if alias == basename { - return command - } - } - } - - return defaultCommand -} - -// NewHyperKubeCommand is the entry point for hyperkube -func NewHyperKubeCommand() (*cobra.Command, []func() *cobra.Command) { - // these have to be functions since the command is polymorphic. Cobra wants you to be top level - // command to get executed - apiserver := func() *cobra.Command { return kubeapiserver.NewAPIServerCommand() } - controller := func() *cobra.Command { return kubecontrollermanager.NewControllerManagerCommand() } - proxy := func() *cobra.Command { return kubeproxy.NewProxyCommand() } - scheduler := func() *cobra.Command { return kubescheduler.NewSchedulerCommand() } - kubectlCmd := func() *cobra.Command { return kubectl.NewDefaultKubectlCommand() } - kubelet := func() *cobra.Command { return kubelet.NewKubeletCommand() } - cloudController := func() *cobra.Command { - cmd := cloudcontrollermanager.NewCloudControllerManagerCommand() - cmd.Deprecated = "please use the cloud controller manager specific " + - "to your external cloud provider" - return cmd - } - - commandFns := []func() *cobra.Command{ - apiserver, - controller, - proxy, - scheduler, - kubectlCmd, - kubelet, - cloudController, - } - - cmd := &cobra.Command{ - Use: "hyperkube", - Short: "Request a new project", - Run: func(cmd *cobra.Command, args []string) { - if len(args) != 0 { - cmd.Help() - os.Exit(1) - } - - }, - } - - for i := range commandFns { - cmd.AddCommand(commandFns[i]()) - } - - return cmd, commandFns -}