From 393e0952e9681c0a62fbff05d85909186945d451 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Wed, 6 May 2020 09:19:48 -0400 Subject: [PATCH] New go-runner image for distroless scenarios Signed-off-by: Davanum Srinivas --- build/BUILD | 1 + build/go-runner/BUILD | 29 ++++++++ build/go-runner/Dockerfile | 45 ++++++++++++ build/go-runner/Makefile | 71 +++++++++++++++++++ build/go-runner/OWNERS | 6 ++ build/go-runner/README.md | 30 ++++++++ build/go-runner/cloudbuild.yaml | 22 ++++++ build/go-runner/go-runner.go | 122 ++++++++++++++++++++++++++++++++ build/go-runner/go.mod | 5 ++ build/go-runner/go.sum | 2 + 10 files changed, 333 insertions(+) create mode 100644 build/go-runner/BUILD create mode 100644 build/go-runner/Dockerfile create mode 100644 build/go-runner/Makefile create mode 100644 build/go-runner/OWNERS create mode 100644 build/go-runner/README.md create mode 100644 build/go-runner/cloudbuild.yaml create mode 100644 build/go-runner/go-runner.go create mode 100644 build/go-runner/go.mod create mode 100644 build/go-runner/go.sum diff --git a/build/BUILD b/build/BUILD index 70e3874b441..9fedd4328af 100644 --- a/build/BUILD +++ b/build/BUILD @@ -20,6 +20,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//build/go-runner:all-srcs", "//build/release-tars:all-srcs", "//build/visible_to:all-srcs", ], diff --git a/build/go-runner/BUILD b/build/go-runner/BUILD new file mode 100644 index 00000000000..1a3f5dc40bd --- /dev/null +++ b/build/go-runner/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["go-runner.go"], + importpath = "k8s.io/kubernetes/build/go-runner", + visibility = ["//visibility:private"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) + +go_binary( + name = "go-runner", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/build/go-runner/Dockerfile b/build/go-runner/Dockerfile new file mode 100644 index 00000000000..a9ad0ef3cd4 --- /dev/null +++ b/build/go-runner/Dockerfile @@ -0,0 +1,45 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the manager binary +FROM golang:1.13 as builder +WORKDIR /workspace + +# Run this with docker build --build_arg goproxy=$(go env GOPROXY) to override the goproxy +ARG goproxy=https://proxy.golang.org +# Run this with docker build --build_arg package=./controlplane/kubeadm or --build_arg package=./bootstrap/kubeadm +ENV GOPROXY=$goproxy + +# Copy the sources +COPY ./ ./ + +# Cache the go build +RUN go build . + +# Build +ARG package=. +ARG ARCH + +# Do not force rebuild of up-to-date packages (do not use -a) +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ + go build -ldflags '-s -w -buildid= -extldflags "-static"' \ + -o go-runner ${package} + +# Production image +FROM gcr.io/distroless/static:latest +LABEL maintainers="Kubernetes Authors" +LABEL description="go based runner for distroless scenarios" +WORKDIR / +COPY --from=builder /workspace/go-runner . +ENTRYPOINT ["/go-runner"] diff --git a/build/go-runner/Makefile b/build/go-runner/Makefile new file mode 100644 index 00000000000..32c56183d76 --- /dev/null +++ b/build/go-runner/Makefile @@ -0,0 +1,71 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set default shell +SHELL=/bin/bash -o pipefail + +TAG ?= 0.1.0 +REGISTRY ?= k8s.gcr.io + +IMGNAME = go-runner +IMAGE = $(REGISTRY)/$(IMGNAME) + +PLATFORMS = linux/amd64 linux/arm64 linux/arm linux/ppc64le linux/s390x + +HOST_GOOS ?= $(shell go env GOOS) +HOST_GOARCH ?= $(shell go env GOARCH) +GO_BUILD ?= go build + +.PHONY: all build clean + +.PHONY: all +all: build + +.PHONY: build +build: + $(GO_BUILD) + +.PHONY: clean +clean: + rm go-runner + +.PHONY: container +container: init-docker-buildx + # https://github.com/docker/buildx/issues/59 + $(foreach PLATFORM,$(PLATFORMS), \ + DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build \ + --load \ + --progress plain \ + --platform $(PLATFORM) \ + --tag $(IMAGE)-$(PLATFORM):$(TAG) .;) + +.PHONY: push +push: container + $(foreach PLATFORM,$(PLATFORMS), \ + docker push $(IMAGE)-$(PLATFORM):$(TAG);) + +.PHONY: manifest +manifest: push + docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(PLATFORMS) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g") + @for arch in $(PLATFORMS); do docker manifest annotate --arch "$${arch##*/}" ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done + docker manifest push --purge $(IMAGE):$(TAG) + +.PHONY: init-docker-buildx +init-docker-buildx: +ifneq ($(shell docker buildx 2>&1 >/dev/null; echo $?),) + $(error "buildx not vailable. Docker 19.03 or higher is required") +endif + docker run --rm --privileged linuxkit/binfmt:4ea3b9b0938cbd19834c096aa31ff475cc75d281 + docker buildx create --name multiarch-go-runner --use || true + docker buildx inspect --bootstrap diff --git a/build/go-runner/OWNERS b/build/go-runner/OWNERS new file mode 100644 index 00000000000..8760e69dd46 --- /dev/null +++ b/build/go-runner/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - build-image-approvers +reviewers: + - build-image-reviewers diff --git a/build/go-runner/README.md b/build/go-runner/README.md new file mode 100644 index 00000000000..0073d6f5267 --- /dev/null +++ b/build/go-runner/README.md @@ -0,0 +1,30 @@ +# Kubernetes go-runner image + +The Kubernetes go-runner image wraps the gcr.io/distroless/static image and provides a go based +binary that can run commands and wrap stdout/stderr etc. + +Why do we need this? Some of our images like kube-apiserver currently use bash for collecting +logs, so we are not able to switch to distroless images directly for these images. The klog's +`--log-file` was supposed to fix this problem, but we ran into trouble in scalability CI jobs +around log rotation and picked this option instead. we essentially publish a multi-arch +manifest with support for various platforms. This can be used as a base for other kubernetes +components. + +For example instead of running kube-apiserver like this: +```bash +"/bin/sh", + "-c", + "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1" +``` + +we would use go-runner like so: +```bash +"/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true", + "/usr/local/bin/kube-apiserver", + "--allow-privileged={{pillar['allow_privileged']}}", + {{params}} +``` + +The go-runner would then ensure that we run the `/usr/local/bin/kube-apiserver` with the +specified parameters and redirect stdout ONLY to the log file specified and ensure anything +logged to stderr also ends up in the log file. diff --git a/build/go-runner/cloudbuild.yaml b/build/go-runner/cloudbuild.yaml new file mode 100644 index 00000000000..088cd12ded9 --- /dev/null +++ b/build/go-runner/cloudbuild.yaml @@ -0,0 +1,22 @@ +# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md for more details on image pushing process + +# this must be specified in seconds. If omitted, defaults to 600s (10 mins) +timeout: 1200s +# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE + machineType: 'N1_HIGHCPU_8' +steps: + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200422-b25d964' + entrypoint: 'bash' + dir: ./build/go-runner + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - REGISTRY=gcr.io/$PROJECT_ID + - HOME=/root + args: + - '-c' + - | + gcloud auth configure-docker \ + && make manifest diff --git a/build/go-runner/go-runner.go b/build/go-runner/go-runner.go new file mode 100644 index 00000000000..feb141201c1 --- /dev/null +++ b/build/go-runner/go-runner.go @@ -0,0 +1,122 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" + + "github.com/pkg/errors" +) + +var ( + logFilePath = flag.String("log-file", "", "If non-empty, save stdout to this file") + alsoToStdOut = flag.Bool("also-stdout", false, "useful with log-file, log to standard output as well as the log file") + redirectStderr = flag.Bool("redirect-stderr", true, "treat stderr same as stdout") +) + +func main() { + flag.Parse() + + if err := configureAndRun(); err != nil { + log.Fatal(err) + } +} + +func configureAndRun() error { + var ( + outputStream io.Writer = os.Stdout + errStream io.Writer = os.Stderr + ) + + args := flag.Args() + if len(args) == 0 { + return errors.Errorf("not enough arguments to run") + } + + if logFilePath != nil && *logFilePath != "" { + logFile, err := os.OpenFile(*logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return errors.Wrapf(err, "failed to create log file %v", *logFilePath) + } + if *alsoToStdOut { + outputStream = io.MultiWriter(os.Stdout, logFile) + } else { + outputStream = logFile + } + } + + if *redirectStderr { + errStream = outputStream + } + + exe := args[0] + var exeArgs []string + if len(args) > 1 { + exeArgs = args[1:] + } + cmd := exec.Command(exe, exeArgs...) + cmd.Stdout = outputStream + cmd.Stderr = errStream + + log.Printf("Running command:\n%v", cmdInfo(cmd)) + err := cmd.Start() + if err != nil { + return errors.Wrap(err, "starting command") + } + + // Handle signals and shutdown process gracefully. + go setupSigHandler(cmd.Process) + return errors.Wrap(cmd.Wait(), "running command") +} + +// cmdInfo generates a useful look at what the command is for printing/debug. +func cmdInfo(cmd *exec.Cmd) string { + return fmt.Sprintf( + `Command env: (log-file=%v, also-stdout=%v, redirect-stderr=%v) +Run from directory: %v +Executable path: %v +Args (comma-delimited): %v`, *logFilePath, *alsoToStdOut, *redirectStderr, + cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","), + ) +} + +// setupSigHandler will forward any termination signals to the process +func setupSigHandler(process *os.Process) { + // terminationSignals are signals that cause the program to exit in the + // supported platforms (linux, darwin, windows). + terminationSignals := []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT} + + c := make(chan os.Signal, 1) + signal.Notify(c, terminationSignals...) + + // Block until a signal is received. + log.Println("Now listening for interrupts") + s := <-c + log.Printf("Got signal: %v. Sending down to process (PID: %v)", s, process.Pid) + if err := process.Signal(s); err != nil { + log.Fatalf("Failed to signal process: %v", err) + } + log.Printf("Signalled process %v successfully.", process.Pid) +} diff --git a/build/go-runner/go.mod b/build/go-runner/go.mod new file mode 100644 index 00000000000..90ef3eab359 --- /dev/null +++ b/build/go-runner/go.mod @@ -0,0 +1,5 @@ +module k8s.io/kubernetes/build/go-runner + +go 1.13 + +require github.com/pkg/errors v0.9.1 diff --git a/build/go-runner/go.sum b/build/go-runner/go.sum new file mode 100644 index 00000000000..7c401c3f58b --- /dev/null +++ b/build/go-runner/go.sum @@ -0,0 +1,2 @@ +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=