Merge pull request #91171 from dims/switch-etcd-to-bash-static

Switch to static bash and distroless image for etcd

cluster/BUILD

@@ -17,6 +17,7 @@ filegroup(
         "//cluster/gce:all-srcs",
         "//cluster/images/conformance:all-srcs",
         "//cluster/images/etcd-version-monitor:all-srcs",
+        "//cluster/images/etcd/cp:all-srcs",
         "//cluster/images/etcd/migrate:all-srcs",
         "//cluster/images/kubemark:all-srcs",
     ],

cluster/images/etcd/Dockerfile

@@ -12,8 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM BASEIMAGE
+FROM BASEIMAGE as builder
+
+# This image needs bash for running "migrate-if-needed.sh". Instead of a full debian image
+# we use just the bash-static and we wrap bash-static into a distroless image instead of
+# a full debian image
+RUN apt-get update -y \
+	&& apt-get -yy -q install --no-install-recommends --no-install-suggests --fix-missing \
+		bash-static
+
+RUN cp /bin/bash-static /sh
+
+FROM RUNNERIMAGE
+WORKDIR /
+
+COPY --from=builder /sh /bin/
 
 EXPOSE 2379 2380 4001 7001
 COPY etcd* etcdctl* /usr/local/bin/
+COPY cp* /bin/
 COPY migrate-if-needed.sh migrate /usr/local/bin/

cluster/images/etcd/Makefile

@@ -34,7 +34,7 @@ LATEST_ETCD_VERSION?=3.4.7
 # REVISION provides a version number fo this image and all it's bundled
 # artifacts. It should start at zero for each LATEST_ETCD_VERSION and increment
 # for each revision of this image at that etcd version.
-REVISION?=1
+REVISION?=2
 
 # IMAGE_TAG Uniquely identifies k8s.gcr.io/etcd docker image with a tag of the form "<etcd-version>-<revision>".
 IMAGE_TAG=$(LATEST_ETCD_VERSION)-$(REVISION)
@@ -82,6 +82,8 @@ ifeq ($(ARCH),s390x)
     BASEIMAGE?=us.gcr.io/k8s-artifacts-prod/build-image/debian-base-s390x:v2.1.0
 endif
 
+RUNNERIMAGE?=gcr.io/distroless/static:latest
+
 build:
 	# Explicitly copy files to the temp directory
 	$(BIN_INSTALL) migrate-if-needed.sh $(TEMP_DIR)
@@ -91,7 +93,10 @@ build:
 	migrate_tmp_dir=$(shell mktemp -d); \
 	docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes$(DOCKER_VOL_OPTS) -v $${migrate_tmp_dir}:/build$(DOCKER_VOL_OPTS) -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
 		/bin/bash -c "CGO_ENABLED=0 go build -o /build/migrate k8s.io/kubernetes/cluster/images/etcd/migrate"; \
-	$(BIN_INSTALL) $${migrate_tmp_dir}/migrate $(TEMP_DIR)
+	$(BIN_INSTALL) $${migrate_tmp_dir}/migrate $(TEMP_DIR); \
+	docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes$(DOCKER_VOL_OPTS) -v $${migrate_tmp_dir}:/build$(DOCKER_VOL_OPTS) -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
+		/bin/bash -c "CGO_ENABLED=0 go build -o /build/cp k8s.io/kubernetes/cluster/images/etcd/cp"; \
+	$(BIN_INSTALL) $${migrate_tmp_dir}/cp $(TEMP_DIR);
 
 ifeq ($(ARCH),amd64)
 
@@ -140,6 +145,7 @@ endif
 
 	# Replace BASEIMAGE with the real base image
 	cd $(TEMP_DIR) && sed -i.bak 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile
+	cd $(TEMP_DIR) && sed -i.bak 's|RUNNERIMAGE|$(RUNNERIMAGE)|g' Dockerfile
 
 	# And build the image
 	docker build --pull -t $(REGISTRY)/etcd-$(ARCH):$(IMAGE_TAG) $(TEMP_DIR)

cluster/images/etcd/cloudbuild.yaml

@@ -0,0 +1,22 @@
+# See https://cloud.google.com/cloud-build/docs/build-config
+timeout: 1200s
+options:
+  substitution_option: ALLOW_LOOSE
+  machineType: 'N1_HIGHCPU_8'
+steps:
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200422-b25d964'
+    entrypoint: 'bash'
+    dir: ./cluster/images/etcd
+    env:
+      - DOCKER_CLI_EXPERIMENTAL=enabled
+      - REGISTRY=gcr.io/$PROJECT_ID
+      - PUSH_REGISTRY=gcr.io/$PROJECT_ID
+      - IMAGE=gcr.io/$PROJECT_ID/etcd
+      - BUILD_IMAGE=debian-build
+      - TMPDIR=/workspace
+    args:
+      - '-c'
+      - |
+        gcloud auth configure-docker \
+        && docker run --rm --privileged linuxkit/binfmt:4ea3b9b0938cbd19834c096aa31ff475cc75d281 \
+        && make all-push

cluster/images/etcd/cp/BUILD

@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["cp.go"],
+    importpath = "k8s.io/kubernetes/cluster/images/etcd/cp",
+    visibility = ["//visibility:private"],
+)
+
+go_binary(
+    name = "cp",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cluster/images/etcd/cp/cp.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		log.Fatal("Usage: cp SOURCE DEST")
+	}
+
+	sf, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatalf("unable to open source file [%s]: %q", os.Args[1], err)
+	}
+	defer sf.Close()
+	fi, err := sf.Stat()
+	if err != nil {
+		log.Fatalf("unable to stat source file [%s]: %q", os.Args[1], err)
+	}
+
+	dir := filepath.Dir(os.Args[2])
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		log.Fatalf("unable to create directory [%s]: %q", dir, err)
+	}
+	df, err := os.Create(os.Args[2])
+	if err != nil {
+		log.Fatalf("unable to create destination file [%s]: %q", os.Args[1], err)
+	}
+	defer df.Close()
+
+	_, err = io.Copy(df, sf)
+	if err != nil {
+		log.Fatalf("unable to copy [%s] to [%s]: %q", os.Args[1], os.Args[2], err)
+	}
+
+	if err := df.Close(); err != nil {
+		log.Fatalf("unable to close destination file: %q", err)
+	}
+	if err := os.Chmod(os.Args[2], fi.Mode()); err != nil {
+		log.Fatalf("unable to close destination file: %q", err)
+	}
+}

cluster/images/etcd/migrate-if-needed.sh

@@ -45,7 +45,8 @@ set -o nounset
 # etcd image (to make this script work correctly).
 BUNDLED_VERSIONS="3.0.17, 3.1.12, 3.2.24, 3.3.17, 3.4.7"
 
-ETCD_NAME="${ETCD_NAME:-etcd-$(hostname)}"
+# shellcheck disable=SC2039
+ETCD_NAME="${ETCD_NAME:-etcd-$HOSTNAME}"
 if [ -z "${DATA_DIRECTORY:-}" ]; then
   echo "DATA_DIRECTORY variable unset - unexpected failure"
   exit 1
@@ -87,8 +88,8 @@ ETCD_CREDS="${ETCD_CREDS:-}"
 
 # Correctly support upgrade and rollback to non-default version.
 if [ "${DO_NOT_MOVE_BINARIES:-}" != "true" ]; then
-  cp "/usr/local/bin/etcd-${TARGET_VERSION}" "/usr/local/bin/etcd"
-  cp "/usr/local/bin/etcdctl-${TARGET_VERSION}" "/usr/local/bin/etcdctl"
+  /bin/cp "/usr/local/bin/etcd-${TARGET_VERSION}" "/usr/local/bin/etcd"
+  /bin/cp "/usr/local/bin/etcdctl-${TARGET_VERSION}" "/usr/local/bin/etcdctl"
 fi
 
 /usr/local/bin/migrate \

cluster/images/etcd/migrate/BUILD

@@ -1,16 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_binary",
-    "go_library",
-    "go_test",
-)
-
-go_binary(
-    name = "migrate",
-    embed = [":go_default_library"],
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -20,9 +8,12 @@ go_library(
         "migrate_client.go",
         "migrate_server.go",
         "migrator.go",
+        "util_others.go",
+        "utils_windows.go",
         "versions.go",
     ],
     importpath = "k8s.io/kubernetes/cluster/images/etcd/migrate",
+    visibility = ["//visibility:private"],
     deps = [
         "//vendor/github.com/blang/semver:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
@@ -30,7 +21,59 @@ go_library(
         "//vendor/go.etcd.io/etcd/clientv3:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:ios": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//vendor/github.com/mrunalp/fileutils:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_binary(
+    name = "migrate",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "data_dir_test.go",
+        "versions_test.go",
     ],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = ["//vendor/github.com/blang/semver:go_default_library"],
 )
 
 filegroup(
@@ -44,15 +87,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "data_dir_test.go",
-        "versions_test.go",
-    ],
-    data = glob(["testdata/**"]),
-    embed = [":go_default_library"],
-    deps = ["//vendor/github.com/blang/semver:go_default_library"],
+    visibility = ["//visibility:public"],
 )

cluster/images/etcd/migrate/data_dir.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -88,7 +87,7 @@ func (d *DataDirectory) Backup() error {
 	if err != nil {
 		return err
 	}
-	err = exec.Command("cp", "-r", d.path, backupDir).Run()
+	err = copyDirectory(d.path, backupDir)
 	if err != nil {
 		return err
 	}

cluster/images/etcd/migrate/data_dir_test.go

@@ -132,6 +132,10 @@ func TestBackup(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to open data dir: %v", err)
 	}
+	_, err = os.Create(filepath.Join(path, "data-dir", "empty.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
 	err = d.Backup()
 	if err != nil {
 		t.Fatalf("Failed to backup data directory %s: %v", d.path, err)

cluster/images/etcd/migrate/migrator.go

@@ -19,7 +19,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"time"
 
 	"github.com/blang/semver"
@@ -159,7 +158,7 @@ func (m *Migrator) rollbackEtcd3MinorVersion(current *EtcdVersionPair, target *E
 	if err != nil {
 		return nil, err
 	}
-	err = exec.Command("mv", m.dataDirectory.path, backupDir).Run()
+	err = os.Rename(m.dataDirectory.path, backupDir)
 	if err != nil {
 		return nil, err
 	}

cluster/images/etcd/migrate/util_others.go

@@ -0,0 +1,27 @@
+// +build !windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/mrunalp/fileutils"
+)
+
+func copyDirectory(source string, dest string) error {
+	return fileutils.CopyDirectory(source, dest)
+}

cluster/images/etcd/migrate/utils_windows.go

@@ -0,0 +1,25 @@
+// +build windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import "fmt"
+
+func copyDirectory(source string, dest string) error {
+	return fmt.Errorf("no support for windows")
+}

go.mod

@@ -70,7 +70,7 @@ require (
 	github.com/miekg/dns v1.1.4
 	github.com/moby/ipvs v1.0.1
 	github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb // indirect
-	github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618 // indirect
+	github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/mvdan/xurls v1.1.0
 	github.com/onsi/ginkgo v1.11.0