Switch core master base images from debian to distroless

Signed-off-by: Davanum Srinivas <davanum@gmail.com>

build/BUILD

@@ -31,7 +31,7 @@ filegroup(
 # in build/common.sh.
 DOCKERIZED_BINARIES = {
     "kube-apiserver": {
-        "base": "@debian-base-{ARCH}//image",
+        "base": "@go-runner-linux-{ARCH}//image",
         "target": "//cmd/kube-apiserver:kube-apiserver",
     },
     "kube-controller-manager": {
@@ -39,7 +39,7 @@ DOCKERIZED_BINARIES = {
         "target": "//cmd/kube-controller-manager:kube-controller-manager",
     },
     "kube-scheduler": {
-        "base": "@debian-base-{ARCH}//image",
+        "base": "@go-runner-linux-{ARCH}//image",
         "target": "//cmd/kube-scheduler:kube-scheduler",
     },
     "kube-proxy": {

build/common.sh

@@ -96,12 +96,13 @@ kube::build::get_docker_wrapped_binaries() {
   local arch=$1
   local debian_base_version=v2.1.0
   local debian_iptables_version=v12.1.0
+  local go_runner_version=v0.1.1
   ### If you change any of these lists, please also update DOCKERIZED_BINARIES
   ### in build/BUILD. And kube::golang::server_image_targets
   local targets=(
-    "kube-apiserver,${KUBE_BASE_IMAGE_REGISTRY}/debian-base-${arch}:${debian_base_version}"
+    "kube-apiserver,${KUBE_BASE_IMAGE_REGISTRY}/go-runner:${go_runner_version}"
     "kube-controller-manager,${KUBE_BASE_IMAGE_REGISTRY}/debian-base-${arch}:${debian_base_version}"
-    "kube-scheduler,${KUBE_BASE_IMAGE_REGISTRY}/debian-base-${arch}:${debian_base_version}"
+    "kube-scheduler,${KUBE_BASE_IMAGE_REGISTRY}/go-runner:${go_runner_version}"
     "kube-proxy,${KUBE_BASE_IMAGE_REGISTRY}/debian-iptables-${arch}:${debian_iptables_version}"
   )
 

build/dependencies.yaml

@@ -117,6 +117,20 @@ dependencies:
     - path: build/workspace.bzl
       match: tag =
 
+  - name: "k8s.gcr.io/go-runner"
+    version: 0.1.1
+    refPaths:
+    - path: build/go-runner/Makefile
+      match: TAG \?=
+
+  - name: "k8s.gcr.io/go-runner: dependents"
+    version: 0.1.1
+    refPaths:
+    - path: build/common.sh
+      match: go_runner_version=
+    - path: build/workspace.bzl
+      match: tag =
+
   - name: "k8s.gcr.io/pause"
     version: 3.3
     refPaths:

build/workspace.bzl

@@ -50,7 +50,7 @@ _ETCD_TARBALL_ARCH_SHA256 = {
 def release_dependencies():
     cni_tarballs()
     cri_tarballs()
-    debian_image_dependencies()
+    image_dependencies()
     etcd_tarballs()
 
 def cni_tarballs():
@@ -99,14 +99,37 @@ _DEBIAN_IPTABLES_DIGEST = {
     "s390x": "sha256:1b91a2788750552913377bf1bc99a095544dfb523d80a55674003c974c8e0905",
 }
 
+# Use skopeo to find these values: https://github.com/containers/skopeo
+#
+# Example
+# Manifest: skopeo inspect docker://gcr.io/k8s-staging-build-image/go-runner:v0.1.1
+# Arches: skopeo inspect --raw docker://gcr.io/k8s-staging-build-image/go-runner:v0.1.1
+_GO_RUNNER_DIGEST = {
+    "manifest": "sha256:4892faa2de0533bc1af72b9b233936f21a9e7362063345d170de1a8f464f2ad8",
+    "amd64": "sha256:821e48a96d46aa53d2f7f5ef9d9093ed69979957a0a7092d1c09c44d81028a9d",
+    "arm": "sha256:2cc042179887b6baa0792e156b53f4cb94181b1a99153790402bd8e517e8cf56",
+    "arm64": "sha256:00ca7f34275349330a5d8ddffd15e2980fe5b2cbdd410f063f4e7617e0e71c29",
+    "ppc64le": "sha256:3e25e0d0e9d17033f3e86d4af5787c7fc5f1173e174d77eebdc14df1a06f1c99",
+    "s390x": "sha256:3e34e290cd35a90285991a575e2e79fddfb161c66f13bc5662a1cc0a4ade32e0",
+}
+
 def _digest(d, arch):
     if arch not in d:
         print("WARNING: %s not found in %r" % (arch, d))
         return d["manifest"]
     return d[arch]
 
-def debian_image_dependencies():
+def image_dependencies():
     for arch in SERVER_PLATFORMS["linux"]:
+        container_pull(
+            name = "go-runner-linux-" + arch,
+            architecture = arch,
+            digest = _digest(_GO_RUNNER_DIGEST, arch),
+            registry = "us.gcr.io/k8s-artifacts-prod/build-image",
+            repository = "go-runner",
+            tag = "v0.1.1",  # ignored, but kept here for documentation
+        )
+
         container_pull(
             name = "debian-base-" + arch,
             architecture = arch,

cluster/gce/gci/apiserver_etcd_test.go

@@ -71,14 +71,14 @@ func TestServerOverride(t *testing.T) {
 
 			c.mustInvokeFunc(
 				tc.env,
-				kubeAPIServerConfigScriptName,
+				[]string{"configure-helper.sh", kubeAPIServerConfigScriptName},
 				"etcd.template",
 				"testdata/kube-apiserver/base.template",
 				"testdata/kube-apiserver/etcd.template",
 			)
 			c.mustLoadPodFromManifest()
 
-			execArgs := c.pod.Spec.Containers[0].Command[2]
+			execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ")
 			for _, f := range tc.want {
 				if !strings.Contains(execArgs, f) {
 					t.Fatalf("Got %q, want it to contain %q", execArgs, f)
@@ -127,14 +127,14 @@ func TestStorageOptions(t *testing.T) {
 
 			c.mustInvokeFunc(
 				tc.env,
-				kubeAPIServerConfigScriptName,
+				[]string{"configure-helper.sh", kubeAPIServerConfigScriptName},
 				"etcd.template",
 				"testdata/kube-apiserver/base.template",
 				"testdata/kube-apiserver/etcd.template",
 			)
 			c.mustLoadPodFromManifest()
 
-			execArgs := c.pod.Spec.Containers[0].Command[2]
+			execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ")
 			for _, f := range tc.want {
 				if !strings.Contains(execArgs, f) {
 					t.Fatalf("Got %q, want it to contain %q", execArgs, f)
@@ -191,14 +191,14 @@ func TestTLSFlags(t *testing.T) {
 
 			c.mustInvokeFunc(
 				tc.env,
-				kubeAPIServerConfigScriptName,
+				[]string{"configure-helper.sh", kubeAPIServerConfigScriptName},
 				"etcd.template",
 				"testdata/kube-apiserver/base.template",
 				"testdata/kube-apiserver/etcd.template",
 			)
 			c.mustLoadPodFromManifest()
 
-			execArgs := c.pod.Spec.Containers[0].Command[2]
+			execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ")
 			for _, f := range tc.want {
 				if !strings.Contains(execArgs, f) {
 					t.Fatalf("Got %q, want it to contain %q", execArgs, f)

cluster/gce/gci/apiserver_kms_test.go

@@ -45,11 +45,6 @@ type kubeAPIServerEnv struct {
 
 func TestEncryptionProviderFlag(t *testing.T) {
 	var (
-		//	command": [
-		//   "/bin/sh", - Index 0
-		//   "-c",      - Index 1
-		//   "exec /usr/local/bin/kube-apiserver " - Index 2
-		execArgsIndex        = 2
 		encryptionConfigFlag = "--encryption-provider-config"
 	)
 
@@ -83,13 +78,13 @@ func TestEncryptionProviderFlag(t *testing.T) {
 
 			c.mustInvokeFunc(
 				e,
-				kubeAPIServerConfigScriptName,
+				[]string{"configure-helper.sh", kubeAPIServerConfigScriptName},
 				"kms.template",
 				"testdata/kube-apiserver/base.template",
 				"testdata/kube-apiserver/kms.template")
 			c.mustLoadPodFromManifest()
 
-			execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex]
+			execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ")
 			flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag)
 			flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath)
 
@@ -118,7 +113,7 @@ func TestEncryptionProviderConfig(t *testing.T) {
 
 	c.mustInvokeFunc(
 		e,
-		kubeAPIServerConfigScriptName,
+		[]string{"configure-helper.sh", kubeAPIServerConfigScriptName},
 		"kms.template",
 
 		"testdata/kube-apiserver/base.template",
@@ -189,7 +184,7 @@ func TestKMSIntegration(t *testing.T) {
 
 			c.mustInvokeFunc(
 				e,
-				kubeAPIServerConfigScriptName,
+				[]string{"configure-helper.sh", kubeAPIServerConfigScriptName},
 				"kms.template",
 
 				"testdata/kube-apiserver/base.template",

cluster/gce/gci/audit_policy_test.go

@@ -54,7 +54,7 @@ func TestCreateMasterAuditPolicy(t *testing.T) {
 	// Initialize required environment variables.
 	c.mustInvokeFunc(
 		kubeAPIServerEnv{KubeHome: c.kubeHome},
-		"configure-helper.sh",
+		[]string{"configure-helper.sh"},
 		"base.template",
 		"testdata/kube-apiserver/base.template",
 	)

cluster/gce/gci/configure-helper.sh

@@ -25,6 +25,24 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+function convert-manifest-params {
+  # A helper function to convert the manifest args from a string to a list of
+  # flag arguments.
+  # Old format:
+  #   command=["/bin/sh", "-c", "exec KUBE_EXEC_BINARY --param1=val1 --param2-val2"].
+  # New format:
+  #   command=["KUBE_EXEC_BINARY"]  # No shell dependencies.
+  #   args=["--param1=val1", "--param2-val2"]
+  IFS=' ' read -ra FLAGS <<< "$1"
+  params=""
+  for flag in "${FLAGS[@]}"; do
+    params+="\n\"$flag\","
+  done
+  if [ ! -z $params ]; then
+    echo "${params::-1}"  #  drop trailing comma
+  fi
+}
+
 function setup-os-params {
   # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to
   # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for
@@ -1927,6 +1945,8 @@ function start-kube-scheduler {
     params+=" --use-legacy-policy-config"
     params+=" --policy-config-file=/etc/srv/kubernetes/kube-scheduler/policy-config"
   fi
+
+  params="$(convert-manifest-params "${params}")"
   local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag")
 
   # Remove salt comments and replace variables with values.

cluster/gce/gci/configure-kubeapiserver.sh

@@ -354,6 +354,7 @@ function start-kube-apiserver {
   # params is passed by reference, so no "$"
   setup-etcd-encryption "${src_file}" params
 
+  params="$(convert-manifest-params "${params}")"
   # Evaluate variables.
   local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"

cluster/gce/gci/configure_helper_test.go

@@ -106,15 +106,19 @@ func (c *ManifestTestCase) mustCreateManifestDstDir() {
 	}
 }
 
-func (c *ManifestTestCase) mustInvokeFunc(env interface{}, scriptName, targetTemplate string, templates ...string) {
+func (c *ManifestTestCase) mustInvokeFunc(env interface{}, scriptNames []string, targetTemplate string, templates ...string) {
 	envScriptPath := c.mustCreateEnv(env, targetTemplate, templates...)
-	args := fmt.Sprintf("source %q ; source %q; %s", envScriptPath, scriptName, c.manifestFuncName)
+	args := fmt.Sprintf("source %q ;", envScriptPath)
+	for _, script := range scriptNames {
+		args += fmt.Sprintf("source %q ;", script)
+	}
+	args += c.manifestFuncName
 	cmd := exec.Command("bash", "-c", args)
 
 	bs, err := cmd.CombinedOutput()
 	if err != nil {
 		c.t.Logf("%q", bs)
-		c.t.Fatalf("Failed to run %q: %v", scriptName, err)
+		c.t.Fatalf("Failed to run %q: %v", cmd.Args, err)
 	}
 	c.t.Logf("%s", string(bs))
 }

cluster/gce/manifests/kube-apiserver.manifest

@@ -26,9 +26,10 @@
       }
     },
     "command": [
-                 "/bin/sh",
-                 "-c",
-                 "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
+                 "/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true",
+                 "/usr/local/bin/kube-apiserver",
+                 "--allow-privileged={{pillar['allow_privileged']}}",
+                 {{params}}
                ],
     {{container_env}}
     "livenessProbe": {

cluster/gce/manifests/kube-scheduler.manifest

@@ -38,9 +38,9 @@
       }
     },
     "command": [
-                 "/bin/sh",
-                 "-c",
-                 "exec /usr/local/bin/kube-scheduler {{params}} 1>>/var/log/kube-scheduler.log 2>&1"
+                 "/go-runner", "--log-file=/var/log/kube-scheduler.log", "--also-stdout=false", "--redirect-stderr=true",
+                 "/usr/local/bin/kube-scheduler",
+                 {{params}}
                ],
     "livenessProbe": {
       "httpGet": {