diff --git a/build/BUILD b/build/BUILD index 9fedd4328af..77c8fc20364 100644 --- a/build/BUILD +++ b/build/BUILD @@ -31,7 +31,7 @@ filegroup( # in build/common.sh. DOCKERIZED_BINARIES = { "kube-apiserver": { - "base": "@debian-base-{ARCH}//image", + "base": "@go-runner-linux-{ARCH}//image", "target": "//cmd/kube-apiserver:kube-apiserver", }, "kube-controller-manager": { @@ -39,7 +39,7 @@ DOCKERIZED_BINARIES = { "target": "//cmd/kube-controller-manager:kube-controller-manager", }, "kube-scheduler": { - "base": "@debian-base-{ARCH}//image", + "base": "@go-runner-linux-{ARCH}//image", "target": "//cmd/kube-scheduler:kube-scheduler", }, "kube-proxy": { diff --git a/build/common.sh b/build/common.sh index 5368f98f6f0..6862ac38427 100755 --- a/build/common.sh +++ b/build/common.sh @@ -96,12 +96,13 @@ kube::build::get_docker_wrapped_binaries() { local arch=$1 local debian_base_version=v2.1.0 local debian_iptables_version=v12.1.0 + local go_runner_version=v0.1.1 ### If you change any of these lists, please also update DOCKERIZED_BINARIES ### in build/BUILD. And kube::golang::server_image_targets local targets=( - "kube-apiserver,${KUBE_BASE_IMAGE_REGISTRY}/debian-base-${arch}:${debian_base_version}" + "kube-apiserver,${KUBE_BASE_IMAGE_REGISTRY}/go-runner:${go_runner_version}" "kube-controller-manager,${KUBE_BASE_IMAGE_REGISTRY}/debian-base-${arch}:${debian_base_version}" - "kube-scheduler,${KUBE_BASE_IMAGE_REGISTRY}/debian-base-${arch}:${debian_base_version}" + "kube-scheduler,${KUBE_BASE_IMAGE_REGISTRY}/go-runner:${go_runner_version}" "kube-proxy,${KUBE_BASE_IMAGE_REGISTRY}/debian-iptables-${arch}:${debian_iptables_version}" ) diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 5332f76df32..31e54376458 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -117,6 +117,20 @@ dependencies: - path: build/workspace.bzl match: tag = + - name: "k8s.gcr.io/go-runner" + version: 0.1.1 + refPaths: + - path: build/go-runner/Makefile + match: TAG \?= + + - name: "k8s.gcr.io/go-runner: dependents" + version: 0.1.1 + refPaths: + - path: build/common.sh + match: go_runner_version= + - path: build/workspace.bzl + match: tag = + - name: "k8s.gcr.io/pause" version: 3.3 refPaths: diff --git a/build/workspace.bzl b/build/workspace.bzl index 49947235d5d..020098cecbe 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -50,7 +50,7 @@ _ETCD_TARBALL_ARCH_SHA256 = { def release_dependencies(): cni_tarballs() cri_tarballs() - debian_image_dependencies() + image_dependencies() etcd_tarballs() def cni_tarballs(): @@ -99,14 +99,37 @@ _DEBIAN_IPTABLES_DIGEST = { "s390x": "sha256:1b91a2788750552913377bf1bc99a095544dfb523d80a55674003c974c8e0905", } +# Use skopeo to find these values: https://github.com/containers/skopeo +# +# Example +# Manifest: skopeo inspect docker://gcr.io/k8s-staging-build-image/go-runner:v0.1.1 +# Arches: skopeo inspect --raw docker://gcr.io/k8s-staging-build-image/go-runner:v0.1.1 +_GO_RUNNER_DIGEST = { + "manifest": "sha256:4892faa2de0533bc1af72b9b233936f21a9e7362063345d170de1a8f464f2ad8", + "amd64": "sha256:821e48a96d46aa53d2f7f5ef9d9093ed69979957a0a7092d1c09c44d81028a9d", + "arm": "sha256:2cc042179887b6baa0792e156b53f4cb94181b1a99153790402bd8e517e8cf56", + "arm64": "sha256:00ca7f34275349330a5d8ddffd15e2980fe5b2cbdd410f063f4e7617e0e71c29", + "ppc64le": "sha256:3e25e0d0e9d17033f3e86d4af5787c7fc5f1173e174d77eebdc14df1a06f1c99", + "s390x": "sha256:3e34e290cd35a90285991a575e2e79fddfb161c66f13bc5662a1cc0a4ade32e0", +} + def _digest(d, arch): if arch not in d: print("WARNING: %s not found in %r" % (arch, d)) return d["manifest"] return d[arch] -def debian_image_dependencies(): +def image_dependencies(): for arch in SERVER_PLATFORMS["linux"]: + container_pull( + name = "go-runner-linux-" + arch, + architecture = arch, + digest = _digest(_GO_RUNNER_DIGEST, arch), + registry = "us.gcr.io/k8s-artifacts-prod/build-image", + repository = "go-runner", + tag = "v0.1.1", # ignored, but kept here for documentation + ) + container_pull( name = "debian-base-" + arch, architecture = arch, diff --git a/cluster/gce/gci/apiserver_etcd_test.go b/cluster/gce/gci/apiserver_etcd_test.go index 8036e5ea896..c90936720e4 100644 --- a/cluster/gce/gci/apiserver_etcd_test.go +++ b/cluster/gce/gci/apiserver_etcd_test.go @@ -71,14 +71,14 @@ func TestServerOverride(t *testing.T) { c.mustInvokeFunc( tc.env, - kubeAPIServerConfigScriptName, + []string{"configure-helper.sh", kubeAPIServerConfigScriptName}, "etcd.template", "testdata/kube-apiserver/base.template", "testdata/kube-apiserver/etcd.template", ) c.mustLoadPodFromManifest() - execArgs := c.pod.Spec.Containers[0].Command[2] + execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ") for _, f := range tc.want { if !strings.Contains(execArgs, f) { t.Fatalf("Got %q, want it to contain %q", execArgs, f) @@ -127,14 +127,14 @@ func TestStorageOptions(t *testing.T) { c.mustInvokeFunc( tc.env, - kubeAPIServerConfigScriptName, + []string{"configure-helper.sh", kubeAPIServerConfigScriptName}, "etcd.template", "testdata/kube-apiserver/base.template", "testdata/kube-apiserver/etcd.template", ) c.mustLoadPodFromManifest() - execArgs := c.pod.Spec.Containers[0].Command[2] + execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ") for _, f := range tc.want { if !strings.Contains(execArgs, f) { t.Fatalf("Got %q, want it to contain %q", execArgs, f) @@ -191,14 +191,14 @@ func TestTLSFlags(t *testing.T) { c.mustInvokeFunc( tc.env, - kubeAPIServerConfigScriptName, + []string{"configure-helper.sh", kubeAPIServerConfigScriptName}, "etcd.template", "testdata/kube-apiserver/base.template", "testdata/kube-apiserver/etcd.template", ) c.mustLoadPodFromManifest() - execArgs := c.pod.Spec.Containers[0].Command[2] + execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ") for _, f := range tc.want { if !strings.Contains(execArgs, f) { t.Fatalf("Got %q, want it to contain %q", execArgs, f) diff --git a/cluster/gce/gci/apiserver_kms_test.go b/cluster/gce/gci/apiserver_kms_test.go index 84f5f5020c3..80817282186 100644 --- a/cluster/gce/gci/apiserver_kms_test.go +++ b/cluster/gce/gci/apiserver_kms_test.go @@ -45,11 +45,6 @@ type kubeAPIServerEnv struct { func TestEncryptionProviderFlag(t *testing.T) { var ( - // command": [ - // "/bin/sh", - Index 0 - // "-c", - Index 1 - // "exec /usr/local/bin/kube-apiserver " - Index 2 - execArgsIndex = 2 encryptionConfigFlag = "--encryption-provider-config" ) @@ -83,13 +78,13 @@ func TestEncryptionProviderFlag(t *testing.T) { c.mustInvokeFunc( e, - kubeAPIServerConfigScriptName, + []string{"configure-helper.sh", kubeAPIServerConfigScriptName}, "kms.template", "testdata/kube-apiserver/base.template", "testdata/kube-apiserver/kms.template") c.mustLoadPodFromManifest() - execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex] + execArgs := strings.Join(c.pod.Spec.Containers[0].Command, " ") flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag) flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath) @@ -118,7 +113,7 @@ func TestEncryptionProviderConfig(t *testing.T) { c.mustInvokeFunc( e, - kubeAPIServerConfigScriptName, + []string{"configure-helper.sh", kubeAPIServerConfigScriptName}, "kms.template", "testdata/kube-apiserver/base.template", @@ -189,7 +184,7 @@ func TestKMSIntegration(t *testing.T) { c.mustInvokeFunc( e, - kubeAPIServerConfigScriptName, + []string{"configure-helper.sh", kubeAPIServerConfigScriptName}, "kms.template", "testdata/kube-apiserver/base.template", diff --git a/cluster/gce/gci/audit_policy_test.go b/cluster/gce/gci/audit_policy_test.go index 2366cc37a3f..34e4abd27c2 100644 --- a/cluster/gce/gci/audit_policy_test.go +++ b/cluster/gce/gci/audit_policy_test.go @@ -54,7 +54,7 @@ func TestCreateMasterAuditPolicy(t *testing.T) { // Initialize required environment variables. c.mustInvokeFunc( kubeAPIServerEnv{KubeHome: c.kubeHome}, - "configure-helper.sh", + []string{"configure-helper.sh"}, "base.template", "testdata/kube-apiserver/base.template", ) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index cb4d7bc7a01..31beef02235 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -25,6 +25,24 @@ set -o errexit set -o nounset set -o pipefail +function convert-manifest-params { + # A helper function to convert the manifest args from a string to a list of + # flag arguments. + # Old format: + # command=["/bin/sh", "-c", "exec KUBE_EXEC_BINARY --param1=val1 --param2-val2"]. + # New format: + # command=["KUBE_EXEC_BINARY"] # No shell dependencies. + # args=["--param1=val1", "--param2-val2"] + IFS=' ' read -ra FLAGS <<< "$1" + params="" + for flag in "${FLAGS[@]}"; do + params+="\n\"$flag\"," + done + if [ ! -z $params ]; then + echo "${params::-1}" # drop trailing comma + fi +} + function setup-os-params { # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for @@ -1927,6 +1945,8 @@ function start-kube-scheduler { params+=" --use-legacy-policy-config" params+=" --policy-config-file=/etc/srv/kubernetes/kube-scheduler/policy-config" fi + + params="$(convert-manifest-params "${params}")" local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag") # Remove salt comments and replace variables with values. diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index 1e38baf0a8a..3f96028aa7e 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -354,6 +354,7 @@ function start-kube-apiserver { # params is passed by reference, so no "$" setup-etcd-encryption "${src_file}" params + params="$(convert-manifest-params "${params}")" # Evaluate variables. local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" sed -i -e "s@{{params}}@${params}@g" "${src_file}" diff --git a/cluster/gce/gci/configure_helper_test.go b/cluster/gce/gci/configure_helper_test.go index 47b82bb7a3b..3eba79b810e 100644 --- a/cluster/gce/gci/configure_helper_test.go +++ b/cluster/gce/gci/configure_helper_test.go @@ -106,15 +106,19 @@ func (c *ManifestTestCase) mustCreateManifestDstDir() { } } -func (c *ManifestTestCase) mustInvokeFunc(env interface{}, scriptName, targetTemplate string, templates ...string) { +func (c *ManifestTestCase) mustInvokeFunc(env interface{}, scriptNames []string, targetTemplate string, templates ...string) { envScriptPath := c.mustCreateEnv(env, targetTemplate, templates...) - args := fmt.Sprintf("source %q ; source %q; %s", envScriptPath, scriptName, c.manifestFuncName) + args := fmt.Sprintf("source %q ;", envScriptPath) + for _, script := range scriptNames { + args += fmt.Sprintf("source %q ;", script) + } + args += c.manifestFuncName cmd := exec.Command("bash", "-c", args) bs, err := cmd.CombinedOutput() if err != nil { c.t.Logf("%q", bs) - c.t.Fatalf("Failed to run %q: %v", scriptName, err) + c.t.Fatalf("Failed to run %q: %v", cmd.Args, err) } c.t.Logf("%s", string(bs)) } diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 4c2dd381904..04f400797ba 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -26,9 +26,10 @@ } }, "command": [ - "/bin/sh", - "-c", - "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1" + "/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true", + "/usr/local/bin/kube-apiserver", + "--allow-privileged={{pillar['allow_privileged']}}", + {{params}} ], {{container_env}} "livenessProbe": { diff --git a/cluster/gce/manifests/kube-scheduler.manifest b/cluster/gce/manifests/kube-scheduler.manifest index 6b75c7a412a..616d8457635 100644 --- a/cluster/gce/manifests/kube-scheduler.manifest +++ b/cluster/gce/manifests/kube-scheduler.manifest @@ -38,9 +38,9 @@ } }, "command": [ - "/bin/sh", - "-c", - "exec /usr/local/bin/kube-scheduler {{params}} 1>>/var/log/kube-scheduler.log 2>&1" + "/go-runner", "--log-file=/var/log/kube-scheduler.log", "--also-stdout=false", "--redirect-stderr=true", + "/usr/local/bin/kube-scheduler", + {{params}} ], "livenessProbe": { "httpGet": {