From ccbb88fc539552cb3a12f3a83c602c9201ccf579 Mon Sep 17 00:00:00 2001 From: Yuwen Ma Date: Tue, 28 May 2019 22:52:24 -0700 Subject: [PATCH] Revert "Revert "[Re-Apply][Distroless] Convert the GCE manifests for master containers."" --- cluster/gce/gci/apiserver_manifest_test.go | 24 ++++++---- cluster/gce/gci/configure-helper.sh | 44 ++++++++++++++++--- cluster/gce/manifests/kube-apiserver.manifest | 10 +++-- .../kube-controller-manager.manifest | 9 ++-- cluster/gce/manifests/kube-scheduler.manifest | 9 ++-- 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/cluster/gce/gci/apiserver_manifest_test.go b/cluster/gce/gci/apiserver_manifest_test.go index 6bbdfce9908..3c7ce508428 100644 --- a/cluster/gce/gci/apiserver_manifest_test.go +++ b/cluster/gce/gci/apiserver_manifest_test.go @@ -94,11 +94,12 @@ func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv s func TestEncryptionProviderFlag(t *testing.T) { var ( - // command": [ - // "/bin/sh", - Index 0 - // "-c", - Index 1 - // "exec /usr/local/bin/kube-apiserver " - Index 2 - execArgsIndex = 2 + // command": [ + // "/usr/local/bin/kube-apiserver " - Index 0, + // "--flag1=val1", - Index 1, + // "--flag2=val2", - Index 2, + // ... + // "--flagN=valN", - Index N, encryptionConfigFlag = "--encryption-provider-config" ) @@ -132,10 +133,15 @@ func TestEncryptionProviderFlag(t *testing.T) { c.invokeTest(e, deployHelperEnv) - execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex] - flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag) - flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath) - + var flagIsInArg bool + var flag, execArgs string + for _, execArgs = range c.pod.Spec.Containers[0].Args[1:] { + if strings.Contains(execArgs, encryptionConfigFlag) { + flagIsInArg = true + flag = fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath) + break + } + } switch { case tc.wantFlag && !flagIsInArg: t.Fatalf("Got %q,\n want flags to contain %q", execArgs, flag) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c883a0ca3b2..99efe9ef6e2 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -25,6 +25,24 @@ set -o errexit set -o nounset set -o pipefail +function convert-manifest-params { + # A helper function to convert the manifest args from a string to a list of + # flag arguments. + # Old format: + # command=["/bin/sh", "-c", "exec KUBE_EXEC_BINARY --param1=val1 --param2-val2"]. + # New format: + # command=["KUBE_EXEC_BINARY"] # No shell dependencies. + # args=["--param1=val1", "--param2-val2"] + IFS=' ' read -ra FLAGS <<< "$1" + params="" + for flag in "${FLAGS[@]}"; do + params+="\n\"$flag\"," + done + if [ ! -z $params ]; then + echo "${params::-1}" # drop trailing comma + fi +} + function setup-os-params { # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for @@ -1842,6 +1860,10 @@ function start-kube-apiserver { # params is passed by reference, so no "$" setup-etcd-encryption "${src_file}" params + params+=" --log-file=${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" + params+=" --logtostderr=false" + params+=" --log-file-max-size=0" + params="$(convert-manifest-params "${params}")" # Evaluate variables. local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" sed -i -e "s@{{params}}@${params}@g" "${src_file}" @@ -2023,7 +2045,8 @@ function apply-encryption-config() { function start-kube-controller-manager { echo "Start kubernetes controller-manager" create-kubeconfig "kube-controller-manager" ${KUBE_CONTROLLER_MANAGER_TOKEN} - prepare-log-file /var/log/kube-controller-manager.log + local LOG_PATH=/var/log/kube-controller-manager.log + prepare-log-file "${LOG_PATH}" # Calculate variables and assemble the command line. local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" params+=" --use-service-account-credentials" @@ -2051,7 +2074,7 @@ function start-kube-controller-manager { params+=" --concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}" fi if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then - params+=" --allocate-node-cidrs=true" + params+=" --allocate-node-cidrs" elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" fi @@ -2082,9 +2105,14 @@ function start-kube-controller-manager { params+=" --pv-recycler-pod-template-filepath-hostpath=$PV_RECYCLER_OVERRIDE_TEMPLATE" fi if [[ -n "${RUN_CONTROLLERS:-}" ]]; then - params+=" --controllers=${RUN_CONTROLLERS}" + # Trim the `RUN_CONTROLLERS` value. This field is quoted which is + # incompatible with the `convert-manifest-params` format. + params+=" --controllers=${RUN_CONTROLLERS//\'}" fi - + params+=" --log-file=${LOG_PATH}" + params+=" --logtostderr=false" + params+=" --log-file-max-size=0" + params="$(convert-manifest-params "${params}")" local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) local container_env="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then @@ -2119,7 +2147,8 @@ function start-kube-controller-manager { function start-kube-scheduler { echo "Start kubernetes scheduler" create-kubeconfig "kube-scheduler" ${KUBE_SCHEDULER_TOKEN} - prepare-log-file /var/log/kube-scheduler.log + local LOG_PATH=/var/log/kube-scheduler.log + prepare-log-file "${LOG_PATH}" # Calculate variables and set them in the manifest. params="${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"} ${SCHEDULER_TEST_ARGS:-}" @@ -2135,6 +2164,11 @@ function start-kube-scheduler { params+=" --use-legacy-policy-config" params+=" --policy-config-file=/etc/srv/kubernetes/kube-scheduler/policy-config" fi + + params+=" --log-file=${LOG_PATH}" + params+=" --logtostderr=false" + params+=" --log-file-max-size=0" + params="$(convert-manifest-params "${params}")" local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag") # Remove salt comments and replace variables with values. diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 2ae818a0fcd..2b93941b8ba 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -25,10 +25,12 @@ } }, "command": [ - "/bin/sh", - "-c", - "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1" - ], + "/usr/local/bin/kube-apiserver" + ], + "args": [ + "--allow-privileged={{pillar['allow_privileged']}}", + {{params}} + ], {{container_env}} "livenessProbe": { "httpGet": { diff --git a/cluster/gce/manifests/kube-controller-manager.manifest b/cluster/gce/manifests/kube-controller-manager.manifest index be19cab1ce1..55850431276 100644 --- a/cluster/gce/manifests/kube-controller-manager.manifest +++ b/cluster/gce/manifests/kube-controller-manager.manifest @@ -25,10 +25,11 @@ } }, "command": [ - "/bin/sh", - "-c", - "exec /usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1" - ], + "/usr/local/bin/kube-controller-manager" + ], + "args": [ + {{params}} + ], {{container_env}} "livenessProbe": { "httpGet": { diff --git a/cluster/gce/manifests/kube-scheduler.manifest b/cluster/gce/manifests/kube-scheduler.manifest index a0648f9c189..e4acaacd029 100644 --- a/cluster/gce/manifests/kube-scheduler.manifest +++ b/cluster/gce/manifests/kube-scheduler.manifest @@ -25,10 +25,11 @@ } }, "command": [ - "/bin/sh", - "-c", - "exec /usr/local/bin/kube-scheduler {{params}} 1>>/var/log/kube-scheduler.log 2>&1" - ], + "/usr/local/bin/kube-scheduler" + ], + "args": [ + {{params}} + ], "livenessProbe": { "httpGet": { "host": "127.0.0.1",