From 2939ebdba5d5909f92461a7986c99043244325c0 Mon Sep 17 00:00:00 2001 From: Amey Deshpande Date: Tue, 23 Aug 2016 19:09:56 -0700 Subject: [PATCH] gci: decouple from the built-in kubelet version Prior to this change, configure.sh would: (1) compare versions of built-in kubelet and downloaded kubelet, and (2) bind-mount downloaded kubelet at /usr/bin/kubelet in case of version mismatch With this change, configure.sh: (1) compares the two versions only on test clusters, and (2) uses the actual file paths to start kubelet w/o any bind-mounting To allow (2), this change also provides its own version of kubelet systemd service file. Effectively with this change we will always use the downloaded kubelet binary along with its own systemd service file on non-test clusters. The main advantage is this change does not rely on the kubelet being built in to the OS image. --- cluster/gce/gci/configure-helper.sh | 46 ++++++++++++++++++++++++----- cluster/gce/gci/configure.sh | 32 ++++++-------------- cluster/gce/gci/master.yaml | 3 ++ cluster/gce/gci/node.yaml | 3 ++ 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 57430cb9149..decc290732b 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -436,13 +436,26 @@ function load-docker-images { fi } -# A kubelet systemd service is built in GCI image, but by default it is not started -# when an instance is up. To start kubelet, the command line flags should be written -# to /etc/default/kubelet in the format "KUBELET_OPTS=", and then start kubelet -# using systemctl. This function assembles the command line and start the kubelet -# systemd service. +# This function assembles the kubelet systemd service file and starts it +# using systemctl. function start-kubelet { echo "Start kubelet" + local kubelet_bin="${KUBE_HOME}/bin/kubelet" + local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")" + local -r builtin_kubelet="/usr/bin/kubelet" + if [[ "${TEST_CLUSTER:-}" == "true" ]]; then + # Determine which binary to use on test clusters. We use the built-in + # version only if the downloaded version is the same as the built-in + # version. This allows GCI to run some of the e2e tests to qualify the + # built-in kubelet. + if [[ -x "${builtin_kubelet}" ]]; then + local -r builtin_version="$("${builtin_kubelet}" --version=true | cut -f2 -d " ")" + if [[ "${builtin_version}" == "${version}" ]]; then + kubelet_bin="${builtin_kubelet}" + fi + fi + fi + echo "Using kubelet binary at ${kubelet_bin}" local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}" flags+=" --allow-privileged=true" flags+=" --babysit-daemons=true" @@ -506,7 +519,26 @@ function start-kubelet { if [[ -n "${FEATURE_GATES:-}" ]]; then flags+=" --feature-gates=${FEATURE_GATES}" fi - echo "KUBELET_OPTS=\"${flags}\"" > /etc/default/kubelet + + local -r kubelet_env_file="/etc/default/kubelet" + echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}" + + # Write the systemd service file for kubelet. + cat </etc/systemd/system/kubelet.service +[Unit] +Description=Kubernetes kubelet +Requires=network-online.target +After=network-online.target + +[Service] +Restart=always +RestartSec=10 +EnvironmentFile=${kubelet_env_file} +ExecStart=${kubelet_bin} \$KUBELET_OPTS + +[Install] +WantedBy=multi-user.target +EOF # Delete docker0 to avoid interference iptables -t nat -F || true @@ -1070,7 +1102,7 @@ function start-rescheduler { function reset-motd { # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl) - local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")" + local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")" # This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1), # or the git hash that's in the build info. local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")" diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 3e29dce5a51..c29bc9aeaa8 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -128,32 +128,19 @@ function install-kube-binary-config { cp -r "${KUBE_HOME}/kubernetes/addons" "${dst_dir}" fi local -r kube_bin="${KUBE_HOME}/bin" - # If the built-in binary version is different from the expected version, we use - # the downloaded binary. The simplest implementation is to always use the downloaded - # binary without checking the version. But we have another version guardian in GKE. - # So, we compare the versions to ensure this run-time binary replacement is only - # applied for OSS kubernetes. - cp "${src_dir}/kubelet" "${kube_bin}" - local -r builtin_version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")" - local -r required_version="$(/home/kubernetes/bin/kubelet --version=true | cut -f2 -d " ")" - if [[ "${TEST_CLUSTER:-}" == "true" ]] || \ - [[ "${builtin_version}" != "${required_version}" ]]; then - cp "${src_dir}/kubectl" "${kube_bin}" - chmod 755 "${kube_bin}/kubelet" - chmod 755 "${kube_bin}/kubectl" - mount --bind "${kube_bin}/kubelet" /usr/bin/kubelet - mount --bind "${kube_bin}/kubectl" /usr/bin/kubectl - else - rm -f "${kube_bin}/kubelet" - fi + mv "${src_dir}/kubelet" "${kube_bin}" + mv "${src_dir}/kubectl" "${kube_bin}" + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \ [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then #TODO(andyzheng0831): We should make the cni version number as a k8s env variable. local -r cni_tar="cni-8a936732094c0941e1543ef5d292a1f4fffa1ac5.tar.gz" download-or-bust "" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}" - tar xzf "${KUBE_HOME}/${cni_tar}" -C "${kube_bin}" --overwrite - mv "${kube_bin}/bin"/* "${kube_bin}" - rmdir "${kube_bin}/bin" + local -r cni_dir="${KUBE_HOME}/cni" + mkdir -p "${cni_dir}" + tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}" --overwrite + mv "${cni_dir}/bin"/* "${kube_bin}" + rmdir "${cni_dir}/bin" rm -f "${KUBE_HOME}/${cni_tar}" fi @@ -184,8 +171,7 @@ function install-kube-binary-config { fi cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh" - chmod 544 "${KUBE_HOME}/bin/configure-helper.sh" - chmod 544 "${KUBE_HOME}/bin/health-monitor.sh" + chmod -R 755 "${kube_bin}" # Clean up. rm -rf "${KUBE_HOME}/kubernetes" diff --git a/cluster/gce/gci/master.yaml b/cluster/gce/gci/master.yaml index b3c283f5f7b..845973df84b 100644 --- a/cluster/gce/gci/master.yaml +++ b/cluster/gce/gci/master.yaml @@ -33,6 +33,7 @@ write_files: [Service] Type=oneshot RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh ExecStart=/home/kubernetes/bin/configure-helper.sh [Install] @@ -51,6 +52,7 @@ write_files: RestartSec=10 RemainAfterExit=yes RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh ExecStart=/home/kubernetes/bin/health-monitor.sh docker [Install] @@ -69,6 +71,7 @@ write_files: RestartSec=10 RemainAfterExit=yes RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet [Install] diff --git a/cluster/gce/gci/node.yaml b/cluster/gce/gci/node.yaml index 803f8409bfa..de59fcac27b 100644 --- a/cluster/gce/gci/node.yaml +++ b/cluster/gce/gci/node.yaml @@ -33,6 +33,7 @@ write_files: [Service] Type=oneshot RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh ExecStart=/home/kubernetes/bin/configure-helper.sh [Install] @@ -51,6 +52,7 @@ write_files: RestartSec=10 RemainAfterExit=yes RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh ExecStart=/home/kubernetes/bin/health-monitor.sh docker [Install] @@ -69,6 +71,7 @@ write_files: RestartSec=10 RemainAfterExit=yes RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet [Install]