gci: decouple from the built-in kubelet version

Prior to this change, configure.sh would:
(1) compare versions of built-in kubelet and downloaded kubelet, and
(2) bind-mount downloaded kubelet at /usr/bin/kubelet in case of
version mismatch

With this change, configure.sh:
(1) compares the two versions only on test clusters, and
(2) uses the actual file paths to start kubelet w/o any bind-mounting

To allow (2), this change also provides its own version of kubelet
systemd service file.

Effectively with this change we will always use the downloaded kubelet
binary along with its own systemd service file on non-test clusters.  The
main advantage is this change does not rely on the kubelet being built in to
the OS image.
This commit is contained in:
Amey Deshpande 2016-08-23 19:09:56 -07:00
parent f2e464fa2b
commit 2939ebdba5
4 changed files with 54 additions and 30 deletions

View File

@ -436,13 +436,26 @@ function load-docker-images {
fi
}
# A kubelet systemd service is built in GCI image, but by default it is not started
# when an instance is up. To start kubelet, the command line flags should be written
# to /etc/default/kubelet in the format "KUBELET_OPTS=<flags>", and then start kubelet
# using systemctl. This function assembles the command line and start the kubelet
# systemd service.
# This function assembles the kubelet systemd service file and starts it
# using systemctl.
function start-kubelet {
echo "Start kubelet"
local kubelet_bin="${KUBE_HOME}/bin/kubelet"
local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")"
local -r builtin_kubelet="/usr/bin/kubelet"
if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
# Determine which binary to use on test clusters. We use the built-in
# version only if the downloaded version is the same as the built-in
# version. This allows GCI to run some of the e2e tests to qualify the
# built-in kubelet.
if [[ -x "${builtin_kubelet}" ]]; then
local -r builtin_version="$("${builtin_kubelet}" --version=true | cut -f2 -d " ")"
if [[ "${builtin_version}" == "${version}" ]]; then
kubelet_bin="${builtin_kubelet}"
fi
fi
fi
echo "Using kubelet binary at ${kubelet_bin}"
local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
flags+=" --allow-privileged=true"
flags+=" --babysit-daemons=true"
@ -506,7 +519,26 @@ function start-kubelet {
if [[ -n "${FEATURE_GATES:-}" ]]; then
flags+=" --feature-gates=${FEATURE_GATES}"
fi
echo "KUBELET_OPTS=\"${flags}\"" > /etc/default/kubelet
local -r kubelet_env_file="/etc/default/kubelet"
echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
# Write the systemd service file for kubelet.
cat <<EOF >/etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes kubelet
Requires=network-online.target
After=network-online.target
[Service]
Restart=always
RestartSec=10
EnvironmentFile=${kubelet_env_file}
ExecStart=${kubelet_bin} \$KUBELET_OPTS
[Install]
WantedBy=multi-user.target
EOF
# Delete docker0 to avoid interference
iptables -t nat -F || true
@ -1070,7 +1102,7 @@ function start-rescheduler {
function reset-motd {
# kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
# This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1),
# or the git hash that's in the build info.
local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")"

View File

@ -128,32 +128,19 @@ function install-kube-binary-config {
cp -r "${KUBE_HOME}/kubernetes/addons" "${dst_dir}"
fi
local -r kube_bin="${KUBE_HOME}/bin"
# If the built-in binary version is different from the expected version, we use
# the downloaded binary. The simplest implementation is to always use the downloaded
# binary without checking the version. But we have another version guardian in GKE.
# So, we compare the versions to ensure this run-time binary replacement is only
# applied for OSS kubernetes.
cp "${src_dir}/kubelet" "${kube_bin}"
local -r builtin_version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
local -r required_version="$(/home/kubernetes/bin/kubelet --version=true | cut -f2 -d " ")"
if [[ "${TEST_CLUSTER:-}" == "true" ]] || \
[[ "${builtin_version}" != "${required_version}" ]]; then
cp "${src_dir}/kubectl" "${kube_bin}"
chmod 755 "${kube_bin}/kubelet"
chmod 755 "${kube_bin}/kubectl"
mount --bind "${kube_bin}/kubelet" /usr/bin/kubelet
mount --bind "${kube_bin}/kubectl" /usr/bin/kubectl
else
rm -f "${kube_bin}/kubelet"
fi
mv "${src_dir}/kubelet" "${kube_bin}"
mv "${src_dir}/kubectl" "${kube_bin}"
if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \
[[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
#TODO(andyzheng0831): We should make the cni version number as a k8s env variable.
local -r cni_tar="cni-8a936732094c0941e1543ef5d292a1f4fffa1ac5.tar.gz"
download-or-bust "" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}"
tar xzf "${KUBE_HOME}/${cni_tar}" -C "${kube_bin}" --overwrite
mv "${kube_bin}/bin"/* "${kube_bin}"
rmdir "${kube_bin}/bin"
local -r cni_dir="${KUBE_HOME}/cni"
mkdir -p "${cni_dir}"
tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}" --overwrite
mv "${cni_dir}/bin"/* "${kube_bin}"
rmdir "${cni_dir}/bin"
rm -f "${KUBE_HOME}/${cni_tar}"
fi
@ -184,8 +171,7 @@ function install-kube-binary-config {
fi
cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
chmod 544 "${KUBE_HOME}/bin/configure-helper.sh"
chmod 544 "${KUBE_HOME}/bin/health-monitor.sh"
chmod -R 755 "${kube_bin}"
# Clean up.
rm -rf "${KUBE_HOME}/kubernetes"

View File

@ -33,6 +33,7 @@ write_files:
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStart=/home/kubernetes/bin/configure-helper.sh
[Install]
@ -51,6 +52,7 @@ write_files:
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
[Install]
@ -69,6 +71,7 @@ write_files:
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
[Install]

View File

@ -33,6 +33,7 @@ write_files:
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStart=/home/kubernetes/bin/configure-helper.sh
[Install]
@ -51,6 +52,7 @@ write_files:
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
[Install]
@ -69,6 +71,7 @@ write_files:
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
[Install]