github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-10 12:32:03 +00:00
Code Issues Projects Releases Wiki Activity

Fix docker run flags and kubelet flags for containized kubelet:

Browse Source 
- /var/lib/docker should be mount read-write, because kubelet needs to
  update /etc/resolv.conf file in container, see #29378
- /var/lib/kubelet should be mount rslave, because mountpoints under
  this directory must be propagated to other containers
- `--pid=host` is needed to run `nsenter --mount=/rootfs/proc/1/ns/mnt -- /usr/bin/systemd-run --scope` correctly
- configure cluster dns for kubelet running in a container
- should propagate sub-mounts of / to kubelet for local volume
- Use rslave to mount /var/lib/docker/ so we can see what the host
  mounts under this path.
- Keep containerized kubelet flags same as non-containerized and check
  kubelet is running or not.
- Use `all_kubelet_flags` to share kubelet flags.
This commit is contained in:
Yecheng Fu 2018-04-03 16:34:53 +08:00
parent 8b94183e90
commit 0f9b0beee7
1 changed files with 103 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
hack/local-up-cluster.sh
View File

@ -664,7 +664,6 @@ function start_kubelet {
fi fi
mkdir -p "/var/lib/kubelet" &>/dev/null || sudo mkdir -p "/var/lib/kubelet" mkdir -p "/var/lib/kubelet" &>/dev/null || sudo mkdir -p "/var/lib/kubelet"
if [[ -z "${DOCKERIZE_KUBELET}" ]]; then
# Enable dns # Enable dns
if [[ "${ENABLE_CLUSTER_DNS}" = true ]]; then if [[ "${ENABLE_CLUSTER_DNS}" = true ]]; then
dns_args="--cluster-dns=${DNS_SERVER_IP} --cluster-domain=${DNS_DOMAIN}" dns_args="--cluster-dns=${DNS_SERVER_IP} --cluster-domain=${DNS_DOMAIN}"
@ -674,7 +673,6 @@ function start_kubelet {
# DNS server for real world hostnames. # DNS server for real world hostnames.
dns_args="--cluster-dns=8.8.8.8" dns_args="--cluster-dns=8.8.8.8"
fi fi
net_plugin_args="" net_plugin_args=""
if [[ -n "${NET_PLUGIN}" ]]; then if [[ -n "${NET_PLUGIN}" ]]; then
net_plugin_args="--network-plugin=${NET_PLUGIN}" net_plugin_args="--network-plugin=${NET_PLUGIN}"
@ -711,42 +709,41 @@ function start_kubelet {
image_service_endpoint_args="--image-service-endpoint=${IMAGE_SERVICE_ENDPOINT}" image_service_endpoint_args="--image-service-endpoint=${IMAGE_SERVICE_ENDPOINT}"
fi fi
sudo -E "${GO_OUT}/hyperkube" kubelet ${priv_arg}\ all_kubelet_flags=(
--v=${LOG_LEVEL} \ ${priv_arg}
--vmodule="${LOG_SPEC}" \ --v="${LOG_LEVEL}"
--chaos-chance="${CHAOS_CHANCE}" \ --vmodule="${LOG_SPEC}"
--container-runtime="${CONTAINER_RUNTIME}" \ --chaos-chance="${CHAOS_CHANCE}"
--hostname-override="${HOSTNAME_OVERRIDE}" \ --container-runtime="${CONTAINER_RUNTIME}"
${cloud_config_arg} \ --hostname-override="${HOSTNAME_OVERRIDE}"
--address="${KUBELET_HOST}" \ ${cloud_config_arg}
--kubeconfig "$CERT_DIR"/kubelet.kubeconfig \ --address="${KUBELET_HOST}"
--feature-gates="${FEATURE_GATES}" \ --kubeconfig "$CERT_DIR"/kubelet.kubeconfig
--cpu-cfs-quota=${CPU_CFS_QUOTA} \ --feature-gates="${FEATURE_GATES}"
--enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" \ --cpu-cfs-quota="${CPU_CFS_QUOTA}"
--cgroups-per-qos=${CGROUPS_PER_QOS} \ --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}"
--cgroup-driver=${CGROUP_DRIVER} \ --cgroups-per-qos="${CGROUPS_PER_QOS}"
--keep-terminated-pod-volumes=${KEEP_TERMINATED_POD_VOLUMES} \ --cgroup-driver="${CGROUP_DRIVER}"
--eviction-hard=${EVICTION_HARD} \ --keep-terminated-pod-volumes="${KEEP_TERMINATED_POD_VOLUMES}"
--eviction-soft=${EVICTION_SOFT} \ --eviction-hard="${EVICTION_HARD}"
--eviction-pressure-transition-period=${EVICTION_PRESSURE_TRANSITION_PERIOD} \ --eviction-soft="${EVICTION_SOFT}"
--pod-manifest-path="${POD_MANIFEST_PATH}" \ --eviction-pressure-transition-period="${EVICTION_PRESSURE_TRANSITION_PERIOD}"
--fail-swap-on="${FAIL_SWAP_ON}" \ --pod-manifest-path="${POD_MANIFEST_PATH}"
${auth_args} \ --fail-swap-on="${FAIL_SWAP_ON}"
${dns_args} \ ${auth_args}
${cni_conf_dir_args} \ ${dns_args}
${cni_bin_dir_args} \ ${cni_conf_dir_args}
${net_plugin_args} \ ${cni_bin_dir_args}
${container_runtime_endpoint_args} \ ${net_plugin_args}
${image_service_endpoint_args} \ ${container_runtime_endpoint_args}
--port="$KUBELET_PORT" \ ${image_service_endpoint_args}
${KUBELET_FLAGS} >"${KUBELET_LOG}" 2>&1 & --port="$KUBELET_PORT"
${KUBELET_FLAGS}
)
if [[ -z "${DOCKERIZE_KUBELET}" ]]; then
sudo -E "${GO_OUT}/hyperkube" kubelet "${all_kubelet_flags[@]}" >"${KUBELET_LOG}" 2>&1 &
KUBELET_PID=$! KUBELET_PID=$!
# Quick check that kubelet is running.
if ps -p $KUBELET_PID > /dev/null ; then
echo "kubelet ( $KUBELET_PID ) is running."
else
cat ${KUBELET_LOG} ; exit 1
fi
else else
# Docker won't run a container with a cidfile (container id file) # Docker won't run a container with a cidfile (container id file)
# unless that file does not already exist; clean up an existing # unless that file does not already exist; clean up an existing
@ -767,22 +764,42 @@ function start_kubelet {
if [[ -n "${cloud_cred}" ]]; then if [[ -n "${cloud_cred}" ]]; then
cred_bind="--volume=${cloud_cred}:${cloud_cred}:ro" cred_bind="--volume=${cloud_cred}:${cloud_cred}:ro"
fi fi
all_kubelet_flags+=(--containerized)
docker run \ docker run --rm --name kubelet \
--volume=/:/rootfs:ro \ --volume=/:/rootfs:ro,rslave \
--volume=/var/run:/var/run:rw \ --volume=/var/run:/var/run:rw \
--volume=/sys:/sys:ro \ --volume=/sys:/sys:ro \
--volume=/var/lib/docker/:/var/lib/docker:ro \ --volume=/var/lib/docker/:/var/lib/docker:rslave \
--volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ --volume=/var/lib/kubelet/:/var/lib/kubelet:rslave \
--volume=/dev:/dev \ --volume=/dev:/dev \
--volume=/run/xtables.lock:/run/xtables.lock:rw \ --volume=/run/xtables.lock:/run/xtables.lock:rw \
${cred_bind} \ ${cred_bind} \
--net=host \ --net=host \
--pid=host \
--privileged=true \ --privileged=true \
-i \ -i \
--cidfile=$KUBELET_CIDFILE \ --cidfile=$KUBELET_CIDFILE \
k8s.gcr.io/kubelet \ k8s.gcr.io/kubelet \
/kubelet --v=${LOG_LEVEL} --containerized ${priv_arg}--chaos-chance="${CHAOS_CHANCE}" --pod-manifest-path="${POD_MANIFEST_PATH}" --hostname-override="${HOSTNAME_OVERRIDE}" ${cloud_config_arg} \ --address="127.0.0.1" --kubeconfig "$CERT_DIR"/kubelet.kubeconfig --port="$KUBELET_PORT" --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" &> $KUBELET_LOG & /kubelet "${all_kubelet_flags[@]}" >"${KUBELET_LOG}" 2>&1 &
# Get PID of kubelet container.
for i in {1..3}; do
echo -n "Trying to get PID of kubelet container..."
KUBELET_PID=$(docker inspect kubelet -f '{{.State.Pid}}' 2>/dev/null || true)
if [ -n "$KUBELET_PID" ]; then
echo " ok, $KUBELET_PID."
break
else
echo " failed, retry in 1 second."
sleep 1
fi
done
fi
# Quick check that kubelet is running.
if [ -n "$KUBELET_PID" ] && ps -p $KUBELET_PID > /dev/null; then
echo "kubelet ( $KUBELET_PID ) is running."
else
cat ${KUBELET_LOG} ; exit 1
fi fi
} }