diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index 4504c77941e..00d79c8326d 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -227,6 +227,47 @@ install_kube_binary_config() { rm "/run/kube-manifests/${manifests_tar}" } +# Assembles kubelet command line flags. +# It should be called by master and nodes before running kubelet process. The caller +# needs to source the config file /etc/kube-env. This function sets the following +# variable that will be used in kubelet command line. +# KUBELET_CMD_FLAGS +assemble_kubelet_flags() { + KUBELET_CMD_FLAGS="--v=2" + if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}" + fi + if [ -n "${KUBELET_PORT:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}" + fi + if [ -n "${KUBELET_TEST_ARGS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}" + fi + if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}" + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false" + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30" + else + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=${MASTER_IP_RANGE}" + fi + if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}" + fi + if [ "${KUBERNETES_MASTER:-}" = "true" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none" + elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \ + [ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \ + [ "${HAIRPIN_MODE:-}" = "none" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}" + fi + if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" + fi + if [ -n "${NODE_LABELS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}" + fi +} + restart_docker_daemon() { # Assemble docker deamon options DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" @@ -317,7 +358,7 @@ mount_master_pd() { # $2: token file add_token_entry() { current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "${tcurrent_token},$1,$1" >> $2 + echo "${current_token},$1,$1" >> $2 } # After the first boot and on upgrade, these files exists on the master-pd @@ -388,7 +429,7 @@ create_master_kubelet_auth() { # $4: value for variable 'cpulimit' # $5: pod name, which should be either etcd or etcd-events prepare_etcd_manifest() { - readonly etcd_temp_file="/tmp/$5" + etcd_temp_file="/tmp/$5" cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}" sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}" sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" @@ -418,11 +459,8 @@ start_etcd_servers() { prepare_log_file /var/log/etcd.log prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest" - # Switch on the second etcd instance if there are more than 50 nodes. - if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -gt 50 ]; then - prepare_log_file /var/log/etcd-events.log - prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" - fi + prepare_log_file /var/log/etcd-events.log + prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" } # Calculates the following variables based on env variables, which will be used @@ -459,9 +497,7 @@ start_kube_apiserver() { # Calculate variables and assemble the command line. params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true" - if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -gt 50 ]; then - params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002" - fi + params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002" if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi diff --git a/cluster/gce/trusty/master.yaml b/cluster/gce/trusty/master.yaml index 990e86f692b..120fcd2aa3e 100644 --- a/cluster/gce/trusty/master.yaml +++ b/cluster/gce/trusty/master.yaml @@ -1,4 +1,4 @@ -From nobody Tue Feb 1 11:33:00 2016 +From nobody Thu Mar 3 15:33:00 2016 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -122,30 +122,15 @@ script set -o nounset echo "Start kubelet upstart job" - + . /etc/kube-configure.sh . /etc/kube-env BINARY_PATH="/usr/bin/kubelet" if [ "${TEST_CLUSTER:-}" = "true" ]; then BINARY_PATH="/usr/local/bin/kubelet" fi - # Assemble command line flags based on env variables. - ARGS="--v=2" - if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then - ARGS="${KUBELET_TEST_LOG_LEVEL}" - fi - if [ -n "${KUBELET_TEST_ARGS:-}" ]; then - ARGS="${ARGS} ${KUBELET_TEST_ARGS}" - fi - if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then - ARGS="${ARGS} --api-servers=https://${KUBELET_APISERVER}" - ARGS="${ARGS} --register-schedulable=false --reconcile-cidr=false" - ARGS="${ARGS} --pod-cidr=10.123.45.0/30" - else - ARGS="${ARGS} --pod-cidr=${MASTER_IP_RANGE}" - fi - if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then - ARGS="${ARGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}" - fi + # Assemble command line flags based on env variables, which will put the string + # of flags in variable KUBELET_CMD_FLAGS + assemble_kubelet_flags ${BINARY_PATH} \ --enable-debugging-handlers=false \ @@ -159,8 +144,8 @@ script --system-cgroups=/system \ --runtime-cgroups=/docker-daemon \ --kubelet-cgroups=/kubelet \ - --nosystemd=true \ - ${ARGS} 1>>/var/log/kubelet.log 2>&1 + --babysit-daemons=true \ + ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1 end script # Wait for 10s to start kubelet again. @@ -236,7 +221,6 @@ script set -o nounset . /etc/kube-env - export HOME="/root" if [ "${TEST_CLUSTER:-}" = "true" ]; then export KUBECTL_BIN="/usr/local/bin/kubectl" @@ -265,6 +249,8 @@ description "Kubenetes master health monitoring" start on stopped kube-docker +respawn + script set -o errexit set -o nounset @@ -274,4 +260,7 @@ script health_monitoring end script +# Wait for 10s to start it again. +post-stop exec sleep 10 + --====================================-- diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index 364b2e1cbc4..e21f42d19d6 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -1,4 +1,4 @@ -From nobody Tue Feb 1 11:33:00 2016 +From nobody Thu Mar 3 15:33:00 2016 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -119,18 +119,17 @@ script set -o errexit set -o nounset + echo "Start kubelet upstart job" + . /etc/kube-configure.sh . /etc/kube-env - ARGS="--v=2" - if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then - ARGS="${KUBELET_TEST_LOG_LEVEL}" - fi - if [ -n "${KUBELET_TEST_ARGS:-}" ]; then - ARGS="${ARGS} ${KUBELET_TEST_ARGS}" - fi BINARY_PATH="/usr/bin/kubelet" if [ "${TEST_CLUSTER:-}" = "true" ]; then BINARY_PATH="/usr/local/bin/kubelet" fi + # Assemble command line flags based on env variables, which will put the string + # of flags in variable KUBELET_CMD_FLAGS. + assemble_kubelet_flags + ${BINARY_PATH} \ --api-servers=https://${KUBERNETES_MASTER_NAME} \ --enable-debugging-handlers=true \ @@ -144,8 +143,8 @@ script --system-cgroups=/system \ --runtime-cgroups=/docker-daemon \ --kubelet-cgroups=/kubelet \ - --nosystemd=true \ - ${ARGS} 1>>/var/log/kubelet.log 2>&1 + --babysit-daemons=true \ + ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1 end script # Wait for 10s to start kubelet again. @@ -272,14 +271,17 @@ respawn script set -o nounset + set -o errexit # Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization. # TODO(andyzheng0831): replace it with a more reliable method if possible. sleep 60 - . /etc/kube-configure.sh . /etc/kube-env health_monitoring end script +# Wait for 10s to start it again. +post-stop exec sleep 10 + --====================================--