diff --git a/build/common.sh b/build/common.sh index 54d9866e234..1a98e0aa54b 100755 --- a/build/common.sh +++ b/build/common.sh @@ -864,9 +864,9 @@ function kube::release::package_salt_tarball() { # such as Ubuntu Trusty. # # There are two sources of manifests files: (1) some manifests in the directory -# cluster/saltbase/salt can be directly used on instances without salt, so we copy -# them from there; (2) for the ones containing salt config, we cannot directly -# use them. Therefore, we will maintain separate copies in cluster/gce/kube-manifests. +# cluster/saltbase/salt can be used directly or after minor revision, so we copy +# them from there; (2) otherwise, we will maintain separate copies in +# cluster/gce/kube-manifests. function kube::release::package_kube_manifests_tarball() { kube::log::status "Building tarball: manifests" @@ -880,9 +880,11 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/" cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/" cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/" + cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/" + # Source 2: manifests from cluster/gce/kube-manifests. # TODO(andyzheng0831): Enable the following line after finishing issue #16702. - # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/*" "${release_stage}/" + # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/"* "${release_stage}/" kube::release::clean_cruft diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index c7bafebfb74..146333c4e50 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -45,8 +45,10 @@ config_ip_firewall() { create_dirs() { # Create required directories. mkdir -p /var/lib/kubelet - mkdir -p /var/lib/kube-proxy mkdir -p /etc/kubernetes/manifests + if [ "${KUBERNETES_MASTER:-}" = "false" ]; then + mkdir -p /var/lib/kube-proxy + fi } download_kube_env() { @@ -65,7 +67,6 @@ for k,v in yaml.load(sys.stdin).iteritems(): create_kubelet_kubeconfig() { # Create the kubelet kubeconfig file. - . /etc/kube-env if [ -z "${KUBELET_CA_CERT:-}" ]; then KUBELET_CA_CERT="${CA_CERT}" fi @@ -158,38 +159,45 @@ download_or_bust() { # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, # and places them into suitable directories. install_kube_binary_config() { - . /etc/kube-env - # For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries, - # and place them in /usr/local/bin. For a non-test cluster, we use the binaries - # pre-installed in the image, or pull and place them in /usr/bin if they are - # not pre-installed. + # In anyway we have to download the release tarball as docker_tag files and + # kube-proxy image file are there. + cd /tmp + k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" + echo "Downloading k8s tar sha1 file ${k8s_sha1}" + download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1" + k8s_tar="${SERVER_BINARY_TAR_URL##*/}" + echo "Downloading k8s tar file ${k8s_tar}" + download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}" + # Validate hash. + actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true + if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then + echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} ==" + else + echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" + fi + tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite + # Copy docker_tag and image files to /run/kube-docker-files. + mkdir -p /run/kube-docker-files + cp /tmp/kubernetes/server/bin/*.docker_tag /run/kube-docker-files/ + if [ "${KUBERNETES_MASTER:-}" = "false" ]; then + cp /tmp/kubernetes/server/bin/kube-proxy.tar /run/kube-docker-files/ + fi + # For a testing cluster, we use kubelet, kube-proxy, and kubectl binaries + # from the release tarball and place them in /usr/local/bin. For a non-test + # cluster, we use the binaries pre-installed in the image, or pull and place + # them in /usr/bin if they are not pre-installed. BINARY_PATH="/usr/bin/" if [ "${TEST_CLUSTER:-}" = "true" ]; then BINARY_PATH="/usr/local/bin/" fi if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null || [ "${TEST_CLUSTER:-}" = "true" ]; then - cd /tmp - k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" - echo "Downloading k8s tar sha1 file ${k8s_sha1}" - download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1" - k8s_tar="${SERVER_BINARY_TAR_URL##*/}" - echo "Downloading k8s tar file ${k8s_tar}" - download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}" - # Validate hash. - actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true - if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then - echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} ==" - else - echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" - fi - tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite cp /tmp/kubernetes/server/bin/kubelet ${BINARY_PATH} - cp /tmp/kubernetes/server/bin/kube-proxy ${BINARY_PATH} cp /tmp/kubernetes/server/bin/kubectl ${BINARY_PATH} - rm -rf "/tmp/kubernetes" - rm "/tmp/${k8s_tar}" - rm "/tmp/${k8s_sha1}" fi + # Clean up. + rm -rf "/tmp/kubernetes" + rm "/tmp/${k8s_tar}" + rm "/tmp/${k8s_sha1}" # Put kube-system pods manifests in /etc/kube-manifests/. mkdir -p /run/kube-manifests @@ -213,7 +221,6 @@ install_kube_binary_config() { } restart_docker_daemon() { - . /etc/kube-env # Assemble docker deamon options DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" if [ "${TEST_CLUSTER:-}" = "true" ]; then @@ -230,3 +237,12 @@ restart_docker_daemon() { ifconfig docker0 down brctl delbr docker0 } + +# Create the log file and set its properties. +# +# $1 is the file to create +prepare_log_file() { + touch $1 + chmod 644 $1 + chown root:root $1 +} diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index 46c9f0ea1d8..36164625658 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -1,4 +1,4 @@ -From nobody Tue Aug 11 10:13:54 2015 +From nobody Tue Dec 22 10:13:54 2015 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -29,10 +29,11 @@ script config_hostname echo "Configuring IP firewall rules" config_ip_firewall - echo "Creating required directories" - create_dirs echo "Downloading kube-env file" download_kube_env + echo "Creating required directories" + . /etc/kube-env + create_dirs echo "Creating kubelet kubeconfig file" create_kubelet_kubeconfig echo "Creating kube-proxy kubeconfig file" @@ -96,6 +97,7 @@ script set -o nounset . /etc/kube-configure.sh + . /etc/kube-env install_kube_binary_config end script @@ -144,42 +146,6 @@ end script # Wait for 10s to start kubelet again. post-stop exec sleep 10 ---==================================== -MIME-Version: 1.0 -Content-Type: text/upstart-job; charset="us-ascii" -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-proxy.conf" - -#upstart-job - -description "Start kube-proxy service" - -start on stopped kube-install-minion and stopped kube-install-packages - -respawn - -script - set -o errexit - set -o nounset - - . /etc/kube-env - ARGS="--v=2" - if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then - ARGS="${KUBEPROXY_TEST_ARGS}" - fi - BINARY_PATH="/usr/bin/kube-proxy" - if [ "${TEST_CLUSTER:-}" = "true" ]; then - BINARY_PATH="/usr/local/bin/kube-proxy" - fi - ${BINARY_PATH} \ - --master=https://${KUBERNETES_MASTER_NAME} \ - --kubeconfig=/var/lib/kube-proxy/kubeconfig \ - ${ARGS} -end script - -# Wait for 10s to start kube-proxy again. -post-stop exec sleep 10 - --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" @@ -200,9 +166,63 @@ script set -o nounset . /etc/kube-configure.sh + . /etc/kube-env restart_docker_daemon end script +--==================================== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-proxy.conf" + +#upstart-job + +description "Start kube-proxy static pod" + +start on stopped kube-docker + +script + set -o errexit + set -o nounset + + . /etc/kube-configure.sh + . /etc/kube-env + prepare_log_file "/var/log/kube-proxy.log" + # Load the docker image from file /run/kube-docker-files/kube-proxy.tar. + echo "Try to load docker image file kube-proxy.tar" + timeout 30 docker load -i /run/kube-docker-files/kube-proxy.tar + # Copy the manifest to /tmp to manipulate + tmp_file="/tmp/kube-proxy.manifest" + cp -f /run/kube-manifests/kubernetes/kube-proxy.manifest ${tmp_file} + # Remove the lines of salt configuration and replace variables with values. + # NOTE: Changes to variable names in cluster/saltbase/salt/kube-proxy/kube-proxy.manifest + # may break this upstart job. + sed -i "/^ *{%/d" ${tmp_file} + kubeconfig="--kubeconfig=\/var\/lib\/kube-proxy\/kubeconfig" + kube_docker_registry="gcr.io\/google_containers" + if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then + kube_docker_registry=${KUBE_DOCKER_REGISTRY} + fi + kube_proxy_docker_tag=$(cat /run/kube-docker-files/kube-proxy.docker_tag) + test_args="" + log_level="--v=2" + if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then + test_args="${KUBEPROXY_TEST_ARGS}" + # test_args should already contain log level setting. + log_level="" + fi + api_servers="--master=https:\/\/${KUBERNETES_MASTER_NAME}" + sed -i -e "s/{{kubeconfig}}/${kubeconfig}/g" ${tmp_file} + sed -i -e "s/{{pillar\['kube_docker_registry'\]}}/${kube_docker_registry}/g" ${tmp_file} + sed -i -e "s/{{pillar\['kube-proxy_docker_tag'\]}}/${kube_proxy_docker_tag}/g" ${tmp_file} + sed -i -e "s/{{test_args}}/${test_args}/g" ${tmp_file} + sed -i -e "s/{{log_level}}/${log_level}/g" ${tmp_file} + sed -i -e "s/{{api_servers_with_port}}/${api_servers}/g" ${tmp_file} + + mv -f ${tmp_file} /etc/kubernetes/manifests/ +end script + --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" @@ -243,7 +263,8 @@ Content-Disposition: attachment; filename="kube-node-health-monitoring.conf" description "Kubenetes node health monitoring" -start on stopped kube-docker and started kube-proxy +# The termination of kube-proxy upstart job indicates that kubelet and docker are ready. +start on stopped kube-proxy respawn @@ -254,6 +275,7 @@ script # TODO(andyzheng0831): replace it with a more reliable method if possible. sleep 60 + . /etc/kube-env sleep_seconds=10 max_seconds=10 # We simply kill the process when there is a failure. Another upstart job will automatically @@ -263,14 +285,15 @@ script echo "Docker daemon failed!" pkill docker fi - . /etc/kube-env if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then echo "Kubelet is unhealthy!" pkill kubelet fi if ! curl -m ${max_seconds} -f -s http://127.0.0.1:10249/healthz > /dev/null; then echo "Kube-proxy is unhealthy!" - pkill kube-proxy + # Get the ID of kube-proxy container and then kill it. + container=$(docker ps -q --filter name='k8s_kube-proxy') + docker kill ${container} fi sleep ${sleep_seconds} done