diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index a5ee96a80b1..11d819abfeb 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -341,7 +341,7 @@ create_master_kubelet_auth() { # $5: pod name, which should be either etcd or etcd-events prepare_etcd_manifest() { etcd_temp_file="/tmp/$5" - cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}" + cp /home/kubernetes/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}" sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}" sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}" @@ -414,7 +414,7 @@ start_kube_apiserver() { prepare_log_file /var/log/kube-apiserver.log # Load the docker image from file. echo "Try to load docker image file kube-apiserver.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-apiserver.tar + timeout 30 docker load -i /home/kubernetes/kube-docker-files/kube-apiserver.tar # Calculate variables and assemble the command line. params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true" @@ -444,9 +444,9 @@ start_kube_apiserver() { readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" fi - readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag) + readonly kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag) - src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest" + src_file="/home/kubernetes/kube-manifests/kubernetes/trusty/kube-apiserver.manifest" remove_salt_config_comments "${src_file}" # Evaluate variables sed -i -e "s@{{params}}@${params}@g" "${src_file}" @@ -476,7 +476,7 @@ start_kube_controller_manager() { prepare_log_file /var/log/kube-controller-manager.log # Load the docker image from file. echo "Try to load docker image file kube-controller-manager.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-controller-manager.tar + timeout 30 docker load -i /home/kubernetes/kube-docker-files/kube-controller-manager.tar # Calculate variables and assemble the command line. params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key" @@ -503,9 +503,9 @@ start_kube_controller_manager() { if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then params="${params} ${CONTROLLER_MANAGER_TEST_ARGS}" fi - readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag) + readonly kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) - src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest" + src_file="/home/kubernetes/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest" remove_salt_config_comments "${src_file}" # Evaluate variables sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" @@ -527,9 +527,10 @@ start_kube_controller_manager() { # DOCKER_REGISTRY start_kube_scheduler() { prepare_log_file /var/log/kube-scheduler.log + kube_home="home/kubernetes" # Load the docker image from file. echo "Try to load docker image file kube-scheduler.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-scheduler.tar + timeout 30 docker load -i "${kube_home}/kube-docker-files/kube-scheduler.tar" # Calculate variables and set them in the manifest. params="" @@ -541,10 +542,10 @@ start_kube_scheduler() { if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then params="${params} ${SCHEDULER_TEST_ARGS}" fi - readonly kube_scheduler_docker_tag=$(cat /run/kube-docker-files/kube-scheduler.docker_tag) + readonly kube_scheduler_docker_tag=$(cat "${kube_home}/kube-docker-files/kube-scheduler.docker_tag") # Remove salt comments and replace variables with values - src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest" + src_file="${kube_home}/kube-manifests/kubernetes/trusty/kube-scheduler.manifest" remove_salt_config_comments "${src_file}" sed -i -e "s@{{params}}@${params}@g" "${src_file}" sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" @@ -556,9 +557,9 @@ start_kube_scheduler() { start_fluentd() { if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then - cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/ + cp /home/kubernetes/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/ elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then - cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/ + cp /home/kubernetes/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/ fi fi } @@ -568,7 +569,7 @@ start_fluentd() { # $1: addon category under /etc/kubernetes # $2: manifest source dir setup_addon_manifests() { - src_dir="/run/kube-manifests/kubernetes/trusty/$2" + src_dir="/home/kubernetes/kube-manifests/kubernetes/trusty/$2" dst_dir="/etc/kubernetes/$1/$2" if [ ! -d "${dst_dir}" ]; then mkdir -p "${dst_dir}" @@ -592,7 +593,7 @@ setup_addon_manifests() { # Prepares the manifests of k8s addons static pods. prepare_kube_addons() { - addon_src_dir="/run/kube-manifests/kubernetes/trusty" + addon_src_dir="/home/kubernetes/kube-manifests/kubernetes/trusty" addon_dst_dir="/etc/kubernetes/addons" # Set up manifests of other addons. if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \ diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index a5863f3b762..2329340d9cf 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -81,7 +81,7 @@ download_or_bust() { } # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, -# and places them into suitable directories. +# and places them into suitable directories. Files are placed in /home/kubernetes. install_kube_binary_config() { # Upstart does not support shell array well. Put urls in a temp file with one # url at a line, and we will use 'read' command to get them one-by-one. @@ -90,7 +90,9 @@ install_kube_binary_config() { tmp_manifests_urls=$(mktemp /tmp/kube-temp.XXXXXX) echo "${KUBE_MANIFESTS_TAR_URL}" | tr "," "\n" > "${tmp_manifests_urls}" - cd /tmp + kube_home="/home/kubernetes" + mkdir -p "${kube_home}" + cd "${kube_home}" read -r server_binary_tar_url < "${tmp_binary_urls}" readonly server_binary_tar="${server_binary_tar_url##*/}" if [ -n "${SERVER_BINARY_TAR_HASH:-}" ]; then @@ -102,37 +104,40 @@ install_kube_binary_config() { fi echo "Downloading binary release tar" download_or_bust "${server_binary_tar_hash}" "${tmp_binary_urls}" - tar xzf "/tmp/${server_binary_tar}" -C /tmp/ --overwrite - # Copy docker_tag and image files to /run/kube-docker-files. - mkdir -p /run/kube-docker-files - cp /tmp/kubernetes/server/bin/*.docker_tag /run/kube-docker-files/ + tar xzf "${kube_home}/${server_binary_tar}" -C "${kube_home}" --overwrite + # Copy docker_tag and image files to /home/kubernetes/kube-docker-files. + src_dir="${kube_home}/kubernetes/server/bin" + dst_dir="${kube_home}/kube-docker-files" + mkdir -p "${dst_dir}" + cp "${src_dir}/"*.docker_tag "${dst_dir}" if [ "${KUBERNETES_MASTER:-}" = "false" ]; then - cp /tmp/kubernetes/server/bin/kube-proxy.tar /run/kube-docker-files/ + cp "${src_dir}/kube-proxy.tar" "${dst_dir}" else - cp /tmp/kubernetes/server/bin/kube-apiserver.tar /run/kube-docker-files/ - cp /tmp/kubernetes/server/bin/kube-controller-manager.tar /run/kube-docker-files/ - cp /tmp/kubernetes/server/bin/kube-scheduler.tar /run/kube-docker-files/ - cp -r /tmp/kubernetes/addons /run/kube-docker-files/ + cp "${src_dir}/kube-apiserver.tar" "${dst_dir}" + cp "${src_dir}/kube-controller-manager.tar" "${dst_dir}" + cp "${src_dir}/kube-scheduler.tar" "${dst_dir}" + cp -r "${kube_home}/kubernetes/addons" "${dst_dir}" fi # Use the binary from the release tarball if they are not preinstalled, or if this is # a test cluster. readonly BIN_PATH="/usr/bin" if ! which kubelet > /dev/null || ! which kubectl > /dev/null; then - cp /tmp/kubernetes/server/bin/kubelet "${BIN_PATH}" - cp /tmp/kubernetes/server/bin/kubectl "${BIN_PATH}" + cp "${src_dir}/kubelet" "${BIN_PATH}" + cp "${src_dir}/kubectl" "${BIN_PATH}" elif [ "${TEST_CLUSTER:-}" = "true" ]; then - mkdir -p /home/kubernetes/bin - cp /tmp/kubernetes/server/bin/kubelet /home/kubernetes/bin - cp /tmp/kubernetes/server/bin/kubectl /home/kubernetes/bin - mount --bind /home/kubernetes/bin/kubelet "${BIN_PATH}/kubelet" + kube_bin="${kube_home}/bin" + mkdir -p "${kube_bin}" + cp "${src_dir}/kubelet" "${kube_bin}" + cp "${src_dir}/kubectl" "${kube_bin}" + mount --bind "${kube_bin}/kubelet" "${BIN_PATH}/kubelet" mount --bind -o remount,ro,^noexec "${BIN_PATH}/kubelet" "${BIN_PATH}/kubelet" - mount --bind /home/kubernetes/bin/kubectl "${BIN_PATH}/kubectl" + mount --bind "${kube_bin}/kubectl" "${BIN_PATH}/kubectl" mount --bind -o remount,ro,^noexec "${BIN_PATH}/kubectl" "${BIN_PATH}/kubectl" fi - # Put kube-system pods manifests in /etc/kube-manifests/. - mkdir -p /run/kube-manifests - cd /run/kube-manifests + # Put kube-system pods manifests in /home/kubernetes/kube-manifests/. + dst_dir="${kube_home}/kube-manifests" + mkdir -p "${dst_dir}" read -r manifests_tar_url < "${tmp_manifests_urls}" readonly manifests_tar="${manifests_tar_url##*/}" if [ -n "${KUBE_MANIFESTS_TAR_HASH:-}" ]; then @@ -144,22 +149,22 @@ install_kube_binary_config() { fi echo "Downloading k8s manifests tar" download_or_bust "${manifests_tar_hash}" "${tmp_manifests_urls}" - tar xzf "/run/kube-manifests/${manifests_tar}" -C /run/kube-manifests/ --overwrite + tar xzf "${kube_home}/${manifests_tar}" -C "${dst_dir}" --overwrite readonly kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}" if [ "${kube_addon_registry}" != "gcr.io/google_containers" ]; then - find /run/kube-manifests -name \*.yaml -or -name \*.yaml.in | \ + find "${dst_dir}" -name \*.yaml -or -name \*.yaml.in | \ xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@" - find /run/kube-manifests -name \*.manifest -or -name \*.json | \ + find "${dst_dir}" -name \*.manifest -or -name \*.json | \ xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" fi - cp /run/kube-manifests/kubernetes/trusty/configure-helper.sh /etc/kube-configure-helper.sh + cp "${dst_dir}/kubernetes/trusty/configure-helper.sh" /etc/kube-configure-helper.sh # Clean up. - rm -rf /tmp/kubernetes - rm -f "/tmp/${server_binary_tar}" - rm -f "/tmp/${server_binary_tar}.sha1" - rm -f "/run/kube-manifests/${manifests_tar}" - rm -f "/run/kube-manifests/${manifests_tar}.sha1" + rm -rf "${kube_home}/kubernetes" + rm -f "${kube_home}/${server_binary_tar}" + rm -f "${kube_home}/${server_binary_tar}.sha1" + rm -f "${kube_home}/${manifests_tar}" + rm -f "${kube_home}/${manifests_tar}.sha1" rm -f "${tmp_binary_urls}" rm -f "${tmp_manifests_urls}" } diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index 75f0dd71f15..3cba3fa25af 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -6,7 +6,7 @@ MIME-Version: 1.0 MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-install-minion.conf" +Content-Disposition: attachment; filename="kube-install-node.conf" #upstart-job @@ -41,7 +41,7 @@ Content-Disposition: attachment; filename="kube-env.conf" description "Prepare kube node environment" -start on stopped kube-install-minion +start on stopped kube-install-node script set -o errexit @@ -71,7 +71,7 @@ Content-Disposition: attachment; filename="kube-install-packages.conf" description "Install packages needed to run kubernetes" -start on stopped kube-install-minion +start on stopped kube-install-node script set -o errexit @@ -185,12 +185,12 @@ script . /etc/kube-configure-helper.sh . /etc/kube-env prepare_log_file "/var/log/kube-proxy.log" - # Load the docker image from file /run/kube-docker-files/kube-proxy.tar. + # Load the docker image from file /home/kubernetes/kube-docker-files/kube-proxy.tar. echo "Try to load docker image file kube-proxy.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-proxy.tar + timeout 30 docker load -i /home/kubernetes/kube-docker-files/kube-proxy.tar # Copy the manifest to /tmp to manipulate tmp_file="/tmp/kube-proxy.manifest" - cp -f /run/kube-manifests/kubernetes/kube-proxy.manifest ${tmp_file} + cp -f /home/kubernetes/kube-manifests/kubernetes/kube-proxy.manifest ${tmp_file} # Remove the lines of salt configuration and replace variables with values. # NOTE: Changes to variable names in cluster/saltbase/salt/kube-proxy/kube-proxy.manifest # may break this upstart job. @@ -200,7 +200,7 @@ script if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then kube_docker_registry=${KUBE_DOCKER_REGISTRY} fi - kube_proxy_docker_tag=$(cat /run/kube-docker-files/kube-proxy.docker_tag) + kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag) test_args="" if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then test_args="${KUBEPROXY_TEST_ARGS}" @@ -237,14 +237,14 @@ script set -o errexit set -o nounset - # Kube-system pod manifest files are located at /run/kube-manifests/kubernetes. . /etc/kube-configure-helper.sh . /etc/kube-env # Fluentd start_fluentd # Kube-registry-proxy if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then - cp /run/kube-manifests/kubernetes/kube-registry-proxy.yaml /etc/kubernetes/manifests/ + cp /home/kubernetes/kube-manifests/kubernetes/kube-registry-proxy.yaml \ + /etc/kubernetes/manifests/ fi end script