diff --git a/cluster/gce/templates/README.md b/cluster/gce/templates/README.md new file mode 100644 index 00000000000..4400fc62518 --- /dev/null +++ b/cluster/gce/templates/README.md @@ -0,0 +1,12 @@ +# Updating Salt debs + +We are caching all of the salt debs in GCS for speed and reliability. + +To update them, follow this simple N step process: + +1. Start up a new base image without salt installed. SSH into this image. +2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo Csh -s -- -M -X` +3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download` +4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/` +5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/` +6. Test things well :) diff --git a/cluster/gce/templates/common.sh b/cluster/gce/templates/common.sh new file mode 100644 index 00000000000..784f58685d6 --- /dev/null +++ b/cluster/gce/templates/common.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Retry a download until we get it. +# +# $1 is the URL to download +download-or-bust() { + until [[ -e "${1##*/}" ]]; do + echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" + curl --ipv4 -LO --connect-timeout 20 --retry 6 --retry-delay 10 "$1" + done +} + +# Install salt from GCS. See README.md for instructions on how to update these +# debs. +# +# $1 If set to --master, also install the master +install-salt() { + apt-get update + + mkdir -p /var/cache/salt-install + cd /var/cache/salt-install + + TARS=( + libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb + python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb + salt-common_2014.1.13+ds-1~bpo70+1_all.deb + salt-minion_2014.1.13+ds-1~bpo70+1_all.deb + ) + if [[ ${1-} == '--master' ]]; then + TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb) + fi + URL_BASE="https://storage.googleapis.com/kubernetes-release/salt" + + for tar in "${TARS[@]}"; do + download-or-bust "${URL_BASE}/${tar}" + dpkg -i "${tar}" + done + + # This will install any of the unmet dependencies from above. + apt-get install -f -y + +} diff --git a/cluster/gce/templates/download-release.sh b/cluster/gce/templates/download-release.sh index 3982dd0f50e..5fadefdb26c 100755 --- a/cluster/gce/templates/download-release.sh +++ b/cluster/gce/templates/download-release.sh @@ -22,10 +22,10 @@ echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" -gsutil cp "$SERVER_BINARY_TAR_URL" . +download-or-bust "$SERVER_BINARY_TAR_URL" echo "Downloading binary release tar ($SALT_TAR_URL)" -gsutil cp "$SALT_TAR_URL" . +download-or-bust "$SALT_TAR_URL" echo "Unpacking Salt tree" rm -rf kubernetes diff --git a/cluster/gce/templates/salt-master.sh b/cluster/gce/templates/salt-master.sh index 94929554fad..38f93116211 100755 --- a/cluster/gce/templates/salt-master.sh +++ b/cluster/gce/templates/salt-master.sh @@ -21,6 +21,11 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf +cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF + cat </etc/salt/minion.d/grains.conf grains: roles: @@ -41,12 +46,16 @@ reactor: - /srv/reactor/highstate-new.sls EOF -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -# -# -M installs the master -set +x -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 http://bootstrap.saltstack.com | sh -s -- -M -X -set -x +cat </etc/salt/master.d/log-level-debug.d +log_level: debug +log_level_logfile: debug +EOF + +install-salt --master + +# Wait a few minutes and trigger another Salt run to better recover from +# any transient errors. +echo "Sleeping 180" +sleep 180 +salt-call state.highstate || true + diff --git a/cluster/gce/templates/salt-minion.sh b/cluster/gce/templates/salt-minion.sh index 7cc8176f32c..6e6e7d140c4 100755 --- a/cluster/gce/templates/salt-minion.sh +++ b/cluster/gce/templates/salt-minion.sh @@ -22,8 +22,10 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf -# Turn on debugging for salt-minion -# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion +cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF # Our minions will have a pool role to distinguish them from the master. cat </etc/salt/minion.d/grains.conf @@ -34,8 +36,10 @@ grains: cloud: gce EOF -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X +install-salt + +# Wait a few minutes and trigger another Salt run to better recover from +# any transient errors. +echo "Sleeping 180" +sleep 180 +salt-call state.highstate || true diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 805fd108fc2..a3e4bebb909 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -121,10 +121,16 @@ function upload-server-tars() { local -r staging_path="${staging_bucket}/devel" echo "+++ Staging server tars to Google Storage: ${staging_path}" - SERVER_BINARY_TAR_URL="${staging_path}/${SERVER_BINARY_TAR##*/}" - gsutil -q cp "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_URL}" - SALT_TAR_URL="${staging_path}/${SALT_TAR##*/}" - gsutil -q cp "${SALT_TAR}" "${SALT_TAR_URL}" + local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" + gsutil -q -h "Cache-Control:private, max-age=0" cp "${SERVER_BINARY_TAR}" "${server_binary_gs_url}" + gsutil acl ch -g all:R "${server_binary_gs_url}" >/dev/null 2>&1 + local salt_gs_url="${staging_path}/${SALT_TAR##*/}" + gsutil -q -h "Cache-Control:private, max-age=0" cp "${SALT_TAR}" "${salt_gs_url}" + gsutil acl ch -g all:R "${salt_gs_url}" >/dev/null 2>&1 + + # Convert from gs:// URL to an https:// URL + SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" + SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" } # Detect the information about the minions @@ -287,6 +293,7 @@ function kube-up { echo "readonly PORTAL_NET='${PORTAL_NET}'" echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'" echo "readonly FLUENTD_GCP='${FLUENTD_GCP:-false}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh" @@ -315,6 +322,7 @@ function kube-up { echo "#! /bin/bash" echo "MASTER_NAME='${MASTER_NAME}'" echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh" ) > "${KUBE_TEMP}/minion-start-${i}.sh" @@ -489,6 +497,7 @@ function kube-push { echo "cd /var/cache/kubernetes-install" echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" echo "echo Executing configuration" echo "sudo salt '*' mine.update" @@ -586,7 +595,7 @@ function setup-monitoring { fi fi - kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null && + kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null && kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-service.json" > /dev/null && kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/heapster-pod.json" > /dev/null if [ $? -ne 0 ]; then