From 887c9b18d6e96bd3a1e476e9f1723a63d2df08cb Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Fri, 7 Nov 2014 16:16:45 -0800 Subject: [PATCH] Install Salt from debs on GCS. Also make downloading more reliable and run 'highstate' after install for good measure. As part of this we no longer use gsutil to download and have to make 'staged' binaries in GCS publicly readable. --- cluster/gce/templates/README.md | 12 +++++ cluster/gce/templates/common.sh | 56 +++++++++++++++++++++++ cluster/gce/templates/download-release.sh | 4 +- cluster/gce/templates/salt-master.sh | 27 +++++++---- cluster/gce/templates/salt-minion.sh | 18 +++++--- cluster/gce/util.sh | 19 ++++++-- 6 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 cluster/gce/templates/README.md create mode 100644 cluster/gce/templates/common.sh diff --git a/cluster/gce/templates/README.md b/cluster/gce/templates/README.md new file mode 100644 index 00000000000..4400fc62518 --- /dev/null +++ b/cluster/gce/templates/README.md @@ -0,0 +1,12 @@ +# Updating Salt debs + +We are caching all of the salt debs in GCS for speed and reliability. + +To update them, follow this simple N step process: + +1. Start up a new base image without salt installed. SSH into this image. +2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo Csh -s -- -M -X` +3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download` +4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/` +5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/` +6. Test things well :) diff --git a/cluster/gce/templates/common.sh b/cluster/gce/templates/common.sh new file mode 100644 index 00000000000..784f58685d6 --- /dev/null +++ b/cluster/gce/templates/common.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Retry a download until we get it. +# +# $1 is the URL to download +download-or-bust() { + until [[ -e "${1##*/}" ]]; do + echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" + curl --ipv4 -LO --connect-timeout 20 --retry 6 --retry-delay 10 "$1" + done +} + +# Install salt from GCS. See README.md for instructions on how to update these +# debs. +# +# $1 If set to --master, also install the master +install-salt() { + apt-get update + + mkdir -p /var/cache/salt-install + cd /var/cache/salt-install + + TARS=( + libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb + python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb + salt-common_2014.1.13+ds-1~bpo70+1_all.deb + salt-minion_2014.1.13+ds-1~bpo70+1_all.deb + ) + if [[ ${1-} == '--master' ]]; then + TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb) + fi + URL_BASE="https://storage.googleapis.com/kubernetes-release/salt" + + for tar in "${TARS[@]}"; do + download-or-bust "${URL_BASE}/${tar}" + dpkg -i "${tar}" + done + + # This will install any of the unmet dependencies from above. + apt-get install -f -y + +} diff --git a/cluster/gce/templates/download-release.sh b/cluster/gce/templates/download-release.sh index 3982dd0f50e..5fadefdb26c 100755 --- a/cluster/gce/templates/download-release.sh +++ b/cluster/gce/templates/download-release.sh @@ -22,10 +22,10 @@ echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" -gsutil cp "$SERVER_BINARY_TAR_URL" . +download-or-bust "$SERVER_BINARY_TAR_URL" echo "Downloading binary release tar ($SALT_TAR_URL)" -gsutil cp "$SALT_TAR_URL" . +download-or-bust "$SALT_TAR_URL" echo "Unpacking Salt tree" rm -rf kubernetes diff --git a/cluster/gce/templates/salt-master.sh b/cluster/gce/templates/salt-master.sh index 94929554fad..38f93116211 100755 --- a/cluster/gce/templates/salt-master.sh +++ b/cluster/gce/templates/salt-master.sh @@ -21,6 +21,11 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf +cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF + cat </etc/salt/minion.d/grains.conf grains: roles: @@ -41,12 +46,16 @@ reactor: - /srv/reactor/highstate-new.sls EOF -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -# -# -M installs the master -set +x -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 http://bootstrap.saltstack.com | sh -s -- -M -X -set -x +cat </etc/salt/master.d/log-level-debug.d +log_level: debug +log_level_logfile: debug +EOF + +install-salt --master + +# Wait a few minutes and trigger another Salt run to better recover from +# any transient errors. +echo "Sleeping 180" +sleep 180 +salt-call state.highstate || true + diff --git a/cluster/gce/templates/salt-minion.sh b/cluster/gce/templates/salt-minion.sh index 7cc8176f32c..6e6e7d140c4 100755 --- a/cluster/gce/templates/salt-minion.sh +++ b/cluster/gce/templates/salt-minion.sh @@ -22,8 +22,10 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf -# Turn on debugging for salt-minion -# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion +cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF # Our minions will have a pool role to distinguish them from the master. cat </etc/salt/minion.d/grains.conf @@ -34,8 +36,10 @@ grains: cloud: gce EOF -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X +install-salt + +# Wait a few minutes and trigger another Salt run to better recover from +# any transient errors. +echo "Sleeping 180" +sleep 180 +salt-call state.highstate || true diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 805fd108fc2..a3e4bebb909 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -121,10 +121,16 @@ function upload-server-tars() { local -r staging_path="${staging_bucket}/devel" echo "+++ Staging server tars to Google Storage: ${staging_path}" - SERVER_BINARY_TAR_URL="${staging_path}/${SERVER_BINARY_TAR##*/}" - gsutil -q cp "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_URL}" - SALT_TAR_URL="${staging_path}/${SALT_TAR##*/}" - gsutil -q cp "${SALT_TAR}" "${SALT_TAR_URL}" + local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" + gsutil -q -h "Cache-Control:private, max-age=0" cp "${SERVER_BINARY_TAR}" "${server_binary_gs_url}" + gsutil acl ch -g all:R "${server_binary_gs_url}" >/dev/null 2>&1 + local salt_gs_url="${staging_path}/${SALT_TAR##*/}" + gsutil -q -h "Cache-Control:private, max-age=0" cp "${SALT_TAR}" "${salt_gs_url}" + gsutil acl ch -g all:R "${salt_gs_url}" >/dev/null 2>&1 + + # Convert from gs:// URL to an https:// URL + SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" + SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" } # Detect the information about the minions @@ -287,6 +293,7 @@ function kube-up { echo "readonly PORTAL_NET='${PORTAL_NET}'" echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'" echo "readonly FLUENTD_GCP='${FLUENTD_GCP:-false}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh" @@ -315,6 +322,7 @@ function kube-up { echo "#! /bin/bash" echo "MASTER_NAME='${MASTER_NAME}'" echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh" ) > "${KUBE_TEMP}/minion-start-${i}.sh" @@ -489,6 +497,7 @@ function kube-push { echo "cd /var/cache/kubernetes-install" echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" echo "echo Executing configuration" echo "sudo salt '*' mine.update" @@ -586,7 +595,7 @@ function setup-monitoring { fi fi - kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null && + kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null && kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-service.json" > /dev/null && kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/heapster-pod.json" > /dev/null if [ $? -ne 0 ]; then