Install Salt from debs on GCS.

Also make downloading more reliable and run 'highstate' after install for good measure.  As part of this we no longer use gsutil to download and have to make 'staged' binaries in GCS publicly readable.

cluster/gce/templates/README.md

@@ -0,0 +1,12 @@
+# Updating Salt debs
+
+We are caching all of the salt debs in GCS for speed and reliability.
+
+To update them, follow this simple N step process:
+
+1. Start up a new base image without salt installed.  SSH into this image.
+2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo Csh -s -- -M -X`
+3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download`
+4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/`
+5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/`
+6. Test things well :)

cluster/gce/templates/common.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retry a download until we get it.
+#
+# $1 is the URL to download
+download-or-bust() {
+  until [[ -e "${1##*/}" ]]; do
+    echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
+    curl --ipv4 -LO --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
+  done
+}
+
+# Install salt from GCS.  See README.md for instructions on how to update these
+# debs.
+#
+# $1 If set to --master, also install the master
+install-salt() {
+  apt-get update
+
+  mkdir -p /var/cache/salt-install
+  cd /var/cache/salt-install
+
+  TARS=(
+    libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
+    python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
+    salt-common_2014.1.13+ds-1~bpo70+1_all.deb
+    salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
+  )
+  if [[ ${1-} == '--master' ]]; then
+    TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb)
+  fi
+  URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
+
+  for tar in "${TARS[@]}"; do
+    download-or-bust "${URL_BASE}/${tar}"
+    dpkg -i "${tar}"
+  done
+
+  # This will install any of the unmet dependencies from above.
+  apt-get install -f -y
+
+}

cluster/gce/templates/download-release.sh

@@ -22,10 +22,10 @@
 
 
 echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
-gsutil cp "$SERVER_BINARY_TAR_URL" .
+download-or-bust "$SERVER_BINARY_TAR_URL"
 
 echo "Downloading binary release tar ($SALT_TAR_URL)"
-gsutil cp "$SALT_TAR_URL" .
+download-or-bust "$SALT_TAR_URL"
 
 echo "Unpacking Salt tree"
 rm -rf kubernetes

cluster/gce/templates/salt-master.sh

@@ -21,6 +21,11 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
+
 cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
   roles:
@@ -41,12 +46,16 @@ reactor:
     - /srv/reactor/highstate-new.sls
 EOF
 
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-#
-# -M installs the master
-set +x
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 http://bootstrap.saltstack.com | sh -s -- -M -X
-set -x
+cat <<EOF >/etc/salt/master.d/log-level-debug.d
+log_level: debug
+log_level_logfile: debug
+EOF
+
+install-salt --master
+
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true
+

cluster/gce/templates/salt-minion.sh

@@ -22,8 +22,10 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
-# Turn on debugging for salt-minion
-# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
 
 # Our minions will have a pool role to distinguish them from the master.
 cat <<EOF >/etc/salt/minion.d/grains.conf
@@ -34,8 +36,10 @@ grains:
   cloud: gce
 EOF
 
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X
+install-salt
+
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true

cluster/gce/util.sh

@@ -121,10 +121,16 @@ function upload-server-tars() {
   local -r staging_path="${staging_bucket}/devel"
 
   echo "+++ Staging server tars to Google Storage: ${staging_path}"
-  SERVER_BINARY_TAR_URL="${staging_path}/${SERVER_BINARY_TAR##*/}"
-  gsutil -q cp "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_URL}"
-  SALT_TAR_URL="${staging_path}/${SALT_TAR##*/}"
-  gsutil -q cp "${SALT_TAR}" "${SALT_TAR_URL}"
+  local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
+  gsutil -q -h "Cache-Control:private, max-age=0" cp "${SERVER_BINARY_TAR}" "${server_binary_gs_url}"
+  gsutil acl ch -g all:R "${server_binary_gs_url}" >/dev/null 2>&1
+  local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
+  gsutil -q -h "Cache-Control:private, max-age=0" cp "${SALT_TAR}" "${salt_gs_url}"
+  gsutil acl ch -g all:R "${salt_gs_url}" >/dev/null 2>&1
+
+  # Convert from gs:// URL to an https:// URL
+  SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
+  SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}"
 }
 
 # Detect the information about the minions
@@ -287,6 +293,7 @@ function kube-up {
     echo "readonly PORTAL_NET='${PORTAL_NET}'"
     echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'"
     echo "readonly FLUENTD_GCP='${FLUENTD_GCP:-false}'"
+    grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh"
@@ -315,6 +322,7 @@ function kube-up {
       echo "#! /bin/bash"
       echo "MASTER_NAME='${MASTER_NAME}'"
       echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
+      grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
       grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh"
     ) > "${KUBE_TEMP}/minion-start-${i}.sh"
 
@@ -489,6 +497,7 @@ function kube-push {
     echo "cd /var/cache/kubernetes-install"
     echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
     echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
+    grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
     echo "echo Executing configuration"
     echo "sudo salt '*' mine.update"
@@ -586,7 +595,7 @@ function setup-monitoring {
 	    fi
 	fi
 
-	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null && 
+	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null &&
 	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-service.json" > /dev/null &&
 	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/heapster-pod.json" > /dev/null
 	if [ $? -ne 0 ]; then