Merge pull request #2245 from jbeda/gcs-deps

Start pulling external dependencies from GCS where possible

cluster/gce/templates/README.md

@@ -0,0 +1,12 @@
+# Updating Salt debs
+
+We are caching all of the salt debs in GCS for speed and reliability.
+
+To update them, follow this simple N step process:
+
+1. Start up a new base image without salt installed.  SSH into this image.
+2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo Csh -s -- -M -X`
+3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download`
+4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/`
+5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/`
+6. Test things well :)

cluster/gce/templates/common.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retry a download until we get it.
+#
+# $1 is the URL to download
+download-or-bust() {
+  until [[ -e "${1##*/}" ]]; do
+    echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
+    curl --ipv4 -LO --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
+  done
+}
+
+# Install salt from GCS.  See README.md for instructions on how to update these
+# debs.
+#
+# $1 If set to --master, also install the master
+install-salt() {
+  apt-get update
+
+  mkdir -p /var/cache/salt-install
+  cd /var/cache/salt-install
+
+  TARS=(
+    libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
+    python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
+    salt-common_2014.1.13+ds-1~bpo70+1_all.deb
+    salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
+  )
+  if [[ ${1-} == '--master' ]]; then
+    TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb)
+  fi
+  URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
+
+  for tar in "${TARS[@]}"; do
+    download-or-bust "${URL_BASE}/${tar}"
+    dpkg -i "${tar}"
+  done
+
+  # This will install any of the unmet dependencies from above.
+  apt-get install -f -y
+
+}

cluster/gce/templates/download-release.sh

@@ -22,10 +22,10 @@
 
 
 echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
-gsutil cp "$SERVER_BINARY_TAR_URL" .
+download-or-bust "$SERVER_BINARY_TAR_URL"
 
 echo "Downloading binary release tar ($SALT_TAR_URL)"
-gsutil cp "$SALT_TAR_URL" .
+download-or-bust "$SALT_TAR_URL"
 
 echo "Unpacking Salt tree"
 rm -rf kubernetes

cluster/gce/templates/salt-master.sh

@@ -21,6 +21,11 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
+
 cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
   roles:
@@ -41,12 +46,16 @@ reactor:
     - /srv/reactor/highstate-new.sls
 EOF
 
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-#
-# -M installs the master
-set +x
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 http://bootstrap.saltstack.com | sh -s -- -M -X
-set -x
+cat <<EOF >/etc/salt/master.d/log-level-debug.d
+log_level: debug
+log_level_logfile: debug
+EOF
+
+install-salt --master
+
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true
+

cluster/gce/templates/salt-minion.sh

@@ -22,8 +22,10 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
-# Turn on debugging for salt-minion
-# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
 
 # Our minions will have a pool role to distinguish them from the master.
 cat <<EOF >/etc/salt/minion.d/grains.conf
@@ -34,8 +36,10 @@ grains:
   cloud: gce
 EOF
 
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X
+install-salt
+
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true

cluster/gce/util.sh

@@ -121,10 +121,16 @@ function upload-server-tars() {
   local -r staging_path="${staging_bucket}/devel"
 
   echo "+++ Staging server tars to Google Storage: ${staging_path}"
-  SERVER_BINARY_TAR_URL="${staging_path}/${SERVER_BINARY_TAR##*/}"
-  gsutil -q cp "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_URL}"
-  SALT_TAR_URL="${staging_path}/${SALT_TAR##*/}"
-  gsutil -q cp "${SALT_TAR}" "${SALT_TAR_URL}"
+  local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
+  gsutil -q -h "Cache-Control:private, max-age=0" cp "${SERVER_BINARY_TAR}" "${server_binary_gs_url}"
+  gsutil acl ch -g all:R "${server_binary_gs_url}" >/dev/null 2>&1
+  local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
+  gsutil -q -h "Cache-Control:private, max-age=0" cp "${SALT_TAR}" "${salt_gs_url}"
+  gsutil acl ch -g all:R "${salt_gs_url}" >/dev/null 2>&1
+
+  # Convert from gs:// URL to an https:// URL
+  SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
+  SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}"
 }
 
 # Detect the information about the minions
@@ -287,6 +293,7 @@ function kube-up {
     echo "readonly PORTAL_NET='${PORTAL_NET}'"
     echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'"
     echo "readonly FLUENTD_GCP='${FLUENTD_GCP:-false}'"
+    grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh"
@@ -315,6 +322,7 @@ function kube-up {
       echo "#! /bin/bash"
       echo "MASTER_NAME='${MASTER_NAME}'"
       echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
+      grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
       grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh"
     ) > "${KUBE_TEMP}/minion-start-${i}.sh"
 
@@ -489,6 +497,7 @@ function kube-push {
     echo "cd /var/cache/kubernetes-install"
     echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
     echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
+    grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
     echo "echo Executing configuration"
     echo "sudo salt '*' mine.update"
@@ -586,7 +595,7 @@ function setup-monitoring {
 	    fi
 	fi
 
-	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null && 
+	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" > /dev/null &&
 	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-service.json" > /dev/null &&
 	kubectl.sh create -f "${KUBE_ROOT}/examples/monitoring/heapster-pod.json" > /dev/null
 	if [ $? -ne 0 ]; then

cluster/saltbase/salt/docker/init.sls

@@ -60,7 +60,7 @@ purge-old-docker:
 #    curl https://get.docker.com/ubuntu/dists/docker/main/binary-amd64/Packages
 # 2. Download based on that:
 #    curl -O https://get.docker.com/ubuntu/pool/main/<...>
-# 3. Upload to GCS (the cache control makes :
+# 3. Upload to GCS:
 #    gsutil cp <deb> gs://kubernetes-release/docker/<deb>
 # 4. Make it world readable:
 #    gsutil acl ch -R -g all:R gs://kubernetes-release/docker/<deb>

cluster/saltbase/salt/etcd/init.sls

@@ -1,7 +1,19 @@
+# We are caching the etcd tar file in GCS for reliability and speed.  To
+# update this to a new version, do the following:
+# 2. Download tar file:
+#    curl -LO https://github.com/coreos/etcd/releases/download/<ver>/etcd-<ver>-linux-amd64.tar.gz
+# 3. Upload to GCS (the cache control makes :
+#    gsutil cp <tar> gs://kubernetes-release/etcd/<tar>
+# 4. Make it world readable:
+#    gsutil -m acl ch -R -g all:R gs://kubernetes-release/etcd/
+# 5. Get a hash of the tar:
+#    shasum <tar>
+# 6. Update this file with new tar version and new hash
+
 {% set etcd_version="v0.4.6" %}
-{% set etcd_tar_url="https://github.com/coreos/etcd/releases/download/%s/etcd-%s-linux-amd64.tar.gz"
-  | format(etcd_version, etcd_version)  %}
-{% set etcd_tar_hash="md5=661d58424ff33dd837b8ee988dd79ae3" %}
+{% set etcd_tar_url="https://storage.googleapis.com/kubernetes-release/etcd/etcd-%s-linux-amd64.tar.gz"
+  | format(etcd_version)  %}
+{% set etcd_tar_hash="sha1=5db514e30b9f340eda00671230d5136855ae14d7" %}
 
 etcd-tar:
   archive: