From ed567736ea3f3b4bc9e489f7bd06b65fe0aa490a Mon Sep 17 00:00:00 2001
From: Zach Loafman <zml@google.com>
Date: Tue, 24 Mar 2015 16:11:40 -0700
Subject: [PATCH] Retry apt operations to death

Nodes are probably broken if update or install fails. Don't proceed
if we can't get past these. Also, instead of ignoring the error off
dpkg, use --force depends, which changes the errors to be kinder
warnings for anyone looking through the logs.
---
 cluster/gce/configure-vm.sh | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh
index cef4b85d906..260d361b652 100644
--- a/cluster/gce/configure-vm.sh
+++ b/cluster/gce/configure-vm.sh
@@ -115,12 +115,16 @@ download-or-bust() {
 # Install salt from GCS.  See README.md for instructions on how to update these
 # debs.
 install-salt() {
-  apt-get update
+  echo "== Refreshing package database =="
+  until apt-get update; do
+    echo "== apt-get update failed, retrying =="
+    echo sleep 5
+  done
 
   mkdir -p /var/cache/salt-install
   cd /var/cache/salt-install
 
-  TARS=(
+  DEBS=(
     libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
     python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
     salt-common_2014.1.13+ds-1~bpo70+1_all.deb
@@ -128,6 +132,10 @@ install-salt() {
   )
   URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
 
+  for deb in "${DEBS[@]}"; do
+    download-or-bust "${URL_BASE}/${deb}"
+  done
+
   # Based on
   # https://major.io/2014/06/26/install-debian-packages-without-starting-daemons/
   # We do this to prevent Salt from starting the salt-minion
@@ -141,13 +149,17 @@ exit 101
 EOF
   chmod 0755 /usr/sbin/policy-rc.d
 
-  for tar in "${TARS[@]}"; do
-    download-or-bust "${URL_BASE}/${tar}"
-    dpkg -i "${tar}" || true
+  for deb in "${DEBS[@]}"; do
+    echo "== Installing ${deb}, ignore dependency complaints (will fix later) =="
+    dpkg --force-depends -i "${deb}"
   done
 
   # This will install any of the unmet dependencies from above.
-  apt-get install -f -y
+  echo "== Installing unmet dependencies =="
+  until apt-get install -f -y; do
+    echo "== apt-get install failed, retrying =="
+    echo sleep 5
+  done
 
   rm /usr/sbin/policy-rc.d
 }