Merge pull request #23992 from mml/docker-checker

Automatic merge from submit-queue

Rewrite docker-checker.sh to make it less kill-happy.

cluster/saltbase/salt/supervisor/docker-checker.sh

@@ -18,6 +18,45 @@
 # it detects a failure.  It then exits, and supervisord restarts it
 # which in turn restarts docker.
 
+main() {
+  if ! healthy 60; then
+    stop_docker
+    start_docker
+    echo "waiting 30s for startup"
+    sleep 30
+    healthy 60
+  fi
+
+  while healthy; do
+    sleep 10
+  done
+
+  echo "Docker failed!"
+  exit 2
+}
+
+# Performs health check on docker.  If a parameter is passed, it is treated as
+# the number of seconds to keep trying for a healthy result.  If none is passed
+# we make only one attempt.
+healthy() {
+  max_retry_sec="$1"
+  shift
+
+  starttime=$(date +%s)
+  while ! timeout 60 docker ps > /dev/null; do
+    if [[ -z "$max_retry_sec" || $(( $(date +%s) - starttime )) -gt "$max_retry_sec" ]]; then
+      echo "docker ps did not succeed"
+      return 2
+    else
+      echo "waiting 5s before retry"
+      sleep 5
+    fi
+  done
+  echo "docker is healthy"
+  return 0
+}
+
+stop_docker() {
   /etc/init.d/docker stop
   # Make sure docker gracefully terminated before start again
   starttime=`date +%s`
@@ -33,7 +72,9 @@ while pidof docker > /dev/null; do
         sleep 10
       fi
   done
+}
 
+start_docker() {
   echo "docker is not running. starting docker"
 
   # cleanup docker network checkpoint to avoid running into known issue
@@ -41,15 +82,6 @@ echo "docker is not running. starting docker"
   rm -rf /var/lib/docker/network
 
   /etc/init.d/docker start
+}
 
-echo "waiting 30s for startup"
+main
-sleep 30
-
-while true; do
-  if ! timeout 60 docker ps > /dev/null; then
-    echo "Docker failed!"
-    exit 2
-  fi
-  sleep 10
-done
-