Merge pull request #10998 from dchen1107/cleanup

Introduce a simple alarm utility to timeout docker load hang issue
This commit is contained in:
David Oppenheimer 2015-07-17 16:26:27 -07:00
commit 1e3f1e5acb

View File

@ -15,28 +15,48 @@
# limitations under the License. # limitations under the License.
# loadedImageFlags is a bit-flag to track which docker images loaded successfully. # loadedImageFlags is a bit-flag to track which docker images loaded successfully.
let loadedImageFlags=0; let loadedImageFlags=0
while true; do while true; do
if which docker 1>/dev/null 2>&1; then restart_docker=false
if docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1; then
let loadedImageFlags="$loadedImageFlags|1";
fi;
if docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1; then
let loadedImageFlags="$loadedImageFlags|2";
fi;
if docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1; then
let loadedImageFlags="$loadedImageFlags|4";
fi;
fi;
# required docker images got installed. exit while loop. if which docker 1>/dev/null 2>&1; then
if [ $loadedImageFlags == 7 ]; then break; fi;
# sleep for 5 seconds before attempting to load docker images again. timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
sleep 5; rc=$?
if [[ $rc == 0 ]]; then
let loadedImageFlags="$loadedImageFlags|1"
elif [[ $rc == 124 ]]; then
restart_docker=true
fi
done; timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
rc=$?
if [[ $rc == 0 ]]; then
let loadedImageFlags="$loadedImageFlags|2"
elif [[ $rc == 124 ]]; then
restart_docker=true
fi
timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
rc=$?
if [[ $rc == 0 ]]; then
let loadedImageFlags="$loadedImageFlags|4"
elif [[ $rc == 124 ]]; then
restart_docker=true
fi
fi
# required docker images got installed. exit while loop.
if [[ $loadedImageFlags == 7 ]]; then break; fi
# Sometimes docker load hang, restart docker daemon resolve the issue
if [[ $restart_docker ]]; then service docker restart; fi
# sleep for 15 seconds before attempting to load docker images again
sleep 15
done
# Now exit. After kube-push, salt will notice that the service is down and it # Now exit. After kube-push, salt will notice that the service is down and it
# will start it and new docker images will be loaded. # will start it and new docker images will be loaded.