From 0705ef94954011cb8daca3834db58f5046291c16 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 13 Aug 2015 20:32:07 +0000 Subject: [PATCH 1/2] add time out exit when process not started correctly on master or minion --- cluster/ubuntu/util.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster/ubuntu/util.sh b/cluster/ubuntu/util.sh index 61e5e8dc85b..7e29ab5a317 100755 --- a/cluster/ubuntu/util.sh +++ b/cluster/ubuntu/util.sh @@ -155,6 +155,8 @@ function verify-master(){ printf "Validating master" local -a required_daemon=("kube-apiserver" "kube-controller-manager" "kube-scheduler") local validated="1" + local try_count=1 + local max_try_count=30 until [[ "$validated" == "0" ]]; do validated="0" local daemon @@ -162,6 +164,11 @@ function verify-master(){ ssh $SSH_OPTS "$MASTER" "pgrep -f ${daemon}" >/dev/null 2>&1 || { printf "." validated="1" + ((try_count=try_count+1)) + if [[ ${try_count} -gt ${max_try_count} ]]; then + printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + exit 1 + fi sleep 2 } done @@ -175,6 +182,8 @@ function verify-minion(){ printf "Validating ${1}" local -a required_daemon=("kube-proxy" "kubelet" "docker") local validated="1" + local try_count=1 + local max_try_count=30 until [[ "$validated" == "0" ]]; do validated="0" local daemon @@ -182,6 +191,11 @@ function verify-minion(){ ssh $SSH_OPTS "$1" "pgrep -f $daemon" >/dev/null 2>&1 || { printf "." validated="1" + ((try_count=try_count+1)) + if [[ ${try_count} -gt ${max_try_count} ]]; then + printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + exit 1 + fi sleep 2 } done From b1ecd645acbb4364410ee5ccb6768b7e156817e2 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 25 Aug 2015 21:09:56 +0000 Subject: [PATCH 2/2] ubuntu deploy scripts: change process start failed message in verify-master() and verify-minion() --- cluster/ubuntu/util.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster/ubuntu/util.sh b/cluster/ubuntu/util.sh index 7e29ab5a317..951ad0cadb9 100755 --- a/cluster/ubuntu/util.sh +++ b/cluster/ubuntu/util.sh @@ -166,7 +166,7 @@ function verify-master(){ validated="1" ((try_count=try_count+1)) if [[ ${try_count} -gt ${max_try_count} ]]; then - printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + printf "\nWarning: Process \"${daemon}\" failed to run on ${MASTER}, please check.\n" exit 1 fi sleep 2 @@ -193,7 +193,7 @@ function verify-minion(){ validated="1" ((try_count=try_count+1)) if [[ ${try_count} -gt ${max_try_count} ]]; then - printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + printf "\nWarning: Process \"${daemon}\" failed to run on ${1}, please check.\n" exit 1 fi sleep 2