From 78dabbdb7f5813d257d054fd8e3d57903e5034ff Mon Sep 17 00:00:00 2001 From: Brendan Burns Date: Tue, 21 Apr 2015 15:27:38 -0700 Subject: [PATCH] Fix the ssh-to-node to actually fail on failures. --- cluster/gce/util.sh | 6 +++++- cluster/gke/util.sh | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index c5f3fce8f27..c2267902cce 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -967,11 +967,15 @@ function test-teardown { function ssh-to-node { local node="$1" local cmd="$2" + # Loop until we can successfully ssh into the box for try in $(seq 1 5); do - if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"; then + if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test"; then break fi + sleep 5 done + # Then actually try the command. + gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}" } # Restart the kube-proxy on a node ($1) diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index c4451adad26..ea66997dac8 100755 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -238,8 +238,15 @@ function ssh-to-node() { local node="$1" local cmd="$2" - "${GCLOUD}" compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" \ - --zone="${ZONE}" "${node}" --command "${cmd}" + # Loop until we can successfully ssh into the box + for try in $(seq 1 5); do + if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test"; then + break + fi + sleep 5 + done + # Then actually try the command. + gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}" } # Restart the kube-proxy on a node ($1)