Merge pull request #22670 from spxtr/ssh-timeout

Auto commit by PR queue bot
This commit is contained in:
k8s-merge-robot 2016-03-09 09:18:18 -08:00
commit ea4280a814
5 changed files with 21 additions and 6 deletions

View File

@ -1511,10 +1511,12 @@ function ssh-to-node {
fi
for try in $(seq 1 5); do
if ssh -oLogLevel=quiet -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${ip} "${cmd}"; then
if ssh -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${ip} "echo test > /dev/null"; then
break
fi
sleep 5
done
ssh -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${ip} "${cmd}"
}
# Restart the kube-proxy on a node ($1)

View File

@ -1319,13 +1319,13 @@ function ssh-to-node {
local cmd="$2"
# Loop until we can successfully ssh into the box
for try in $(seq 1 5); do
if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test > /dev/null"; then
if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test > /dev/null"; then
break
fi
sleep 5
done
# Then actually try the command.
gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
}
# Restart the kube-proxy on a node ($1)

View File

@ -263,13 +263,13 @@ function ssh-to-node() {
local cmd="$2"
# Loop until we can successfully ssh into the box
for try in $(seq 1 5); do
if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test > /dev/null"; then
if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "echo test > /dev/null"; then
break
fi
sleep 5
done
# Then actually try the command.
gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
}
# Restart the kube-proxy on a node ($1)

View File

@ -345,7 +345,7 @@ function ssh-to-node {
if [[ -z "$machine" ]]; then
echo "$node is an unknown machine to ssh to" >&2
fi
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlMaster=no "core@$machine" "$cmd"
ssh -o ConnectTimeout=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlMaster=no "core@$machine" "$cmd"
}
# Restart the kube-proxy on a node ($1)

View File

@ -30,6 +30,13 @@ source "${KUBE_ROOT}/cluster/kube-util.sh"
readonly report_dir="${1:-_artifacts}"
echo "Dumping master and node logs to ${report_dir}"
# Attempts to SSH to a node ($1) and run a simple command. Returns 0 on
# success and 1 on error.
function test-ssh() {
local -r node_name="${1}"
return $(ssh-to-node "${node_name}" "echo test > /dev/null" &> /dev/null)
}
# Saves a single output of running a given command ($2) on a given node ($1)
# into a given local file ($3). Does not fail if the ssh command fails for any
# reason, just prints an error to stderr.
@ -70,6 +77,8 @@ if [[ ! "${master_ssh_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
echo "Master SSH not supported for ${KUBERNETES_PROVIDER}"
elif ! $(detect-master &> /dev/null); then
echo "Master not detected. Is the cluster up?"
elif ! test-ssh "${MASTER_NAME}"; then
echo "Could not SSH to ${MASTER_NAME}" >&2
else
echo "Master Name: ${MASTER_NAME}"
readonly master_prefix="${report_dir}/${MASTER_NAME}"
@ -88,6 +97,10 @@ elif [[ "${#NODE_NAMES[@]}" -eq 0 ]]; then
else
echo "Node Names: ${NODE_NAMES[*]}"
for node_name in "${NODE_NAMES[@]}"; do
if ! test-ssh "${node_name}"; then
echo "Could not SSH to ${node_name}" >&2
continue
fi
node_prefix="${report_dir}/${node_name}"
save-log "${node_name}" "cat /var/log/kube-proxy.log" "${node_prefix}-kube-proxy.log"
save-common-logs "${node_name}" "${node_prefix}"