From 6e34409f2bb5b122d6d2e9b61f3b2a2c70ce180a Mon Sep 17 00:00:00 2001 From: Antoni Zawodny Date: Fri, 3 Apr 2020 17:05:58 +0200 Subject: [PATCH 1/2] Enable timeout for SSH-ing to non-logexported nodes --- cluster/log-dump/log-dump.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index 04ad09097d1..38fade250e4 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -419,6 +419,7 @@ function dump_nodes() { all_selected_nodes+=( "${windows_node_names[@]}" ) proc=${max_dump_processes} + start="$(date +%s)" for i in "${!all_selected_nodes[@]}"; do node_name="${all_selected_nodes[$i]}" node_dir="${report_dir}/${node_name}" @@ -438,6 +439,10 @@ function dump_nodes() { if [[ proc -eq 0 ]]; then proc=${max_dump_processes} wait + now="$(date +%s)" + if [[ ! -z "${LOG_DUMP_SSH_TIMEOUT}" && $((now - start)) -gt ${LOG_DUMP_SSH_TIMEOUT} ]]; then + break + fi fi done # Wait for any remaining processes. From 15160168e533d0892262332673b0ee0bd5bae683 Mon Sep 17 00:00:00 2001 From: Antoni Zawodny Date: Mon, 6 Apr 2020 10:06:51 +0200 Subject: [PATCH 2/2] Apply PR fixes --- cluster/log-dump/log-dump.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index 38fade250e4..a9fefc297ff 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -420,6 +420,11 @@ function dump_nodes() { proc=${max_dump_processes} start="$(date +%s)" + # log_dump_ssh_timeout is the maximal number of seconds the log dumping over + # SSH operation can take. Please note that the logic enforcing the timeout + # is only a best effort. The actual time of the operation may be longer + # due to waiting for all the child processes below. + log_dump_ssh_timeout_seconds="${LOG_DUMP_SSH_TIMEOUT_SECONDS:-}" for i in "${!all_selected_nodes[@]}"; do node_name="${all_selected_nodes[$i]}" node_dir="${report_dir}/${node_name}" @@ -440,7 +445,8 @@ function dump_nodes() { proc=${max_dump_processes} wait now="$(date +%s)" - if [[ ! -z "${LOG_DUMP_SSH_TIMEOUT}" && $((now - start)) -gt ${LOG_DUMP_SSH_TIMEOUT} ]]; then + if [[ -n "${log_dump_ssh_timeout_seconds}" && $((now - start)) -gt ${log_dump_ssh_timeout_seconds} ]]; then + echo "WARNING: Hit timeout after ${log_dump_ssh_timeout_seconds} seconds, finishing log dumping over SSH shortly" break fi fi