github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 02:11:09 +00:00
Code Issues Projects Releases Wiki Activity

Return non-zero code on logexporter failures

Browse Source
This commit is contained in:
Antoni Zawodny 2020-10-14 18:26:06 +02:00
parent 6d01c5a589
commit e42a8dbe38
1 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cluster/log-dump/log-dump.sh
View File

@ -63,6 +63,14 @@ readonly windows_node_otherfiles="C:\\Windows\\MEMORY.dmp"
# file descriptors for large clusters.
readonly max_dump_processes=25
# Indicator variable whether we experienced a significant failure during
# logexporter creation or execution.
logexporter_failed=0
# Percentage of nodes that must be logexported successfully (otherwise the
# process will exit with a non-zero exit code).
readonly log_dump_expected_success_percentage="${LOG_DUMP_EXPECTED_SUCCESS_PERCENTAGE:-0}"
# TODO: Get rid of all the sourcing of bash dependencies eventually.
function setup() {
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
@ -546,6 +554,7 @@ function dump_nodes_with_logexporter() {
echo 'Failed to create logexporter daemonset.. falling back to logdump through SSH'
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true
dump_nodes "${NODE_NAMES[@]}"
logexporter_failed=1
return
fi
@ -595,6 +604,7 @@ function dump_nodes_with_logexporter() {
echo 'Final attempt to list marker files failed.. falling back to logdump through SSH'
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true
dump_nodes "${NODE_NAMES[@]}"
logexporter_failed=1
return
fi
sleep 2
@ -611,6 +621,11 @@ function dump_nodes_with_logexporter() {
done
fi
# If less than a certain ratio of the nodes got logexported, report an error.
if [[ $(((${#NODE_NAMES[@]} - ${#failed_nodes[@]}) * 100)) -lt $((${#NODE_NAMES[@]} * log_dump_expected_success_percentage )) ]]; then
logexporter_failed=1
fi
# Delete the logexporter resources and dump logs for the failed nodes (if any) through SSH.
"${KUBECTL}" get pods --namespace "${logexporter_namespace}" || true
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true
@ -675,6 +690,9 @@ function main() {
fi
detect_node_failures
if [[ ${logexporter_failed} -ne 0 && ${log_dump_expected_success_percentage} -gt 0 ]]; then
return 1
fi
}
main