mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-30 15:05:27 +00:00
Revert "fix cluster/log-dump/log-dump.sh shellcheck failures"
This reverts commit 742701622a.
It broke log dumping, see https://github.com/kubernetes/kubernetes/pull/88349#issuecomment-688385378
This commit is contained in:
Matt Matejczyk
parent
b16070f3ce
commit
67f9d69eaf
@ -67,7 +67,7 @@ readonly max_dump_processes=25
|
||||
function setup() {
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
|
||||
if [[ -z "${use_custom_instance_list}" ]]; then
|
||||
: "${KUBE_CONFIG_FILE:="config-test.sh"}"
|
||||
: ${KUBE_CONFIG_FILE:="config-test.sh"}
|
||||
echo "Sourcing kube-util.sh"
|
||||
source "${KUBE_ROOT}/cluster/kube-util.sh"
|
||||
echo "Detecting project"
|
||||
@ -89,7 +89,7 @@ function setup() {
|
||||
}
|
||||
|
||||
function log-dump-ssh() {
|
||||
if [[ "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
|
||||
if [[ "${gcloud_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
|
||||
ssh-to-node "$@"
|
||||
return
|
||||
fi
|
||||
@ -106,7 +106,7 @@ function log-dump-ssh() {
|
||||
function copy-logs-from-node() {
|
||||
local -r node="${1}"
|
||||
local -r dir="${2}"
|
||||
while IFS= read -r line; do files+=("$line"); done <<< "$3"
|
||||
local files=( ${3} )
|
||||
# Append "*"
|
||||
# The * at the end is needed to also copy rotated logs (which happens
|
||||
# in large clusters and long runs).
|
||||
@ -116,13 +116,12 @@ function copy-logs-from-node() {
|
||||
# Comma delimit (even the singleton, or scp does the wrong thing), surround by braces.
|
||||
local -r scp_files="{$(printf "%s," "${files[@]}")}"
|
||||
|
||||
if [[ "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
|
||||
if [[ "${gcloud_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
|
||||
# get-serial-port-output lets you ask for ports 1-4, but currently (11/21/2016) only port 1 contains useful information
|
||||
gcloud compute instances get-serial-port-output --project "${PROJECT}" --zone "${ZONE}" --port 1 "${node}" > "${dir}/serial-1.log" || true
|
||||
gcloud compute scp --recurse --project "${PROJECT}" --zone "${ZONE}" "${node}:${scp_files}" "${dir}" > /dev/null || true
|
||||
elif [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
|
||||
local ip
|
||||
ip=$(get_ssh_hostname "${node}")
|
||||
local ip=$(get_ssh_hostname "${node}")
|
||||
scp -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" "${SSH_USER}@${ip}:${scp_files}" "${dir}" > /dev/null || true
|
||||
elif [[ -n "${use_custom_instance_list}" ]]; then
|
||||
scp -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${LOG_DUMP_SSH_KEY}" "${LOG_DUMP_SSH_USER}@${node}:${scp_files}" "${dir}" > /dev/null || true
|
||||
@ -139,27 +138,26 @@ function copy-logs-from-node() {
|
||||
function save-logs() {
|
||||
local -r node_name="${1}"
|
||||
local -r dir="${2}"
|
||||
local save_files="${3}"
|
||||
local files="${3}"
|
||||
local opt_systemd_services="${4:-""}"
|
||||
local on_master="${5:-"false"}"
|
||||
|
||||
save_files="${save_files} ${extra_log_files}"
|
||||
files="${files} ${extra_log_files}"
|
||||
if [[ -n "${use_custom_instance_list}" ]]; then
|
||||
if [[ -n "${LOG_DUMP_SAVE_LOGS:-}" ]]; then
|
||||
save_files="${save_files} ${LOG_DUMP_SAVE_LOGS:-}"
|
||||
files="${files} ${LOG_DUMP_SAVE_LOGS:-}"
|
||||
fi
|
||||
else
|
||||
case "${KUBERNETES_PROVIDER}" in
|
||||
gce|gke)
|
||||
save_files="${save_files} ${gce_logfiles}"
|
||||
files="${files} ${gce_logfiles}"
|
||||
;;
|
||||
aws)
|
||||
save_files="${save_files} ${aws_logfiles}"
|
||||
files="${files} ${aws_logfiles}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
read -r -a services <<< "${systemd_services} ${opt_systemd_services} ${extra_systemd_services}"
|
||||
local -r services=( ${systemd_services} ${opt_systemd_services} ${extra_systemd_services} )
|
||||
|
||||
if log-dump-ssh "${node_name}" "command -v journalctl" &> /dev/null; then
|
||||
if [[ "${on_master}" == "true" ]]; then
|
||||
@ -179,7 +177,7 @@ function save-logs() {
|
||||
log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise" > "${dir}/systemd.log" || true
|
||||
fi
|
||||
else
|
||||
save_files="${kern_logfile} ${save_files} ${initd_logfiles} ${supervisord_logfiles}"
|
||||
files="${kern_logfile} ${files} ${initd_logfiles} ${supervisord_logfiles}"
|
||||
fi
|
||||
|
||||
# Try dumping coverage profiles, if it looks like coverage is enabled in the first place.
|
||||
@ -200,8 +198,8 @@ function save-logs() {
|
||||
echo "Changing logfiles to be world-readable for download"
|
||||
log-dump-ssh "${node_name}" "sudo chmod -R a+r /var/log" || true
|
||||
|
||||
echo "Copying '${save_files}' from ${node_name}"
|
||||
copy-logs-from-node "${node_name}" "${dir}" "${save_files}"
|
||||
echo "Copying '${files}' from ${node_name}"
|
||||
copy-logs-from-node "${node_name}" "${dir}" "${files}"
|
||||
}
|
||||
|
||||
# Saves a copy of the Windows Docker event log to ${WINDOWS_LOGS_DIR}\docker.log
|
||||
@ -247,9 +245,8 @@ function save-windows-logs-via-diagnostics-tool() {
|
||||
local node="${1}"
|
||||
local dest_dir="${2}"
|
||||
|
||||
gcloud compute instances add-metadata "${node}" --metadata enable-diagnostics=true "--project=${PROJECT}" "--zone=${ZONE}"
|
||||
local logs_archive_in_gcs
|
||||
logs_archive_in_gcs=$(gcloud alpha compute diagnose export-logs "${node}" "--zone=${ZONE}" "--project=${PROJECT}" | tail -n 1)
|
||||
gcloud compute instances add-metadata ${node} --metadata enable-diagnostics=true --project=${PROJECT} --zone=${ZONE}
|
||||
local logs_archive_in_gcs=$(gcloud alpha compute diagnose export-logs ${node} --zone=${ZONE} --project=${PROJECT} | tail -n 1)
|
||||
local temp_local_path="${node}.zip"
|
||||
for retry in {1..20}; do
|
||||
if gsutil mv "${logs_archive_in_gcs}" "${temp_local_path}" > /dev/null 2>&1; then
|
||||
@ -261,8 +258,8 @@ function save-windows-logs-via-diagnostics-tool() {
|
||||
done
|
||||
|
||||
if [[ -f "${temp_local_path}" ]]; then
|
||||
unzip "${temp_local_path}" -d "${dest_dir}" > /dev/null
|
||||
rm -f "${temp_local_path}"
|
||||
unzip ${temp_local_path} -d "${dest_dir}" > /dev/null
|
||||
rm -f ${temp_local_path}
|
||||
fi
|
||||
}
|
||||
|
||||
@ -275,14 +272,14 @@ function save-windows-logs-via-ssh() {
|
||||
export-windows-docker-images-list "${node}"
|
||||
|
||||
local remote_files=()
|
||||
for file in "${windows_node_logfiles[@]}"; do
|
||||
for file in ${windows_node_logfiles[@]}; do
|
||||
remote_files+=( "${WINDOWS_LOGS_DIR}\\${file}" )
|
||||
done
|
||||
remote_files+=( "${windows_node_otherfiles[@]}" )
|
||||
|
||||
# TODO(pjh, yujuhong): handle rotated logs and copying multiple files at the
|
||||
# same time.
|
||||
for remote_file in "${remote_files[@]}"; do
|
||||
for remote_file in ${remote_files[@]}; do
|
||||
# Retry up to 3 times to allow ssh keys to be properly propagated and
|
||||
# stored.
|
||||
for retry in {1..3}; do
|
||||
@ -304,7 +301,7 @@ function save-logs-windows() {
|
||||
local -r node="${1}"
|
||||
local -r dest_dir="${2}"
|
||||
|
||||
if [[ ! "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
|
||||
if [[ ! "${gcloud_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
|
||||
echo "Not saving logs for ${node}, Windows log dumping requires gcloud support"
|
||||
return
|
||||
fi
|
||||
@ -326,14 +323,14 @@ function run-in-docker-container() {
|
||||
local node_name="$1"
|
||||
local container="$2"
|
||||
shift 2
|
||||
log-dump-ssh "${node_name}" "docker exec \"\$(docker ps -f label=io.kubernetes.container.name=${container} --format \"{{.ID}}\")\" $*"
|
||||
log-dump-ssh "${node_name}" "docker exec \"\$(docker ps -f label=io.kubernetes.container.name=${container} --format \"{{.ID}}\")\" $@"
|
||||
}
|
||||
|
||||
function dump_masters() {
|
||||
local master_names
|
||||
if [[ -n "${use_custom_instance_list}" ]]; then
|
||||
while IFS='' read -r line; do master_names+=("$line"); done < <(log_dump_custom_get_instances master)
|
||||
elif [[ ! "${master_ssh_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
|
||||
master_names=( $(log_dump_custom_get_instances master) )
|
||||
elif [[ ! "${master_ssh_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
|
||||
echo "Master SSH not supported for ${KUBERNETES_PROVIDER}"
|
||||
return
|
||||
elif [[ -n "${KUBEMARK_MASTER_NAME:-}" ]]; then
|
||||
@ -384,8 +381,8 @@ function dump_nodes() {
|
||||
node_names=( "$@" )
|
||||
elif [[ -n "${use_custom_instance_list}" ]]; then
|
||||
echo "Dumping logs for nodes provided by log_dump_custom_get_instances() function"
|
||||
while IFS='' read -r line; do node_names+=("$line"); done < <(log_dump_custom_get_instances node)
|
||||
elif [[ ! "${node_ssh_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
|
||||
node_names=( $(log_dump_custom_get_instances node) )
|
||||
elif [[ ! "${node_ssh_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
|
||||
echo "Node SSH not supported for ${KUBERNETES_PROVIDER}"
|
||||
return
|
||||
else
|
||||
@ -412,7 +409,7 @@ function dump_nodes() {
|
||||
linux_nodes_selected_for_logs=()
|
||||
if [[ -n "${LOGDUMP_ONLY_N_RANDOM_NODES:-}" ]]; then
|
||||
# We randomly choose 'LOGDUMP_ONLY_N_RANDOM_NODES' many nodes for fetching logs.
|
||||
for index in $(shuf -i 0-$(( ${#node_names[*]} - 1 )) -n "${LOGDUMP_ONLY_N_RANDOM_NODES}")
|
||||
for index in `shuf -i 0-$(( ${#node_names[*]} - 1 )) -n ${LOGDUMP_ONLY_N_RANDOM_NODES}`
|
||||
do
|
||||
linux_nodes_selected_for_logs+=("${node_names[$index]}")
|
||||
done
|
||||
@ -478,7 +475,7 @@ function find_non_logexported_nodes() {
|
||||
echo "Successfully listed marker files for successful nodes"
|
||||
NON_LOGEXPORTED_NODES=()
|
||||
for node in "${NODE_NAMES[@]}"; do
|
||||
if [[ ! "${succeeded_nodes}" =~ ${node} ]]; then
|
||||
if [[ ! "${succeeded_nodes}" =~ "${node}" ]]; then
|
||||
NON_LOGEXPORTED_NODES+=("${node}")
|
||||
fi
|
||||
done
|
||||
@ -489,8 +486,7 @@ function find_non_logexported_nodes() {
|
||||
function dump_nodes_with_logexporter() {
|
||||
if [[ -n "${use_custom_instance_list}" ]]; then
|
||||
echo "Dumping logs for nodes provided by log_dump_custom_get_instances() function"
|
||||
NODE_NAMES=()
|
||||
while IFS='' read -r line; do NODE_NAMES+=("$line"); done < <(log_dump_custom_get_instances node)
|
||||
NODE_NAMES=( $(log_dump_custom_get_instances node) )
|
||||
else
|
||||
echo "Detecting nodes in the cluster"
|
||||
detect-node-names &> /dev/null
|
||||
@ -502,7 +498,7 @@ function dump_nodes_with_logexporter() {
|
||||
fi
|
||||
|
||||
# Obtain parameters required by logexporter.
|
||||
local -r service_account_credentials="$(base64 <<< "${GOOGLE_APPLICATION_CREDENTIALS}" | tr -d '\n')"
|
||||
local -r service_account_credentials="$(cat ${GOOGLE_APPLICATION_CREDENTIALS} | base64 | tr -d '\n')"
|
||||
local -r cloud_provider="${KUBERNETES_PROVIDER}"
|
||||
local -r enable_hollow_node_logs="${ENABLE_HOLLOW_NODE_LOGS:-false}"
|
||||
local -r logexport_sleep_seconds="$(( 90 + NUM_NODES / 3 ))"
|
||||
@ -563,6 +559,7 @@ function dump_nodes_with_logexporter() {
|
||||
done; wait)
|
||||
|
||||
# List registry of marker files (of nodes whose logexporter succeeded) from GCS.
|
||||
local nodes_succeeded
|
||||
for retry in {1..10}; do
|
||||
if find_non_logexported_nodes; then
|
||||
break
|
||||
@ -592,29 +589,28 @@ function dump_nodes_with_logexporter() {
|
||||
"${KUBECTL}" get pods --namespace "${logexporter_namespace}" || true
|
||||
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true
|
||||
if [[ "${#failed_nodes[@]}" != 0 ]]; then
|
||||
echo -e "Dumping logs through SSH for the following nodes:\n${failed_nodes[*]}"
|
||||
echo -e "Dumping logs through SSH for the following nodes:\n${failed_nodes[@]}"
|
||||
dump_nodes "${failed_nodes[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
function detect_node_failures() {
|
||||
if ! [[ "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
|
||||
if ! [[ "${gcloud_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
detect-node-names
|
||||
if [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then
|
||||
local all_instance_groups=("${INSTANCE_GROUPS[@]}" "${WINDOWS_INSTANCE_GROUPS[@]}")
|
||||
local all_instance_groups=(${INSTANCE_GROUPS[@]} ${WINDOWS_INSTANCE_GROUPS[@]})
|
||||
else
|
||||
local all_instance_groups=("${INSTANCE_GROUPS[@]}")
|
||||
local all_instance_groups=(${INSTANCE_GROUPS[@]})
|
||||
fi
|
||||
|
||||
if [ -z "${all_instance_groups:-}" ]; then
|
||||
return
|
||||
fi
|
||||
for group in "${all_instance_groups[@]}"; do
|
||||
local creation_timestamp
|
||||
creation_timestamp=$(gcloud compute instance-groups managed describe \
|
||||
local creation_timestamp=$(gcloud compute instance-groups managed describe \
|
||||
"${group}" \
|
||||
--project "${PROJECT}" \
|
||||
--zone "${ZONE}" \
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
./cluster/gce/gci/configure.sh
|
||||
./cluster/gce/gci/master-helper.sh
|
||||
./cluster/gce/util.sh
|
||||
./cluster/log-dump/log-dump.sh
|
||||
|
||||
Loading…
Reference in New Issue
Block a user