AWS kube-up: collect logs from e2e

By implementing detect-node-names and a few other helpers, we are able
to collect the logs from nodes.
This commit is contained in:
Justin Santa Barbara
2016-03-15 20:26:46 -04:00
parent 4e29c580a8
commit bb68f740cd
2 changed files with 39 additions and 12 deletions

View File

@@ -45,7 +45,7 @@ function copy-logs-from-node() {
local -r scp_files="{$(echo ${files[*]} | tr ' ' ',')}"
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
local ip=$(get_instance_public_ip "${node}")
local ip=$(get_ssh_hostname "${node}")
scp -i "${AWS_SSH_KEY}" "${SSH_USER}@${ip}:${scp_files}" "${dir}" > /dev/null || true
else
gcloud compute copy-files --project "${PROJECT}" --zone "${ZONE}" "${node}:${scp_files}" "${dir}" > /dev/null || true
@@ -62,10 +62,14 @@ function save-logs() {
if [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then
files="${files} ${gce_logfiles}"
fi
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
files="${files} ${aws_logfiles}"
fi
if ssh-to-node "${node_name}" "sudo systemctl status kubelet.service" &> /dev/null; then
ssh-to-node "${node_name}" "sudo journalctl --output=cat -u kubelet.service" > "${dir}/kubelet.log" || true
ssh-to-node "${node_name}" "sudo journalctl --output=cat -u docker.service" > "${dir}/docker.log" || true
else
files="${files} ${supervisord_logfiles}"
files="${files} ${initd_logfiles} ${supervisord_logfiles}"
fi
copy-logs-from-node "${node_name}" "${dir}" "${files}"
}
@@ -75,8 +79,10 @@ readonly node_ssh_supported_providers="gce gke aws"
readonly master_logfiles="kube-apiserver kube-scheduler kube-controller-manager etcd"
readonly node_logfiles="kube-proxy"
readonly aws_logfiles="cloud-init-output"
readonly gce_logfiles="startupscript"
readonly common_logfiles="kern docker"
readonly common_logfiles="kern"
readonly initd_logfiles="docker"
readonly supervisord_logfiles="kubelet supervisor/supervisord supervisor/kubelet-stdout supervisor/kubelet-stderr"
# Limit the number of concurrent node connections so that we don't run out of
@@ -85,7 +91,7 @@ readonly max_scp_processes=25
if [[ ! "${master_ssh_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then
echo "Master SSH not supported for ${KUBERNETES_PROVIDER}"
elif ! $(detect-master &> /dev/null); then
elif ! (detect-master &> /dev/null); then
echo "Master not detected. Is the cluster up?"
else
readonly master_dir="${report_dir}/${MASTER_NAME}"