From 14f8ae13405d7f77ad741c658375d8b59ed4ff17 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 17 Feb 2016 15:23:17 -0800 Subject: [PATCH 1/3] Log kubelet /healthz failure reasons to supervisord log file --- cluster/saltbase/salt/supervisor/kubelet-checker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/saltbase/salt/supervisor/kubelet-checker.sh b/cluster/saltbase/salt/supervisor/kubelet-checker.sh index 7dda87d5449..41fb061a125 100755 --- a/cluster/saltbase/salt/supervisor/kubelet-checker.sh +++ b/cluster/saltbase/salt/supervisor/kubelet-checker.sh @@ -34,6 +34,7 @@ max_seconds=10 while true; do if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:{{kubelet_port}}/healthz > /dev/null; then echo "kubelet failed!" + curl --insecure -s http://127.0.0.1:{{kubelet_port}}/healthz exit 2 fi sleep 10 From c46ee395b9fa00110217dbd2ae80cc117a2c531b Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 17 Feb 2016 16:50:45 -0800 Subject: [PATCH 2/3] Collect kubelet healthz failure reason log on nodes. Also collect etcd log on master node. --- test/e2e/core.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e/core.go b/test/e2e/core.go index ce6a6bd8c32..37697e90822 100644 --- a/test/e2e/core.go +++ b/test/e2e/core.go @@ -55,6 +55,7 @@ func CoreDump(dir string) { {"cat /var/log/kube-apiserver.log", "kube-apiserver"}, {"cat /var/log/kube-scheduler.log", "kube-scheduler"}, {"cat /var/log/kube-controller-manager.log", "kube-controller-manager"}, + {"cat /var/log/etcd.log", "kube-etcd"}, } if isUsingSystemdKubelet(provider, master) { cmds = append(cmds, command{"sudo journalctl --output=cat -u kubelet.service", "kubelet"}) @@ -62,6 +63,8 @@ func CoreDump(dir string) { cmds = append(cmds, []command{ {"cat /var/log/kubelet.log", "kubelet"}, {"cat /var/log/supervisor/supervisord.log", "supervisord"}, + {"cat /var/log/supervisor/kubelet-stdout.log", "supervisord-kubelet-stdout"}, + {"cat /var/log/supervisor/kubelet-stderr.log", "supervisord-kubelet-stderr"}, {"cat /var/log/kern.log", "kern.log"}, }...) } @@ -88,6 +91,8 @@ func CoreDump(dir string) { cmds = append(cmds, []command{ {"cat /var/log/kubelet.log", "kubelet"}, {"cat /var/log/supervisor/supervisord.log", "supervisord"}, + {"cat /var/log/supervisor/kubelet-stdout.log", "supervisord-kubelet-stdout"}, + {"cat /var/log/supervisor/kubelet-stderr.log", "supervisord-kubelet-stderr"}, {"cat /var/log/kern.log", "kern.log"}, }...) } From b17412a1d934c85dbd1cfef9944d39bab6c75c5b Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 17 Feb 2016 17:06:07 -0800 Subject: [PATCH 3/3] collect docker.log --- test/e2e/core.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/core.go b/test/e2e/core.go index 37697e90822..9c28703708a 100644 --- a/test/e2e/core.go +++ b/test/e2e/core.go @@ -66,6 +66,7 @@ func CoreDump(dir string) { {"cat /var/log/supervisor/kubelet-stdout.log", "supervisord-kubelet-stdout"}, {"cat /var/log/supervisor/kubelet-stderr.log", "supervisord-kubelet-stderr"}, {"cat /var/log/kern.log", "kern.log"}, + {"cat /var/log/docker.log", "docker.log"}, }...) } @@ -94,6 +95,7 @@ func CoreDump(dir string) { {"cat /var/log/supervisor/kubelet-stdout.log", "supervisord-kubelet-stdout"}, {"cat /var/log/supervisor/kubelet-stderr.log", "supervisord-kubelet-stderr"}, {"cat /var/log/kern.log", "kern.log"}, + {"cat /var/log/docker.log", "docker.log"}, }...) }