From 8d1d368401ead5d5991d026cd0013eaa82bde78c Mon Sep 17 00:00:00 2001 From: Joe Finney Date: Wed, 2 Mar 2016 14:39:44 -0800 Subject: [PATCH] Call cluster/log-dump.sh from test/e2e/core.go. --- test/e2e/core.go | 141 +++-------------------------------------------- 1 file changed, 9 insertions(+), 132 deletions(-) diff --git a/test/e2e/core.go b/test/e2e/core.go index 9c28703708a..ccd8fdab828 100644 --- a/test/e2e/core.go +++ b/test/e2e/core.go @@ -17,140 +17,17 @@ limitations under the License. package e2e import ( - "fmt" - "io/ioutil" - "net" - "strings" - "sync" + "log" + "os" + "os/exec" + "path" ) -type command struct { - cmd string - component string -} - func CoreDump(dir string) { - c, err := loadClient() - if err != nil { - fmt.Printf("Error creating client: %v", err) - return + cmd := exec.Command(path.Join(testContext.RepoRoot, "cluster", "log-dump.sh"), dir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Printf("Error running cluster/log-dump.sh: %v", err) } - provider := testContext.Provider - - // requires ssh to master machine - if !providerIs(providersWithMasterSSH...) { - fmt.Printf("Skipping SSH core dump, which is not implemented for %s", provider) - return - } - - // I wish there was a better way to get the master IP... - config, err := loadConfig() - if err != nil { - fmt.Printf("Error loading config: %v", err) - } - ix := strings.LastIndex(config.Host, "/") - master := net.JoinHostPort(config.Host[ix+1:], "22") - - cmds := []command{ - {"cat /var/log/kube-apiserver.log", "kube-apiserver"}, - {"cat /var/log/kube-scheduler.log", "kube-scheduler"}, - {"cat /var/log/kube-controller-manager.log", "kube-controller-manager"}, - {"cat /var/log/etcd.log", "kube-etcd"}, - } - if isUsingSystemdKubelet(provider, master) { - cmds = append(cmds, command{"sudo journalctl --output=cat -u kubelet.service", "kubelet"}) - } else { - cmds = append(cmds, []command{ - {"cat /var/log/kubelet.log", "kubelet"}, - {"cat /var/log/supervisor/supervisord.log", "supervisord"}, - {"cat /var/log/supervisor/kubelet-stdout.log", "supervisord-kubelet-stdout"}, - {"cat /var/log/supervisor/kubelet-stderr.log", "supervisord-kubelet-stderr"}, - {"cat /var/log/kern.log", "kern.log"}, - {"cat /var/log/docker.log", "docker.log"}, - }...) - } - - logCore(cmds, []string{master}, dir, provider) - - // requires ssh - if !providerIs(providersWithSSH...) { - fmt.Printf("Skipping SSH core dump for nodes, which is not implemented for %s", provider) - return - } - - // Get all nodes' external IPs. - hosts, err := NodeSSHHosts(c) - if err != nil { - fmt.Printf("Error getting node hostnames: %v", err) - return - } - - cmds = []command{{"cat /var/log/kube-proxy.log", "kube-proxy"}} - if isUsingSystemdKubelet(provider, hosts...) { - cmds = append(cmds, command{"sudo journalctl --output=cat -u kubelet.service", "kubelet"}) - } else { - cmds = append(cmds, []command{ - {"cat /var/log/kubelet.log", "kubelet"}, - {"cat /var/log/supervisor/supervisord.log", "supervisord"}, - {"cat /var/log/supervisor/kubelet-stdout.log", "supervisord-kubelet-stdout"}, - {"cat /var/log/supervisor/kubelet-stderr.log", "supervisord-kubelet-stderr"}, - {"cat /var/log/kern.log", "kern.log"}, - {"cat /var/log/docker.log", "docker.log"}, - }...) - } - - logCore(cmds, hosts, dir, provider) -} - -func logCore(cmds []command, hosts []string, dir, provider string) { - wg := &sync.WaitGroup{} - // Run commands on all nodes via SSH. - for _, cmd := range cmds { - fmt.Printf("SSH'ing to all nodes and running %s\n", cmd.cmd) - for _, host := range hosts { - wg.Add(1) - go func(cmd command, host string) { - defer wg.Done() - logfile := fmt.Sprintf("%s/%s-%s.log", dir, host, cmd.component) - fmt.Printf("Writing to %s.\n", logfile) - result, err := SSH(cmd.cmd, host, provider) - if err != nil { - fmt.Printf("Error running command: %v\n", err) - } - if err := ioutil.WriteFile(logfile, []byte(result.Stdout+result.Stderr), 0777); err != nil { - fmt.Printf("Error writing logfile: %v\n", err) - } - }(cmd, host) - } - } - wg.Wait() -} - -func isUsingSystemdKubelet(provider string, hosts ...string) bool { - wg := &sync.WaitGroup{} - results := make([]bool, len(hosts)) - cmd := "sudo systemctl status kubelet.service" - - wg.Add(len(hosts)) - for i := range hosts { - go func(i int) { - defer wg.Done() - result, err := SSH(cmd, hosts[i], provider) - if err != nil { - fmt.Printf("Error running command: %v\n", err) - return - } - if result.Code == 0 { - results[i] = true - } - }(i) - } - wg.Wait() - - for _, r := range results { - if r { - return true - } - } - return false }