Add a 'core dump' to failed tests.

test/e2e/core.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"strings"
+	"sync"
+)
+
+type command struct {
+	cmd       string
+	component string
+}
+
+func coreDump(dir string) {
+	c, err := loadClient()
+	if err != nil {
+		fmt.Printf("Error creating client: %v", err)
+		return
+	}
+	provider := testContext.Provider
+	if !providerIs("gce", "gke") {
+		fmt.Printf("Skipping SSH core dump, which is not implemented for %s", provider)
+		return
+	}
+
+	// Get all nodes' external IPs.
+	hosts, err := NodeSSHHosts(c)
+	if err != nil {
+		fmt.Printf("Error getting node hostnames: %v", err)
+		return
+	}
+
+	cmds := []command{
+		{"cat /var/log/kubelet.log", "kubelet"},
+		{"cat /var/log/kube-proxy.log", "kube-proxy"},
+	}
+	logCore(cmds, hosts, dir, provider)
+
+	// I wish there was a better way to get the master IP...
+	config, err := loadConfig()
+	if err != nil {
+		fmt.Printf("Error loading config: %v")
+	}
+	ix := strings.LastIndex(config.Host, "/")
+	master := net.JoinHostPort(config.Host[ix+1:], "22")
+	cmds = []command{
+		{"cat /var/log/kube-apiserver.log", "kube-apiserver"},
+		{"cat /var/log/kube-scheduler.log", "kube-scheduler"},
+		{"cat /var/log/kube-controller-manager.log", "kube-controller-manager"},
+	}
+	logCore(cmds, []string{master}, dir, provider)
+}
+
+func logCore(cmds []command, hosts []string, dir, provider string) {
+	wg := &sync.WaitGroup{}
+	// Run commands on all nodes via SSH.
+	for _, cmd := range cmds {
+		fmt.Printf("SSH'ing to all nodes and running %s\n", cmd.cmd)
+		for _, host := range hosts {
+			go func() {
+				defer wg.Done()
+				wg.Add(1)
+
+				logfile := fmt.Sprintf("%s/%s-%s.log", dir, host, cmd.component)
+				fmt.Printf("Writing to %s.\n", logfile)
+				stdout, stderr, _, err := SSH(cmd.cmd, host, provider)
+				if err != nil {
+					fmt.Printf("Error running command: %v\n", err)
+				}
+				if err := ioutil.WriteFile(logfile, []byte(stdout+stderr), 0777); err != nil {
+					fmt.Printf("Error writing logfile: %v\n", err)
+				}
+			}()
+		}
+	}
+	wg.Wait()
+}

test/e2e/e2e_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/ginkgo/types"
 	"github.com/onsi/gomega"
 )
 
@@ -66,6 +67,22 @@ var (
 	reportDir = flag.String("report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
 )
 
+type failReporter struct {
+	failed bool
+}
+
+func (f *failReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) {
+}
+func (f *failReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
+func (f *failReporter) SpecWillRun(specSummary *types.SpecSummary)         {}
+func (f *failReporter) SpecDidComplete(specSummary *types.SpecSummary) {
+	if specSummary.Failed() {
+		f.failed = true
+	}
+}
+func (f *failReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}
+func (f *failReporter) SpecSuiteDidEnd(summary *types.SuiteSummary)       {}
+
 func init() {
 	// Turn on verbose by default to get spec names
 	config.DefaultReporterConfig.Verbose = true
@@ -119,7 +136,6 @@ func TestE2E(t *testing.T) {
 	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
 		config.GinkgoConfig.SkipString = "Skipped"
 	}
-
 	gomega.RegisterFailHandler(ginkgo.Fail)
 
 	// Ensure all pods are running and ready before starting tests (otherwise,
@@ -134,6 +150,13 @@ func TestE2E(t *testing.T) {
 	var r []ginkgo.Reporter
 	if *reportDir != "" {
 		r = append(r, reporters.NewJUnitReporter(path.Join(*reportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode))))
+		failReport := &failReporter{}
+		r = append(r, failReport)
+		defer func() {
+			if failReport.failed {
+				coreDump(*reportDir)
+			}
+		}()
 	}
 	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
 }