diff --git a/test/e2e/core.go b/test/e2e/core.go new file mode 100644 index 00000000000..c4e3a3a1b7c --- /dev/null +++ b/test/e2e/core.go @@ -0,0 +1,95 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "io/ioutil" + "net" + "strings" + "sync" +) + +type command struct { + cmd string + component string +} + +func coreDump(dir string) { + c, err := loadClient() + if err != nil { + fmt.Printf("Error creating client: %v", err) + return + } + provider := testContext.Provider + if !providerIs("gce", "gke") { + fmt.Printf("Skipping SSH core dump, which is not implemented for %s", provider) + return + } + + // Get all nodes' external IPs. + hosts, err := NodeSSHHosts(c) + if err != nil { + fmt.Printf("Error getting node hostnames: %v", err) + return + } + + cmds := []command{ + {"cat /var/log/kubelet.log", "kubelet"}, + {"cat /var/log/kube-proxy.log", "kube-proxy"}, + } + logCore(cmds, hosts, dir, provider) + + // I wish there was a better way to get the master IP... + config, err := loadConfig() + if err != nil { + fmt.Printf("Error loading config: %v") + } + ix := strings.LastIndex(config.Host, "/") + master := net.JoinHostPort(config.Host[ix+1:], "22") + cmds = []command{ + {"cat /var/log/kube-apiserver.log", "kube-apiserver"}, + {"cat /var/log/kube-scheduler.log", "kube-scheduler"}, + {"cat /var/log/kube-controller-manager.log", "kube-controller-manager"}, + } + logCore(cmds, []string{master}, dir, provider) +} + +func logCore(cmds []command, hosts []string, dir, provider string) { + wg := &sync.WaitGroup{} + // Run commands on all nodes via SSH. + for _, cmd := range cmds { + fmt.Printf("SSH'ing to all nodes and running %s\n", cmd.cmd) + for _, host := range hosts { + go func() { + defer wg.Done() + wg.Add(1) + + logfile := fmt.Sprintf("%s/%s-%s.log", dir, host, cmd.component) + fmt.Printf("Writing to %s.\n", logfile) + stdout, stderr, _, err := SSH(cmd.cmd, host, provider) + if err != nil { + fmt.Printf("Error running command: %v\n", err) + } + if err := ioutil.WriteFile(logfile, []byte(stdout+stderr), 0777); err != nil { + fmt.Printf("Error writing logfile: %v\n", err) + } + }() + } + } + wg.Wait() +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 465e61f018e..62554130761 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -32,6 +32,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" + "github.com/onsi/ginkgo/types" "github.com/onsi/gomega" ) @@ -66,6 +67,22 @@ var ( reportDir = flag.String("report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") ) +type failReporter struct { + failed bool +} + +func (f *failReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { +} +func (f *failReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} +func (f *failReporter) SpecWillRun(specSummary *types.SpecSummary) {} +func (f *failReporter) SpecDidComplete(specSummary *types.SpecSummary) { + if specSummary.Failed() { + f.failed = true + } +} +func (f *failReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {} +func (f *failReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {} + func init() { // Turn on verbose by default to get spec names config.DefaultReporterConfig.Verbose = true @@ -119,7 +136,6 @@ func TestE2E(t *testing.T) { if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { config.GinkgoConfig.SkipString = "Skipped" } - gomega.RegisterFailHandler(ginkgo.Fail) // Ensure all pods are running and ready before starting tests (otherwise, @@ -134,6 +150,13 @@ func TestE2E(t *testing.T) { var r []ginkgo.Reporter if *reportDir != "" { r = append(r, reporters.NewJUnitReporter(path.Join(*reportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode)))) + failReport := &failReporter{} + r = append(r, failReport) + defer func() { + if failReport.failed { + coreDump(*reportDir) + } + }() } ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r) }