From 5fcb8e5153c5eee7d369751fdb566d0559e0d469 Mon Sep 17 00:00:00 2001 From: Dave Chen Date: Tue, 2 Aug 2022 23:57:30 +0800 Subject: [PATCH] e2e: trim junit report for Spyglass Including the full information for successful tests makes the resulting XML file too large for the 200GB limit in Spyglass when running large jobs (like scale testing). Co-authored-by: Patrick Ohly Co-authored-by: Dave Chen --- test/e2e/e2e_test.go | 60 ++++++++++++++++++++++++++---- test/e2e/framework/test_context.go | 9 ----- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index b59232597b6..258e8557079 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -21,11 +21,14 @@ import ( "fmt" "math/rand" "os" + "path" "path/filepath" "testing" "time" "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/reporters" + "github.com/onsi/ginkgo/v2/types" "gopkg.in/yaml.v2" // Never, ever remove the line with "/ginkgo". Without it, @@ -143,10 +146,9 @@ var _ = ginkgo.ReportAfterEach(func(report ginkgo.SpecReport) { progressReporter.ProcessSpecReport(report) }) -var _ = ginkgo.ReportAfterSuite("Kubernetes e2e suite report", func(report ginkgo.Report) { - var err error - // The DetailsRepoerter will output details about every test (name, files, lines, etc) which helps - // when documenting our tests. +// writeSpecSummaryOutput will output details about every test (name, files, lines, etc) which helps +// when documenting our tests. +func writeSpecSummaryOutput(report ginkgo.Report) { if len(framework.TestContext.SpecSummaryOutput) <= 0 { return } @@ -169,16 +171,58 @@ var _ = ginkgo.ReportAfterSuite("Kubernetes e2e suite report", func(report ginkg klog.Errorf("Error in detail reporter: %v", err) return } - _, err = f.Write(b) - if err != nil { + if _, err = f.Write(b); err != nil { klog.Errorf("Error saving test details in detail reporter: %v", err) return } // Printing newline between records for easier viewing in various tools. - _, err = fmt.Fprintln(f, "") - if err != nil { + if _, err = fmt.Fprintln(f, ""); err != nil { klog.Errorf("Error saving test details in detail reporter: %v", err) return } } +} + +// writeJUnitReport generates a JUnit file in the e2e report directory that is +// shorter than the one normally written by `ginkgo --junit-report`. This is +// needed because the full report can become too large for tools like Spyglass +// (https://github.com/kubernetes/kubernetes/issues/111510). +// +// Users who want the full report can use `--junit-report`. +func writeJUnitReport(report ginkgo.Report) { + if framework.TestContext.ReportDir == "" { + return + } + + trimmedReport := report + trimmedReport.SpecReports = nil + for _, specReport := range report.SpecReports { + // Remove details for any spec that hasn't failed. In Prow, + // the test output captured in build-log.txt has all of this + // information, so we don't need it in the XML. + if specReport.State != types.SpecStateFailed { + // strip the "system-error" if the testcase is not failed. + specReport.CapturedGinkgoWriterOutput = "" + // strip the "system-out" if the testcase is not failed. + specReport.CapturedStdOutErr = "" + // strip some details for tracing each steps executed by Ginkgo, this is used to build the "system-out" + // while "system-out" is not shown by Spyglass if the testcase is not failed. + specReport.ReportEntries = nil + + } + + trimmedReport.SpecReports = append(trimmedReport.SpecReports, specReport) + } + + // With Ginkgo v1, we used to write one file per parallel node. Now + // Ginkgo v2 automatically merges all results into a report for us. The + // 01 suffix is kept in case that users expect files to be called + // "junit_.xml". + junitReport := path.Join(framework.TestContext.ReportDir, "junit_"+framework.TestContext.ReportPrefix+"01.xml") + reporters.GenerateJUnitReport(trimmedReport, junitReport) +} + +var _ = ginkgo.ReportAfterSuite("Kubernetes e2e suite report", func(report ginkgo.Report) { + writeSpecSummaryOutput(report) + writeJUnitReport(report) }) diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index adbda516285..d5f343c947a 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -24,7 +24,6 @@ import ( "fmt" "math" "os" - "path" "sort" "strings" "time" @@ -330,14 +329,6 @@ func CreateGinkgoConfig() (types.SuiteConfig, types.ReporterConfig) { suiteConfig.RandomizeAllSpecs = true // Turn on verbose by default to get spec names reporterConfig.Verbose = true - // Enable JUnit output to the result directory, but only if not already specified - // via -junit-report. - if reporterConfig.JUnitReport == "" && TestContext.ReportDir != "" { - // With Ginkgo v1, we used to write one file per parallel node. Now Ginkgo v2 automatically - // merges all results into a single file for us. The 01 suffix is kept in case that users - // expect files to be called "junit_.xml". - reporterConfig.JUnitReport = path.Join(TestContext.ReportDir, "junit_"+TestContext.ReportPrefix+"01.xml") - } // Disable skipped tests unless they are explicitly requested. if len(suiteConfig.FocusStrings) == 0 && len(suiteConfig.SkipStrings) == 0 { suiteConfig.SkipStrings = []string{`\[Flaky\]|\[Feature:.+\]`}