From 8fe3e8df46a0d36e22a8a118e9be32085e0305da Mon Sep 17 00:00:00 2001 From: Dave Chen Date: Thu, 22 Dec 2022 10:39:55 +0800 Subject: [PATCH 1/2] e2e: bring back total test spec for `Ginkgo` v2 Bring back the number of test spec which was dropped earlier. It's now available in the reporting node of `ReportBeforeSuite` by extracting the number from report.PreRunStats.SpecsThatWillBeRun. Signed-off-by: Dave Chen --- test/e2e/e2e_test.go | 4 ++++ test/e2e/reporters/progress.go | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 67131b1465b..088e611b0ca 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -151,6 +151,10 @@ var _ = ginkgo.ReportAfterEach(func(report ginkgo.SpecReport) { progressReporter.ProcessSpecReport(report) }) +var _ = ginkgo.ReportBeforeSuite(func(report ginkgo.Report) { + progressReporter.SetTestsTotal(report.PreRunStats.SpecsThatWillRun) +}) + var _ = ginkgo.ReportAfterSuite("Kubernetes e2e suite report", func(report ginkgo.Report) { var err error // The DetailsRepoerter will output details about every test (name, files, lines, etc) which helps diff --git a/test/e2e/reporters/progress.go b/test/e2e/reporters/progress.go index e06a5532e30..3547ee5f8d5 100644 --- a/test/e2e/reporters/progress.go +++ b/test/e2e/reporters/progress.go @@ -33,11 +33,10 @@ import ( // ProgressReporter is a ginkgo reporter which tracks the total number of tests to be run/passed/failed/skipped. // As new tests are completed it updates the values and prints them to stdout and optionally, sends the updates // to the configured URL. -// TODO: Number of test specs is not available now, we can add it back when this is fixed in the Ginkgo V2. -// pls see: https://github.com/kubernetes/kubernetes/issues/109744 type ProgressReporter struct { LastMsg string `json:"msg"` + TestsTotal int `json:"total"` TestsCompleted int `json:"completed"` TestsSkipped int `json:"skipped"` TestsFailed int `json:"failed"` @@ -109,6 +108,11 @@ func (reporter *ProgressReporter) SetStartMsg() { reporter.SendUpdates() } +func (reporter *ProgressReporter) SetTestsTotal(totalSpec int) { + reporter.TestsTotal = totalSpec + reporter.SendUpdates() +} + // ProcessSpecReport summarizes the report state and sends the state to the configured endpoint if set. func (reporter *ProgressReporter) ProcessSpecReport(report ginkgo.SpecReport) { testName := strings.Join(report.ContainerHierarchyTexts, " ") From 7800dabb62dd390d5bf2e344e564eb6956845857 Mon Sep 17 00:00:00 2001 From: Dave Chen Date: Fri, 23 Dec 2022 12:03:08 +0800 Subject: [PATCH 2/2] e2e: comment the known limitation of the `ProgressReporter` Signed-off-by: Dave Chen --- test/e2e/reporters/progress.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/e2e/reporters/progress.go b/test/e2e/reporters/progress.go index 3547ee5f8d5..28f296fb499 100644 --- a/test/e2e/reporters/progress.go +++ b/test/e2e/reporters/progress.go @@ -33,6 +33,23 @@ import ( // ProgressReporter is a ginkgo reporter which tracks the total number of tests to be run/passed/failed/skipped. // As new tests are completed it updates the values and prints them to stdout and optionally, sends the updates // to the configured URL. + +// One known limitation of the ProgressReporter it this reporter will not consolidate the reporter from each sub-process +// if the tests are run in parallel. +// As what's observed the reporter sent before test suite is started will be assembled correctly but each process will report +// on its own after each test or suite are completed. +// Here is a sample report that 5 testcases are executed totally, and run in parallel with 3 procs, +// 3 of them are failed and other 2 passed. +// {"msg":"","total":5,"completed":0,"skipped":0,"failed":0} +// {"msg":"Test Suite starting","total":5,"completed":0,"skipped":0,"failed":0} +// {"msg":"FAILED [sig-node] NoExecuteTaintManager Single Pod doesn't evict pod with tolerations from tainted nodes","total":0,"completed":0,"skipped":1332,"failed":1,"failures":["[sig-node] NoExecuteTaintManager..."]} +// {"msg":"FAILED [sig-node] NoExecuteTaintManager Single Pod evicts pods from tainted nodes","total":5,"completed":0,"skipped":2524,"failed":1,"failures":["[sig-node] NoExecuteTaintManager Single Pod evicts pods from tainted nodes"]} +// {"msg":"PASSED [sig-node] NoExecuteTaintManager Single Pod removing taint cancels eviction [Disruptive] [Conformance]","total":0,"completed":1,"skipped":1181,"failed":0} +// {"msg":"Test Suite completed","total":0,"completed":1,"skipped":2592,"failed":0} +// {"msg":"PASSED [sig-node] NoExecuteTaintManager Single Pod eventually evict pod with finite tolerations from tainted nodes","total":0,"completed":1,"skipped":1399,"failed":1,"failures":["[sig-node] NoExecuteTaintManager..."]} +// {"msg":"Test Suite completed","total":0,"completed":1,"skipped":1399,"failed":1,"failures":["[sig-node] NoExecuteTaintManager Single Pod doesn't evict pod with tolerations from tainted nodes"]} +// {"msg":"FAILED [sig-node] NoExecuteTaintManager Single Pod pods evicted from tainted nodes...","total":5,"completed":0,"skipped":3076,"failed":2,"failures":["[sig-node] NoExecuteTaintManager...","[sig-node] NoExecuteTaintManager..."]} +// {"msg":"Test Suite completed","total":5,"completed":0,"skipped":3076,"failed":2,"failures":["[sig-node] NoExecuteTaintManager Single Pod evicts pods from tainted nodes","[sig-node] NoExecuteTaintManager..."]} type ProgressReporter struct { LastMsg string `json:"msg"`