mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 02:41:25 +00:00
e2e framework: verify JUnit result, not the in-memory structs
The old tests were no longer passing with Ginkgo v2.5.0. Instead of keeping the old approach of checking recorded spec results, now the tests actually cover what we care about most: the results recorded in JUnit. This also gets rid of having to repeat the stack backtrace twice (once as part of the output, once for the separate backtrace field).
This commit is contained in:
Patrick Ohly
parent
f6cdd37046
commit
766e7e07da
@ -17,84 +17,103 @@ limitations under the License.
|
||||
package output
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io/ioutil"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/ginkgo/v2/reporters"
|
||||
"github.com/onsi/gomega"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/e2e/framework/internal/junit"
|
||||
)
|
||||
|
||||
func TestGinkgoOutput(t *testing.T, expected SuiteResults, runSpecsArgs ...interface{}) {
|
||||
// Run the Ginkgo suite with spec results collected via ReportAfterEach
|
||||
// in adddition to the default one. To see what the full
|
||||
// Ginkgo output looks like, run this test with "go test -v".
|
||||
// TestGinkgoOutput runs the current suite and verifies that the generated
|
||||
// JUnit file matches the expected result.
|
||||
//
|
||||
// The Ginkgo output on the console (aka the test suite log) does not get
|
||||
// checked. It is usually less important for the CI and more relevant when
|
||||
// using test suite interactively. To see what that Ginkgo output looks like,
|
||||
// run tests with "go test -v".
|
||||
func TestGinkgoOutput(t *testing.T, expected TestResult, runSpecsArgs ...interface{}) {
|
||||
tmpdir := t.TempDir()
|
||||
junitFile := path.Join(tmpdir, "junit.xml")
|
||||
gomega.RegisterFailHandler(framework.Fail)
|
||||
var report []ginkgo.SpecReport
|
||||
ginkgo.ReportAfterEach(func(spec ginkgo.SpecReport) {
|
||||
report = append(report, spec)
|
||||
ginkgo.ReportAfterSuite("write JUnit file", func(report ginkgo.Report) {
|
||||
junit.WriteJUnitReport(report, junitFile)
|
||||
})
|
||||
fakeT := &testing.T{}
|
||||
ginkgo.RunSpecs(fakeT, "Logging Suite", runSpecsArgs...)
|
||||
|
||||
// Now check the output.
|
||||
actual := normalizeReport(report)
|
||||
var actual reporters.JUnitTestSuites
|
||||
data, err := ioutil.ReadFile(junitFile)
|
||||
require.NoError(t, err)
|
||||
err = xml.Unmarshal(data, &actual)
|
||||
require.NoError(t, err)
|
||||
|
||||
if assert.Equal(t, len(expected), len(actual), "Should have %d test results, got: %v", len(expected), actual) {
|
||||
for i := 0; i < len(expected); i++ {
|
||||
assert.Equal(t, expected[i].Name, actual[i].Name, "name from test #%d", i)
|
||||
output := actual[i].Output
|
||||
if expected[i].NormalizeOutput != nil {
|
||||
output = expected[i].NormalizeOutput(output)
|
||||
}
|
||||
assert.Equal(t, expected[i].Output, output, "output from test #%d (%s)", i, expected[i].Name)
|
||||
assert.Equal(t, expected[i].Stack, actual[i].Stack, "stack from test #%d (%s)", i, expected[i].Name)
|
||||
failure := actual[i].Failure
|
||||
if expected[i].NormalizeFailure != nil {
|
||||
failure = expected[i].NormalizeFailure(failure)
|
||||
}
|
||||
assert.Equal(t, expected[i].Failure, failure, "failure from test #%d (%s)", i, expected[i].Name)
|
||||
}
|
||||
if len(actual.TestSuites) != 1 {
|
||||
t.Fatalf("expected one test suite, got %d, JUnit content:\n%s", len(actual.TestSuites), string(data))
|
||||
}
|
||||
diff := cmp.Diff(expected.Suite, actual.TestSuites[0],
|
||||
// Time varies.
|
||||
// Name and Classname are "Logging Suite".
|
||||
// Package includes a varying path, not interesting.
|
||||
// Properties also too complicated to compare.
|
||||
cmpopts.IgnoreFields(reporters.JUnitTestSuite{}, "Time", "Timestamp", "Name", "Package", "Properties"),
|
||||
cmpopts.IgnoreFields(reporters.JUnitTestCase{}, "Time", "Classname"),
|
||||
cmpopts.SortSlices(func(tc1, tc2 reporters.JUnitTestCase) bool {
|
||||
return tc1.Name < tc2.Name
|
||||
}),
|
||||
cmpopts.AcyclicTransformer("simplify", func(in string) any {
|
||||
out := simplify(in, expected)
|
||||
// Sometimes cmp.Diff does not print the full string when it is long.
|
||||
// Uncommenting this here may help debug differences.
|
||||
// if len(out) > 100 {
|
||||
// t.Logf("%s\n---------------------------------------\n%s\n", in, out)
|
||||
// }
|
||||
|
||||
// Same idea as in
|
||||
// https://github.com/google/go-cmp/issues/192#issuecomment-605346277:
|
||||
// it forces cmp.Diff to diff strings line-by-line,
|
||||
// even when it normally wouldn't. The downside is
|
||||
// that the output is harder to turn back into the
|
||||
// expected reference string.
|
||||
// if len(out) > 50 {
|
||||
// return strings.Split(out, "\n")
|
||||
// }
|
||||
|
||||
return out
|
||||
}),
|
||||
)
|
||||
if diff != "" {
|
||||
t.Fatalf("Simplified JUnit report not as expected (-want, +got):\n%s\n\nFull XML:\n%s", diff, string(data))
|
||||
}
|
||||
}
|
||||
|
||||
// TestResult is the outcome of one It spec.
|
||||
// TestResult is the expected outcome of the suite, with additional parameters that
|
||||
// determine equality.
|
||||
type TestResult struct {
|
||||
// Name is the full string for a Ginkgo It, including the "[Top Level]" prefix.
|
||||
Name string
|
||||
// Output written to GinkgoWriter during test.
|
||||
Output string
|
||||
// Failure is SpecSummary.Failure.Message with varying parts stripped.
|
||||
Failure string
|
||||
// Stack is a normalized version (just file names, function parameters stripped) of
|
||||
// Ginkgo's FullStackTrace of a failure. Empty if no failure.
|
||||
Stack string
|
||||
// Called to normalize the actual output string before comparison if non-nil.
|
||||
// Called to normalize all output strings before comparison if non-nil.
|
||||
NormalizeOutput func(string) string
|
||||
// Called to normalize the actual failure string before comparison if non-nil.
|
||||
NormalizeFailure func(string) string
|
||||
|
||||
// All test cases and overall suite results.
|
||||
Suite reporters.JUnitTestSuite
|
||||
}
|
||||
|
||||
type SuiteResults []TestResult
|
||||
|
||||
func normalizeReport(report []ginkgo.SpecReport) SuiteResults {
|
||||
var results SuiteResults
|
||||
for _, spec := range report {
|
||||
results = append(results, TestResult{
|
||||
Name: strings.Join(spec.ContainerHierarchyTexts, " ") + " " + spec.LeafNodeText,
|
||||
Output: normalizeLocation(stripAddresses(stripTimes(spec.CapturedGinkgoWriterOutput))),
|
||||
Failure: stripAddresses(stripTimes(spec.Failure.Message)),
|
||||
Stack: normalizeLocation(spec.Failure.Location.FullStackTrace),
|
||||
})
|
||||
func simplify(in string, expected TestResult) string {
|
||||
out := normalizeLocation(in)
|
||||
out = stripTimes(out)
|
||||
out = stripAddresses(out)
|
||||
if expected.NormalizeOutput != nil {
|
||||
out = expected.NormalizeOutput(out)
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return strings.Compare(results[i].Name, results[j].Name) < 0
|
||||
})
|
||||
return results
|
||||
return out
|
||||
}
|
||||
|
||||
// timePrefix matches "Jul 17 08:08:25.950: " at the beginning of each line.
|
||||
@ -103,13 +122,13 @@ var timePrefix = regexp.MustCompile(`(?m)^[[:alpha:]]{3} +[[:digit:]]{1,2} +[[:d
|
||||
// elapsedSuffix matches "Elapsed: 16.189µs"
|
||||
var elapsedSuffix = regexp.MustCompile(`Elapsed: [[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m)`)
|
||||
|
||||
// timeSuffix matches "09/06/22 15:36:43.445" as printed by Ginkgo v2 for log output.
|
||||
var timeSuffix = regexp.MustCompile(`(?m)[[:space:]][[:digit:]]{2}/[[:digit:]]{2}/[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}(\.[[:digit:]]{1,3})?$`)
|
||||
// timeSuffix matches "@ 09/06/22 15:36:43.44 (5.001s)" as printed by Ginkgo v2 for log output, with the duration being optional.
|
||||
var timeSuffix = regexp.MustCompile(`(?m)@[[:space:]][[:digit:]]{2}/[[:digit:]]{2}/[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}(\.[[:digit:]]{1,3})?( \([[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m)\))?$`)
|
||||
|
||||
func stripTimes(in string) string {
|
||||
out := timePrefix.ReplaceAllString(in, "")
|
||||
out = elapsedSuffix.ReplaceAllString(out, "Elapsed: <elapsed>")
|
||||
out = timeSuffix.ReplaceAllString(out, "")
|
||||
out = timeSuffix.ReplaceAllString(out, "<time>")
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/ginkgo/v2/reporters"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/klog/v2"
|
||||
@ -46,7 +47,6 @@ import (
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
// This must be line #50.
|
||||
|
||||
func init() {
|
||||
@ -100,7 +100,7 @@ var _ = ginkgo.Describe("e2e", func() {
|
||||
fail := func(ctx context.Context, name string) error {
|
||||
return fmt.Errorf("fake error for %q", name)
|
||||
}
|
||||
ginkgo.DeferCleanup(framework.IgnoreNotFound(fail), "failure")
|
||||
ginkgo.DeferCleanup(framework.IgnoreNotFound(fail), "failure") // Without a failure the output would not be shown in JUnit.
|
||||
|
||||
// More test cases can be added here without affeccting line numbering
|
||||
// of existing tests.
|
||||
@ -108,50 +108,52 @@ var _ = ginkgo.Describe("e2e", func() {
|
||||
})
|
||||
|
||||
const (
|
||||
ginkgoOutput = `[BeforeEach] e2e
|
||||
cleanup_test.go:63
|
||||
ginkgoOutput = `> Enter [BeforeEach] e2e - cleanup_test.go:63 <time>
|
||||
INFO: before
|
||||
[BeforeEach] e2e
|
||||
set up framework | framework.go:xxx
|
||||
STEP: Creating a kubernetes client
|
||||
< Exit [BeforeEach] e2e - cleanup_test.go:63 <time>
|
||||
> Enter [BeforeEach] e2e - set up framework | framework.go:xxx <time>
|
||||
STEP: Creating a kubernetes client - framework.go:xxx <time>
|
||||
INFO: >>> kubeConfig: yyy/kube.config
|
||||
STEP: Building a namespace api object, basename test-namespace
|
||||
STEP: Building a namespace api object, basename test-namespace - framework.go:xxx <time>
|
||||
INFO: Skipping waiting for service account
|
||||
[BeforeEach] e2e
|
||||
cleanup_test.go:56
|
||||
< Exit [BeforeEach] e2e - set up framework | framework.go:xxx <time>
|
||||
> Enter [BeforeEach] e2e - cleanup_test.go:56 <time>
|
||||
INFO: extension before
|
||||
[BeforeEach] e2e
|
||||
cleanup_test.go:71
|
||||
< Exit [BeforeEach] e2e - cleanup_test.go:56 <time>
|
||||
> Enter [BeforeEach] e2e - cleanup_test.go:71 <time>
|
||||
INFO: before #1
|
||||
[BeforeEach] e2e
|
||||
cleanup_test.go:75
|
||||
< Exit [BeforeEach] e2e - cleanup_test.go:71 <time>
|
||||
> Enter [BeforeEach] e2e - cleanup_test.go:75 <time>
|
||||
INFO: before #2
|
||||
[It] works
|
||||
cleanup_test.go:90
|
||||
[AfterEach] e2e
|
||||
cleanup_test.go:57
|
||||
< Exit [BeforeEach] e2e - cleanup_test.go:75 <time>
|
||||
> Enter [It] works - cleanup_test.go:90 <time>
|
||||
< Exit [It] works - cleanup_test.go:90 <time>
|
||||
> Enter [AfterEach] e2e - cleanup_test.go:57 <time>
|
||||
INFO: extension after
|
||||
[AfterEach] e2e
|
||||
cleanup_test.go:79
|
||||
< Exit [AfterEach] e2e - cleanup_test.go:57 <time>
|
||||
> Enter [AfterEach] e2e - cleanup_test.go:79 <time>
|
||||
INFO: after #1
|
||||
[AfterEach] e2e
|
||||
cleanup_test.go:86
|
||||
< Exit [AfterEach] e2e - cleanup_test.go:79 <time>
|
||||
> Enter [AfterEach] e2e - cleanup_test.go:86 <time>
|
||||
INFO: after #2
|
||||
[DeferCleanup (Each)] e2e
|
||||
cleanup_test.go:103
|
||||
[DeferCleanup (Each)] e2e
|
||||
cleanup_test.go:99
|
||||
[DeferCleanup (Each)] e2e
|
||||
cleanup_test.go:95
|
||||
< Exit [AfterEach] e2e - cleanup_test.go:86 <time>
|
||||
> Enter [DeferCleanup (Each)] e2e - cleanup_test.go:103 <time>
|
||||
[FAILED] DeferCleanup callback returned error: fake error for "failure"
|
||||
In [DeferCleanup (Each)] at: cleanup_test.go:103 <time>
|
||||
< Exit [DeferCleanup (Each)] e2e - cleanup_test.go:103 <time>
|
||||
> Enter [DeferCleanup (Each)] e2e - cleanup_test.go:99 <time>
|
||||
< Exit [DeferCleanup (Each)] e2e - cleanup_test.go:99 <time>
|
||||
> Enter [DeferCleanup (Each)] e2e - cleanup_test.go:95 <time>
|
||||
INFO: cleanup first
|
||||
[DeferCleanup (Each)] e2e
|
||||
cleanup_test.go:92
|
||||
< Exit [DeferCleanup (Each)] e2e - cleanup_test.go:95 <time>
|
||||
> Enter [DeferCleanup (Each)] e2e - cleanup_test.go:92 <time>
|
||||
INFO: cleanup last
|
||||
[DeferCleanup (Each)] e2e
|
||||
dump namespaces | framework.go:xxx
|
||||
[DeferCleanup (Each)] e2e
|
||||
tear down framework | framework.go:xxx
|
||||
STEP: Destroying namespace "test-namespace-zzz" for this suite.
|
||||
< Exit [DeferCleanup (Each)] e2e - cleanup_test.go:92 <time>
|
||||
> Enter [DeferCleanup (Each)] e2e - dump namespaces | framework.go:xxx <time>
|
||||
< Exit [DeferCleanup (Each)] e2e - dump namespaces | framework.go:xxx <time>
|
||||
> Enter [DeferCleanup (Each)] e2e - tear down framework | framework.go:xxx <time>
|
||||
STEP: Destroying namespace "test-namespace-zzz" for this suite. - framework.go:xxx <time>
|
||||
< Exit [DeferCleanup (Each)] e2e - tear down framework | framework.go:xxx <time>
|
||||
`
|
||||
)
|
||||
|
||||
@ -195,15 +197,28 @@ func TestCleanup(t *testing.T) {
|
||||
framework.AfterReadingAllFlags(&framework.TestContext)
|
||||
suiteConfig, reporterConfig := framework.CreateGinkgoConfig()
|
||||
|
||||
expected := output.SuiteResults{
|
||||
output.TestResult{
|
||||
Name: "e2e works",
|
||||
NormalizeOutput: normalizeOutput,
|
||||
Output: ginkgoOutput,
|
||||
// It would be nice to get the cleanup failure into the
|
||||
// output, but that depends on Ginkgo enhancements:
|
||||
// https://github.com/onsi/ginkgo/issues/1041#issuecomment-1274611444
|
||||
Failure: `DeferCleanup callback returned error: fake error for "failure"`,
|
||||
expected := output.TestResult{
|
||||
NormalizeOutput: normalizeOutput,
|
||||
Suite: reporters.JUnitTestSuite{
|
||||
Tests: 1,
|
||||
Failures: 1,
|
||||
Errors: 0,
|
||||
Disabled: 0,
|
||||
Skipped: 0,
|
||||
|
||||
TestCases: []reporters.JUnitTestCase{
|
||||
{
|
||||
Name: "[It] e2e works",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] DeferCleanup callback returned error: fake error for "failure"
|
||||
In [DeferCleanup (Each)] at: cleanup_test.go:103 <time>
|
||||
`,
|
||||
},
|
||||
SystemErr: ginkgoOutput,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -23,6 +23,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/ginkgo/v2/reporters"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
@ -76,122 +77,212 @@ var _ = ginkgo.Describe("log", func() {
|
||||
})
|
||||
|
||||
func TestFailureOutput(t *testing.T) {
|
||||
// output from AfterEach
|
||||
commonOutput := `
|
||||
expected := output.TestResult{
|
||||
Suite: reporters.JUnitTestSuite{
|
||||
Tests: 6,
|
||||
Failures: 6,
|
||||
Errors: 0,
|
||||
Disabled: 0,
|
||||
Skipped: 0,
|
||||
|
||||
TestCases: []reporters.JUnitTestCase{
|
||||
{
|
||||
Name: "[It] log fails",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] I'm failing.
|
||||
In [It] at: log_test.go:57 <time>
|
||||
|
||||
There were additional failures detected after the initial failure. These are visible in the timeline
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [BeforeEach] log - log_test.go:48 <time>
|
||||
INFO: before
|
||||
< Exit [BeforeEach] log - log_test.go:48 <time>
|
||||
> Enter [It] fails - log_test.go:55 <time>
|
||||
[FAILED] I'm failing.
|
||||
In [It] at: log_test.go:57 <time>
|
||||
< Exit [It] fails - log_test.go:55 <time>
|
||||
> Enter [AfterEach] log - log_test.go:51 <time>
|
||||
INFO: after
|
||||
FAIL: true is never false either
|
||||
[FAILED] true is never false either
|
||||
Expected
|
||||
<bool>: true
|
||||
to equal
|
||||
<bool>: false
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.2()
|
||||
log_test.go:52
|
||||
`
|
||||
|
||||
// Sorted by name!
|
||||
expected := output.SuiteResults{
|
||||
output.TestResult{
|
||||
Name: "log asserts",
|
||||
Output: `INFO: before
|
||||
FAIL: false is never true
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
< Exit [AfterEach] log - log_test.go:51 <time>
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "[It] log asserts",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] false is never true
|
||||
Expected
|
||||
<bool>: false
|
||||
to equal
|
||||
<bool>: true
|
||||
In [It] at: log_test.go:61 <time>
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.4()
|
||||
log_test.go:60` + commonOutput,
|
||||
Failure: `false is never true
|
||||
There were additional failures detected after the initial failure. These are visible in the timeline
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [BeforeEach] log - log_test.go:48 <time>
|
||||
INFO: before
|
||||
< Exit [BeforeEach] log - log_test.go:48 <time>
|
||||
> Enter [It] asserts - log_test.go:60 <time>
|
||||
[FAILED] false is never true
|
||||
Expected
|
||||
<bool>: false
|
||||
to equal
|
||||
<bool>: true`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework_test.glob..func1.4()
|
||||
log_test.go:60`,
|
||||
},
|
||||
output.TestResult{
|
||||
Name: "log equal",
|
||||
Output: `INFO: before
|
||||
FAIL: of course it's not equal...
|
||||
<bool>: true
|
||||
In [It] at: log_test.go:61 <time>
|
||||
< Exit [It] asserts - log_test.go:60 <time>
|
||||
> Enter [AfterEach] log - log_test.go:51 <time>
|
||||
INFO: after
|
||||
[FAILED] true is never false either
|
||||
Expected
|
||||
<int>: 0
|
||||
<bool>: true
|
||||
to equal
|
||||
<int>: 1
|
||||
<bool>: false
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
< Exit [AfterEach] log - log_test.go:51 <time>
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "[It] log error",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] hard-coded error: an error with a long, useless description
|
||||
In [It] at: log_test.go:65 <time>
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.6()
|
||||
log_test.go:67` + commonOutput,
|
||||
Failure: `of course it's not equal...
|
||||
Expected
|
||||
<int>: 0
|
||||
to equal
|
||||
<int>: 1`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework_test.glob..func1.6()
|
||||
log_test.go:67`,
|
||||
},
|
||||
output.TestResult{
|
||||
Name: "log error",
|
||||
Output: `INFO: before
|
||||
There were additional failures detected after the initial failure. These are visible in the timeline
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [BeforeEach] log - log_test.go:48 <time>
|
||||
INFO: before
|
||||
< Exit [BeforeEach] log - log_test.go:48 <time>
|
||||
> Enter [It] error - log_test.go:63 <time>
|
||||
INFO: Unexpected error: hard-coded error:
|
||||
<*errors.errorString>: {
|
||||
s: "an error with a long, useless description",
|
||||
}
|
||||
FAIL: hard-coded error: an error with a long, useless description
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.5()
|
||||
log_test.go:64` + commonOutput,
|
||||
Failure: `hard-coded error: an error with a long, useless description`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework_test.glob..func1.5()
|
||||
log_test.go:64`,
|
||||
},
|
||||
output.TestResult{
|
||||
Name: "log fails",
|
||||
Output: `INFO: before
|
||||
FAIL: I'm failing.
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.3.1()
|
||||
log_test.go:56
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.3()
|
||||
log_test.go:57` + commonOutput,
|
||||
Failure: "I'm failing.",
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework_test.glob..func1.3.1()
|
||||
log_test.go:56
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.3()
|
||||
log_test.go:57`,
|
||||
},
|
||||
output.TestResult{
|
||||
Name: "log fails with helper",
|
||||
Output: `INFO: before
|
||||
FAIL: I'm failing with helper.
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework_test.failHelper()
|
||||
log_test.go:43
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.7()
|
||||
log_test.go:70` + commonOutput,
|
||||
Failure: "I'm failing with helper.",
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework_test.failHelper()
|
||||
log_test.go:43
|
||||
k8s.io/kubernetes/test/e2e/framework_test.glob..func1.7()
|
||||
log_test.go:70`,
|
||||
},
|
||||
output.TestResult{
|
||||
Name: "log redirects klog",
|
||||
Output: `INFO: before
|
||||
<klog> log_test.go:73] hello world
|
||||
<klog> log_test.go:74] <nil>not really an error` + commonOutput,
|
||||
Failure: `true is never false either
|
||||
[FAILED] hard-coded error: an error with a long, useless description
|
||||
In [It] at: log_test.go:65 <time>
|
||||
< Exit [It] error - log_test.go:63 <time>
|
||||
> Enter [AfterEach] log - log_test.go:51 <time>
|
||||
INFO: after
|
||||
[FAILED] true is never false either
|
||||
Expected
|
||||
<bool>: true
|
||||
to equal
|
||||
<bool>: false`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework_test.glob..func1.2()
|
||||
log_test.go:52`,
|
||||
<bool>: false
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
< Exit [AfterEach] log - log_test.go:51 <time>
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "[It] log equal",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] of course it's not equal...
|
||||
Expected
|
||||
<int>: 0
|
||||
to equal
|
||||
<int>: 1
|
||||
In [It] at: log_test.go:68 <time>
|
||||
|
||||
There were additional failures detected after the initial failure. These are visible in the timeline
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [BeforeEach] log - log_test.go:48 <time>
|
||||
INFO: before
|
||||
< Exit [BeforeEach] log - log_test.go:48 <time>
|
||||
> Enter [It] equal - log_test.go:67 <time>
|
||||
[FAILED] of course it's not equal...
|
||||
Expected
|
||||
<int>: 0
|
||||
to equal
|
||||
<int>: 1
|
||||
In [It] at: log_test.go:68 <time>
|
||||
< Exit [It] equal - log_test.go:67 <time>
|
||||
> Enter [AfterEach] log - log_test.go:51 <time>
|
||||
INFO: after
|
||||
[FAILED] true is never false either
|
||||
Expected
|
||||
<bool>: true
|
||||
to equal
|
||||
<bool>: false
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
< Exit [AfterEach] log - log_test.go:51 <time>
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "[It] log fails with helper",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] I'm failing with helper.
|
||||
In [It] at: log_test.go:44 <time>
|
||||
|
||||
There were additional failures detected after the initial failure. These are visible in the timeline
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [BeforeEach] log - log_test.go:48 <time>
|
||||
INFO: before
|
||||
< Exit [BeforeEach] log - log_test.go:48 <time>
|
||||
> Enter [It] fails with helper - log_test.go:70 <time>
|
||||
[FAILED] I'm failing with helper.
|
||||
In [It] at: log_test.go:44 <time>
|
||||
< Exit [It] fails with helper - log_test.go:70 <time>
|
||||
> Enter [AfterEach] log - log_test.go:51 <time>
|
||||
INFO: after
|
||||
[FAILED] true is never false either
|
||||
Expected
|
||||
<bool>: true
|
||||
to equal
|
||||
<bool>: false
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
< Exit [AfterEach] log - log_test.go:51 <time>
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "[It] log redirects klog",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] true is never false either
|
||||
Expected
|
||||
<bool>: true
|
||||
to equal
|
||||
<bool>: false
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [BeforeEach] log - log_test.go:48 <time>
|
||||
INFO: before
|
||||
< Exit [BeforeEach] log - log_test.go:48 <time>
|
||||
> Enter [It] redirects klog - log_test.go:73 <time>
|
||||
<klog> log_test.go:74] hello world
|
||||
<klog> log_test.go:75] <nil>not really an error
|
||||
< Exit [It] redirects klog - log_test.go:73 <time>
|
||||
> Enter [AfterEach] log - log_test.go:51 <time>
|
||||
INFO: after
|
||||
[FAILED] true is never false either
|
||||
Expected
|
||||
<bool>: true
|
||||
to equal
|
||||
<bool>: false
|
||||
In [AfterEach] at: log_test.go:53 <time>
|
||||
< Exit [AfterEach] log - log_test.go:51 <time>
|
||||
`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -23,6 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/ginkgo/v2/reporters"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@ -46,10 +47,7 @@ import (
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
// This must be line #52.
|
||||
// This must be line #50.
|
||||
|
||||
var _ = ginkgo.Describe("pod", func() {
|
||||
ginkgo.It("not found", func(ctx context.Context) {
|
||||
@ -72,14 +70,32 @@ var (
|
||||
)
|
||||
|
||||
func TestFailureOutput(t *testing.T) {
|
||||
// Sorted by name!
|
||||
expected := output.SuiteResults{
|
||||
output.TestResult{
|
||||
Name: "pod not found",
|
||||
// "Ignoring NotFound..." will normally occur every two seconds,
|
||||
// but we reduce it to one line because it might occur less often
|
||||
// on a loaded system.
|
||||
Output: `INFO: Waiting up to 5s for pod "no-such-pod" in namespace "default" to be "running"
|
||||
|
||||
expected := output.TestResult{
|
||||
// "INFO: Ignoring ..." or "INFO: Pod ..." will normally occur
|
||||
// every two seconds, but we reduce it to one line because it
|
||||
// might occur less often on a loaded system.
|
||||
NormalizeOutput: func(output string) string {
|
||||
return trimDuplicateLines(output, "INFO: ")
|
||||
},
|
||||
Suite: reporters.JUnitTestSuite{
|
||||
Tests: 2,
|
||||
Failures: 2,
|
||||
Errors: 0,
|
||||
Disabled: 0,
|
||||
Skipped: 0,
|
||||
TestCases: []reporters.JUnitTestCase{
|
||||
{
|
||||
Name: "[It] pod not found",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Type: "failed",
|
||||
Description: `[FAILED] error while waiting for pod default/no-such-pod to be running: pods "no-such-pod" not found
|
||||
In [It] at: wait_test.go:54 <time>
|
||||
`,
|
||||
},
|
||||
SystemErr: `> Enter [It] not found - wait_test.go:53 <time>
|
||||
INFO: Waiting up to 5s for pod "no-such-pod" in namespace "default" to be "running"
|
||||
INFO: Ignoring NotFound error while getting pod default/no-such-pod
|
||||
INFO: Unexpected error:
|
||||
<*fmt.wrapError>: {
|
||||
@ -101,25 +117,22 @@ INFO: Unexpected error:
|
||||
},
|
||||
},
|
||||
}
|
||||
FAIL: error while waiting for pod default/no-such-pod to be running: pods "no-such-pod" not found
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework/pod_test.glob..func1.1()
|
||||
wait_test.go:56
|
||||
[FAILED] error while waiting for pod default/no-such-pod to be running: pods "no-such-pod" not found
|
||||
In [It] at: wait_test.go:54 <time>
|
||||
< Exit [It] not found - wait_test.go:53 <time>
|
||||
`,
|
||||
NormalizeOutput: func(output string) string {
|
||||
return trimDuplicateLines(output, "INFO: Ignoring NotFound error while getting pod default/no-such-pod")
|
||||
},
|
||||
Failure: `error while waiting for pod default/no-such-pod to be running: pods "no-such-pod" not found`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework/pod_test.glob..func1.1()
|
||||
wait_test.go:56`,
|
||||
},
|
||||
output.TestResult{
|
||||
Name: "pod not running",
|
||||
// "INFO: Pod ..." will normally occur every two seconds,
|
||||
// but we reduce it to one line because it might occur less often
|
||||
// on a loaded system.
|
||||
Output: `INFO: Waiting up to 5s for pod "pending-pod" in namespace "default" to be "running"
|
||||
},
|
||||
{
|
||||
Name: "[It] pod not running",
|
||||
Status: "failed",
|
||||
Failure: &reporters.JUnitFailure{
|
||||
Description: `[FAILED] wait for pod pending-pod running: timed out while waiting for pod default/pending-pod to be running
|
||||
In [It] at: wait_test.go:58 <time>
|
||||
`,
|
||||
Type: "failed",
|
||||
},
|
||||
SystemErr: `> Enter [It] not running - wait_test.go:57 <time>
|
||||
INFO: Waiting up to 5s for pod "pending-pod" in namespace "default" to be "running"
|
||||
INFO: Pod "pending-pod": Phase="", Reason="", readiness=false. Elapsed: <elapsed>
|
||||
INFO: Unexpected error: wait for pod pending-pod running:
|
||||
<*pod.timeoutError>: {
|
||||
@ -205,21 +218,14 @@ INFO: Unexpected error: wait for pod pending-pod running:
|
||||
},
|
||||
],
|
||||
}
|
||||
FAIL: wait for pod pending-pod running: timed out while waiting for pod default/pending-pod to be running
|
||||
|
||||
Full Stack Trace
|
||||
k8s.io/kubernetes/test/e2e/framework/pod_test.glob..func1.2()
|
||||
wait_test.go:60
|
||||
[FAILED] wait for pod pending-pod running: timed out while waiting for pod default/pending-pod to be running
|
||||
In [It] at: wait_test.go:58 <time>
|
||||
< Exit [It] not running - wait_test.go:57 <time>
|
||||
`,
|
||||
NormalizeOutput: func(output string) string {
|
||||
return trimDuplicateLines(output, `INFO: Pod "pending-pod": Phase="", Reason="", readiness=false. Elapsed: <elapsed>`)
|
||||
},
|
||||
},
|
||||
Failure: `wait for pod pending-pod running: timed out while waiting for pod default/pending-pod to be running`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework/pod_test.glob..func1.2()
|
||||
wait_test.go:60`,
|
||||
},
|
||||
}
|
||||
|
||||
output.TestGinkgoOutput(t, expected)
|
||||
}
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/ginkgo/v2/reporters"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/e2e/framework/internal/output"
|
||||
@ -70,17 +71,24 @@ func TestSkip(t *testing.T) {
|
||||
}
|
||||
framework.AfterReadingAllFlags(&framework.TestContext)
|
||||
suiteConfig, reporterConfig := framework.CreateGinkgoConfig()
|
||||
reporterConfig.Verbose = true
|
||||
|
||||
expected := output.SuiteResults{
|
||||
output.TestResult{
|
||||
Name: "e2e skips",
|
||||
Output: `[It] skips
|
||||
skipper_test.go:53
|
||||
INFO: skipping 1, 3, 4
|
||||
`,
|
||||
Failure: `skipping 1, 3, 4`,
|
||||
Stack: `k8s.io/kubernetes/test/e2e/framework/skipper_test.glob..func1.1()
|
||||
skipper_test.go:54`,
|
||||
expected := output.TestResult{
|
||||
Suite: reporters.JUnitTestSuite{
|
||||
Tests: 1,
|
||||
Failures: 0,
|
||||
Errors: 0,
|
||||
Disabled: 0,
|
||||
Skipped: 1,
|
||||
TestCases: []reporters.JUnitTestCase{
|
||||
{
|
||||
Name: "[It] e2e skips",
|
||||
Status: "skipped",
|
||||
Skipped: &reporters.JUnitSkipped{
|
||||
Message: `skipped - skipping 1, 3, 4`,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user