diff --git a/test/e2e/framework/.import-restrictions b/test/e2e/framework/.import-restrictions index 894b9749b0a..660e7453fa7 100644 --- a/test/e2e/framework/.import-restrictions +++ b/test/e2e/framework/.import-restrictions @@ -4,7 +4,7 @@ rules: # The following packages are okay to use: # # public API - - selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils)/|^[a-z]+(/|$)|github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils) + - selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils) allowedPrefixes: [ "" ] # stdlib @@ -16,7 +16,7 @@ rules: allowedPrefixes: [ "" ] # Ginkgo + Gomega - - selectorRegexp: github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils) + - selectorRegexp: ^github.com/onsi/(ginkgo|gomega) allowedPrefixes: [ "" ] # kube-openapi @@ -33,8 +33,10 @@ rules: # Third party deps - selectorRegexp: ^github.com/|^gopkg.in - allowedPrefixes: [ + allowedPrefixes: [ "gopkg.in/inf.v0", + "gopkg.in/yaml.v2", + "github.com/blang/semver/", "github.com/davecgh/go-spew/spew", "github.com/evanphx/json-patch", "github.com/go-logr/logr", @@ -48,6 +50,10 @@ rules: "github.com/google/gofuzz", "github.com/google/uuid", "github.com/imdario/mergo", + "github.com/prometheus/client_golang/", + "github.com/prometheus/client_model/", + "github.com/prometheus/common/", + "github.com/prometheus/procfs", "github.com/spf13/cobra", "github.com/spf13/pflag", "github.com/stretchr/testify/assert", diff --git a/test/e2e/framework/ginkgowrapper.go b/test/e2e/framework/ginkgowrapper.go index e35fc4ae982..5826f7885c5 100644 --- a/test/e2e/framework/ginkgowrapper.go +++ b/test/e2e/framework/ginkgowrapper.go @@ -17,13 +17,71 @@ limitations under the License. package framework import ( + "fmt" "path" "reflect" + "strings" "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +// Feature is the name of a certain feature that the cluster under test must have. +// Such features are different from feature gates. +type Feature string + +// Environment is the name for the environment in which a test can run, like +// "Linux" or "Windows". +type Environment string + +// NodeFeature is the name of a feature that a node must support. To be +// removed, see +// https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features#nodefeature. +type NodeFeature string + +type Valid[T comparable] struct { + items sets.Set[T] + frozen bool +} + +// Add registers a new valid item name. The expected usage is +// +// var SomeFeature = framework.ValidFeatures.Add("Some") +// +// during the init phase of an E2E suite. Individual tests should not register +// their own, to avoid uncontrolled proliferation of new items. E2E suites can, +// but don't have to, enforce that by freezing the set of valid names. +func (v *Valid[T]) Add(item T) T { + if v.frozen { + RecordBug(NewBug(fmt.Sprintf(`registry %T is already frozen, "%v" must not be added anymore`, *v, item), 1)) + } + if v.items == nil { + v.items = sets.New[T]() + } + if v.items.Has(item) { + RecordBug(NewBug(fmt.Sprintf(`registry %T already contains "%v", it must not be added again`, *v, item), 1)) + } + v.items.Insert(item) + return item +} + +func (v *Valid[T]) Freeze() { + v.frozen = true +} + +// These variables contain the parameters that [WithFeature], [WithEnvironment] +// and [WithNodeFeatures] accept. The framework itself has no pre-defined +// constants. Test suites and tests may define their own and then add them here +// before calling these With functions. +var ( + ValidFeatures Valid[Feature] + ValidEnvironments Valid[Environment] + ValidNodeFeatures Valid[NodeFeature] ) var errInterface = reflect.TypeOf((*error)(nil)).Elem() @@ -67,6 +125,322 @@ func AnnotatedLocationWithOffset(annotation string, offset int) types.CodeLocati // ConformanceIt is wrapper function for ginkgo It. Adds "[Conformance]" tag and makes static analysis easier. func ConformanceIt(text string, args ...interface{}) bool { - args = append(args, ginkgo.Offset(1)) - return ginkgo.It(text+" [Conformance]", args...) + args = append(args, ginkgo.Offset(1), WithConformance()) + return It(text, args...) +} + +// It is a wrapper around [ginkgo.It] which supports framework With* labels as +// optional arguments in addition to those already supported by ginkgo itself, +// like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func It(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.It, text, args) +} + +// It is a shorthand for the corresponding package function. +func (f *Framework) It(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.It, text, args) +} + +// Describe is a wrapper around [ginkgo.Describe] which supports framework +// With* labels as optional arguments in addition to those already supported by +// ginkgo itself, like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func Describe(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Describe, text, args) +} + +// Describe is a shorthand for the corresponding package function. +func (f *Framework) Describe(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Describe, text, args) +} + +// Context is a wrapper around [ginkgo.Context] which supports framework With* +// labels as optional arguments in addition to those already supported by +// ginkgo itself, like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func Context(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Context, text, args) +} + +// Context is a shorthand for the corresponding package function. +func (f *Framework) Context(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Context, text, args) +} + +// registerInSuite is the common implementation of all wrapper functions. It +// expects to be called through one intermediate wrapper. +func registerInSuite(ginkgoCall func(text string, args ...interface{}) bool, text string, args []interface{}) bool { + var ginkgoArgs []interface{} + var offset ginkgo.Offset + var texts []string + if text != "" { + texts = append(texts, text) + } + + addLabel := func(label string) { + texts = append(texts, fmt.Sprintf("[%s]", label)) + ginkgoArgs = append(ginkgoArgs, ginkgo.Label(label)) + } + + haveEmptyStrings := false + for _, arg := range args { + switch arg := arg.(type) { + case label: + fullLabel := strings.Join(arg.parts, ": ") + addLabel(fullLabel) + if arg.extra != "" { + addLabel(arg.extra) + } + if fullLabel == "Serial" { + ginkgoArgs = append(ginkgoArgs, ginkgo.Serial) + } + case ginkgo.Offset: + offset = arg + case string: + if arg == "" { + haveEmptyStrings = true + } + texts = append(texts, arg) + default: + ginkgoArgs = append(ginkgoArgs, arg) + } + } + offset += 2 // This function and its direct caller. + + // Now that we have the final offset, we can record bugs. + if haveEmptyStrings { + RecordBug(NewBug("empty strings as separators are unnecessary and need to be removed", int(offset))) + } + + // Enforce that text snippets to not start or end with spaces because + // those lead to double spaces when concatenating below. + for _, text := range texts { + if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") { + RecordBug(NewBug(fmt.Sprintf("trailing or leading spaces are unnecessary and need to be removed: %q", text), int(offset))) + } + } + + ginkgoArgs = append(ginkgoArgs, offset) + text = strings.Join(texts, " ") + return ginkgoCall(text, ginkgoArgs...) +} + +// WithEnvironment specifies that a certain test or group of tests only works +// with a feature available. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The feature must be listed in ValidFeatures. +func WithFeature(name Feature) interface{} { + return withFeature(name) +} + +// WithFeature is a shorthand for the corresponding package function. +func (f *Framework) WithFeature(name Feature) interface{} { + return withFeature(name) +} + +func withFeature(name Feature) interface{} { + if !ValidFeatures.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithFeature: unknown feature %q", name), 2)) + } + return newLabel("Feature", string(name)) +} + +// WithFeatureGate specifies that a certain test or group of tests depends on a +// feature gate being enabled. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The feature gate must be listed in +// [k8s.io/apiserver/pkg/util/feature.DefaultMutableFeatureGate]. Once a +// feature gate gets removed from there, the WithFeatureGate calls using it +// also need to be removed. +func WithFeatureGate(featureGate featuregate.Feature) interface{} { + return withFeatureGate(featureGate) +} + +// WithFeatureGate is a shorthand for the corresponding package function. +func (f *Framework) WithFeatureGate(featureGate featuregate.Feature) interface{} { + return withFeatureGate(featureGate) +} + +func withFeatureGate(featureGate featuregate.Feature) interface{} { + spec, ok := utilfeature.DefaultMutableFeatureGate.GetAll()[featureGate] + if !ok { + RecordBug(NewBug(fmt.Sprintf("WithFeatureGate: the feature gate %q is unknown", featureGate), 2)) + } + + // We use mixed case (i.e. Beta instead of BETA). GA feature gates have no level string. + var level string + if spec.PreRelease != "" { + level = string(spec.PreRelease) + level = strings.ToUpper(level[0:1]) + strings.ToLower(level[1:]) + } + + l := newLabel("FeatureGate", string(featureGate)) + l.extra = level + return l +} + +// WithEnvironment specifies that a certain test or group of tests only works +// in a certain environment. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The environment must be listed in ValidEnvironments. +func WithEnvironment(name Environment) interface{} { + return withEnvironment(name) +} + +// WithEnvironment is a shorthand for the corresponding package function. +func (f *Framework) WithEnvironment(name Environment) interface{} { + return withEnvironment(name) +} + +func withEnvironment(name Environment) interface{} { + if !ValidEnvironments.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithEnvironment: unknown environment %q", name), 2)) + } + return newLabel("Environment", string(name)) +} + +// WithNodeFeature specifies that a certain test or group of tests only works +// if the node supports a certain feature. The return value must be passed as +// additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +// +// The environment must be listed in ValidNodeFeatures. +func WithNodeFeature(name NodeFeature) interface{} { + return withNodeFeature(name) +} + +// WithNodeFeature is a shorthand for the corresponding package function. +func (f *Framework) WithNodeFeature(name NodeFeature) interface{} { + return withNodeFeature(name) +} + +func withNodeFeature(name NodeFeature) interface{} { + if !ValidNodeFeatures.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithNodeFeature: unknown environment %q", name), 2)) + } + return newLabel(string(name)) +} + +// WithConformace specifies that a certain test or group of tests must pass in +// all conformant Kubernetes clusters. The return value must be passed as +// additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +func WithConformance() interface{} { + return withConformance() +} + +// WithConformance is a shorthand for the corresponding package function. +func (f *Framework) WithConformance() interface{} { + return withConformance() +} + +func withConformance() interface{} { + return newLabel("Conformance") +} + +// WithNodeConformance specifies that a certain test or group of tests for node +// functionality that does not depend on runtime or Kubernetes distro specific +// behavior. The return value must be passed as additional argument to +// [framework.It], [framework.Describe], [framework.Context]. +func WithNodeConformance() interface{} { + return withNodeConformance() +} + +// WithNodeConformance is a shorthand for the corresponding package function. +func (f *Framework) WithNodeConformance() interface{} { + return withNodeConformance() +} + +func withNodeConformance() interface{} { + return newLabel("NodeConformance") +} + +// WithDisruptive specifies that a certain test or group of tests temporarily +// affects the functionality of the Kubernetes cluster. The return value must +// be passed as additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +func WithDisruptive() interface{} { + return withDisruptive() +} + +// WithDisruptive is a shorthand for the corresponding package function. +func (f *Framework) WithDisruptive() interface{} { + return withDisruptive() +} + +func withDisruptive() interface{} { + return newLabel("Disruptive") +} + +// WithSerial specifies that a certain test or group of tests must not run in +// parallel with other tests. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// Starting with ginkgo v2, serial and parallel tests can be executed in the +// same invocation. Ginkgo itself will ensure that the serial tests run +// sequentially. +func WithSerial() interface{} { + return withSerial() +} + +// WithSerial is a shorthand for the corresponding package function. +func (f *Framework) WithSerial() interface{} { + return withSerial() +} + +func withSerial() interface{} { + return newLabel("Serial") +} + +// WithSlow specifies that a certain test or group of tests must not run in +// parallel with other tests. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +func WithSlow() interface{} { + return withSlow() +} + +// WithSlow is a shorthand for the corresponding package function. +func (f *Framework) WithSlow() interface{} { + return WithSlow() +} + +func withSlow() interface{} { + return newLabel("Slow") +} + +// WithLabel is a wrapper around [ginkgo.Label]. Besides adding an arbitrary +// label to a test, it also injects the label in square brackets into the test +// name. +func WithLabel(label string) interface{} { + return withLabel(label) +} + +// WithLabel is a shorthand for the corresponding package function. +func (f *Framework) WithLabel(label string) interface{} { + return withLabel(label) +} + +func withLabel(label string) interface{} { + return newLabel(label) +} + +type label struct { + // parts get concatenated with ": " to build the full label. + parts []string + // extra is an optional fully-formed extra label. + extra string +} + +func newLabel(parts ...string) label { + return label{parts: parts} } diff --git a/test/e2e/framework/internal/unittests/bugs/bugs.go b/test/e2e/framework/internal/unittests/bugs/bugs.go new file mode 100644 index 00000000000..842aab8f8da --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/bugs.go @@ -0,0 +1,167 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bugs + +import ( + "bytes" + "testing" + + "github.com/onsi/ginkgo/v2" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs/features" +) + +// The line number of the following code is checked in BugOutput below. +// Be careful when moving it around or changing the import statements above. +// Here are some intentionally blank lines that can be removed to compensate +// for future additional import statements. +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// This must be line #50. + +func helper() { + framework.RecordBug(framework.NewBug("new bug", 0)) + framework.RecordBug(framework.NewBug("parent", 1)) +} + +func RecordBugs() { + helper() + framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"}) + framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"}) +} + +var ( + validFeature = framework.ValidFeatures.Add("feature-foo") + validEnvironment = framework.ValidEnvironments.Add("Linux") + validNodeFeature = framework.ValidNodeFeatures.Add("node-feature-foo") +) + +func Describe() { + // Normally a single line would be better, but this is an extreme example and + // thus uses multiple. + framework.Describe("abc", + // Bugs in parameters will be attributed to the Describe call, not the line of the parameter. + "", // buggy: not needed + " space1", // buggy: leading white space + "space2 ", // buggy: trailing white space + framework.WithFeature("no-such-feature"), + framework.WithFeature(validFeature), + framework.WithEnvironment("no-such-env"), + framework.WithEnvironment(validEnvironment), + framework.WithNodeFeature("no-such-node-env"), + framework.WithNodeFeature(validNodeFeature), + framework.WithFeatureGate("no-such-feature-gate"), + framework.WithFeatureGate(features.Alpha), + framework.WithFeatureGate(features.Beta), + framework.WithFeatureGate(features.GA), + framework.WithConformance(), + framework.WithNodeConformance(), + framework.WithSlow(), + framework.WithSerial(), + framework.WithDisruptive(), + framework.WithLabel("custom-label"), + "xyz", // okay, becomes part of the final text + func() { + f := framework.NewDefaultFramework("abc") + + framework.Context("y", framework.WithLabel("foo"), func() { + framework.It("should", f.WithLabel("bar"), func() { + }) + }) + + f.Context("x", f.WithLabel("foo"), func() { + f.It("should", f.WithLabel("bar"), func() { + }) + }) + }, + ) +} + +const ( + numBugs = 3 + bugOutput = `ERROR: bugs.go:53: new bug +ERROR: bugs.go:58: parent +ERROR: bugs.go:72: empty strings as separators are unnecessary and need to be removed +ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: " space1" +ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: "space2 " +ERROR: bugs.go:77: WithFeature: unknown feature "no-such-feature" +ERROR: bugs.go:79: WithEnvironment: unknown environment "no-such-env" +ERROR: bugs.go:81: WithNodeFeature: unknown environment "no-such-node-env" +ERROR: bugs.go:83: WithFeatureGate: the feature gate "no-such-feature-gate" is unknown +ERROR: buggy/buggy.go:100: hello world +ERROR: some/relative/path/buggy.go:200: with spaces +` + // Used by unittests/list-tests. It's sorted by test name, not source code location. + ListTestsOutput = `The following spec names can be used with 'ginkgo run --focus/skip': + ../bugs/bugs.go:103: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz x [foo] should [bar] + ../bugs/bugs.go:98: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz y [foo] should [bar] + +` + + // Used by unittests/list-labels. + ListLabelsOutput = `The following labels can be used with 'gingko run --label-filter': + Alpha + Beta + Conformance + Disruptive + Environment: Linux + Environment: no-such-env + Feature: feature-foo + Feature: no-such-feature + FeatureGate: TestAlphaFeature + FeatureGate: TestBetaFeature + FeatureGate: TestGAFeature + FeatureGate: no-such-feature-gate + NodeConformance + Serial + Slow + bar + custom-label + foo + no-such-node-env + node-feature-foo + +` +) + +func GetGinkgoOutput(t *testing.T) string { + var buffer bytes.Buffer + ginkgo.GinkgoWriter.TeeTo(&buffer) + t.Cleanup(ginkgo.GinkgoWriter.ClearTeeWriters) + + suiteConfig, reporterConfig := framework.CreateGinkgoConfig() + fakeT := &testing.T{} + ginkgo.RunSpecs(fakeT, "Buggy Suite", suiteConfig, reporterConfig) + + return buffer.String() +} diff --git a/test/e2e/framework/internal/unittests/bugs/bugs_test.go b/test/e2e/framework/internal/unittests/bugs/bugs_test.go index f314b117392..dd8a66c4e18 100644 --- a/test/e2e/framework/internal/unittests/bugs/bugs_test.go +++ b/test/e2e/framework/internal/unittests/bugs/bugs_test.go @@ -20,60 +20,22 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/kubernetes/test/e2e/framework" -) - -// The line number of the following code is checked in BugOutput below. -// Be careful when moving it around or changing the import statements above. -// Here are some intentionally blank lines that can be removed to compensate -// for future additional import statements. -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// This must be line #50. - -func helper() { - framework.RecordBug(framework.NewBug("new bug", 0)) - framework.RecordBug(framework.NewBug("parent", 1)) -} - -func recordBugs() { - helper() - framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"}) - framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"}) -} - -const ( - numBugs = 3 - bugOutput = `ERROR: bugs_test.go:53: new bug -ERROR: bugs_test.go:58: parent -ERROR: buggy/buggy.go:100: hello world -ERROR: some/relative/path/buggy.go:200: with spaces -` + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" ) func TestBugs(t *testing.T) { assert.NoError(t, framework.FormatBugs()) - recordBugs() + RecordBugs() + Describe() + err := framework.FormatBugs() - if assert.Error(t, err) { - assert.Equal(t, bugOutput, err.Error()) - } + require.Error(t, err) + require.Equal(t, bugOutput, err.Error()) + + output, code := unittests.GetFrameworkOutput(t, nil) + assert.Equal(t, 1, code) + assert.Equal(t, "ERROR: E2E suite initialization was faulty, these errors must be fixed:\n"+bugOutput, output) } diff --git a/test/e2e/framework/internal/unittests/bugs/features/features.go b/test/e2e/framework/internal/unittests/bugs/features/features.go new file mode 100644 index 00000000000..092ea8a8bf0 --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/features/features.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/apimachinery/pkg/util/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +const ( + Alpha featuregate.Feature = "TestAlphaFeature" + Beta featuregate.Feature = "TestBetaFeature" + GA featuregate.Feature = "TestGAFeature" +) + +func init() { + runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(testFeatureGates)) +} + +var testFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + Alpha: {PreRelease: featuregate.Alpha}, + Beta: {PreRelease: featuregate.Beta}, + GA: {PreRelease: featuregate.GA}, +} diff --git a/test/e2e/framework/internal/unittests/framework_test.go b/test/e2e/framework/internal/unittests/framework_test.go index 30c8d8d0311..efb56ba6556 100644 --- a/test/e2e/framework/internal/unittests/framework_test.go +++ b/test/e2e/framework/internal/unittests/framework_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework_test +package unittests_test import ( "reflect" diff --git a/test/e2e/framework/internal/unittests/helpers.go b/test/e2e/framework/internal/unittests/helpers.go new file mode 100644 index 00000000000..a3dcc3581ae --- /dev/null +++ b/test/e2e/framework/internal/unittests/helpers.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unittests + +import ( + "bytes" + "flag" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kubernetes/test/e2e/framework" +) + +// GetFrameworkOutput captures writes to framework.Output during a test suite setup +// and returns it together with any explicit Exit call code, -1 if none. +// May only be called once per test binary. +func GetFrameworkOutput(t *testing.T, flags map[string]string) (output string, finalExitCode int) { + // This simulates how test/e2e uses the framework and how users + // invoke test/e2e. + framework.RegisterCommonFlags(flag.CommandLine) + framework.RegisterClusterFlags(flag.CommandLine) + for flagname, value := range flags { + require.NoError(t, flag.Set(flagname, value), "set %s", flagname) + } + var buffer bytes.Buffer + framework.Output = &buffer + framework.Exit = func(code int) { + panic(exitCode(code)) + } + finalExitCode = -1 + defer func() { + if r := recover(); r != nil { + if code, ok := r.(exitCode); ok { + finalExitCode = int(code) + } else { + panic(r) + } + } + output = buffer.String() + }() + framework.AfterReadingAllFlags(&framework.TestContext) + + // Results set by defer. + return +} + +type exitCode int diff --git a/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go b/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go new file mode 100644 index 00000000000..95b0416d9a4 --- /dev/null +++ b/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package listlabels + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs" +) + +func TestListTests(t *testing.T) { + bugs.Describe() + framework.CheckForBugs = false + output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-labels": "true"}) + assert.Equal(t, 0, code) + assert.Equal(t, bugs.ListLabelsOutput, output) +} diff --git a/test/e2e/framework/internal/unittests/list-tests/listtests_test.go b/test/e2e/framework/internal/unittests/list-tests/listtests_test.go new file mode 100644 index 00000000000..4981bd0aeb9 --- /dev/null +++ b/test/e2e/framework/internal/unittests/list-tests/listtests_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package listtests + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs" +) + +func TestListTests(t *testing.T) { + bugs.Describe() + framework.CheckForBugs = false + output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-tests": "true"}) + assert.Equal(t, 0, code) + assert.Equal(t, bugs.ListTestsOutput, output) +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 97ca535f0e4..a95f50c7dc1 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -60,6 +60,14 @@ var ( // Output is used for output when not running tests, for example in -list-tests. // Test output should go to ginkgo.GinkgoWriter. Output io.Writer = os.Stdout + + // Exit is called when the framework detects fatal errors or when + // it is done with the execution of e.g. -list-tests. + Exit = os.Exit + + // CheckForBugs determines whether the framework bails out when + // test initialization found any bugs. + CheckForBugs = true ) // TestContextType contains test settings and global state. Due to @@ -495,7 +503,7 @@ func AfterReadingAllFlags(t *TestContextType) { for _, v := range image.GetImageConfigs() { fmt.Println(v.GetE2EImage()) } - os.Exit(0) + Exit(0) } // Reconfigure gomega defaults. The poll interval should be suitable @@ -509,15 +517,15 @@ func AfterReadingAllFlags(t *TestContextType) { // ginkgo.PreviewSpecs will expand all nodes and thus may find new bugs. report := ginkgo.PreviewSpecs("Kubernetes e2e test statistics") - if err := FormatBugs(); err != nil { + if err := FormatBugs(); CheckForBugs && err != nil { // Refuse to do anything if the E2E suite is buggy. fmt.Fprint(Output, "ERROR: E2E suite initialization was faulty, these errors must be fixed:") fmt.Fprint(Output, "\n"+err.Error()) - os.Exit(1) + Exit(1) } if t.listLabels || t.listTests { listTestInformation(report) - os.Exit(0) + Exit(0) } // Only set a default host if one won't be supplied via kubeconfig @@ -579,7 +587,7 @@ func AfterReadingAllFlags(t *TestContextType) { } else { klog.Errorf("Failed to setup provider config for %q: %v", TestContext.Provider, err) } - os.Exit(1) + Exit(1) } if TestContext.ReportDir != "" { @@ -589,13 +597,13 @@ func AfterReadingAllFlags(t *TestContextType) { // in parallel, so we will get "exists" error in most of them. if err := os.MkdirAll(TestContext.ReportDir, 0777); err != nil && !os.IsExist(err) { klog.Errorf("Create report dir: %v", err) - os.Exit(1) + Exit(1) } ginkgoDir := path.Join(TestContext.ReportDir, "ginkgo") if TestContext.ReportCompleteGinkgo || TestContext.ReportCompleteJUnit { if err := os.MkdirAll(ginkgoDir, 0777); err != nil && !os.IsExist(err) { klog.Errorf("Create /ginkgo: %v", err) - os.Exit(1) + Exit(1) } }