github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-08 11:38:15 +00:00
Code Issues Projects Releases Wiki Activity

e2e: add wrapper functions to annotate tests

Browse Source 
These wrapper functions set labels in addition to injecting the annotation into
the test text. It then becomes possible to select tests in different ways:

    ginkgo -v --focus="should respect internalTrafficPolicy.*\[FeatureGate:ServiceInternalTrafficPolicy\]"

    ginkgo -v --label-filter="FeatureGate:ServiceInternalTrafficPolicy"

    ginkgo -v --label-filter="Beta"

When a test runs, ginkgo shows it as:

    [It] should respect internalTrafficPolicy=Local Pod to Pod [FeatureGate:ServiceInternalTrafficPolicy] [Beta] [FeatureGate:ServiceInternalTrafficPolicy, Beta]

The test name and the labels at the end are in different colors. Embedding the
annotations inside the text is redundant and only done because users of the e2e
suite might expect it. Also, our tooling that consumes test results currently
doesn't know about ginkgo labels.

Environments, features and node features as described by
https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features
are also supported.

The framework and thus (at the moment) test/e2e do not have any pre-defined
environments and features. Adding those and modifying tests will follow in
a separate commit.
This commit is contained in:
Patrick Ohly 2022-10-06 11:38:13 +02:00
parent 535ab74346
commit 39b6916cbc
10 changed files with 749 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test/e2e/framework/.import-restrictions
View File

@ -4,7 +4,7 @@ rules:
# The following packages are okay to use:
#
# public API
- selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils)/|^[a-z]+(/|$)|github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils)
- selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils)
allowedPrefixes: [ "" ]
# stdlib
@ -16,7 +16,7 @@ rules:
allowedPrefixes: [ "" ]
# Ginkgo + Gomega
- selectorRegexp: github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils)
- selectorRegexp: ^github.com/onsi/(ginkgo|gomega)
allowedPrefixes: [ "" ]
# kube-openapi
@ -35,6 +35,8 @@ rules:
- selectorRegexp: ^github.com/|^gopkg.in
allowedPrefixes: [
"gopkg.in/inf.v0",
"gopkg.in/yaml.v2",
"github.com/blang/semver/",
"github.com/davecgh/go-spew/spew",
"github.com/evanphx/json-patch",
"github.com/go-logr/logr",
@ -48,6 +50,10 @@ rules:
"github.com/google/gofuzz",
"github.com/google/uuid",
"github.com/imdario/mergo",
"github.com/prometheus/client_golang/",
"github.com/prometheus/client_model/",
"github.com/prometheus/common/",
"github.com/prometheus/procfs",
"github.com/spf13/cobra",
"github.com/spf13/pflag",
"github.com/stretchr/testify/assert",

378
test/e2e/framework/ginkgowrapper.go
View File

@ -17,13 +17,71 @@ limitations under the License.
package framework
import (
"fmt"
"path"
"reflect"
"strings"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/ginkgo/v2/types"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
)
// Feature is the name of a certain feature that the cluster under test must have.
// Such features are different from feature gates.
type Feature string
// Environment is the name for the environment in which a test can run, like
// "Linux" or "Windows".
type Environment string
// NodeFeature is the name of a feature that a node must support. To be
// removed, see
// https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features#nodefeature.
type NodeFeature string
type Valid[T comparable] struct {
items sets.Set[T]
frozen bool
}
// Add registers a new valid item name. The expected usage is
//
// var SomeFeature = framework.ValidFeatures.Add("Some")
//
// during the init phase of an E2E suite. Individual tests should not register
// their own, to avoid uncontrolled proliferation of new items. E2E suites can,
// but don't have to, enforce that by freezing the set of valid names.
func (v *Valid[T]) Add(item T) T {
if v.frozen {
RecordBug(NewBug(fmt.Sprintf(`registry %T is already frozen, "%v" must not be added anymore`, *v, item), 1))
}
if v.items == nil {
v.items = sets.New[T]()
}
if v.items.Has(item) {
RecordBug(NewBug(fmt.Sprintf(`registry %T already contains "%v", it must not be added again`, *v, item), 1))
}
v.items.Insert(item)
return item
}
func (v *Valid[T]) Freeze() {
v.frozen = true
}
// These variables contain the parameters that [WithFeature], [WithEnvironment]
// and [WithNodeFeatures] accept. The framework itself has no pre-defined
// constants. Test suites and tests may define their own and then add them here
// before calling these With functions.
var (
ValidFeatures Valid[Feature]
ValidEnvironments Valid[Environment]
ValidNodeFeatures Valid[NodeFeature]
)
var errInterface = reflect.TypeOf((*error)(nil)).Elem()
@ -67,6 +125,322 @@ func AnnotatedLocationWithOffset(annotation string, offset int) types.CodeLocati
// ConformanceIt is wrapper function for ginkgo It. Adds "[Conformance]" tag and makes static analysis easier.
func ConformanceIt(text string, args ...interface{}) bool {
args = append(args, ginkgo.Offset(1))
return ginkgo.It(text+" [Conformance]", args...)
args = append(args, ginkgo.Offset(1), WithConformance())
return It(text, args...)
}
// It is a wrapper around [ginkgo.It] which supports framework With* labels as
// optional arguments in addition to those already supported by ginkgo itself,
// like [ginkgo.Label] and [gingko.Offset].
//
// Text and arguments may be mixed. The final text is a concatenation
// of the text arguments and special tags from the With functions.
func It(text string, args ...interface{}) bool {
return registerInSuite(ginkgo.It, text, args)
}
// It is a shorthand for the corresponding package function.
func (f *Framework) It(text string, args ...interface{}) bool {
return registerInSuite(ginkgo.It, text, args)
}
// Describe is a wrapper around [ginkgo.Describe] which supports framework
// With* labels as optional arguments in addition to those already supported by
// ginkgo itself, like [ginkgo.Label] and [gingko.Offset].
//
// Text and arguments may be mixed. The final text is a concatenation
// of the text arguments and special tags from the With functions.
func Describe(text string, args ...interface{}) bool {
return registerInSuite(ginkgo.Describe, text, args)
}
// Describe is a shorthand for the corresponding package function.
func (f *Framework) Describe(text string, args ...interface{}) bool {
return registerInSuite(ginkgo.Describe, text, args)
}
// Context is a wrapper around [ginkgo.Context] which supports framework With*
// labels as optional arguments in addition to those already supported by
// ginkgo itself, like [ginkgo.Label] and [gingko.Offset].
//
// Text and arguments may be mixed. The final text is a concatenation
// of the text arguments and special tags from the With functions.
func Context(text string, args ...interface{}) bool {
return registerInSuite(ginkgo.Context, text, args)
}
// Context is a shorthand for the corresponding package function.
func (f *Framework) Context(text string, args ...interface{}) bool {
return registerInSuite(ginkgo.Context, text, args)
}
// registerInSuite is the common implementation of all wrapper functions. It
// expects to be called through one intermediate wrapper.
func registerInSuite(ginkgoCall func(text string, args ...interface{}) bool, text string, args []interface{}) bool {
var ginkgoArgs []interface{}
var offset ginkgo.Offset
var texts []string
if text != "" {
texts = append(texts, text)
}
addLabel := func(label string) {
texts = append(texts, fmt.Sprintf("[%s]", label))
ginkgoArgs = append(ginkgoArgs, ginkgo.Label(label))
}
haveEmptyStrings := false
for _, arg := range args {
switch arg := arg.(type) {
case label:
fullLabel := strings.Join(arg.parts, ": ")
addLabel(fullLabel)
if arg.extra != "" {
addLabel(arg.extra)
}
if fullLabel == "Serial" {
ginkgoArgs = append(ginkgoArgs, ginkgo.Serial)
}
case ginkgo.Offset:
offset = arg
case string:
if arg == "" {
haveEmptyStrings = true
}
texts = append(texts, arg)
default:
ginkgoArgs = append(ginkgoArgs, arg)
}
}
offset += 2 // This function and its direct caller.
// Now that we have the final offset, we can record bugs.
if haveEmptyStrings {
RecordBug(NewBug("empty strings as separators are unnecessary and need to be removed", int(offset)))
}
// Enforce that text snippets to not start or end with spaces because
// those lead to double spaces when concatenating below.
for _, text := range texts {
if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") {
RecordBug(NewBug(fmt.Sprintf("trailing or leading spaces are unnecessary and need to be removed: %q", text), int(offset)))
}
}
ginkgoArgs = append(ginkgoArgs, offset)
text = strings.Join(texts, " ")
return ginkgoCall(text, ginkgoArgs...)
}
// WithEnvironment specifies that a certain test or group of tests only works
// with a feature available. The return value must be passed as additional
// argument to [framework.It], [framework.Describe], [framework.Context].
//
// The feature must be listed in ValidFeatures.
func WithFeature(name Feature) interface{} {
return withFeature(name)
}
// WithFeature is a shorthand for the corresponding package function.
func (f *Framework) WithFeature(name Feature) interface{} {
return withFeature(name)
}
func withFeature(name Feature) interface{} {
if !ValidFeatures.items.Has(name) {
RecordBug(NewBug(fmt.Sprintf("WithFeature: unknown feature %q", name), 2))
}
return newLabel("Feature", string(name))
}
// WithFeatureGate specifies that a certain test or group of tests depends on a
// feature gate being enabled. The return value must be passed as additional
// argument to [framework.It], [framework.Describe], [framework.Context].
//
// The feature gate must be listed in
// [k8s.io/apiserver/pkg/util/feature.DefaultMutableFeatureGate]. Once a
// feature gate gets removed from there, the WithFeatureGate calls using it
// also need to be removed.
func WithFeatureGate(featureGate featuregate.Feature) interface{} {
return withFeatureGate(featureGate)
}
// WithFeatureGate is a shorthand for the corresponding package function.
func (f *Framework) WithFeatureGate(featureGate featuregate.Feature) interface{} {
return withFeatureGate(featureGate)
}
func withFeatureGate(featureGate featuregate.Feature) interface{} {
spec, ok := utilfeature.DefaultMutableFeatureGate.GetAll()[featureGate]
if !ok {
RecordBug(NewBug(fmt.Sprintf("WithFeatureGate: the feature gate %q is unknown", featureGate), 2))
}
// We use mixed case (i.e. Beta instead of BETA). GA feature gates have no level string.
var level string
if spec.PreRelease != "" {
level = string(spec.PreRelease)
level = strings.ToUpper(level[0:1]) + strings.ToLower(level[1:])
}
l := newLabel("FeatureGate", string(featureGate))
l.extra = level
return l
}
// WithEnvironment specifies that a certain test or group of tests only works
// in a certain environment. The return value must be passed as additional
// argument to [framework.It], [framework.Describe], [framework.Context].
//
// The environment must be listed in ValidEnvironments.
func WithEnvironment(name Environment) interface{} {
return withEnvironment(name)
}
// WithEnvironment is a shorthand for the corresponding package function.
func (f *Framework) WithEnvironment(name Environment) interface{} {
return withEnvironment(name)
}
func withEnvironment(name Environment) interface{} {
if !ValidEnvironments.items.Has(name) {
RecordBug(NewBug(fmt.Sprintf("WithEnvironment: unknown environment %q", name), 2))
}
return newLabel("Environment", string(name))
}
// WithNodeFeature specifies that a certain test or group of tests only works
// if the node supports a certain feature. The return value must be passed as
// additional argument to [framework.It], [framework.Describe],
// [framework.Context].
//
// The environment must be listed in ValidNodeFeatures.
func WithNodeFeature(name NodeFeature) interface{} {
return withNodeFeature(name)
}
// WithNodeFeature is a shorthand for the corresponding package function.
func (f *Framework) WithNodeFeature(name NodeFeature) interface{} {
return withNodeFeature(name)
}
func withNodeFeature(name NodeFeature) interface{} {
if !ValidNodeFeatures.items.Has(name) {
RecordBug(NewBug(fmt.Sprintf("WithNodeFeature: unknown environment %q", name), 2))
}
return newLabel(string(name))
}
// WithConformace specifies that a certain test or group of tests must pass in
// all conformant Kubernetes clusters. The return value must be passed as
// additional argument to [framework.It], [framework.Describe],
// [framework.Context].
func WithConformance() interface{} {
return withConformance()
}
// WithConformance is a shorthand for the corresponding package function.
func (f *Framework) WithConformance() interface{} {
return withConformance()
}
func withConformance() interface{} {
return newLabel("Conformance")
}
// WithNodeConformance specifies that a certain test or group of tests for node
// functionality that does not depend on runtime or Kubernetes distro specific
// behavior. The return value must be passed as additional argument to
// [framework.It], [framework.Describe], [framework.Context].
func WithNodeConformance() interface{} {
return withNodeConformance()
}
// WithNodeConformance is a shorthand for the corresponding package function.
func (f *Framework) WithNodeConformance() interface{} {
return withNodeConformance()
}
func withNodeConformance() interface{} {
return newLabel("NodeConformance")
}
// WithDisruptive specifies that a certain test or group of tests temporarily
// affects the functionality of the Kubernetes cluster. The return value must
// be passed as additional argument to [framework.It], [framework.Describe],
// [framework.Context].
func WithDisruptive() interface{} {
return withDisruptive()
}
// WithDisruptive is a shorthand for the corresponding package function.
func (f *Framework) WithDisruptive() interface{} {
return withDisruptive()
}
func withDisruptive() interface{} {
return newLabel("Disruptive")
}
// WithSerial specifies that a certain test or group of tests must not run in
// parallel with other tests. The return value must be passed as additional
// argument to [framework.It], [framework.Describe], [framework.Context].
//
// Starting with ginkgo v2, serial and parallel tests can be executed in the
// same invocation. Ginkgo itself will ensure that the serial tests run
// sequentially.
func WithSerial() interface{} {
return withSerial()
}
// WithSerial is a shorthand for the corresponding package function.
func (f *Framework) WithSerial() interface{} {
return withSerial()
}
func withSerial() interface{} {
return newLabel("Serial")
}
// WithSlow specifies that a certain test or group of tests must not run in
// parallel with other tests. The return value must be passed as additional
// argument to [framework.It], [framework.Describe], [framework.Context].
func WithSlow() interface{} {
return withSlow()
}
// WithSlow is a shorthand for the corresponding package function.
func (f *Framework) WithSlow() interface{} {
return WithSlow()
}
func withSlow() interface{} {
return newLabel("Slow")
}
// WithLabel is a wrapper around [ginkgo.Label]. Besides adding an arbitrary
// label to a test, it also injects the label in square brackets into the test
// name.
func WithLabel(label string) interface{} {
return withLabel(label)
}
// WithLabel is a shorthand for the corresponding package function.
func (f *Framework) WithLabel(label string) interface{} {
return withLabel(label)
}
func withLabel(label string) interface{} {
return newLabel(label)
}
type label struct {
// parts get concatenated with ": " to build the full label.
parts []string
// extra is an optional fully-formed extra label.
extra string
}
func newLabel(parts ...string) label {
return label{parts: parts}
}

167
test/e2e/framework/internal/unittests/bugs/bugs.go Normal file
View File

@ -0,0 +1,167 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package bugs
import (
"bytes"
"testing"
"github.com/onsi/ginkgo/v2"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs/features"
)
// The line number of the following code is checked in BugOutput below.
// Be careful when moving it around or changing the import statements above.
// Here are some intentionally blank lines that can be removed to compensate
// for future additional import statements.
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
// This must be line #50.
func helper() {
framework.RecordBug(framework.NewBug("new bug", 0))
framework.RecordBug(framework.NewBug("parent", 1))
}
func RecordBugs() {
helper()
framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"})
framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"})
}
var (
validFeature = framework.ValidFeatures.Add("feature-foo")
validEnvironment = framework.ValidEnvironments.Add("Linux")
validNodeFeature = framework.ValidNodeFeatures.Add("node-feature-foo")
)
func Describe() {
// Normally a single line would be better, but this is an extreme example and
// thus uses multiple.
framework.Describe("abc",
// Bugs in parameters will be attributed to the Describe call, not the line of the parameter.
"", // buggy: not needed
" space1", // buggy: leading white space
"space2 ", // buggy: trailing white space
framework.WithFeature("no-such-feature"),
framework.WithFeature(validFeature),
framework.WithEnvironment("no-such-env"),
framework.WithEnvironment(validEnvironment),
framework.WithNodeFeature("no-such-node-env"),
framework.WithNodeFeature(validNodeFeature),
framework.WithFeatureGate("no-such-feature-gate"),
framework.WithFeatureGate(features.Alpha),
framework.WithFeatureGate(features.Beta),
framework.WithFeatureGate(features.GA),
framework.WithConformance(),
framework.WithNodeConformance(),
framework.WithSlow(),
framework.WithSerial(),
framework.WithDisruptive(),
framework.WithLabel("custom-label"),
"xyz", // okay, becomes part of the final text
func() {
f := framework.NewDefaultFramework("abc")
framework.Context("y", framework.WithLabel("foo"), func() {
framework.It("should", f.WithLabel("bar"), func() {
})
})
f.Context("x", f.WithLabel("foo"), func() {
f.It("should", f.WithLabel("bar"), func() {
})
})
},
)
}
const (
numBugs = 3
bugOutput = `ERROR: bugs.go:53: new bug
ERROR: bugs.go:58: parent
ERROR: bugs.go:72: empty strings as separators are unnecessary and need to be removed
ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: " space1"
ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: "space2 "
ERROR: bugs.go:77: WithFeature: unknown feature "no-such-feature"
ERROR: bugs.go:79: WithEnvironment: unknown environment "no-such-env"
ERROR: bugs.go:81: WithNodeFeature: unknown environment "no-such-node-env"
ERROR: bugs.go:83: WithFeatureGate: the feature gate "no-such-feature-gate" is unknown
ERROR: buggy/buggy.go:100: hello world
ERROR: some/relative/path/buggy.go:200: with spaces
`
// Used by unittests/list-tests. It's sorted by test name, not source code location.
ListTestsOutput = `The following spec names can be used with 'ginkgo run --focus/skip':
../bugs/bugs.go:103: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz x [foo] should [bar]
../bugs/bugs.go:98: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz y [foo] should [bar]
`
// Used by unittests/list-labels.
ListLabelsOutput = `The following labels can be used with 'gingko run --label-filter':
Alpha
Beta
Conformance
Disruptive
Environment: Linux
Environment: no-such-env
Feature: feature-foo
Feature: no-such-feature
FeatureGate: TestAlphaFeature
FeatureGate: TestBetaFeature
FeatureGate: TestGAFeature
FeatureGate: no-such-feature-gate
NodeConformance
Serial
Slow
bar
custom-label
foo
no-such-node-env
node-feature-foo
`
)
func GetGinkgoOutput(t *testing.T) string {
var buffer bytes.Buffer
ginkgo.GinkgoWriter.TeeTo(&buffer)
t.Cleanup(ginkgo.GinkgoWriter.ClearTeeWriters)
suiteConfig, reporterConfig := framework.CreateGinkgoConfig()
fakeT := &testing.T{}
ginkgo.RunSpecs(fakeT, "Buggy Suite", suiteConfig, reporterConfig)
return buffer.String()
}

60
test/e2e/framework/internal/unittests/bugs/bugs_test.go
View File

@ -20,60 +20,22 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/kubernetes/test/e2e/framework"
)
// The line number of the following code is checked in BugOutput below.
// Be careful when moving it around or changing the import statements above.
// Here are some intentionally blank lines that can be removed to compensate
// for future additional import statements.
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
// This must be line #50.
func helper() {
framework.RecordBug(framework.NewBug("new bug", 0))
framework.RecordBug(framework.NewBug("parent", 1))
}
func recordBugs() {
helper()
framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"})
framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"})
}
const (
numBugs = 3
bugOutput = `ERROR: bugs_test.go:53: new bug
ERROR: bugs_test.go:58: parent
ERROR: buggy/buggy.go:100: hello world
ERROR: some/relative/path/buggy.go:200: with spaces
`
"k8s.io/kubernetes/test/e2e/framework/internal/unittests"
)
func TestBugs(t *testing.T) {
assert.NoError(t, framework.FormatBugs())
recordBugs()
RecordBugs()
Describe()
err := framework.FormatBugs()
if assert.Error(t, err) {
assert.Equal(t, bugOutput, err.Error())
}
require.Error(t, err)
require.Equal(t, bugOutput, err.Error())
output, code := unittests.GetFrameworkOutput(t, nil)
assert.Equal(t, 1, code)
assert.Equal(t, "ERROR: E2E suite initialization was faulty, these errors must be fixed:\n"+bugOutput, output)
}

39
test/e2e/framework/internal/unittests/bugs/features/features.go Normal file
View File

@ -0,0 +1,39 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"k8s.io/apimachinery/pkg/util/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
)
const (
Alpha featuregate.Feature = "TestAlphaFeature"
Beta featuregate.Feature = "TestBetaFeature"
GA featuregate.Feature = "TestGAFeature"
)
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(testFeatureGates))
}
var testFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
Alpha: {PreRelease: featuregate.Alpha},
Beta: {PreRelease: featuregate.Beta},
GA: {PreRelease: featuregate.GA},
}

2
test/e2e/framework/internal/unittests/framework_test.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package framework_test
package unittests_test
import (
"reflect"

61
test/e2e/framework/internal/unittests/helpers.go Normal file
View File

@ -0,0 +1,61 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package unittests
import (
"bytes"
"flag"
"testing"
"github.com/stretchr/testify/require"
"k8s.io/kubernetes/test/e2e/framework"
)
// GetFrameworkOutput captures writes to framework.Output during a test suite setup
// and returns it together with any explicit Exit call code, -1 if none.
// May only be called once per test binary.
func GetFrameworkOutput(t *testing.T, flags map[string]string) (output string, finalExitCode int) {
// This simulates how test/e2e uses the framework and how users
// invoke test/e2e.
framework.RegisterCommonFlags(flag.CommandLine)
framework.RegisterClusterFlags(flag.CommandLine)
for flagname, value := range flags {
require.NoError(t, flag.Set(flagname, value), "set %s", flagname)
}
var buffer bytes.Buffer
framework.Output = &buffer
framework.Exit = func(code int) {
panic(exitCode(code))
}
finalExitCode = -1
defer func() {
if r := recover(); r != nil {
if code, ok := r.(exitCode); ok {
finalExitCode = int(code)
} else {
panic(r)
}
}
output = buffer.String()
}()
framework.AfterReadingAllFlags(&framework.TestContext)
// Results set by defer.
return
}
type exitCode int

35
test/e2e/framework/internal/unittests/list-labels/listlabels_test.go Normal file
View File

@ -0,0 +1,35 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package listlabels
import (
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/internal/unittests"
"k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs"
)
func TestListTests(t *testing.T) {
bugs.Describe()
framework.CheckForBugs = false
output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-labels": "true"})
assert.Equal(t, 0, code)
assert.Equal(t, bugs.ListLabelsOutput, output)
}

35
test/e2e/framework/internal/unittests/list-tests/listtests_test.go Normal file
View File

@ -0,0 +1,35 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package listtests
import (
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/internal/unittests"
"k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs"
)
func TestListTests(t *testing.T) {
bugs.Describe()
framework.CheckForBugs = false
output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-tests": "true"})
assert.Equal(t, 0, code)
assert.Equal(t, bugs.ListTestsOutput, output)
}

22
test/e2e/framework/test_context.go
View File

@ -60,6 +60,14 @@ var (
// Output is used for output when not running tests, for example in -list-tests.
// Test output should go to ginkgo.GinkgoWriter.
Output io.Writer = os.Stdout
// Exit is called when the framework detects fatal errors or when
// it is done with the execution of e.g. -list-tests.
Exit = os.Exit
// CheckForBugs determines whether the framework bails out when
// test initialization found any bugs.
CheckForBugs = true
)
// TestContextType contains test settings and global state. Due to
@ -495,7 +503,7 @@ func AfterReadingAllFlags(t *TestContextType) {
for _, v := range image.GetImageConfigs() {
fmt.Println(v.GetE2EImage())
}
os.Exit(0)
Exit(0)
}
// Reconfigure gomega defaults. The poll interval should be suitable
@ -509,15 +517,15 @@ func AfterReadingAllFlags(t *TestContextType) {
// ginkgo.PreviewSpecs will expand all nodes and thus may find new bugs.
report := ginkgo.PreviewSpecs("Kubernetes e2e test statistics")
if err := FormatBugs(); err != nil {
if err := FormatBugs(); CheckForBugs && err != nil {
// Refuse to do anything if the E2E suite is buggy.
fmt.Fprint(Output, "ERROR: E2E suite initialization was faulty, these errors must be fixed:")
fmt.Fprint(Output, "\n"+err.Error())
os.Exit(1)
Exit(1)
}
if t.listLabels || t.listTests {
listTestInformation(report)
os.Exit(0)
Exit(0)
}
// Only set a default host if one won't be supplied via kubeconfig
@ -579,7 +587,7 @@ func AfterReadingAllFlags(t *TestContextType) {
} else {
klog.Errorf("Failed to setup provider config for %q: %v", TestContext.Provider, err)
}
os.Exit(1)
Exit(1)
}
if TestContext.ReportDir != "" {
@ -589,13 +597,13 @@ func AfterReadingAllFlags(t *TestContextType) {
// in parallel, so we will get "exists" error in most of them.
if err := os.MkdirAll(TestContext.ReportDir, 0777); err != nil && !os.IsExist(err) {
klog.Errorf("Create report dir: %v", err)
os.Exit(1)
Exit(1)
}
ginkgoDir := path.Join(TestContext.ReportDir, "ginkgo")
if TestContext.ReportCompleteGinkgo || TestContext.ReportCompleteJUnit {
if err := os.MkdirAll(ginkgoDir, 0777); err != nil && !os.IsExist(err) {
klog.Errorf("Create <report-dir>/ginkgo: %v", err)
os.Exit(1)
Exit(1)
}
}