From aa4d2fc5edfbf527e5a7cb3cb5deff0bbf5e8aca Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 9 Oct 2023 10:59:09 +0200 Subject: [PATCH 1/6] e2e storage: remove useless info message Always printing "Enabling in-tree volume drivers" whenever the E2E suite is initializing doesn't provide any useful information and makes output of the upcoming -list-tests look weird. --- test/e2e/storage/in_tree_volumes.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 53764bcbc5c..d7dfc4062fe 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -53,8 +53,6 @@ var testDrivers = []func() storageframework.TestDriver{ // This executes testSuites for in-tree volumes. var _ = utils.SIGDescribe("In-tree Volumes", func() { - framework.Logf("Enabling in-tree volume drivers") - gceEnabled := false for _, driver := range framework.TestContext.EnabledVolumeDrivers { switch driver { From 3afdcc03ea544888ebde42b8c3eda982b7d20299 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 19 Jun 2023 12:19:22 +0200 Subject: [PATCH 2/6] e2e framework: list specs and labels -list-tests is a more concise alternative for `ginkgo --dry-run` with one line per test. In contrast to `--dry-run`, it really lists all tests. `--dry-run` without additional parameters uses the default skip expression from the E2E context, which filters out flaky and feature-gated tests. The output includes the source code location where each test is defined. It is sorted by test name (not source code location) because that order is independent of reorganizing the source code and ordering by location can be achieved with "sort". -list-labels has no corresponding feature in Ginkgo. One possible usage is to figure out what values might make sense for -focus/skip/label-filter. Unit tests will follow in a future commit. --- test/e2e/framework/test_context.go | 62 ++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 096411cfd60..763fdd1d82a 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -23,9 +23,11 @@ import ( "errors" "flag" "fmt" + "io" "math" "os" "path" + "path/filepath" "sort" "strings" "time" @@ -36,6 +38,7 @@ import ( "github.com/onsi/gomega" gomegaformat "github.com/onsi/gomega/format" + "k8s.io/apimachinery/pkg/util/sets" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" cliflag "k8s.io/component-base/cli/flag" @@ -53,6 +56,12 @@ const ( DefaultNumNodes = -1 ) +var ( + // Output is used for output when not running tests, for example in -list-tests. + // Test output should go to ginkgo.GinkgoWriter. + Output io.Writer = os.Stdout +) + // TestContextType contains test settings and global state. Due to // historic reasons, it is a mixture of items managed by the test // framework itself, cloud providers and individual tests. @@ -94,6 +103,8 @@ type TestContextType struct { // ListImages will list off all images that are used then quit ListImages bool + listTests, listLabels bool + // ListConformanceTests will list off all conformance tests that are available then quit ListConformanceTests bool @@ -356,6 +367,8 @@ func RegisterCommonFlags(flags *flag.FlagSet) { flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/control-plane`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests.") flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for running tests.") + flags.BoolVar(&TestContext.listLabels, "list-labels", false, "If true, will show the list of labels that can be used to select tests via -ginkgo.label-filter.") + flags.BoolVar(&TestContext.listTests, "list-tests", false, "If true, will show the full names of all tests (aka specs) that can be used to select test via -ginkgo.focus/skip.") flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.") @@ -494,6 +507,11 @@ func AfterReadingAllFlags(t *TestContextType) { gomega.SetDefaultEventuallyTimeout(t.timeouts.PodStart) gomega.SetDefaultConsistentlyDuration(t.timeouts.PodStartShort) + if t.listLabels || t.listTests { + listTestInformation(ginkgo.PreviewSpecs("Kubernetes e2e test statistics")) + os.Exit(0) + } + // Only set a default host if one won't be supplied via kubeconfig if len(t.Host) == 0 && len(t.KubeConfig) == 0 { // Check if we can use the in-cluster config @@ -600,3 +618,47 @@ func AfterReadingAllFlags(t *TestContextType) { }) } } + +func listTestInformation(report ginkgo.Report) { + indent := strings.Repeat(" ", 4) + + if TestContext.listLabels { + labels := sets.New[string]() + for _, spec := range report.SpecReports { + if spec.LeafNodeType == types.NodeTypeIt { + labels.Insert(spec.Labels()...) + } + } + fmt.Fprintf(Output, "The following labels can be used with 'gingko run --label-filter':\n%s%s\n\n", indent, strings.Join(sets.List(labels), "\n"+indent)) + } + if TestContext.listTests { + leafs := make([][]string, 0, len(report.SpecReports)) + wd, _ := os.Getwd() + for _, spec := range report.SpecReports { + if spec.LeafNodeType == types.NodeTypeIt { + leafs = append(leafs, []string{fmt.Sprintf("%s:%d: ", relativePath(wd, spec.LeafNodeLocation.FileName), spec.LeafNodeLocation.LineNumber), spec.FullText()}) + } + } + // Sort by test name, not the source code location, because the test + // name is more stable across code refactoring. + sort.Slice(leafs, func(i, j int) bool { + return leafs[i][1] < leafs[j][1] + }) + fmt.Fprint(Output, "The following spec names can be used with 'ginkgo run --focus/skip':\n") + for _, leaf := range leafs { + fmt.Fprintf(Output, "%s%s%s\n", indent, leaf[0], leaf[1]) + } + fmt.Fprint(Output, "\n") + } +} + +func relativePath(wd, path string) string { + if wd == "" { + return path + } + relpath, err := filepath.Rel(wd, path) + if err != nil { + return path + } + return relpath +} From 535ab743461f8a21aba66f3e48bc71b3596b6f45 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 27 Sep 2023 19:27:33 +0200 Subject: [PATCH 3/6] e2e framework: track and report internal bugs If something goes wrong during the test registration phase, the only solution so far was to panic. This is not user-friendly and only allows to report one problem at a time. If initialization can continue, then a better solution is to record a bug, continue, and then report all bugs together. This also works when just listing tests. The new verify-e2e-suites.sh uses that to check all test suites (identified as "packages that call framework.AfterReadingAllFlags", with some exceptions) as part of pull-kubernetes-verify. Example output for a fake framework.RecordBug(framework.NewBug("fake bug during SIGDescribe", 0)) in test/e2e/storage/volume_metrics.go: ``` $ hack/verify-e2e-suites.sh go version go1.21.1 linux/amd64 ERROR: E2E test suite invocation failed for test/e2e. ERROR: E2E suite initialization was faulty, these errors must be fixed: ERROR: test/e2e/storage/volume_metrics.go:49: fake bug during SIGDescribe E2E suite test/e2e_kubeadm passed. E2E suite test/e2e_node passed. ``` --- hack/verify-e2e-suites.sh | 45 ++++++++ test/e2e/framework/bugs.go | 108 ++++++++++++++++++ .../internal/unittests/bugs/bugs_test.go | 79 +++++++++++++ test/e2e/framework/test_context.go | 10 +- 4 files changed, 241 insertions(+), 1 deletion(-) create mode 100755 hack/verify-e2e-suites.sh create mode 100644 test/e2e/framework/bugs.go create mode 100644 test/e2e/framework/internal/unittests/bugs/bugs_test.go diff --git a/hack/verify-e2e-suites.sh b/hack/verify-e2e-suites.sh new file mode 100755 index 00000000000..1235a838c37 --- /dev/null +++ b/hack/verify-e2e-suites.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script checks that all E2E test suites are sane, i.e. can be +# started without an error. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/util.sh" + +kube::golang::verify_go_version + +cd "${KUBE_ROOT}" + +kube::util::ensure-temp-dir + +for suite in $(git grep -l framework.AfterReadingAllFlags | grep -v -e ^test/e2e/framework -e ^hack | xargs -n 1 dirname | sort -u); do + # Build a binary and run it in the root directory to get paths that are + # relative to that instead of the package directory. + out="" + if (cd "$suite" && go test -c -o "${KUBE_TEMP}/e2e.bin" .) && out=$("${KUBE_TEMP}/e2e.bin" --list-tests); then + echo "E2E suite $suite passed." + else + echo >&2 "ERROR: E2E test suite invocation failed for $suite." + # shellcheck disable=SC2001 + echo "$out" | sed -e 's/^/ /' + fi +done diff --git a/test/e2e/framework/bugs.go b/test/e2e/framework/bugs.go new file mode 100644 index 00000000000..a8202353307 --- /dev/null +++ b/test/e2e/framework/bugs.go @@ -0,0 +1,108 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/onsi/ginkgo/v2/types" +) + +var ( + bugs []Bug + bugMutex sync.Mutex +) + +// RecordBug stores information about a bug in the E2E suite source code that +// cannot be reported through ginkgo.Fail because it was found outside of some +// test, for example during test registration. +// +// This can be used instead of raising a panic. Then all bugs can be reported +// together instead of failing after the first one. +func RecordBug(bug Bug) { + bugMutex.Lock() + defer bugMutex.Unlock() + + bugs = append(bugs, bug) +} + +type Bug struct { + FileName string + LineNumber int + Message string +} + +// NewBug creates a new bug with a location that is obtained by skipping a certain number +// of stack frames. Passing zero will record the source code location of the direct caller +// of NewBug. +func NewBug(message string, skip int) Bug { + location := types.NewCodeLocation(skip + 1) + return Bug{FileName: location.FileName, LineNumber: location.LineNumber, Message: message} +} + +// FormatBugs produces a report that includes all bugs recorded earlier via +// RecordBug. An error is returned with the report if there have been bugs. +func FormatBugs() error { + bugMutex.Lock() + defer bugMutex.Unlock() + + if len(bugs) == 0 { + return nil + } + + lines := make([]string, 0, len(bugs)) + wd, err := os.Getwd() + if err != nil { + return fmt.Errorf("get current directory: %v", err) + } + // Sort by file name, line number, message. For the sake of simplicity + // this uses the full file name even though the output the may use a + // relative path. Usually the result should be the same because full + // paths will all have the same prefix. + sort.Slice(bugs, func(i, j int) bool { + switch strings.Compare(bugs[i].FileName, bugs[j].FileName) { + case -1: + return true + case 1: + return false + } + if bugs[i].LineNumber < bugs[j].LineNumber { + return true + } + if bugs[i].LineNumber > bugs[j].LineNumber { + return false + } + return bugs[i].Message < bugs[j].Message + }) + for _, bug := range bugs { + // Use relative paths, if possible. + path := bug.FileName + if wd != "" { + if relpath, err := filepath.Rel(wd, bug.FileName); err == nil { + path = relpath + } + } + lines = append(lines, fmt.Sprintf("ERROR: %s:%d: %s\n", path, bug.LineNumber, strings.TrimSpace(bug.Message))) + } + return errors.New(strings.Join(lines, "")) +} diff --git a/test/e2e/framework/internal/unittests/bugs/bugs_test.go b/test/e2e/framework/internal/unittests/bugs/bugs_test.go new file mode 100644 index 00000000000..f314b117392 --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/bugs_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bugs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" +) + +// The line number of the following code is checked in BugOutput below. +// Be careful when moving it around or changing the import statements above. +// Here are some intentionally blank lines that can be removed to compensate +// for future additional import statements. +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// This must be line #50. + +func helper() { + framework.RecordBug(framework.NewBug("new bug", 0)) + framework.RecordBug(framework.NewBug("parent", 1)) +} + +func recordBugs() { + helper() + framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"}) + framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"}) +} + +const ( + numBugs = 3 + bugOutput = `ERROR: bugs_test.go:53: new bug +ERROR: bugs_test.go:58: parent +ERROR: buggy/buggy.go:100: hello world +ERROR: some/relative/path/buggy.go:200: with spaces +` +) + +func TestBugs(t *testing.T) { + assert.NoError(t, framework.FormatBugs()) + recordBugs() + err := framework.FormatBugs() + if assert.Error(t, err) { + assert.Equal(t, bugOutput, err.Error()) + } +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 763fdd1d82a..97ca535f0e4 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -507,8 +507,16 @@ func AfterReadingAllFlags(t *TestContextType) { gomega.SetDefaultEventuallyTimeout(t.timeouts.PodStart) gomega.SetDefaultConsistentlyDuration(t.timeouts.PodStartShort) + // ginkgo.PreviewSpecs will expand all nodes and thus may find new bugs. + report := ginkgo.PreviewSpecs("Kubernetes e2e test statistics") + if err := FormatBugs(); err != nil { + // Refuse to do anything if the E2E suite is buggy. + fmt.Fprint(Output, "ERROR: E2E suite initialization was faulty, these errors must be fixed:") + fmt.Fprint(Output, "\n"+err.Error()) + os.Exit(1) + } if t.listLabels || t.listTests { - listTestInformation(ginkgo.PreviewSpecs("Kubernetes e2e test statistics")) + listTestInformation(report) os.Exit(0) } From 39b6916cbcc81c18fd23f69871f7fd43e7f92f2d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 6 Oct 2022 11:38:13 +0200 Subject: [PATCH 4/6] e2e: add wrapper functions to annotate tests These wrapper functions set labels in addition to injecting the annotation into the test text. It then becomes possible to select tests in different ways: ginkgo -v --focus="should respect internalTrafficPolicy.*\[FeatureGate:ServiceInternalTrafficPolicy\]" ginkgo -v --label-filter="FeatureGate:ServiceInternalTrafficPolicy" ginkgo -v --label-filter="Beta" When a test runs, ginkgo shows it as: [It] should respect internalTrafficPolicy=Local Pod to Pod [FeatureGate:ServiceInternalTrafficPolicy] [Beta] [FeatureGate:ServiceInternalTrafficPolicy, Beta] The test name and the labels at the end are in different colors. Embedding the annotations inside the text is redundant and only done because users of the e2e suite might expect it. Also, our tooling that consumes test results currently doesn't know about ginkgo labels. Environments, features and node features as described by https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features are also supported. The framework and thus (at the moment) test/e2e do not have any pre-defined environments and features. Adding those and modifying tests will follow in a separate commit. --- test/e2e/framework/.import-restrictions | 12 +- test/e2e/framework/ginkgowrapper.go | 378 +++++++++++++++++- .../framework/internal/unittests/bugs/bugs.go | 167 ++++++++ .../internal/unittests/bugs/bugs_test.go | 60 +-- .../unittests/bugs/features/features.go | 39 ++ .../internal/unittests/framework_test.go | 2 +- .../framework/internal/unittests/helpers.go | 61 +++ .../unittests/list-labels/listlabels_test.go | 35 ++ .../unittests/list-tests/listtests_test.go | 35 ++ test/e2e/framework/test_context.go | 22 +- 10 files changed, 749 insertions(+), 62 deletions(-) create mode 100644 test/e2e/framework/internal/unittests/bugs/bugs.go create mode 100644 test/e2e/framework/internal/unittests/bugs/features/features.go create mode 100644 test/e2e/framework/internal/unittests/helpers.go create mode 100644 test/e2e/framework/internal/unittests/list-labels/listlabels_test.go create mode 100644 test/e2e/framework/internal/unittests/list-tests/listtests_test.go diff --git a/test/e2e/framework/.import-restrictions b/test/e2e/framework/.import-restrictions index 894b9749b0a..660e7453fa7 100644 --- a/test/e2e/framework/.import-restrictions +++ b/test/e2e/framework/.import-restrictions @@ -4,7 +4,7 @@ rules: # The following packages are okay to use: # # public API - - selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils)/|^[a-z]+(/|$)|github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils) + - selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils) allowedPrefixes: [ "" ] # stdlib @@ -16,7 +16,7 @@ rules: allowedPrefixes: [ "" ] # Ginkgo + Gomega - - selectorRegexp: github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils) + - selectorRegexp: ^github.com/onsi/(ginkgo|gomega) allowedPrefixes: [ "" ] # kube-openapi @@ -33,8 +33,10 @@ rules: # Third party deps - selectorRegexp: ^github.com/|^gopkg.in - allowedPrefixes: [ + allowedPrefixes: [ "gopkg.in/inf.v0", + "gopkg.in/yaml.v2", + "github.com/blang/semver/", "github.com/davecgh/go-spew/spew", "github.com/evanphx/json-patch", "github.com/go-logr/logr", @@ -48,6 +50,10 @@ rules: "github.com/google/gofuzz", "github.com/google/uuid", "github.com/imdario/mergo", + "github.com/prometheus/client_golang/", + "github.com/prometheus/client_model/", + "github.com/prometheus/common/", + "github.com/prometheus/procfs", "github.com/spf13/cobra", "github.com/spf13/pflag", "github.com/stretchr/testify/assert", diff --git a/test/e2e/framework/ginkgowrapper.go b/test/e2e/framework/ginkgowrapper.go index e35fc4ae982..5826f7885c5 100644 --- a/test/e2e/framework/ginkgowrapper.go +++ b/test/e2e/framework/ginkgowrapper.go @@ -17,13 +17,71 @@ limitations under the License. package framework import ( + "fmt" "path" "reflect" + "strings" "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +// Feature is the name of a certain feature that the cluster under test must have. +// Such features are different from feature gates. +type Feature string + +// Environment is the name for the environment in which a test can run, like +// "Linux" or "Windows". +type Environment string + +// NodeFeature is the name of a feature that a node must support. To be +// removed, see +// https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features#nodefeature. +type NodeFeature string + +type Valid[T comparable] struct { + items sets.Set[T] + frozen bool +} + +// Add registers a new valid item name. The expected usage is +// +// var SomeFeature = framework.ValidFeatures.Add("Some") +// +// during the init phase of an E2E suite. Individual tests should not register +// their own, to avoid uncontrolled proliferation of new items. E2E suites can, +// but don't have to, enforce that by freezing the set of valid names. +func (v *Valid[T]) Add(item T) T { + if v.frozen { + RecordBug(NewBug(fmt.Sprintf(`registry %T is already frozen, "%v" must not be added anymore`, *v, item), 1)) + } + if v.items == nil { + v.items = sets.New[T]() + } + if v.items.Has(item) { + RecordBug(NewBug(fmt.Sprintf(`registry %T already contains "%v", it must not be added again`, *v, item), 1)) + } + v.items.Insert(item) + return item +} + +func (v *Valid[T]) Freeze() { + v.frozen = true +} + +// These variables contain the parameters that [WithFeature], [WithEnvironment] +// and [WithNodeFeatures] accept. The framework itself has no pre-defined +// constants. Test suites and tests may define their own and then add them here +// before calling these With functions. +var ( + ValidFeatures Valid[Feature] + ValidEnvironments Valid[Environment] + ValidNodeFeatures Valid[NodeFeature] ) var errInterface = reflect.TypeOf((*error)(nil)).Elem() @@ -67,6 +125,322 @@ func AnnotatedLocationWithOffset(annotation string, offset int) types.CodeLocati // ConformanceIt is wrapper function for ginkgo It. Adds "[Conformance]" tag and makes static analysis easier. func ConformanceIt(text string, args ...interface{}) bool { - args = append(args, ginkgo.Offset(1)) - return ginkgo.It(text+" [Conformance]", args...) + args = append(args, ginkgo.Offset(1), WithConformance()) + return It(text, args...) +} + +// It is a wrapper around [ginkgo.It] which supports framework With* labels as +// optional arguments in addition to those already supported by ginkgo itself, +// like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func It(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.It, text, args) +} + +// It is a shorthand for the corresponding package function. +func (f *Framework) It(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.It, text, args) +} + +// Describe is a wrapper around [ginkgo.Describe] which supports framework +// With* labels as optional arguments in addition to those already supported by +// ginkgo itself, like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func Describe(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Describe, text, args) +} + +// Describe is a shorthand for the corresponding package function. +func (f *Framework) Describe(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Describe, text, args) +} + +// Context is a wrapper around [ginkgo.Context] which supports framework With* +// labels as optional arguments in addition to those already supported by +// ginkgo itself, like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func Context(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Context, text, args) +} + +// Context is a shorthand for the corresponding package function. +func (f *Framework) Context(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Context, text, args) +} + +// registerInSuite is the common implementation of all wrapper functions. It +// expects to be called through one intermediate wrapper. +func registerInSuite(ginkgoCall func(text string, args ...interface{}) bool, text string, args []interface{}) bool { + var ginkgoArgs []interface{} + var offset ginkgo.Offset + var texts []string + if text != "" { + texts = append(texts, text) + } + + addLabel := func(label string) { + texts = append(texts, fmt.Sprintf("[%s]", label)) + ginkgoArgs = append(ginkgoArgs, ginkgo.Label(label)) + } + + haveEmptyStrings := false + for _, arg := range args { + switch arg := arg.(type) { + case label: + fullLabel := strings.Join(arg.parts, ": ") + addLabel(fullLabel) + if arg.extra != "" { + addLabel(arg.extra) + } + if fullLabel == "Serial" { + ginkgoArgs = append(ginkgoArgs, ginkgo.Serial) + } + case ginkgo.Offset: + offset = arg + case string: + if arg == "" { + haveEmptyStrings = true + } + texts = append(texts, arg) + default: + ginkgoArgs = append(ginkgoArgs, arg) + } + } + offset += 2 // This function and its direct caller. + + // Now that we have the final offset, we can record bugs. + if haveEmptyStrings { + RecordBug(NewBug("empty strings as separators are unnecessary and need to be removed", int(offset))) + } + + // Enforce that text snippets to not start or end with spaces because + // those lead to double spaces when concatenating below. + for _, text := range texts { + if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") { + RecordBug(NewBug(fmt.Sprintf("trailing or leading spaces are unnecessary and need to be removed: %q", text), int(offset))) + } + } + + ginkgoArgs = append(ginkgoArgs, offset) + text = strings.Join(texts, " ") + return ginkgoCall(text, ginkgoArgs...) +} + +// WithEnvironment specifies that a certain test or group of tests only works +// with a feature available. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The feature must be listed in ValidFeatures. +func WithFeature(name Feature) interface{} { + return withFeature(name) +} + +// WithFeature is a shorthand for the corresponding package function. +func (f *Framework) WithFeature(name Feature) interface{} { + return withFeature(name) +} + +func withFeature(name Feature) interface{} { + if !ValidFeatures.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithFeature: unknown feature %q", name), 2)) + } + return newLabel("Feature", string(name)) +} + +// WithFeatureGate specifies that a certain test or group of tests depends on a +// feature gate being enabled. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The feature gate must be listed in +// [k8s.io/apiserver/pkg/util/feature.DefaultMutableFeatureGate]. Once a +// feature gate gets removed from there, the WithFeatureGate calls using it +// also need to be removed. +func WithFeatureGate(featureGate featuregate.Feature) interface{} { + return withFeatureGate(featureGate) +} + +// WithFeatureGate is a shorthand for the corresponding package function. +func (f *Framework) WithFeatureGate(featureGate featuregate.Feature) interface{} { + return withFeatureGate(featureGate) +} + +func withFeatureGate(featureGate featuregate.Feature) interface{} { + spec, ok := utilfeature.DefaultMutableFeatureGate.GetAll()[featureGate] + if !ok { + RecordBug(NewBug(fmt.Sprintf("WithFeatureGate: the feature gate %q is unknown", featureGate), 2)) + } + + // We use mixed case (i.e. Beta instead of BETA). GA feature gates have no level string. + var level string + if spec.PreRelease != "" { + level = string(spec.PreRelease) + level = strings.ToUpper(level[0:1]) + strings.ToLower(level[1:]) + } + + l := newLabel("FeatureGate", string(featureGate)) + l.extra = level + return l +} + +// WithEnvironment specifies that a certain test or group of tests only works +// in a certain environment. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The environment must be listed in ValidEnvironments. +func WithEnvironment(name Environment) interface{} { + return withEnvironment(name) +} + +// WithEnvironment is a shorthand for the corresponding package function. +func (f *Framework) WithEnvironment(name Environment) interface{} { + return withEnvironment(name) +} + +func withEnvironment(name Environment) interface{} { + if !ValidEnvironments.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithEnvironment: unknown environment %q", name), 2)) + } + return newLabel("Environment", string(name)) +} + +// WithNodeFeature specifies that a certain test or group of tests only works +// if the node supports a certain feature. The return value must be passed as +// additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +// +// The environment must be listed in ValidNodeFeatures. +func WithNodeFeature(name NodeFeature) interface{} { + return withNodeFeature(name) +} + +// WithNodeFeature is a shorthand for the corresponding package function. +func (f *Framework) WithNodeFeature(name NodeFeature) interface{} { + return withNodeFeature(name) +} + +func withNodeFeature(name NodeFeature) interface{} { + if !ValidNodeFeatures.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithNodeFeature: unknown environment %q", name), 2)) + } + return newLabel(string(name)) +} + +// WithConformace specifies that a certain test or group of tests must pass in +// all conformant Kubernetes clusters. The return value must be passed as +// additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +func WithConformance() interface{} { + return withConformance() +} + +// WithConformance is a shorthand for the corresponding package function. +func (f *Framework) WithConformance() interface{} { + return withConformance() +} + +func withConformance() interface{} { + return newLabel("Conformance") +} + +// WithNodeConformance specifies that a certain test or group of tests for node +// functionality that does not depend on runtime or Kubernetes distro specific +// behavior. The return value must be passed as additional argument to +// [framework.It], [framework.Describe], [framework.Context]. +func WithNodeConformance() interface{} { + return withNodeConformance() +} + +// WithNodeConformance is a shorthand for the corresponding package function. +func (f *Framework) WithNodeConformance() interface{} { + return withNodeConformance() +} + +func withNodeConformance() interface{} { + return newLabel("NodeConformance") +} + +// WithDisruptive specifies that a certain test or group of tests temporarily +// affects the functionality of the Kubernetes cluster. The return value must +// be passed as additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +func WithDisruptive() interface{} { + return withDisruptive() +} + +// WithDisruptive is a shorthand for the corresponding package function. +func (f *Framework) WithDisruptive() interface{} { + return withDisruptive() +} + +func withDisruptive() interface{} { + return newLabel("Disruptive") +} + +// WithSerial specifies that a certain test or group of tests must not run in +// parallel with other tests. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// Starting with ginkgo v2, serial and parallel tests can be executed in the +// same invocation. Ginkgo itself will ensure that the serial tests run +// sequentially. +func WithSerial() interface{} { + return withSerial() +} + +// WithSerial is a shorthand for the corresponding package function. +func (f *Framework) WithSerial() interface{} { + return withSerial() +} + +func withSerial() interface{} { + return newLabel("Serial") +} + +// WithSlow specifies that a certain test or group of tests must not run in +// parallel with other tests. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +func WithSlow() interface{} { + return withSlow() +} + +// WithSlow is a shorthand for the corresponding package function. +func (f *Framework) WithSlow() interface{} { + return WithSlow() +} + +func withSlow() interface{} { + return newLabel("Slow") +} + +// WithLabel is a wrapper around [ginkgo.Label]. Besides adding an arbitrary +// label to a test, it also injects the label in square brackets into the test +// name. +func WithLabel(label string) interface{} { + return withLabel(label) +} + +// WithLabel is a shorthand for the corresponding package function. +func (f *Framework) WithLabel(label string) interface{} { + return withLabel(label) +} + +func withLabel(label string) interface{} { + return newLabel(label) +} + +type label struct { + // parts get concatenated with ": " to build the full label. + parts []string + // extra is an optional fully-formed extra label. + extra string +} + +func newLabel(parts ...string) label { + return label{parts: parts} } diff --git a/test/e2e/framework/internal/unittests/bugs/bugs.go b/test/e2e/framework/internal/unittests/bugs/bugs.go new file mode 100644 index 00000000000..842aab8f8da --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/bugs.go @@ -0,0 +1,167 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bugs + +import ( + "bytes" + "testing" + + "github.com/onsi/ginkgo/v2" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs/features" +) + +// The line number of the following code is checked in BugOutput below. +// Be careful when moving it around or changing the import statements above. +// Here are some intentionally blank lines that can be removed to compensate +// for future additional import statements. +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// This must be line #50. + +func helper() { + framework.RecordBug(framework.NewBug("new bug", 0)) + framework.RecordBug(framework.NewBug("parent", 1)) +} + +func RecordBugs() { + helper() + framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"}) + framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"}) +} + +var ( + validFeature = framework.ValidFeatures.Add("feature-foo") + validEnvironment = framework.ValidEnvironments.Add("Linux") + validNodeFeature = framework.ValidNodeFeatures.Add("node-feature-foo") +) + +func Describe() { + // Normally a single line would be better, but this is an extreme example and + // thus uses multiple. + framework.Describe("abc", + // Bugs in parameters will be attributed to the Describe call, not the line of the parameter. + "", // buggy: not needed + " space1", // buggy: leading white space + "space2 ", // buggy: trailing white space + framework.WithFeature("no-such-feature"), + framework.WithFeature(validFeature), + framework.WithEnvironment("no-such-env"), + framework.WithEnvironment(validEnvironment), + framework.WithNodeFeature("no-such-node-env"), + framework.WithNodeFeature(validNodeFeature), + framework.WithFeatureGate("no-such-feature-gate"), + framework.WithFeatureGate(features.Alpha), + framework.WithFeatureGate(features.Beta), + framework.WithFeatureGate(features.GA), + framework.WithConformance(), + framework.WithNodeConformance(), + framework.WithSlow(), + framework.WithSerial(), + framework.WithDisruptive(), + framework.WithLabel("custom-label"), + "xyz", // okay, becomes part of the final text + func() { + f := framework.NewDefaultFramework("abc") + + framework.Context("y", framework.WithLabel("foo"), func() { + framework.It("should", f.WithLabel("bar"), func() { + }) + }) + + f.Context("x", f.WithLabel("foo"), func() { + f.It("should", f.WithLabel("bar"), func() { + }) + }) + }, + ) +} + +const ( + numBugs = 3 + bugOutput = `ERROR: bugs.go:53: new bug +ERROR: bugs.go:58: parent +ERROR: bugs.go:72: empty strings as separators are unnecessary and need to be removed +ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: " space1" +ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: "space2 " +ERROR: bugs.go:77: WithFeature: unknown feature "no-such-feature" +ERROR: bugs.go:79: WithEnvironment: unknown environment "no-such-env" +ERROR: bugs.go:81: WithNodeFeature: unknown environment "no-such-node-env" +ERROR: bugs.go:83: WithFeatureGate: the feature gate "no-such-feature-gate" is unknown +ERROR: buggy/buggy.go:100: hello world +ERROR: some/relative/path/buggy.go:200: with spaces +` + // Used by unittests/list-tests. It's sorted by test name, not source code location. + ListTestsOutput = `The following spec names can be used with 'ginkgo run --focus/skip': + ../bugs/bugs.go:103: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz x [foo] should [bar] + ../bugs/bugs.go:98: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz y [foo] should [bar] + +` + + // Used by unittests/list-labels. + ListLabelsOutput = `The following labels can be used with 'gingko run --label-filter': + Alpha + Beta + Conformance + Disruptive + Environment: Linux + Environment: no-such-env + Feature: feature-foo + Feature: no-such-feature + FeatureGate: TestAlphaFeature + FeatureGate: TestBetaFeature + FeatureGate: TestGAFeature + FeatureGate: no-such-feature-gate + NodeConformance + Serial + Slow + bar + custom-label + foo + no-such-node-env + node-feature-foo + +` +) + +func GetGinkgoOutput(t *testing.T) string { + var buffer bytes.Buffer + ginkgo.GinkgoWriter.TeeTo(&buffer) + t.Cleanup(ginkgo.GinkgoWriter.ClearTeeWriters) + + suiteConfig, reporterConfig := framework.CreateGinkgoConfig() + fakeT := &testing.T{} + ginkgo.RunSpecs(fakeT, "Buggy Suite", suiteConfig, reporterConfig) + + return buffer.String() +} diff --git a/test/e2e/framework/internal/unittests/bugs/bugs_test.go b/test/e2e/framework/internal/unittests/bugs/bugs_test.go index f314b117392..dd8a66c4e18 100644 --- a/test/e2e/framework/internal/unittests/bugs/bugs_test.go +++ b/test/e2e/framework/internal/unittests/bugs/bugs_test.go @@ -20,60 +20,22 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/kubernetes/test/e2e/framework" -) - -// The line number of the following code is checked in BugOutput below. -// Be careful when moving it around or changing the import statements above. -// Here are some intentionally blank lines that can be removed to compensate -// for future additional import statements. -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// This must be line #50. - -func helper() { - framework.RecordBug(framework.NewBug("new bug", 0)) - framework.RecordBug(framework.NewBug("parent", 1)) -} - -func recordBugs() { - helper() - framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"}) - framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"}) -} - -const ( - numBugs = 3 - bugOutput = `ERROR: bugs_test.go:53: new bug -ERROR: bugs_test.go:58: parent -ERROR: buggy/buggy.go:100: hello world -ERROR: some/relative/path/buggy.go:200: with spaces -` + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" ) func TestBugs(t *testing.T) { assert.NoError(t, framework.FormatBugs()) - recordBugs() + RecordBugs() + Describe() + err := framework.FormatBugs() - if assert.Error(t, err) { - assert.Equal(t, bugOutput, err.Error()) - } + require.Error(t, err) + require.Equal(t, bugOutput, err.Error()) + + output, code := unittests.GetFrameworkOutput(t, nil) + assert.Equal(t, 1, code) + assert.Equal(t, "ERROR: E2E suite initialization was faulty, these errors must be fixed:\n"+bugOutput, output) } diff --git a/test/e2e/framework/internal/unittests/bugs/features/features.go b/test/e2e/framework/internal/unittests/bugs/features/features.go new file mode 100644 index 00000000000..092ea8a8bf0 --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/features/features.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/apimachinery/pkg/util/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +const ( + Alpha featuregate.Feature = "TestAlphaFeature" + Beta featuregate.Feature = "TestBetaFeature" + GA featuregate.Feature = "TestGAFeature" +) + +func init() { + runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(testFeatureGates)) +} + +var testFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + Alpha: {PreRelease: featuregate.Alpha}, + Beta: {PreRelease: featuregate.Beta}, + GA: {PreRelease: featuregate.GA}, +} diff --git a/test/e2e/framework/internal/unittests/framework_test.go b/test/e2e/framework/internal/unittests/framework_test.go index 30c8d8d0311..efb56ba6556 100644 --- a/test/e2e/framework/internal/unittests/framework_test.go +++ b/test/e2e/framework/internal/unittests/framework_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework_test +package unittests_test import ( "reflect" diff --git a/test/e2e/framework/internal/unittests/helpers.go b/test/e2e/framework/internal/unittests/helpers.go new file mode 100644 index 00000000000..a3dcc3581ae --- /dev/null +++ b/test/e2e/framework/internal/unittests/helpers.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unittests + +import ( + "bytes" + "flag" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kubernetes/test/e2e/framework" +) + +// GetFrameworkOutput captures writes to framework.Output during a test suite setup +// and returns it together with any explicit Exit call code, -1 if none. +// May only be called once per test binary. +func GetFrameworkOutput(t *testing.T, flags map[string]string) (output string, finalExitCode int) { + // This simulates how test/e2e uses the framework and how users + // invoke test/e2e. + framework.RegisterCommonFlags(flag.CommandLine) + framework.RegisterClusterFlags(flag.CommandLine) + for flagname, value := range flags { + require.NoError(t, flag.Set(flagname, value), "set %s", flagname) + } + var buffer bytes.Buffer + framework.Output = &buffer + framework.Exit = func(code int) { + panic(exitCode(code)) + } + finalExitCode = -1 + defer func() { + if r := recover(); r != nil { + if code, ok := r.(exitCode); ok { + finalExitCode = int(code) + } else { + panic(r) + } + } + output = buffer.String() + }() + framework.AfterReadingAllFlags(&framework.TestContext) + + // Results set by defer. + return +} + +type exitCode int diff --git a/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go b/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go new file mode 100644 index 00000000000..95b0416d9a4 --- /dev/null +++ b/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package listlabels + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs" +) + +func TestListTests(t *testing.T) { + bugs.Describe() + framework.CheckForBugs = false + output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-labels": "true"}) + assert.Equal(t, 0, code) + assert.Equal(t, bugs.ListLabelsOutput, output) +} diff --git a/test/e2e/framework/internal/unittests/list-tests/listtests_test.go b/test/e2e/framework/internal/unittests/list-tests/listtests_test.go new file mode 100644 index 00000000000..4981bd0aeb9 --- /dev/null +++ b/test/e2e/framework/internal/unittests/list-tests/listtests_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package listtests + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs" +) + +func TestListTests(t *testing.T) { + bugs.Describe() + framework.CheckForBugs = false + output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-tests": "true"}) + assert.Equal(t, 0, code) + assert.Equal(t, bugs.ListTestsOutput, output) +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 97ca535f0e4..a95f50c7dc1 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -60,6 +60,14 @@ var ( // Output is used for output when not running tests, for example in -list-tests. // Test output should go to ginkgo.GinkgoWriter. Output io.Writer = os.Stdout + + // Exit is called when the framework detects fatal errors or when + // it is done with the execution of e.g. -list-tests. + Exit = os.Exit + + // CheckForBugs determines whether the framework bails out when + // test initialization found any bugs. + CheckForBugs = true ) // TestContextType contains test settings and global state. Due to @@ -495,7 +503,7 @@ func AfterReadingAllFlags(t *TestContextType) { for _, v := range image.GetImageConfigs() { fmt.Println(v.GetE2EImage()) } - os.Exit(0) + Exit(0) } // Reconfigure gomega defaults. The poll interval should be suitable @@ -509,15 +517,15 @@ func AfterReadingAllFlags(t *TestContextType) { // ginkgo.PreviewSpecs will expand all nodes and thus may find new bugs. report := ginkgo.PreviewSpecs("Kubernetes e2e test statistics") - if err := FormatBugs(); err != nil { + if err := FormatBugs(); CheckForBugs && err != nil { // Refuse to do anything if the E2E suite is buggy. fmt.Fprint(Output, "ERROR: E2E suite initialization was faulty, these errors must be fixed:") fmt.Fprint(Output, "\n"+err.Error()) - os.Exit(1) + Exit(1) } if t.listLabels || t.listTests { listTestInformation(report) - os.Exit(0) + Exit(0) } // Only set a default host if one won't be supplied via kubeconfig @@ -579,7 +587,7 @@ func AfterReadingAllFlags(t *TestContextType) { } else { klog.Errorf("Failed to setup provider config for %q: %v", TestContext.Provider, err) } - os.Exit(1) + Exit(1) } if TestContext.ReportDir != "" { @@ -589,13 +597,13 @@ func AfterReadingAllFlags(t *TestContextType) { // in parallel, so we will get "exists" error in most of them. if err := os.MkdirAll(TestContext.ReportDir, 0777); err != nil && !os.IsExist(err) { klog.Errorf("Create report dir: %v", err) - os.Exit(1) + Exit(1) } ginkgoDir := path.Join(TestContext.ReportDir, "ginkgo") if TestContext.ReportCompleteGinkgo || TestContext.ReportCompleteJUnit { if err := os.MkdirAll(ginkgoDir, 0777); err != nil && !os.IsExist(err) { klog.Errorf("Create /ginkgo: %v", err) - os.Exit(1) + Exit(1) } } From f2d34426f8691a72fca2e462b07791299dce1282 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 19 Jun 2023 17:13:37 +0200 Subject: [PATCH 5/6] e2e: enhance SIGDescribe framework.SIGDescribe is better because: - Ginkgo uses the source code location of the test, not of the wrapper, when reporting progress. - Additional annotations can be passed. To make this a drop-in replacement, framework.SIGDescribe generates a function that can be used instead of the former SIGDescribe functions. windows.SIGDescribe contained some additional code to ensure that tests are skipped when not running with a suitable node OS. This gets moved into a separate wrapper generator, to allow using framework.SIGDescribe as intended. To ensure that all callers were modified, the windows.sigDescribe isn't exported anymore (wasn't necessary in the first place!). --- test/e2e/README.md | 6 ++---- test/e2e/apimachinery/framework.go | 6 ++---- test/e2e/apps/framework.go | 6 ++---- test/e2e/architecture/framework.go | 6 ++---- test/e2e/auth/framework.go | 6 ++---- test/e2e/autoscaling/framework.go | 6 ++---- test/e2e/cloud/framework.go | 6 ++---- test/e2e/cloud/gcp/apps/framework.go | 6 ++---- test/e2e/cloud/gcp/auth/framework.go | 6 ++---- test/e2e/cloud/gcp/framework.go | 6 ++---- test/e2e/cloud/gcp/network/framework.go | 6 ++---- test/e2e/cloud/gcp/node/framework.go | 6 ++---- test/e2e/common/network/framework.go | 6 ++---- test/e2e/common/node/framework.go | 7 ++----- test/e2e/common/storage/framework.go | 6 ++---- test/e2e/framework/ginkgowrapper.go | 21 +++++++++++++++++++ .../framework/internal/unittests/bugs/bugs.go | 7 ++++--- test/e2e/instrumentation/common/framework.go | 6 ++---- test/e2e/kubectl/framework.go | 6 ++---- test/e2e/lifecycle/framework.go | 6 ++---- test/e2e/network/common/framework.go | 6 ++---- test/e2e/node/framework.go | 6 ++---- test/e2e/scheduling/framework.go | 10 +++------ test/e2e/storage/utils/framework.go | 6 ++---- test/e2e/windows/cpu_limits.go | 4 ++-- test/e2e/windows/density.go | 4 ++-- test/e2e/windows/device_plugin.go | 4 ++-- test/e2e/windows/dns.go | 4 ++-- test/e2e/windows/framework.go | 20 +++++++++++++----- test/e2e/windows/gmsa_full.go | 4 ++-- test/e2e/windows/gmsa_kubelet.go | 4 ++-- test/e2e/windows/host_process.go | 4 ++-- test/e2e/windows/hybrid_network.go | 4 ++-- test/e2e/windows/hyperv.go | 4 ++-- test/e2e/windows/kubelet_stats.go | 9 ++++---- test/e2e/windows/memory_limits.go | 5 ++--- test/e2e/windows/reboot_node.go | 4 ++-- test/e2e/windows/security_context.go | 4 ++-- test/e2e/windows/service.go | 5 ++--- test/e2e/windows/volumes.go | 5 ++--- test/e2e_node/framework.go | 6 ++---- 41 files changed, 120 insertions(+), 139 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index bef1f2cbff2..656c86eceec 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -45,12 +45,10 @@ import ( // test/e2e/lifecycle/framework.go package lifecycle -import "github.com/onsi/ginkgo" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cluster-lifecycle") ``` ```golang // test/e2e/lifecycle/bootstrap/bootstrap_signer.go diff --git a/test/e2e/apimachinery/framework.go b/test/e2e/apimachinery/framework.go index 6b7ee59a919..4edc3411949 100644 --- a/test/e2e/apimachinery/framework.go +++ b/test/e2e/apimachinery/framework.go @@ -16,9 +16,7 @@ limitations under the License. package apimachinery -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-api-machinery] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("api-machinery") diff --git a/test/e2e/apps/framework.go b/test/e2e/apps/framework.go index dde7fa0326b..d940e5f1f2f 100644 --- a/test/e2e/apps/framework.go +++ b/test/e2e/apps/framework.go @@ -16,9 +16,7 @@ limitations under the License. package apps -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-apps] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("apps") diff --git a/test/e2e/architecture/framework.go b/test/e2e/architecture/framework.go index 4d7d819c006..b8b12b950a6 100644 --- a/test/e2e/architecture/framework.go +++ b/test/e2e/architecture/framework.go @@ -16,9 +16,7 @@ limitations under the License. package architecture -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-architecture] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("architecture") diff --git a/test/e2e/auth/framework.go b/test/e2e/auth/framework.go index cf3d006234b..0c0e3bc8408 100644 --- a/test/e2e/auth/framework.go +++ b/test/e2e/auth/framework.go @@ -16,9 +16,7 @@ limitations under the License. package auth -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-auth] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("auth") diff --git a/test/e2e/autoscaling/framework.go b/test/e2e/autoscaling/framework.go index 0392976c4cc..5dd080ee845 100644 --- a/test/e2e/autoscaling/framework.go +++ b/test/e2e/autoscaling/framework.go @@ -16,9 +16,7 @@ limitations under the License. package autoscaling -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-autoscaling] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("autoscaling") diff --git a/test/e2e/cloud/framework.go b/test/e2e/cloud/framework.go index 1d80fbbc937..8eb4e55409d 100644 --- a/test/e2e/cloud/framework.go +++ b/test/e2e/cloud/framework.go @@ -16,9 +16,7 @@ limitations under the License. package cloud -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cloud-provider] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cloud-provider") diff --git a/test/e2e/cloud/gcp/apps/framework.go b/test/e2e/cloud/gcp/apps/framework.go index 5f2edc490e2..7e768e42a2b 100644 --- a/test/e2e/cloud/gcp/apps/framework.go +++ b/test/e2e/cloud/gcp/apps/framework.go @@ -16,9 +16,7 @@ limitations under the License. package apps -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-apps] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("apps") diff --git a/test/e2e/cloud/gcp/auth/framework.go b/test/e2e/cloud/gcp/auth/framework.go index 8245c662f04..f0b0298eb73 100644 --- a/test/e2e/cloud/gcp/auth/framework.go +++ b/test/e2e/cloud/gcp/auth/framework.go @@ -16,9 +16,7 @@ limitations under the License. package auth -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-auth] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("auth") diff --git a/test/e2e/cloud/gcp/framework.go b/test/e2e/cloud/gcp/framework.go index edd24776ca6..abe47298dab 100644 --- a/test/e2e/cloud/gcp/framework.go +++ b/test/e2e/cloud/gcp/framework.go @@ -16,9 +16,7 @@ limitations under the License. package gcp -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cloud-provider-gcp] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cloud-provider-gcp") diff --git a/test/e2e/cloud/gcp/network/framework.go b/test/e2e/cloud/gcp/network/framework.go index 3e3e946d9f7..055cfa3675b 100644 --- a/test/e2e/cloud/gcp/network/framework.go +++ b/test/e2e/cloud/gcp/network/framework.go @@ -16,9 +16,7 @@ limitations under the License. package network -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-network] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("network") diff --git a/test/e2e/cloud/gcp/node/framework.go b/test/e2e/cloud/gcp/node/framework.go index b40fd35c8ca..7a7ee5d5297 100644 --- a/test/e2e/cloud/gcp/node/framework.go +++ b/test/e2e/cloud/gcp/node/framework.go @@ -16,9 +16,7 @@ limitations under the License. package node -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") diff --git a/test/e2e/common/network/framework.go b/test/e2e/common/network/framework.go index 3e3e946d9f7..055cfa3675b 100644 --- a/test/e2e/common/network/framework.go +++ b/test/e2e/common/network/framework.go @@ -16,9 +16,7 @@ limitations under the License. package network -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-network] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("network") diff --git a/test/e2e/common/node/framework.go b/test/e2e/common/node/framework.go index b40fd35c8ca..884f4bf48bf 100644 --- a/test/e2e/common/node/framework.go +++ b/test/e2e/common/node/framework.go @@ -16,9 +16,6 @@ limitations under the License. package node -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" -// SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") diff --git a/test/e2e/common/storage/framework.go b/test/e2e/common/storage/framework.go index d3351a06ff2..a7967e83aac 100644 --- a/test/e2e/common/storage/framework.go +++ b/test/e2e/common/storage/framework.go @@ -16,9 +16,7 @@ limitations under the License. package storage -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-storage] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("storage") diff --git a/test/e2e/framework/ginkgowrapper.go b/test/e2e/framework/ginkgowrapper.go index 5826f7885c5..ed7b9432f05 100644 --- a/test/e2e/framework/ginkgowrapper.go +++ b/test/e2e/framework/ginkgowrapper.go @@ -20,6 +20,7 @@ import ( "fmt" "path" "reflect" + "regexp" "strings" "github.com/onsi/ginkgo/v2" @@ -123,6 +124,26 @@ func AnnotatedLocationWithOffset(annotation string, offset int) types.CodeLocati return codeLocation } +// SIGDescribe returns a wrapper function for ginkgo.Describe which injects +// the SIG name as annotation. The parameter should be lowercase with +// no spaces and no sig- or SIG- prefix. +func SIGDescribe(sig string) func(string, ...interface{}) bool { + if !sigRE.MatchString(sig) || strings.HasPrefix(sig, "sig-") { + panic(fmt.Sprintf("SIG label must be lowercase, no spaces and no sig- prefix, got instead: %q", sig)) + } + return func(text string, args ...interface{}) bool { + args = append(args, ginkgo.Label("sig-"+sig)) + if text == "" { + text = fmt.Sprintf("[sig-%s]", sig) + } else { + text = fmt.Sprintf("[sig-%s] %s", sig, text) + } + return registerInSuite(ginkgo.Describe, text, args) + } +} + +var sigRE = regexp.MustCompile(`^[a-z]+(-[a-z]+)*$`) + // ConformanceIt is wrapper function for ginkgo It. Adds "[Conformance]" tag and makes static analysis easier. func ConformanceIt(text string, args ...interface{}) bool { args = append(args, ginkgo.Offset(1), WithConformance()) diff --git a/test/e2e/framework/internal/unittests/bugs/bugs.go b/test/e2e/framework/internal/unittests/bugs/bugs.go index 842aab8f8da..e84d04d638d 100644 --- a/test/e2e/framework/internal/unittests/bugs/bugs.go +++ b/test/e2e/framework/internal/unittests/bugs/bugs.go @@ -69,7 +69,7 @@ var ( func Describe() { // Normally a single line would be better, but this is an extreme example and // thus uses multiple. - framework.Describe("abc", + framework.SIGDescribe("testing")("abc", // Bugs in parameters will be attributed to the Describe call, not the line of the parameter. "", // buggy: not needed " space1", // buggy: leading white space @@ -123,8 +123,8 @@ ERROR: some/relative/path/buggy.go:200: with spaces ` // Used by unittests/list-tests. It's sorted by test name, not source code location. ListTestsOutput = `The following spec names can be used with 'ginkgo run --focus/skip': - ../bugs/bugs.go:103: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz x [foo] should [bar] - ../bugs/bugs.go:98: abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz y [foo] should [bar] + ../bugs/bugs.go:103: [sig-testing] abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz x [foo] should [bar] + ../bugs/bugs.go:98: [sig-testing] abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz y [foo] should [bar] ` @@ -150,6 +150,7 @@ ERROR: some/relative/path/buggy.go:200: with spaces foo no-such-node-env node-feature-foo + sig-testing ` ) diff --git a/test/e2e/instrumentation/common/framework.go b/test/e2e/instrumentation/common/framework.go index a7386091378..7b557d993f1 100644 --- a/test/e2e/instrumentation/common/framework.go +++ b/test/e2e/instrumentation/common/framework.go @@ -16,9 +16,7 @@ limitations under the License. package common -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-instrumentation] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("instrumentation") diff --git a/test/e2e/kubectl/framework.go b/test/e2e/kubectl/framework.go index 1b95893fb2c..bfb1b95d902 100644 --- a/test/e2e/kubectl/framework.go +++ b/test/e2e/kubectl/framework.go @@ -16,9 +16,7 @@ limitations under the License. package kubectl -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cli] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cli") diff --git a/test/e2e/lifecycle/framework.go b/test/e2e/lifecycle/framework.go index 97026c000ff..c697880ce66 100644 --- a/test/e2e/lifecycle/framework.go +++ b/test/e2e/lifecycle/framework.go @@ -16,9 +16,7 @@ limitations under the License. package lifecycle -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cluster-lifecycle") diff --git a/test/e2e/network/common/framework.go b/test/e2e/network/common/framework.go index b4c77ecfaf0..6efe22e2735 100644 --- a/test/e2e/network/common/framework.go +++ b/test/e2e/network/common/framework.go @@ -16,9 +16,7 @@ limitations under the License. package common -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-network] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("network") diff --git a/test/e2e/node/framework.go b/test/e2e/node/framework.go index eb2a1bb9e7c..126d2d3a8a4 100644 --- a/test/e2e/node/framework.go +++ b/test/e2e/node/framework.go @@ -16,9 +16,7 @@ limitations under the License. package node -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") diff --git a/test/e2e/scheduling/framework.go b/test/e2e/scheduling/framework.go index 507820094d7..20ef5b180e9 100644 --- a/test/e2e/scheduling/framework.go +++ b/test/e2e/scheduling/framework.go @@ -21,8 +21,6 @@ import ( "fmt" "time" - "github.com/onsi/ginkgo/v2" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -33,12 +31,10 @@ import ( var ( timeout = 10 * time.Minute waitTime = 2 * time.Second -) -// SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-scheduling] "+text, body) -} + // SIGDescribe annotates the test with the SIG label. + SIGDescribe = framework.SIGDescribe("scheduling") +) // WaitForStableCluster waits until all existing pods are scheduled and returns their amount. func WaitForStableCluster(c clientset.Interface, workerNodes sets.Set[string]) int { diff --git a/test/e2e/storage/utils/framework.go b/test/e2e/storage/utils/framework.go index 7bd007044f2..2257e03287c 100644 --- a/test/e2e/storage/utils/framework.go +++ b/test/e2e/storage/utils/framework.go @@ -16,9 +16,7 @@ limitations under the License. package utils -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-storage] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("storage") diff --git a/test/e2e/windows/cpu_limits.go b/test/e2e/windows/cpu_limits.go index 3e7003c4018..bc799af3d04 100644 --- a/test/e2e/windows/cpu_limits.go +++ b/test/e2e/windows/cpu_limits.go @@ -35,7 +35,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Cpu Resources [Serial]", func() { +var _ = sigDescribe("[Feature:Windows] Cpu Resources [Serial]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("cpu-resources-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -100,7 +100,7 @@ var _ = SIGDescribe("[Feature:Windows] Cpu Resources [Serial]", func() { } }) }) -}) +})) // newCPUBurnPods creates a list of pods (specification) with a workload that will consume all available CPU resources up to container limit func newCPUBurnPods(numPods int, image imageutils.Config, cpuLimit string, memoryLimit string) []*v1.Pod { diff --git a/test/e2e/windows/density.go b/test/e2e/windows/density.go index b344b663e34..ceb98cbf5c3 100644 --- a/test/e2e/windows/density.go +++ b/test/e2e/windows/density.go @@ -40,7 +40,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Density [Serial] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] Density [Serial] [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("density-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -72,7 +72,7 @@ var _ = SIGDescribe("[Feature:Windows] Density [Serial] [Slow]", func() { } }) -}) +})) type densityTest struct { // number of pods diff --git a/test/e2e/windows/device_plugin.go b/test/e2e/windows/device_plugin.go index dfcaa0babb1..416e1969a74 100644 --- a/test/e2e/windows/device_plugin.go +++ b/test/e2e/windows/device_plugin.go @@ -39,7 +39,7 @@ const ( testSlowMultiplier = 60 ) -var _ = SIGDescribe("[Feature:GPUDevicePlugin] Device Plugin", func() { +var _ = sigDescribe("[Feature:GPUDevicePlugin] Device Plugin", skipUnlessWindows(func() { f := framework.NewDefaultFramework("device-plugin") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -132,4 +132,4 @@ var _ = SIGDescribe("[Feature:GPUDevicePlugin] Device Plugin", func() { _, envVarDirectxGpuNameErr := e2eoutput.LookForStringInPodExec(defaultNs, windowsPod.Name, envVarCommand, envVarDirectxGpuName, time.Minute) framework.ExpectNoError(envVarDirectxGpuNameErr, "failed: didn't find expected environment variable.") }) -}) +})) diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go index bb2dcaadc3b..a9de7723076 100644 --- a/test/e2e/windows/dns.go +++ b/test/e2e/windows/dns.go @@ -31,7 +31,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] DNS", func() { +var _ = sigDescribe("[Feature:Windows] DNS", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") @@ -136,4 +136,4 @@ var _ = SIGDescribe("[Feature:Windows] DNS", func() { // TODO: Add more test cases for other DNSPolicies. }) -}) +})) diff --git a/test/e2e/windows/framework.go b/test/e2e/windows/framework.go index a0ec0cabcd9..dd9bcc396cb 100644 --- a/test/e2e/windows/framework.go +++ b/test/e2e/windows/framework.go @@ -17,19 +17,29 @@ limitations under the License. package windows import ( + "k8s.io/kubernetes/test/e2e/framework" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "github.com/onsi/ginkgo/v2" ) -// SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-windows] "+text, func() { +// sigDescribe annotates the test with the SIG label. +// Use this together with skipUnlessWindows to define +// tests that only run if the node OS is Windows: +// +// sigDescribe("foo", skipUnlessWindows(func() { ... })) +var sigDescribe = framework.SIGDescribe("windows") + +// skipUnlessWindows wraps some other Ginkgo callback such that +// a BeforeEach runs before tests defined by that callback which +// skips those tests unless the node OS is Windows. +func skipUnlessWindows(cb func()) func() { + return func() { ginkgo.BeforeEach(func() { // all tests in this package are Windows specific e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) - body() - }) + cb() + } } diff --git a/test/e2e/windows/gmsa_full.go b/test/e2e/windows/gmsa_full.go index 100c0dde0d2..2a3f94a98d1 100644 --- a/test/e2e/windows/gmsa_full.go +++ b/test/e2e/windows/gmsa_full.go @@ -90,7 +90,7 @@ const ( gmsaSharedFolder = "write_test" ) -var _ = SIGDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("gmsa-full-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -220,7 +220,7 @@ var _ = SIGDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", func() { }) }) -}) +})) func isValidOutput(output string) bool { return strings.Contains(output, expectedQueryOutput) && diff --git a/test/e2e/windows/gmsa_kubelet.go b/test/e2e/windows/gmsa_kubelet.go index e6fe25c14f6..a7d041fcd97 100644 --- a/test/e2e/windows/gmsa_kubelet.go +++ b/test/e2e/windows/gmsa_kubelet.go @@ -39,7 +39,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] GMSA Kubelet [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] GMSA Kubelet [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("gmsa-kubelet-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -133,7 +133,7 @@ var _ = SIGDescribe("[Feature:Windows] GMSA Kubelet [Slow]", func() { }) }) }) -}) +})) func generateDummyCredSpecs(domain string) *string { shortName := strings.ToUpper(strings.Split(domain, ".")[0]) diff --git a/test/e2e/windows/host_process.go b/test/e2e/windows/host_process.go index 156b5d821b3..d9562ec5619 100644 --- a/test/e2e/windows/host_process.go +++ b/test/e2e/windows/host_process.go @@ -85,7 +85,7 @@ var ( User_NTAuthoritySystem = "NT AUTHORITY\\SYSTEM" ) -var _ = SIGDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers", func() { +var _ = sigDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) @@ -799,7 +799,7 @@ var _ = SIGDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersi gomega.Expect(strings.ToLower(logs)).ShouldNot(gomega.ContainSubstring("nt authority"), "Container runs 'whoami' and logs should not contain 'nt authority'") }) -}) +})) func makeTestPodWithVolumeMounts(name string) *v1.Pod { hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go index 20a3ef6d32a..e2d05a3ec3c 100644 --- a/test/e2e/windows/hybrid_network.go +++ b/test/e2e/windows/hybrid_network.go @@ -44,7 +44,7 @@ var ( linuxBusyBoxImage = imageutils.GetE2EImage(imageutils.Nginx) ) -var _ = SIGDescribe("Hybrid cluster network", func() { +var _ = sigDescribe("Hybrid cluster network", skipUnlessWindows(func() { f := framework.NewDefaultFramework("hybrid-network") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -99,7 +99,7 @@ var _ = SIGDescribe("Hybrid cluster network", func() { }) }) -}) +})) var ( warmUpDuration = "30s" diff --git a/test/e2e/windows/hyperv.go b/test/e2e/windows/hyperv.go index fe188d80b30..edeccc5feac 100644 --- a/test/e2e/windows/hyperv.go +++ b/test/e2e/windows/hyperv.go @@ -35,7 +35,7 @@ var ( WindowsHyperVContainerRuntimeClass = "runhcs-wcow-hypervisor" ) -var _ = SIGDescribe("[Feature:WindowsHyperVContainers] HyperV containers", func() { +var _ = sigDescribe("[Feature:WindowsHyperVContainers] HyperV containers", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) @@ -143,4 +143,4 @@ var _ = SIGDescribe("[Feature:WindowsHyperVContainers] HyperV containers", func( gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded), "pod should have succeeded") }) -}) +})) diff --git a/test/e2e/windows/kubelet_stats.go b/test/e2e/windows/kubelet_stats.go index 7d5826e9eb0..8c93c177fb5 100644 --- a/test/e2e/windows/kubelet_stats.go +++ b/test/e2e/windows/kubelet_stats.go @@ -37,7 +37,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats [Serial]", func() { +var _ = sigDescribe("[Feature:Windows] Kubelet-Stats [Serial]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("kubelet-stats-test-windows-serial") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -113,8 +113,9 @@ var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats [Serial]", func() { }) }) }) -}) -var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats", func() { +})) + +var _ = sigDescribe("[Feature:Windows] Kubelet-Stats", skipUnlessWindows(func() { f := framework.NewDefaultFramework("kubelet-stats-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -204,7 +205,7 @@ var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats", func() { }) }) }) -}) +})) // findWindowsNode finds a Windows node that is Ready and Schedulable func findWindowsNode(ctx context.Context, f *framework.Framework) (v1.Node, error) { diff --git a/test/e2e/windows/memory_limits.go b/test/e2e/windows/memory_limits.go index da96baad3fe..d108e2a495b 100644 --- a/test/e2e/windows/memory_limits.go +++ b/test/e2e/windows/memory_limits.go @@ -39,7 +39,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Memory Limits [Serial] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] Memory Limits [Serial] [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("memory-limit-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -60,8 +60,7 @@ var _ = SIGDescribe("[Feature:Windows] Memory Limits [Serial] [Slow]", func() { overrideAllocatableMemoryTest(ctx, f, framework.TestContext.CloudConfig.NumNodes) }) }) - -}) +})) type nodeMemory struct { // capacity diff --git a/test/e2e/windows/reboot_node.go b/test/e2e/windows/reboot_node.go index 51226e377de..a1b7790f60f 100644 --- a/test/e2e/windows/reboot_node.go +++ b/test/e2e/windows/reboot_node.go @@ -34,7 +34,7 @@ import ( admissionapi "k8s.io/pod-security-admission/api" ) -var _ = SIGDescribe("[Feature:Windows] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] RebootHost containers [Serial] [Disruptive] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] RebootHost containers [Serial] [Disruptive] [Slow]", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) @@ -254,4 +254,4 @@ var _ = SIGDescribe("[Feature:Windows] [Excluded:WindowsDocker] [MinimumKubeletV framework.ExpectNoError(err, "Error retrieving pod") gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) }) -}) +})) diff --git a/test/e2e/windows/security_context.go b/test/e2e/windows/security_context.go index 2278b80e2f8..f72dd48ef3e 100644 --- a/test/e2e/windows/security_context.go +++ b/test/e2e/windows/security_context.go @@ -40,7 +40,7 @@ import ( const runAsUserNameContainerName = "run-as-username-container" -var _ = SIGDescribe("[Feature:Windows] SecurityContext", func() { +var _ = sigDescribe("[Feature:Windows] SecurityContext", skipUnlessWindows(func() { f := framework.NewDefaultFramework("windows-run-as-username") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -190,7 +190,7 @@ var _ = SIGDescribe("[Feature:Windows] SecurityContext", func() { expectedEventError := "container's runAsUserName (CONTAINERADMINISTRATOR) which will be regarded as root identity and will break non-root policy" gomega.Expect(event.Message).Should(gomega.ContainSubstring(expectedEventError), "Event error should indicate non-root policy caused container to not start") }) -}) +})) func runAsUserNamePod(username *string) *v1.Pod { podName := "run-as-username-" + string(uuid.NewUUID()) diff --git a/test/e2e/windows/service.go b/test/e2e/windows/service.go index 657c2bc516e..e7c431e2598 100644 --- a/test/e2e/windows/service.go +++ b/test/e2e/windows/service.go @@ -36,7 +36,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("Services", func() { +var _ = sigDescribe("Services", skipUnlessWindows(func() { f := framework.NewDefaultFramework("services") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -85,5 +85,4 @@ var _ = SIGDescribe("Services", func() { assertConsistentConnectivity(ctx, f, testPod.ObjectMeta.Name, windowsOS, windowsCheck(fmt.Sprintf("http://%s", net.JoinHostPort(nodeIP, strconv.Itoa(nodePort))))) }) - -}) +})) diff --git a/test/e2e/windows/volumes.go b/test/e2e/windows/volumes.go index 1d1c8fe81d7..31fb6918cb4 100644 --- a/test/e2e/windows/volumes.go +++ b/test/e2e/windows/volumes.go @@ -44,7 +44,7 @@ var ( image = imageutils.GetE2EImage(imageutils.Pause) ) -var _ = SIGDescribe("[Feature:Windows] Windows volume mounts", func() { +var _ = sigDescribe("[Feature:Windows] Windows volume mounts", skipUnlessWindows(func() { f := framework.NewDefaultFramework("windows-volumes") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged var ( @@ -86,8 +86,7 @@ var _ = SIGDescribe("[Feature:Windows] Windows volume mounts", func() { }) }) - -}) +})) func doReadOnlyTest(ctx context.Context, f *framework.Framework, source v1.VolumeSource, volumePath string) { var ( diff --git a/test/e2e_node/framework.go b/test/e2e_node/framework.go index f2a7c34651d..ad2b27a6110 100644 --- a/test/e2e_node/framework.go +++ b/test/e2e_node/framework.go @@ -16,9 +16,7 @@ limitations under the License. package e2enode -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") From 19ecf93ec3dd343e8416fb90886de2c99117ebf1 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 19 Jun 2023 16:03:46 +0200 Subject: [PATCH 6/6] e2e: define features and node features The list is based on the -list-tests output. --- test/e2e/e2e_test.go | 4 + test/e2e/feature/feature.go | 132 +++++++++++++++++++++++++++ test/e2e/nodefeature/nodefeature.go | 55 +++++++++++ test/e2e_node/.import-restrictions | 2 + test/e2e_node/e2e_node_suite_test.go | 4 + 5 files changed, 197 insertions(+) create mode 100644 test/e2e/feature/feature.go create mode 100644 test/e2e/nodefeature/nodefeature.go diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index af333af5e66..6c2d342e736 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -43,6 +43,10 @@ import ( e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" testfixtures "k8s.io/kubernetes/test/fixtures" + // define and freeze constants + _ "k8s.io/kubernetes/test/e2e/feature" + _ "k8s.io/kubernetes/test/e2e/nodefeature" + // test sources _ "k8s.io/kubernetes/test/e2e/apimachinery" _ "k8s.io/kubernetes/test/e2e/apps" diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go new file mode 100644 index 00000000000..09ebab6b14f --- /dev/null +++ b/test/e2e/feature/feature.go @@ -0,0 +1,132 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package feature contains pre-defined features used by test/e2e and/or +// test/e2e_node. +package feature + +import ( + "k8s.io/kubernetes/test/e2e/framework" +) + +var ( + APIServerIdentity = framework.WithFeature(framework.ValidFeatures.Add("APIServerIdentity")) + AppArmor = framework.WithFeature(framework.ValidFeatures.Add("AppArmor")) + BootstrapTokens = framework.WithFeature(framework.ValidFeatures.Add("BootstrapTokens")) + BoundServiceAccountTokenVolume = framework.WithFeature(framework.ValidFeatures.Add("BoundServiceAccountTokenVolume")) + CloudProvider = framework.WithFeature(framework.ValidFeatures.Add("CloudProvider")) + ClusterAutoscalerScalability1 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability1")) + ClusterAutoscalerScalability2 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability2")) + ClusterAutoscalerScalability3 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability3")) + ClusterAutoscalerScalability4 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability4")) + ClusterAutoscalerScalability5 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability5")) + ClusterAutoscalerScalability6 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability6")) + ClusterDowngrade = framework.WithFeature(framework.ValidFeatures.Add("ClusterDowngrade")) + ClusterSizeAutoscalingGpu = framework.WithFeature(framework.ValidFeatures.Add("ClusterSizeAutoscalingGpu")) + ClusterSizeAutoscalingScaleDown = framework.WithFeature(framework.ValidFeatures.Add("ClusterSizeAutoscalingScaleDown")) + ClusterSizeAutoscalingScaleUp = framework.WithFeature(framework.ValidFeatures.Add("ClusterSizeAutoscalingScaleUp")) + ClusterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("ClusterUpgrade")) + ComprehensiveNamespaceDraining = framework.WithFeature(framework.ValidFeatures.Add("ComprehensiveNamespaceDraining")) + CPUManager = framework.WithFeature(framework.ValidFeatures.Add("CPUManager")) + CustomMetricsAutoscaling = framework.WithFeature(framework.ValidFeatures.Add("CustomMetricsAutoscaling")) + DeviceManager = framework.WithFeature(framework.ValidFeatures.Add("DeviceManager")) + DevicePluginProbe = framework.WithFeature(framework.ValidFeatures.Add("DevicePluginProbe")) + Downgrade = framework.WithFeature(framework.ValidFeatures.Add("Downgrade")) + DynamicResourceAllocation = framework.WithFeature(framework.ValidFeatures.Add("DynamicResourceAllocation")) + EphemeralStorage = framework.WithFeature(framework.ValidFeatures.Add("EphemeralStorage")) + Example = framework.WithFeature(framework.ValidFeatures.Add("Example")) + ExperimentalResourceUsageTracking = framework.WithFeature(framework.ValidFeatures.Add("ExperimentalResourceUsageTracking")) + Flexvolumes = framework.WithFeature(framework.ValidFeatures.Add("Flexvolumes")) + GKENodePool = framework.WithFeature(framework.ValidFeatures.Add("GKENodePool")) + GPUClusterDowngrade = framework.WithFeature(framework.ValidFeatures.Add("GPUClusterDowngrade")) + GPUClusterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("GPUClusterUpgrade")) + GPUDevicePlugin = framework.WithFeature(framework.ValidFeatures.Add("GPUDevicePlugin")) + GPUMasterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("GPUMasterUpgrade")) + GPUUpgrade = framework.WithFeature(framework.ValidFeatures.Add("GPUUpgrade")) + HAMaster = framework.WithFeature(framework.ValidFeatures.Add("HAMaster")) + HPA = framework.WithFeature(framework.ValidFeatures.Add("HPA")) + HugePages = framework.WithFeature(framework.ValidFeatures.Add("HugePages")) + Ingress = framework.WithFeature(framework.ValidFeatures.Add("Ingress")) + IngressScale = framework.WithFeature(framework.ValidFeatures.Add("IngressScale")) + InPlacePodVerticalScaling = framework.WithFeature(framework.ValidFeatures.Add("InPlacePodVerticalScaling")) + IPv6DualStack = framework.WithFeature(framework.ValidFeatures.Add("IPv6DualStack")) + Kind = framework.WithFeature(framework.ValidFeatures.Add("Kind")) + KubeletCredentialProviders = framework.WithFeature(framework.ValidFeatures.Add("KubeletCredentialProviders")) + KubeletSecurity = framework.WithFeature(framework.ValidFeatures.Add("KubeletSecurity")) + KubeProxyDaemonSetDowngrade = framework.WithFeature(framework.ValidFeatures.Add("KubeProxyDaemonSetDowngrade")) + KubeProxyDaemonSetUpgrade = framework.WithFeature(framework.ValidFeatures.Add("KubeProxyDaemonSetUpgrade")) + KubeProxyDaemonSetMigration = framework.WithFeature(framework.ValidFeatures.Add("KubeProxyDaemonSetMigration")) + LabelSelector = framework.WithFeature(framework.ValidFeatures.Add("LabelSelector")) + LocalStorageCapacityIsolation = framework.WithFeature(framework.ValidFeatures.Add("LocalStorageCapacityIsolation")) + LocalStorageCapacityIsolationQuota = framework.WithFeature(framework.ValidFeatures.Add("LocalStorageCapacityIsolationQuota")) + MasterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("MasterUpgrade")) + MemoryManager = framework.WithFeature(framework.ValidFeatures.Add("MemoryManager")) + NEG = framework.WithFeature(framework.ValidFeatures.Add("NEG")) + NetworkingDNS = framework.WithFeature(framework.ValidFeatures.Add("Networking-DNS")) + NetworkingIPv4 = framework.WithFeature(framework.ValidFeatures.Add("Networking-IPv4")) + NetworkingIPv6 = framework.WithFeature(framework.ValidFeatures.Add("Networking-IPv6")) + NetworkingPerformance = framework.WithFeature(framework.ValidFeatures.Add("Networking-Performance")) + NetworkPolicy = framework.WithFeature(framework.ValidFeatures.Add("NetworkPolicy")) + NodeAuthenticator = framework.WithFeature(framework.ValidFeatures.Add("NodeAuthenticator")) + NodeAuthorizer = framework.WithFeature(framework.ValidFeatures.Add("NodeAuthorizer")) + NodeOutOfServiceVolumeDetach = framework.WithFeature(framework.ValidFeatures.Add("NodeOutOfServiceVolumeDetach")) + NoSNAT = framework.WithFeature(framework.ValidFeatures.Add("NoSNAT")) + PerformanceDNS = framework.WithFeature(framework.ValidFeatures.Add("PerformanceDNS")) + PodGarbageCollector = framework.WithFeature(framework.ValidFeatures.Add("PodGarbageCollector")) + PodPriority = framework.WithFeature(framework.ValidFeatures.Add("PodPriority")) + PodReadyToStartContainersCondition = framework.WithFeature(framework.ValidFeatures.Add("PodReadyToStartContainersCondition")) + PodResources = framework.WithFeature(framework.ValidFeatures.Add("PodResources")) + ProbeTerminationGracePeriod = framework.WithFeature(framework.ValidFeatures.Add("ProbeTerminationGracePeriod")) + Reboot = framework.WithFeature(framework.ValidFeatures.Add("Reboot")) + ReclaimPolicy = framework.WithFeature(framework.ValidFeatures.Add("ReclaimPolicy")) + RecoverVolumeExpansionFailure = framework.WithFeature(framework.ValidFeatures.Add("RecoverVolumeExpansionFailure")) + Recreate = framework.WithFeature(framework.ValidFeatures.Add("Recreate")) + RegularResourceUsageTracking = framework.WithFeature(framework.ValidFeatures.Add("RegularResourceUsageTracking")) + ScopeSelectors = framework.WithFeature(framework.ValidFeatures.Add("ScopeSelectors")) + SCTPConnectivity = framework.WithFeature(framework.ValidFeatures.Add("SCTPConnectivity")) + SeccompDefault = framework.WithFeature(framework.ValidFeatures.Add("SeccompDefault")) + SELinux = framework.WithFeature(framework.ValidFeatures.Add("SELinux")) + SELinuxMountReadWriteOncePod = framework.WithFeature(framework.ValidFeatures.Add("SELinuxMountReadWriteOncePod")) + StackdriverAcceleratorMonitoring = framework.WithFeature(framework.ValidFeatures.Add("StackdriverAcceleratorMonitoring")) + StackdriverCustomMetrics = framework.WithFeature(framework.ValidFeatures.Add("StackdriverCustomMetrics")) + StackdriverExternalMetrics = framework.WithFeature(framework.ValidFeatures.Add("StackdriverExternalMetrics")) + StackdriverMetadataAgent = framework.WithFeature(framework.ValidFeatures.Add("StackdriverMetadataAgent")) + StackdriverMonitoring = framework.WithFeature(framework.ValidFeatures.Add("StackdriverMonitoring")) + StandaloneMode = framework.WithFeature(framework.ValidFeatures.Add("StandaloneMode")) + StatefulSet = framework.WithFeature(framework.ValidFeatures.Add("StatefulSet")) + StatefulSetStartOrdinal = framework.WithFeature(framework.ValidFeatures.Add("StatefulSetStartOrdinal")) + StatefulUpgrade = framework.WithFeature(framework.ValidFeatures.Add("StatefulUpgrade")) + StorageProvider = framework.WithFeature(framework.ValidFeatures.Add("StorageProvider")) + StorageVersionAPI = framework.WithFeature(framework.ValidFeatures.Add("StorageVersionAPI")) + TopologyHints = framework.WithFeature(framework.ValidFeatures.Add("Topology Hints")) + UDP = framework.WithFeature(framework.ValidFeatures.Add("UDP")) + Upgrade = framework.WithFeature(framework.ValidFeatures.Add("Upgrade")) + UserNamespacesStatelessPodsSupport = framework.WithFeature(framework.ValidFeatures.Add("UserNamespacesStatelessPodsSupport")) + ValidatingAdmissionPolicy = framework.WithFeature(framework.ValidFeatures.Add("ValidatingAdmissionPolicy")) + Volumes = framework.WithFeature(framework.ValidFeatures.Add("Volumes")) + VolumeSnapshotDataSource = framework.WithFeature(framework.ValidFeatures.Add("VolumeSnapshotDataSource")) + VolumeSourceXFS = framework.WithFeature(framework.ValidFeatures.Add("VolumeSourceXFS")) + Vsphere = framework.WithFeature(framework.ValidFeatures.Add("vsphere")) + WatchList = framework.WithFeature(framework.ValidFeatures.Add("WatchList")) + Windows = framework.WithFeature(framework.ValidFeatures.Add("Windows")) + WindowsHostProcessContainers = framework.WithFeature(framework.ValidFeatures.Add("WindowsHostProcessContainers")) + WindowsHyperVContainers = framework.WithFeature(framework.ValidFeatures.Add("WindowsHyperVContainers")) +) + +func init() { + // This prevents adding additional ad-hoc features in tests. + framework.ValidFeatures.Freeze() +} diff --git a/test/e2e/nodefeature/nodefeature.go b/test/e2e/nodefeature/nodefeature.go new file mode 100644 index 00000000000..ed0b3f091b3 --- /dev/null +++ b/test/e2e/nodefeature/nodefeature.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package feature contains pre-defined node features used by test/e2e and/or +// test/e2e_node. +package nodefeature + +import ( + "k8s.io/kubernetes/test/e2e/framework" +) + +var ( + AppArmor = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("AppArmor")) + CheckpointContainer = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("CheckpointContainer")) + CriticalPod = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("CriticalPod")) + DeviceManager = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DeviceManager")) + DevicePluginProbe = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DevicePluginProbe")) + DownwardAPIHugePages = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DownwardAPIHugePages")) + DynamicResourceAllocation = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DynamicResourceAllocation")) + Eviction = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("Eviction")) + FSGroup = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("FSGroup")) + GarbageCollect = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GarbageCollect")) + GracefulNodeShutdown = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdown")) + GracefulNodeShutdownBasedOnPodPriority = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdownBasedOnPodPriority")) + HostAccess = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("HostAccess")) + ImageID = framework.WithNodeFeature(framework.ValidNodeFeatures.Add(" ImageID")) + LSCIQuotaMonitoring = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("LSCIQuotaMonitoring")) + NodeAllocatable = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeAllocatable")) + NodeProblemDetector = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeProblemDetector")) + OOMScoreAdj = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("OOMScoreAdj")) + PodDisruptionConditions = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("PodDisruptionConditions")) + PodResources = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("PodResources")) + ResourceMetrics = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("ResourceMetrics")) + RuntimeHandler = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("RuntimeHandler")) + SystemNodeCriticalPod = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("SystemNodeCriticalPod")) + TopologyManager = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("TopologyManager")) +) + +func init() { + // This prevents adding additional ad-hoc features in tests. + framework.ValidNodeFeatures.Freeze() +} diff --git a/test/e2e_node/.import-restrictions b/test/e2e_node/.import-restrictions index a4fbc3e2e2d..8eb0fdbaa49 100644 --- a/test/e2e_node/.import-restrictions +++ b/test/e2e_node/.import-restrictions @@ -4,9 +4,11 @@ rules: allowedPrefixes: - k8s.io/kubernetes/test/e2e/common - k8s.io/kubernetes/test/e2e/dra/test-driver/app + - k8s.io/kubernetes/test/e2e/feature - k8s.io/kubernetes/test/e2e/framework - k8s.io/kubernetes/test/e2e/storage/utils - k8s.io/kubernetes/test/e2e/network/common + - k8s.io/kubernetes/test/e2e/nodefeature - k8s.io/kubernetes/test/e2e/perftype - k8s.io/kubernetes/test/e2e/testing-manifests - k8s.io/kubernetes/test/e2e_node diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 5d7f54af18d..95ae7d670e5 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -53,6 +53,10 @@ import ( e2enodetestingmanifests "k8s.io/kubernetes/test/e2e_node/testing-manifests" system "k8s.io/system-validators/validators" + // define and freeze constants + _ "k8s.io/kubernetes/test/e2e/feature" + _ "k8s.io/kubernetes/test/e2e/nodefeature" + // reconfigure framework _ "k8s.io/kubernetes/test/e2e/framework/debug/init" _ "k8s.io/kubernetes/test/e2e/framework/metrics/init"