diff --git a/hack/verify-e2e-suites.sh b/hack/verify-e2e-suites.sh new file mode 100755 index 00000000000..1235a838c37 --- /dev/null +++ b/hack/verify-e2e-suites.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script checks that all E2E test suites are sane, i.e. can be +# started without an error. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/util.sh" + +kube::golang::verify_go_version + +cd "${KUBE_ROOT}" + +kube::util::ensure-temp-dir + +for suite in $(git grep -l framework.AfterReadingAllFlags | grep -v -e ^test/e2e/framework -e ^hack | xargs -n 1 dirname | sort -u); do + # Build a binary and run it in the root directory to get paths that are + # relative to that instead of the package directory. + out="" + if (cd "$suite" && go test -c -o "${KUBE_TEMP}/e2e.bin" .) && out=$("${KUBE_TEMP}/e2e.bin" --list-tests); then + echo "E2E suite $suite passed." + else + echo >&2 "ERROR: E2E test suite invocation failed for $suite." + # shellcheck disable=SC2001 + echo "$out" | sed -e 's/^/ /' + fi +done diff --git a/test/e2e/README.md b/test/e2e/README.md index bef1f2cbff2..656c86eceec 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -45,12 +45,10 @@ import ( // test/e2e/lifecycle/framework.go package lifecycle -import "github.com/onsi/ginkgo" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cluster-lifecycle") ``` ```golang // test/e2e/lifecycle/bootstrap/bootstrap_signer.go diff --git a/test/e2e/apimachinery/framework.go b/test/e2e/apimachinery/framework.go index 6b7ee59a919..4edc3411949 100644 --- a/test/e2e/apimachinery/framework.go +++ b/test/e2e/apimachinery/framework.go @@ -16,9 +16,7 @@ limitations under the License. package apimachinery -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-api-machinery] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("api-machinery") diff --git a/test/e2e/apps/framework.go b/test/e2e/apps/framework.go index dde7fa0326b..d940e5f1f2f 100644 --- a/test/e2e/apps/framework.go +++ b/test/e2e/apps/framework.go @@ -16,9 +16,7 @@ limitations under the License. package apps -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-apps] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("apps") diff --git a/test/e2e/architecture/framework.go b/test/e2e/architecture/framework.go index 4d7d819c006..b8b12b950a6 100644 --- a/test/e2e/architecture/framework.go +++ b/test/e2e/architecture/framework.go @@ -16,9 +16,7 @@ limitations under the License. package architecture -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-architecture] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("architecture") diff --git a/test/e2e/auth/framework.go b/test/e2e/auth/framework.go index cf3d006234b..0c0e3bc8408 100644 --- a/test/e2e/auth/framework.go +++ b/test/e2e/auth/framework.go @@ -16,9 +16,7 @@ limitations under the License. package auth -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-auth] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("auth") diff --git a/test/e2e/autoscaling/framework.go b/test/e2e/autoscaling/framework.go index 0392976c4cc..5dd080ee845 100644 --- a/test/e2e/autoscaling/framework.go +++ b/test/e2e/autoscaling/framework.go @@ -16,9 +16,7 @@ limitations under the License. package autoscaling -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-autoscaling] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("autoscaling") diff --git a/test/e2e/cloud/framework.go b/test/e2e/cloud/framework.go index 1d80fbbc937..8eb4e55409d 100644 --- a/test/e2e/cloud/framework.go +++ b/test/e2e/cloud/framework.go @@ -16,9 +16,7 @@ limitations under the License. package cloud -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cloud-provider] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cloud-provider") diff --git a/test/e2e/cloud/gcp/apps/framework.go b/test/e2e/cloud/gcp/apps/framework.go index 5f2edc490e2..7e768e42a2b 100644 --- a/test/e2e/cloud/gcp/apps/framework.go +++ b/test/e2e/cloud/gcp/apps/framework.go @@ -16,9 +16,7 @@ limitations under the License. package apps -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-apps] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("apps") diff --git a/test/e2e/cloud/gcp/auth/framework.go b/test/e2e/cloud/gcp/auth/framework.go index 8245c662f04..f0b0298eb73 100644 --- a/test/e2e/cloud/gcp/auth/framework.go +++ b/test/e2e/cloud/gcp/auth/framework.go @@ -16,9 +16,7 @@ limitations under the License. package auth -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-auth] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("auth") diff --git a/test/e2e/cloud/gcp/framework.go b/test/e2e/cloud/gcp/framework.go index edd24776ca6..abe47298dab 100644 --- a/test/e2e/cloud/gcp/framework.go +++ b/test/e2e/cloud/gcp/framework.go @@ -16,9 +16,7 @@ limitations under the License. package gcp -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cloud-provider-gcp] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cloud-provider-gcp") diff --git a/test/e2e/cloud/gcp/network/framework.go b/test/e2e/cloud/gcp/network/framework.go index 3e3e946d9f7..055cfa3675b 100644 --- a/test/e2e/cloud/gcp/network/framework.go +++ b/test/e2e/cloud/gcp/network/framework.go @@ -16,9 +16,7 @@ limitations under the License. package network -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-network] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("network") diff --git a/test/e2e/cloud/gcp/node/framework.go b/test/e2e/cloud/gcp/node/framework.go index b40fd35c8ca..7a7ee5d5297 100644 --- a/test/e2e/cloud/gcp/node/framework.go +++ b/test/e2e/cloud/gcp/node/framework.go @@ -16,9 +16,7 @@ limitations under the License. package node -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") diff --git a/test/e2e/common/network/framework.go b/test/e2e/common/network/framework.go index 3e3e946d9f7..055cfa3675b 100644 --- a/test/e2e/common/network/framework.go +++ b/test/e2e/common/network/framework.go @@ -16,9 +16,7 @@ limitations under the License. package network -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-network] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("network") diff --git a/test/e2e/common/node/framework.go b/test/e2e/common/node/framework.go index b40fd35c8ca..884f4bf48bf 100644 --- a/test/e2e/common/node/framework.go +++ b/test/e2e/common/node/framework.go @@ -16,9 +16,6 @@ limitations under the License. package node -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" -// SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") diff --git a/test/e2e/common/storage/framework.go b/test/e2e/common/storage/framework.go index d3351a06ff2..a7967e83aac 100644 --- a/test/e2e/common/storage/framework.go +++ b/test/e2e/common/storage/framework.go @@ -16,9 +16,7 @@ limitations under the License. package storage -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-storage] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("storage") diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index af333af5e66..6c2d342e736 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -43,6 +43,10 @@ import ( e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" testfixtures "k8s.io/kubernetes/test/fixtures" + // define and freeze constants + _ "k8s.io/kubernetes/test/e2e/feature" + _ "k8s.io/kubernetes/test/e2e/nodefeature" + // test sources _ "k8s.io/kubernetes/test/e2e/apimachinery" _ "k8s.io/kubernetes/test/e2e/apps" diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go new file mode 100644 index 00000000000..09ebab6b14f --- /dev/null +++ b/test/e2e/feature/feature.go @@ -0,0 +1,132 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package feature contains pre-defined features used by test/e2e and/or +// test/e2e_node. +package feature + +import ( + "k8s.io/kubernetes/test/e2e/framework" +) + +var ( + APIServerIdentity = framework.WithFeature(framework.ValidFeatures.Add("APIServerIdentity")) + AppArmor = framework.WithFeature(framework.ValidFeatures.Add("AppArmor")) + BootstrapTokens = framework.WithFeature(framework.ValidFeatures.Add("BootstrapTokens")) + BoundServiceAccountTokenVolume = framework.WithFeature(framework.ValidFeatures.Add("BoundServiceAccountTokenVolume")) + CloudProvider = framework.WithFeature(framework.ValidFeatures.Add("CloudProvider")) + ClusterAutoscalerScalability1 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability1")) + ClusterAutoscalerScalability2 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability2")) + ClusterAutoscalerScalability3 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability3")) + ClusterAutoscalerScalability4 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability4")) + ClusterAutoscalerScalability5 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability5")) + ClusterAutoscalerScalability6 = framework.WithFeature(framework.ValidFeatures.Add("ClusterAutoscalerScalability6")) + ClusterDowngrade = framework.WithFeature(framework.ValidFeatures.Add("ClusterDowngrade")) + ClusterSizeAutoscalingGpu = framework.WithFeature(framework.ValidFeatures.Add("ClusterSizeAutoscalingGpu")) + ClusterSizeAutoscalingScaleDown = framework.WithFeature(framework.ValidFeatures.Add("ClusterSizeAutoscalingScaleDown")) + ClusterSizeAutoscalingScaleUp = framework.WithFeature(framework.ValidFeatures.Add("ClusterSizeAutoscalingScaleUp")) + ClusterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("ClusterUpgrade")) + ComprehensiveNamespaceDraining = framework.WithFeature(framework.ValidFeatures.Add("ComprehensiveNamespaceDraining")) + CPUManager = framework.WithFeature(framework.ValidFeatures.Add("CPUManager")) + CustomMetricsAutoscaling = framework.WithFeature(framework.ValidFeatures.Add("CustomMetricsAutoscaling")) + DeviceManager = framework.WithFeature(framework.ValidFeatures.Add("DeviceManager")) + DevicePluginProbe = framework.WithFeature(framework.ValidFeatures.Add("DevicePluginProbe")) + Downgrade = framework.WithFeature(framework.ValidFeatures.Add("Downgrade")) + DynamicResourceAllocation = framework.WithFeature(framework.ValidFeatures.Add("DynamicResourceAllocation")) + EphemeralStorage = framework.WithFeature(framework.ValidFeatures.Add("EphemeralStorage")) + Example = framework.WithFeature(framework.ValidFeatures.Add("Example")) + ExperimentalResourceUsageTracking = framework.WithFeature(framework.ValidFeatures.Add("ExperimentalResourceUsageTracking")) + Flexvolumes = framework.WithFeature(framework.ValidFeatures.Add("Flexvolumes")) + GKENodePool = framework.WithFeature(framework.ValidFeatures.Add("GKENodePool")) + GPUClusterDowngrade = framework.WithFeature(framework.ValidFeatures.Add("GPUClusterDowngrade")) + GPUClusterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("GPUClusterUpgrade")) + GPUDevicePlugin = framework.WithFeature(framework.ValidFeatures.Add("GPUDevicePlugin")) + GPUMasterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("GPUMasterUpgrade")) + GPUUpgrade = framework.WithFeature(framework.ValidFeatures.Add("GPUUpgrade")) + HAMaster = framework.WithFeature(framework.ValidFeatures.Add("HAMaster")) + HPA = framework.WithFeature(framework.ValidFeatures.Add("HPA")) + HugePages = framework.WithFeature(framework.ValidFeatures.Add("HugePages")) + Ingress = framework.WithFeature(framework.ValidFeatures.Add("Ingress")) + IngressScale = framework.WithFeature(framework.ValidFeatures.Add("IngressScale")) + InPlacePodVerticalScaling = framework.WithFeature(framework.ValidFeatures.Add("InPlacePodVerticalScaling")) + IPv6DualStack = framework.WithFeature(framework.ValidFeatures.Add("IPv6DualStack")) + Kind = framework.WithFeature(framework.ValidFeatures.Add("Kind")) + KubeletCredentialProviders = framework.WithFeature(framework.ValidFeatures.Add("KubeletCredentialProviders")) + KubeletSecurity = framework.WithFeature(framework.ValidFeatures.Add("KubeletSecurity")) + KubeProxyDaemonSetDowngrade = framework.WithFeature(framework.ValidFeatures.Add("KubeProxyDaemonSetDowngrade")) + KubeProxyDaemonSetUpgrade = framework.WithFeature(framework.ValidFeatures.Add("KubeProxyDaemonSetUpgrade")) + KubeProxyDaemonSetMigration = framework.WithFeature(framework.ValidFeatures.Add("KubeProxyDaemonSetMigration")) + LabelSelector = framework.WithFeature(framework.ValidFeatures.Add("LabelSelector")) + LocalStorageCapacityIsolation = framework.WithFeature(framework.ValidFeatures.Add("LocalStorageCapacityIsolation")) + LocalStorageCapacityIsolationQuota = framework.WithFeature(framework.ValidFeatures.Add("LocalStorageCapacityIsolationQuota")) + MasterUpgrade = framework.WithFeature(framework.ValidFeatures.Add("MasterUpgrade")) + MemoryManager = framework.WithFeature(framework.ValidFeatures.Add("MemoryManager")) + NEG = framework.WithFeature(framework.ValidFeatures.Add("NEG")) + NetworkingDNS = framework.WithFeature(framework.ValidFeatures.Add("Networking-DNS")) + NetworkingIPv4 = framework.WithFeature(framework.ValidFeatures.Add("Networking-IPv4")) + NetworkingIPv6 = framework.WithFeature(framework.ValidFeatures.Add("Networking-IPv6")) + NetworkingPerformance = framework.WithFeature(framework.ValidFeatures.Add("Networking-Performance")) + NetworkPolicy = framework.WithFeature(framework.ValidFeatures.Add("NetworkPolicy")) + NodeAuthenticator = framework.WithFeature(framework.ValidFeatures.Add("NodeAuthenticator")) + NodeAuthorizer = framework.WithFeature(framework.ValidFeatures.Add("NodeAuthorizer")) + NodeOutOfServiceVolumeDetach = framework.WithFeature(framework.ValidFeatures.Add("NodeOutOfServiceVolumeDetach")) + NoSNAT = framework.WithFeature(framework.ValidFeatures.Add("NoSNAT")) + PerformanceDNS = framework.WithFeature(framework.ValidFeatures.Add("PerformanceDNS")) + PodGarbageCollector = framework.WithFeature(framework.ValidFeatures.Add("PodGarbageCollector")) + PodPriority = framework.WithFeature(framework.ValidFeatures.Add("PodPriority")) + PodReadyToStartContainersCondition = framework.WithFeature(framework.ValidFeatures.Add("PodReadyToStartContainersCondition")) + PodResources = framework.WithFeature(framework.ValidFeatures.Add("PodResources")) + ProbeTerminationGracePeriod = framework.WithFeature(framework.ValidFeatures.Add("ProbeTerminationGracePeriod")) + Reboot = framework.WithFeature(framework.ValidFeatures.Add("Reboot")) + ReclaimPolicy = framework.WithFeature(framework.ValidFeatures.Add("ReclaimPolicy")) + RecoverVolumeExpansionFailure = framework.WithFeature(framework.ValidFeatures.Add("RecoverVolumeExpansionFailure")) + Recreate = framework.WithFeature(framework.ValidFeatures.Add("Recreate")) + RegularResourceUsageTracking = framework.WithFeature(framework.ValidFeatures.Add("RegularResourceUsageTracking")) + ScopeSelectors = framework.WithFeature(framework.ValidFeatures.Add("ScopeSelectors")) + SCTPConnectivity = framework.WithFeature(framework.ValidFeatures.Add("SCTPConnectivity")) + SeccompDefault = framework.WithFeature(framework.ValidFeatures.Add("SeccompDefault")) + SELinux = framework.WithFeature(framework.ValidFeatures.Add("SELinux")) + SELinuxMountReadWriteOncePod = framework.WithFeature(framework.ValidFeatures.Add("SELinuxMountReadWriteOncePod")) + StackdriverAcceleratorMonitoring = framework.WithFeature(framework.ValidFeatures.Add("StackdriverAcceleratorMonitoring")) + StackdriverCustomMetrics = framework.WithFeature(framework.ValidFeatures.Add("StackdriverCustomMetrics")) + StackdriverExternalMetrics = framework.WithFeature(framework.ValidFeatures.Add("StackdriverExternalMetrics")) + StackdriverMetadataAgent = framework.WithFeature(framework.ValidFeatures.Add("StackdriverMetadataAgent")) + StackdriverMonitoring = framework.WithFeature(framework.ValidFeatures.Add("StackdriverMonitoring")) + StandaloneMode = framework.WithFeature(framework.ValidFeatures.Add("StandaloneMode")) + StatefulSet = framework.WithFeature(framework.ValidFeatures.Add("StatefulSet")) + StatefulSetStartOrdinal = framework.WithFeature(framework.ValidFeatures.Add("StatefulSetStartOrdinal")) + StatefulUpgrade = framework.WithFeature(framework.ValidFeatures.Add("StatefulUpgrade")) + StorageProvider = framework.WithFeature(framework.ValidFeatures.Add("StorageProvider")) + StorageVersionAPI = framework.WithFeature(framework.ValidFeatures.Add("StorageVersionAPI")) + TopologyHints = framework.WithFeature(framework.ValidFeatures.Add("Topology Hints")) + UDP = framework.WithFeature(framework.ValidFeatures.Add("UDP")) + Upgrade = framework.WithFeature(framework.ValidFeatures.Add("Upgrade")) + UserNamespacesStatelessPodsSupport = framework.WithFeature(framework.ValidFeatures.Add("UserNamespacesStatelessPodsSupport")) + ValidatingAdmissionPolicy = framework.WithFeature(framework.ValidFeatures.Add("ValidatingAdmissionPolicy")) + Volumes = framework.WithFeature(framework.ValidFeatures.Add("Volumes")) + VolumeSnapshotDataSource = framework.WithFeature(framework.ValidFeatures.Add("VolumeSnapshotDataSource")) + VolumeSourceXFS = framework.WithFeature(framework.ValidFeatures.Add("VolumeSourceXFS")) + Vsphere = framework.WithFeature(framework.ValidFeatures.Add("vsphere")) + WatchList = framework.WithFeature(framework.ValidFeatures.Add("WatchList")) + Windows = framework.WithFeature(framework.ValidFeatures.Add("Windows")) + WindowsHostProcessContainers = framework.WithFeature(framework.ValidFeatures.Add("WindowsHostProcessContainers")) + WindowsHyperVContainers = framework.WithFeature(framework.ValidFeatures.Add("WindowsHyperVContainers")) +) + +func init() { + // This prevents adding additional ad-hoc features in tests. + framework.ValidFeatures.Freeze() +} diff --git a/test/e2e/framework/.import-restrictions b/test/e2e/framework/.import-restrictions index 894b9749b0a..660e7453fa7 100644 --- a/test/e2e/framework/.import-restrictions +++ b/test/e2e/framework/.import-restrictions @@ -4,7 +4,7 @@ rules: # The following packages are okay to use: # # public API - - selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils)/|^[a-z]+(/|$)|github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils) + - selectorRegexp: ^k8s[.]io/(api|apimachinery|client-go|component-base|klog|pod-security-admission|utils) allowedPrefixes: [ "" ] # stdlib @@ -16,7 +16,7 @@ rules: allowedPrefixes: [ "" ] # Ginkgo + Gomega - - selectorRegexp: github.com/onsi/(ginkgo|gomega)|^k8s[.]io/kubernetes/test/(e2e/framework/internal/|utils) + - selectorRegexp: ^github.com/onsi/(ginkgo|gomega) allowedPrefixes: [ "" ] # kube-openapi @@ -33,8 +33,10 @@ rules: # Third party deps - selectorRegexp: ^github.com/|^gopkg.in - allowedPrefixes: [ + allowedPrefixes: [ "gopkg.in/inf.v0", + "gopkg.in/yaml.v2", + "github.com/blang/semver/", "github.com/davecgh/go-spew/spew", "github.com/evanphx/json-patch", "github.com/go-logr/logr", @@ -48,6 +50,10 @@ rules: "github.com/google/gofuzz", "github.com/google/uuid", "github.com/imdario/mergo", + "github.com/prometheus/client_golang/", + "github.com/prometheus/client_model/", + "github.com/prometheus/common/", + "github.com/prometheus/procfs", "github.com/spf13/cobra", "github.com/spf13/pflag", "github.com/stretchr/testify/assert", diff --git a/test/e2e/framework/bugs.go b/test/e2e/framework/bugs.go new file mode 100644 index 00000000000..a8202353307 --- /dev/null +++ b/test/e2e/framework/bugs.go @@ -0,0 +1,108 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/onsi/ginkgo/v2/types" +) + +var ( + bugs []Bug + bugMutex sync.Mutex +) + +// RecordBug stores information about a bug in the E2E suite source code that +// cannot be reported through ginkgo.Fail because it was found outside of some +// test, for example during test registration. +// +// This can be used instead of raising a panic. Then all bugs can be reported +// together instead of failing after the first one. +func RecordBug(bug Bug) { + bugMutex.Lock() + defer bugMutex.Unlock() + + bugs = append(bugs, bug) +} + +type Bug struct { + FileName string + LineNumber int + Message string +} + +// NewBug creates a new bug with a location that is obtained by skipping a certain number +// of stack frames. Passing zero will record the source code location of the direct caller +// of NewBug. +func NewBug(message string, skip int) Bug { + location := types.NewCodeLocation(skip + 1) + return Bug{FileName: location.FileName, LineNumber: location.LineNumber, Message: message} +} + +// FormatBugs produces a report that includes all bugs recorded earlier via +// RecordBug. An error is returned with the report if there have been bugs. +func FormatBugs() error { + bugMutex.Lock() + defer bugMutex.Unlock() + + if len(bugs) == 0 { + return nil + } + + lines := make([]string, 0, len(bugs)) + wd, err := os.Getwd() + if err != nil { + return fmt.Errorf("get current directory: %v", err) + } + // Sort by file name, line number, message. For the sake of simplicity + // this uses the full file name even though the output the may use a + // relative path. Usually the result should be the same because full + // paths will all have the same prefix. + sort.Slice(bugs, func(i, j int) bool { + switch strings.Compare(bugs[i].FileName, bugs[j].FileName) { + case -1: + return true + case 1: + return false + } + if bugs[i].LineNumber < bugs[j].LineNumber { + return true + } + if bugs[i].LineNumber > bugs[j].LineNumber { + return false + } + return bugs[i].Message < bugs[j].Message + }) + for _, bug := range bugs { + // Use relative paths, if possible. + path := bug.FileName + if wd != "" { + if relpath, err := filepath.Rel(wd, bug.FileName); err == nil { + path = relpath + } + } + lines = append(lines, fmt.Sprintf("ERROR: %s:%d: %s\n", path, bug.LineNumber, strings.TrimSpace(bug.Message))) + } + return errors.New(strings.Join(lines, "")) +} diff --git a/test/e2e/framework/ginkgowrapper.go b/test/e2e/framework/ginkgowrapper.go index e35fc4ae982..ed7b9432f05 100644 --- a/test/e2e/framework/ginkgowrapper.go +++ b/test/e2e/framework/ginkgowrapper.go @@ -17,13 +17,72 @@ limitations under the License. package framework import ( + "fmt" "path" "reflect" + "regexp" + "strings" "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +// Feature is the name of a certain feature that the cluster under test must have. +// Such features are different from feature gates. +type Feature string + +// Environment is the name for the environment in which a test can run, like +// "Linux" or "Windows". +type Environment string + +// NodeFeature is the name of a feature that a node must support. To be +// removed, see +// https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features#nodefeature. +type NodeFeature string + +type Valid[T comparable] struct { + items sets.Set[T] + frozen bool +} + +// Add registers a new valid item name. The expected usage is +// +// var SomeFeature = framework.ValidFeatures.Add("Some") +// +// during the init phase of an E2E suite. Individual tests should not register +// their own, to avoid uncontrolled proliferation of new items. E2E suites can, +// but don't have to, enforce that by freezing the set of valid names. +func (v *Valid[T]) Add(item T) T { + if v.frozen { + RecordBug(NewBug(fmt.Sprintf(`registry %T is already frozen, "%v" must not be added anymore`, *v, item), 1)) + } + if v.items == nil { + v.items = sets.New[T]() + } + if v.items.Has(item) { + RecordBug(NewBug(fmt.Sprintf(`registry %T already contains "%v", it must not be added again`, *v, item), 1)) + } + v.items.Insert(item) + return item +} + +func (v *Valid[T]) Freeze() { + v.frozen = true +} + +// These variables contain the parameters that [WithFeature], [WithEnvironment] +// and [WithNodeFeatures] accept. The framework itself has no pre-defined +// constants. Test suites and tests may define their own and then add them here +// before calling these With functions. +var ( + ValidFeatures Valid[Feature] + ValidEnvironments Valid[Environment] + ValidNodeFeatures Valid[NodeFeature] ) var errInterface = reflect.TypeOf((*error)(nil)).Elem() @@ -65,8 +124,344 @@ func AnnotatedLocationWithOffset(annotation string, offset int) types.CodeLocati return codeLocation } +// SIGDescribe returns a wrapper function for ginkgo.Describe which injects +// the SIG name as annotation. The parameter should be lowercase with +// no spaces and no sig- or SIG- prefix. +func SIGDescribe(sig string) func(string, ...interface{}) bool { + if !sigRE.MatchString(sig) || strings.HasPrefix(sig, "sig-") { + panic(fmt.Sprintf("SIG label must be lowercase, no spaces and no sig- prefix, got instead: %q", sig)) + } + return func(text string, args ...interface{}) bool { + args = append(args, ginkgo.Label("sig-"+sig)) + if text == "" { + text = fmt.Sprintf("[sig-%s]", sig) + } else { + text = fmt.Sprintf("[sig-%s] %s", sig, text) + } + return registerInSuite(ginkgo.Describe, text, args) + } +} + +var sigRE = regexp.MustCompile(`^[a-z]+(-[a-z]+)*$`) + // ConformanceIt is wrapper function for ginkgo It. Adds "[Conformance]" tag and makes static analysis easier. func ConformanceIt(text string, args ...interface{}) bool { - args = append(args, ginkgo.Offset(1)) - return ginkgo.It(text+" [Conformance]", args...) + args = append(args, ginkgo.Offset(1), WithConformance()) + return It(text, args...) +} + +// It is a wrapper around [ginkgo.It] which supports framework With* labels as +// optional arguments in addition to those already supported by ginkgo itself, +// like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func It(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.It, text, args) +} + +// It is a shorthand for the corresponding package function. +func (f *Framework) It(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.It, text, args) +} + +// Describe is a wrapper around [ginkgo.Describe] which supports framework +// With* labels as optional arguments in addition to those already supported by +// ginkgo itself, like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func Describe(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Describe, text, args) +} + +// Describe is a shorthand for the corresponding package function. +func (f *Framework) Describe(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Describe, text, args) +} + +// Context is a wrapper around [ginkgo.Context] which supports framework With* +// labels as optional arguments in addition to those already supported by +// ginkgo itself, like [ginkgo.Label] and [gingko.Offset]. +// +// Text and arguments may be mixed. The final text is a concatenation +// of the text arguments and special tags from the With functions. +func Context(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Context, text, args) +} + +// Context is a shorthand for the corresponding package function. +func (f *Framework) Context(text string, args ...interface{}) bool { + return registerInSuite(ginkgo.Context, text, args) +} + +// registerInSuite is the common implementation of all wrapper functions. It +// expects to be called through one intermediate wrapper. +func registerInSuite(ginkgoCall func(text string, args ...interface{}) bool, text string, args []interface{}) bool { + var ginkgoArgs []interface{} + var offset ginkgo.Offset + var texts []string + if text != "" { + texts = append(texts, text) + } + + addLabel := func(label string) { + texts = append(texts, fmt.Sprintf("[%s]", label)) + ginkgoArgs = append(ginkgoArgs, ginkgo.Label(label)) + } + + haveEmptyStrings := false + for _, arg := range args { + switch arg := arg.(type) { + case label: + fullLabel := strings.Join(arg.parts, ": ") + addLabel(fullLabel) + if arg.extra != "" { + addLabel(arg.extra) + } + if fullLabel == "Serial" { + ginkgoArgs = append(ginkgoArgs, ginkgo.Serial) + } + case ginkgo.Offset: + offset = arg + case string: + if arg == "" { + haveEmptyStrings = true + } + texts = append(texts, arg) + default: + ginkgoArgs = append(ginkgoArgs, arg) + } + } + offset += 2 // This function and its direct caller. + + // Now that we have the final offset, we can record bugs. + if haveEmptyStrings { + RecordBug(NewBug("empty strings as separators are unnecessary and need to be removed", int(offset))) + } + + // Enforce that text snippets to not start or end with spaces because + // those lead to double spaces when concatenating below. + for _, text := range texts { + if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") { + RecordBug(NewBug(fmt.Sprintf("trailing or leading spaces are unnecessary and need to be removed: %q", text), int(offset))) + } + } + + ginkgoArgs = append(ginkgoArgs, offset) + text = strings.Join(texts, " ") + return ginkgoCall(text, ginkgoArgs...) +} + +// WithEnvironment specifies that a certain test or group of tests only works +// with a feature available. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The feature must be listed in ValidFeatures. +func WithFeature(name Feature) interface{} { + return withFeature(name) +} + +// WithFeature is a shorthand for the corresponding package function. +func (f *Framework) WithFeature(name Feature) interface{} { + return withFeature(name) +} + +func withFeature(name Feature) interface{} { + if !ValidFeatures.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithFeature: unknown feature %q", name), 2)) + } + return newLabel("Feature", string(name)) +} + +// WithFeatureGate specifies that a certain test or group of tests depends on a +// feature gate being enabled. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The feature gate must be listed in +// [k8s.io/apiserver/pkg/util/feature.DefaultMutableFeatureGate]. Once a +// feature gate gets removed from there, the WithFeatureGate calls using it +// also need to be removed. +func WithFeatureGate(featureGate featuregate.Feature) interface{} { + return withFeatureGate(featureGate) +} + +// WithFeatureGate is a shorthand for the corresponding package function. +func (f *Framework) WithFeatureGate(featureGate featuregate.Feature) interface{} { + return withFeatureGate(featureGate) +} + +func withFeatureGate(featureGate featuregate.Feature) interface{} { + spec, ok := utilfeature.DefaultMutableFeatureGate.GetAll()[featureGate] + if !ok { + RecordBug(NewBug(fmt.Sprintf("WithFeatureGate: the feature gate %q is unknown", featureGate), 2)) + } + + // We use mixed case (i.e. Beta instead of BETA). GA feature gates have no level string. + var level string + if spec.PreRelease != "" { + level = string(spec.PreRelease) + level = strings.ToUpper(level[0:1]) + strings.ToLower(level[1:]) + } + + l := newLabel("FeatureGate", string(featureGate)) + l.extra = level + return l +} + +// WithEnvironment specifies that a certain test or group of tests only works +// in a certain environment. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// The environment must be listed in ValidEnvironments. +func WithEnvironment(name Environment) interface{} { + return withEnvironment(name) +} + +// WithEnvironment is a shorthand for the corresponding package function. +func (f *Framework) WithEnvironment(name Environment) interface{} { + return withEnvironment(name) +} + +func withEnvironment(name Environment) interface{} { + if !ValidEnvironments.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithEnvironment: unknown environment %q", name), 2)) + } + return newLabel("Environment", string(name)) +} + +// WithNodeFeature specifies that a certain test or group of tests only works +// if the node supports a certain feature. The return value must be passed as +// additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +// +// The environment must be listed in ValidNodeFeatures. +func WithNodeFeature(name NodeFeature) interface{} { + return withNodeFeature(name) +} + +// WithNodeFeature is a shorthand for the corresponding package function. +func (f *Framework) WithNodeFeature(name NodeFeature) interface{} { + return withNodeFeature(name) +} + +func withNodeFeature(name NodeFeature) interface{} { + if !ValidNodeFeatures.items.Has(name) { + RecordBug(NewBug(fmt.Sprintf("WithNodeFeature: unknown environment %q", name), 2)) + } + return newLabel(string(name)) +} + +// WithConformace specifies that a certain test or group of tests must pass in +// all conformant Kubernetes clusters. The return value must be passed as +// additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +func WithConformance() interface{} { + return withConformance() +} + +// WithConformance is a shorthand for the corresponding package function. +func (f *Framework) WithConformance() interface{} { + return withConformance() +} + +func withConformance() interface{} { + return newLabel("Conformance") +} + +// WithNodeConformance specifies that a certain test or group of tests for node +// functionality that does not depend on runtime or Kubernetes distro specific +// behavior. The return value must be passed as additional argument to +// [framework.It], [framework.Describe], [framework.Context]. +func WithNodeConformance() interface{} { + return withNodeConformance() +} + +// WithNodeConformance is a shorthand for the corresponding package function. +func (f *Framework) WithNodeConformance() interface{} { + return withNodeConformance() +} + +func withNodeConformance() interface{} { + return newLabel("NodeConformance") +} + +// WithDisruptive specifies that a certain test or group of tests temporarily +// affects the functionality of the Kubernetes cluster. The return value must +// be passed as additional argument to [framework.It], [framework.Describe], +// [framework.Context]. +func WithDisruptive() interface{} { + return withDisruptive() +} + +// WithDisruptive is a shorthand for the corresponding package function. +func (f *Framework) WithDisruptive() interface{} { + return withDisruptive() +} + +func withDisruptive() interface{} { + return newLabel("Disruptive") +} + +// WithSerial specifies that a certain test or group of tests must not run in +// parallel with other tests. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +// +// Starting with ginkgo v2, serial and parallel tests can be executed in the +// same invocation. Ginkgo itself will ensure that the serial tests run +// sequentially. +func WithSerial() interface{} { + return withSerial() +} + +// WithSerial is a shorthand for the corresponding package function. +func (f *Framework) WithSerial() interface{} { + return withSerial() +} + +func withSerial() interface{} { + return newLabel("Serial") +} + +// WithSlow specifies that a certain test or group of tests must not run in +// parallel with other tests. The return value must be passed as additional +// argument to [framework.It], [framework.Describe], [framework.Context]. +func WithSlow() interface{} { + return withSlow() +} + +// WithSlow is a shorthand for the corresponding package function. +func (f *Framework) WithSlow() interface{} { + return WithSlow() +} + +func withSlow() interface{} { + return newLabel("Slow") +} + +// WithLabel is a wrapper around [ginkgo.Label]. Besides adding an arbitrary +// label to a test, it also injects the label in square brackets into the test +// name. +func WithLabel(label string) interface{} { + return withLabel(label) +} + +// WithLabel is a shorthand for the corresponding package function. +func (f *Framework) WithLabel(label string) interface{} { + return withLabel(label) +} + +func withLabel(label string) interface{} { + return newLabel(label) +} + +type label struct { + // parts get concatenated with ": " to build the full label. + parts []string + // extra is an optional fully-formed extra label. + extra string +} + +func newLabel(parts ...string) label { + return label{parts: parts} } diff --git a/test/e2e/framework/internal/unittests/bugs/bugs.go b/test/e2e/framework/internal/unittests/bugs/bugs.go new file mode 100644 index 00000000000..e84d04d638d --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/bugs.go @@ -0,0 +1,168 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bugs + +import ( + "bytes" + "testing" + + "github.com/onsi/ginkgo/v2" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs/features" +) + +// The line number of the following code is checked in BugOutput below. +// Be careful when moving it around or changing the import statements above. +// Here are some intentionally blank lines that can be removed to compensate +// for future additional import statements. +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// This must be line #50. + +func helper() { + framework.RecordBug(framework.NewBug("new bug", 0)) + framework.RecordBug(framework.NewBug("parent", 1)) +} + +func RecordBugs() { + helper() + framework.RecordBug(framework.Bug{FileName: "buggy/buggy.go", LineNumber: 100, Message: "hello world"}) + framework.RecordBug(framework.Bug{FileName: "some/relative/path/buggy.go", LineNumber: 200, Message: " with spaces \n"}) +} + +var ( + validFeature = framework.ValidFeatures.Add("feature-foo") + validEnvironment = framework.ValidEnvironments.Add("Linux") + validNodeFeature = framework.ValidNodeFeatures.Add("node-feature-foo") +) + +func Describe() { + // Normally a single line would be better, but this is an extreme example and + // thus uses multiple. + framework.SIGDescribe("testing")("abc", + // Bugs in parameters will be attributed to the Describe call, not the line of the parameter. + "", // buggy: not needed + " space1", // buggy: leading white space + "space2 ", // buggy: trailing white space + framework.WithFeature("no-such-feature"), + framework.WithFeature(validFeature), + framework.WithEnvironment("no-such-env"), + framework.WithEnvironment(validEnvironment), + framework.WithNodeFeature("no-such-node-env"), + framework.WithNodeFeature(validNodeFeature), + framework.WithFeatureGate("no-such-feature-gate"), + framework.WithFeatureGate(features.Alpha), + framework.WithFeatureGate(features.Beta), + framework.WithFeatureGate(features.GA), + framework.WithConformance(), + framework.WithNodeConformance(), + framework.WithSlow(), + framework.WithSerial(), + framework.WithDisruptive(), + framework.WithLabel("custom-label"), + "xyz", // okay, becomes part of the final text + func() { + f := framework.NewDefaultFramework("abc") + + framework.Context("y", framework.WithLabel("foo"), func() { + framework.It("should", f.WithLabel("bar"), func() { + }) + }) + + f.Context("x", f.WithLabel("foo"), func() { + f.It("should", f.WithLabel("bar"), func() { + }) + }) + }, + ) +} + +const ( + numBugs = 3 + bugOutput = `ERROR: bugs.go:53: new bug +ERROR: bugs.go:58: parent +ERROR: bugs.go:72: empty strings as separators are unnecessary and need to be removed +ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: " space1" +ERROR: bugs.go:72: trailing or leading spaces are unnecessary and need to be removed: "space2 " +ERROR: bugs.go:77: WithFeature: unknown feature "no-such-feature" +ERROR: bugs.go:79: WithEnvironment: unknown environment "no-such-env" +ERROR: bugs.go:81: WithNodeFeature: unknown environment "no-such-node-env" +ERROR: bugs.go:83: WithFeatureGate: the feature gate "no-such-feature-gate" is unknown +ERROR: buggy/buggy.go:100: hello world +ERROR: some/relative/path/buggy.go:200: with spaces +` + // Used by unittests/list-tests. It's sorted by test name, not source code location. + ListTestsOutput = `The following spec names can be used with 'ginkgo run --focus/skip': + ../bugs/bugs.go:103: [sig-testing] abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz x [foo] should [bar] + ../bugs/bugs.go:98: [sig-testing] abc space1 space2 [Feature: no-such-feature] [Feature: feature-foo] [Environment: no-such-env] [Environment: Linux] [no-such-node-env] [node-feature-foo] [FeatureGate: no-such-feature-gate] [FeatureGate: TestAlphaFeature] [Alpha] [FeatureGate: TestBetaFeature] [Beta] [FeatureGate: TestGAFeature] [Conformance] [NodeConformance] [Slow] [Serial] [Disruptive] [custom-label] xyz y [foo] should [bar] + +` + + // Used by unittests/list-labels. + ListLabelsOutput = `The following labels can be used with 'gingko run --label-filter': + Alpha + Beta + Conformance + Disruptive + Environment: Linux + Environment: no-such-env + Feature: feature-foo + Feature: no-such-feature + FeatureGate: TestAlphaFeature + FeatureGate: TestBetaFeature + FeatureGate: TestGAFeature + FeatureGate: no-such-feature-gate + NodeConformance + Serial + Slow + bar + custom-label + foo + no-such-node-env + node-feature-foo + sig-testing + +` +) + +func GetGinkgoOutput(t *testing.T) string { + var buffer bytes.Buffer + ginkgo.GinkgoWriter.TeeTo(&buffer) + t.Cleanup(ginkgo.GinkgoWriter.ClearTeeWriters) + + suiteConfig, reporterConfig := framework.CreateGinkgoConfig() + fakeT := &testing.T{} + ginkgo.RunSpecs(fakeT, "Buggy Suite", suiteConfig, reporterConfig) + + return buffer.String() +} diff --git a/test/e2e/framework/internal/unittests/bugs/bugs_test.go b/test/e2e/framework/internal/unittests/bugs/bugs_test.go new file mode 100644 index 00000000000..dd8a66c4e18 --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/bugs_test.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bugs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" +) + +func TestBugs(t *testing.T) { + assert.NoError(t, framework.FormatBugs()) + RecordBugs() + Describe() + + err := framework.FormatBugs() + require.Error(t, err) + require.Equal(t, bugOutput, err.Error()) + + output, code := unittests.GetFrameworkOutput(t, nil) + assert.Equal(t, 1, code) + assert.Equal(t, "ERROR: E2E suite initialization was faulty, these errors must be fixed:\n"+bugOutput, output) +} diff --git a/test/e2e/framework/internal/unittests/bugs/features/features.go b/test/e2e/framework/internal/unittests/bugs/features/features.go new file mode 100644 index 00000000000..092ea8a8bf0 --- /dev/null +++ b/test/e2e/framework/internal/unittests/bugs/features/features.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/apimachinery/pkg/util/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +const ( + Alpha featuregate.Feature = "TestAlphaFeature" + Beta featuregate.Feature = "TestBetaFeature" + GA featuregate.Feature = "TestGAFeature" +) + +func init() { + runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(testFeatureGates)) +} + +var testFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + Alpha: {PreRelease: featuregate.Alpha}, + Beta: {PreRelease: featuregate.Beta}, + GA: {PreRelease: featuregate.GA}, +} diff --git a/test/e2e/framework/internal/unittests/framework_test.go b/test/e2e/framework/internal/unittests/framework_test.go index 30c8d8d0311..efb56ba6556 100644 --- a/test/e2e/framework/internal/unittests/framework_test.go +++ b/test/e2e/framework/internal/unittests/framework_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework_test +package unittests_test import ( "reflect" diff --git a/test/e2e/framework/internal/unittests/helpers.go b/test/e2e/framework/internal/unittests/helpers.go new file mode 100644 index 00000000000..a3dcc3581ae --- /dev/null +++ b/test/e2e/framework/internal/unittests/helpers.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unittests + +import ( + "bytes" + "flag" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kubernetes/test/e2e/framework" +) + +// GetFrameworkOutput captures writes to framework.Output during a test suite setup +// and returns it together with any explicit Exit call code, -1 if none. +// May only be called once per test binary. +func GetFrameworkOutput(t *testing.T, flags map[string]string) (output string, finalExitCode int) { + // This simulates how test/e2e uses the framework and how users + // invoke test/e2e. + framework.RegisterCommonFlags(flag.CommandLine) + framework.RegisterClusterFlags(flag.CommandLine) + for flagname, value := range flags { + require.NoError(t, flag.Set(flagname, value), "set %s", flagname) + } + var buffer bytes.Buffer + framework.Output = &buffer + framework.Exit = func(code int) { + panic(exitCode(code)) + } + finalExitCode = -1 + defer func() { + if r := recover(); r != nil { + if code, ok := r.(exitCode); ok { + finalExitCode = int(code) + } else { + panic(r) + } + } + output = buffer.String() + }() + framework.AfterReadingAllFlags(&framework.TestContext) + + // Results set by defer. + return +} + +type exitCode int diff --git a/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go b/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go new file mode 100644 index 00000000000..95b0416d9a4 --- /dev/null +++ b/test/e2e/framework/internal/unittests/list-labels/listlabels_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package listlabels + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs" +) + +func TestListTests(t *testing.T) { + bugs.Describe() + framework.CheckForBugs = false + output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-labels": "true"}) + assert.Equal(t, 0, code) + assert.Equal(t, bugs.ListLabelsOutput, output) +} diff --git a/test/e2e/framework/internal/unittests/list-tests/listtests_test.go b/test/e2e/framework/internal/unittests/list-tests/listtests_test.go new file mode 100644 index 00000000000..4981bd0aeb9 --- /dev/null +++ b/test/e2e/framework/internal/unittests/list-tests/listtests_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package listtests + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests" + "k8s.io/kubernetes/test/e2e/framework/internal/unittests/bugs" +) + +func TestListTests(t *testing.T) { + bugs.Describe() + framework.CheckForBugs = false + output, code := unittests.GetFrameworkOutput(t, map[string]string{"list-tests": "true"}) + assert.Equal(t, 0, code) + assert.Equal(t, bugs.ListTestsOutput, output) +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 096411cfd60..a95f50c7dc1 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -23,9 +23,11 @@ import ( "errors" "flag" "fmt" + "io" "math" "os" "path" + "path/filepath" "sort" "strings" "time" @@ -36,6 +38,7 @@ import ( "github.com/onsi/gomega" gomegaformat "github.com/onsi/gomega/format" + "k8s.io/apimachinery/pkg/util/sets" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" cliflag "k8s.io/component-base/cli/flag" @@ -53,6 +56,20 @@ const ( DefaultNumNodes = -1 ) +var ( + // Output is used for output when not running tests, for example in -list-tests. + // Test output should go to ginkgo.GinkgoWriter. + Output io.Writer = os.Stdout + + // Exit is called when the framework detects fatal errors or when + // it is done with the execution of e.g. -list-tests. + Exit = os.Exit + + // CheckForBugs determines whether the framework bails out when + // test initialization found any bugs. + CheckForBugs = true +) + // TestContextType contains test settings and global state. Due to // historic reasons, it is a mixture of items managed by the test // framework itself, cloud providers and individual tests. @@ -94,6 +111,8 @@ type TestContextType struct { // ListImages will list off all images that are used then quit ListImages bool + listTests, listLabels bool + // ListConformanceTests will list off all conformance tests that are available then quit ListConformanceTests bool @@ -356,6 +375,8 @@ func RegisterCommonFlags(flags *flag.FlagSet) { flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/control-plane`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests.") flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for running tests.") + flags.BoolVar(&TestContext.listLabels, "list-labels", false, "If true, will show the list of labels that can be used to select tests via -ginkgo.label-filter.") + flags.BoolVar(&TestContext.listTests, "list-tests", false, "If true, will show the full names of all tests (aka specs) that can be used to select test via -ginkgo.focus/skip.") flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.") @@ -482,7 +503,7 @@ func AfterReadingAllFlags(t *TestContextType) { for _, v := range image.GetImageConfigs() { fmt.Println(v.GetE2EImage()) } - os.Exit(0) + Exit(0) } // Reconfigure gomega defaults. The poll interval should be suitable @@ -494,6 +515,19 @@ func AfterReadingAllFlags(t *TestContextType) { gomega.SetDefaultEventuallyTimeout(t.timeouts.PodStart) gomega.SetDefaultConsistentlyDuration(t.timeouts.PodStartShort) + // ginkgo.PreviewSpecs will expand all nodes and thus may find new bugs. + report := ginkgo.PreviewSpecs("Kubernetes e2e test statistics") + if err := FormatBugs(); CheckForBugs && err != nil { + // Refuse to do anything if the E2E suite is buggy. + fmt.Fprint(Output, "ERROR: E2E suite initialization was faulty, these errors must be fixed:") + fmt.Fprint(Output, "\n"+err.Error()) + Exit(1) + } + if t.listLabels || t.listTests { + listTestInformation(report) + Exit(0) + } + // Only set a default host if one won't be supplied via kubeconfig if len(t.Host) == 0 && len(t.KubeConfig) == 0 { // Check if we can use the in-cluster config @@ -553,7 +587,7 @@ func AfterReadingAllFlags(t *TestContextType) { } else { klog.Errorf("Failed to setup provider config for %q: %v", TestContext.Provider, err) } - os.Exit(1) + Exit(1) } if TestContext.ReportDir != "" { @@ -563,13 +597,13 @@ func AfterReadingAllFlags(t *TestContextType) { // in parallel, so we will get "exists" error in most of them. if err := os.MkdirAll(TestContext.ReportDir, 0777); err != nil && !os.IsExist(err) { klog.Errorf("Create report dir: %v", err) - os.Exit(1) + Exit(1) } ginkgoDir := path.Join(TestContext.ReportDir, "ginkgo") if TestContext.ReportCompleteGinkgo || TestContext.ReportCompleteJUnit { if err := os.MkdirAll(ginkgoDir, 0777); err != nil && !os.IsExist(err) { klog.Errorf("Create /ginkgo: %v", err) - os.Exit(1) + Exit(1) } } @@ -600,3 +634,47 @@ func AfterReadingAllFlags(t *TestContextType) { }) } } + +func listTestInformation(report ginkgo.Report) { + indent := strings.Repeat(" ", 4) + + if TestContext.listLabels { + labels := sets.New[string]() + for _, spec := range report.SpecReports { + if spec.LeafNodeType == types.NodeTypeIt { + labels.Insert(spec.Labels()...) + } + } + fmt.Fprintf(Output, "The following labels can be used with 'gingko run --label-filter':\n%s%s\n\n", indent, strings.Join(sets.List(labels), "\n"+indent)) + } + if TestContext.listTests { + leafs := make([][]string, 0, len(report.SpecReports)) + wd, _ := os.Getwd() + for _, spec := range report.SpecReports { + if spec.LeafNodeType == types.NodeTypeIt { + leafs = append(leafs, []string{fmt.Sprintf("%s:%d: ", relativePath(wd, spec.LeafNodeLocation.FileName), spec.LeafNodeLocation.LineNumber), spec.FullText()}) + } + } + // Sort by test name, not the source code location, because the test + // name is more stable across code refactoring. + sort.Slice(leafs, func(i, j int) bool { + return leafs[i][1] < leafs[j][1] + }) + fmt.Fprint(Output, "The following spec names can be used with 'ginkgo run --focus/skip':\n") + for _, leaf := range leafs { + fmt.Fprintf(Output, "%s%s%s\n", indent, leaf[0], leaf[1]) + } + fmt.Fprint(Output, "\n") + } +} + +func relativePath(wd, path string) string { + if wd == "" { + return path + } + relpath, err := filepath.Rel(wd, path) + if err != nil { + return path + } + return relpath +} diff --git a/test/e2e/instrumentation/common/framework.go b/test/e2e/instrumentation/common/framework.go index a7386091378..7b557d993f1 100644 --- a/test/e2e/instrumentation/common/framework.go +++ b/test/e2e/instrumentation/common/framework.go @@ -16,9 +16,7 @@ limitations under the License. package common -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-instrumentation] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("instrumentation") diff --git a/test/e2e/kubectl/framework.go b/test/e2e/kubectl/framework.go index 1b95893fb2c..bfb1b95d902 100644 --- a/test/e2e/kubectl/framework.go +++ b/test/e2e/kubectl/framework.go @@ -16,9 +16,7 @@ limitations under the License. package kubectl -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cli] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cli") diff --git a/test/e2e/lifecycle/framework.go b/test/e2e/lifecycle/framework.go index 97026c000ff..c697880ce66 100644 --- a/test/e2e/lifecycle/framework.go +++ b/test/e2e/lifecycle/framework.go @@ -16,9 +16,7 @@ limitations under the License. package lifecycle -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("cluster-lifecycle") diff --git a/test/e2e/network/common/framework.go b/test/e2e/network/common/framework.go index b4c77ecfaf0..6efe22e2735 100644 --- a/test/e2e/network/common/framework.go +++ b/test/e2e/network/common/framework.go @@ -16,9 +16,7 @@ limitations under the License. package common -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-network] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("network") diff --git a/test/e2e/node/framework.go b/test/e2e/node/framework.go index eb2a1bb9e7c..126d2d3a8a4 100644 --- a/test/e2e/node/framework.go +++ b/test/e2e/node/framework.go @@ -16,9 +16,7 @@ limitations under the License. package node -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node") diff --git a/test/e2e/nodefeature/nodefeature.go b/test/e2e/nodefeature/nodefeature.go new file mode 100644 index 00000000000..ed0b3f091b3 --- /dev/null +++ b/test/e2e/nodefeature/nodefeature.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package feature contains pre-defined node features used by test/e2e and/or +// test/e2e_node. +package nodefeature + +import ( + "k8s.io/kubernetes/test/e2e/framework" +) + +var ( + AppArmor = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("AppArmor")) + CheckpointContainer = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("CheckpointContainer")) + CriticalPod = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("CriticalPod")) + DeviceManager = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DeviceManager")) + DevicePluginProbe = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DevicePluginProbe")) + DownwardAPIHugePages = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DownwardAPIHugePages")) + DynamicResourceAllocation = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("DynamicResourceAllocation")) + Eviction = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("Eviction")) + FSGroup = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("FSGroup")) + GarbageCollect = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GarbageCollect")) + GracefulNodeShutdown = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdown")) + GracefulNodeShutdownBasedOnPodPriority = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdownBasedOnPodPriority")) + HostAccess = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("HostAccess")) + ImageID = framework.WithNodeFeature(framework.ValidNodeFeatures.Add(" ImageID")) + LSCIQuotaMonitoring = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("LSCIQuotaMonitoring")) + NodeAllocatable = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeAllocatable")) + NodeProblemDetector = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeProblemDetector")) + OOMScoreAdj = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("OOMScoreAdj")) + PodDisruptionConditions = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("PodDisruptionConditions")) + PodResources = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("PodResources")) + ResourceMetrics = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("ResourceMetrics")) + RuntimeHandler = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("RuntimeHandler")) + SystemNodeCriticalPod = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("SystemNodeCriticalPod")) + TopologyManager = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("TopologyManager")) +) + +func init() { + // This prevents adding additional ad-hoc features in tests. + framework.ValidNodeFeatures.Freeze() +} diff --git a/test/e2e/scheduling/framework.go b/test/e2e/scheduling/framework.go index 507820094d7..20ef5b180e9 100644 --- a/test/e2e/scheduling/framework.go +++ b/test/e2e/scheduling/framework.go @@ -21,8 +21,6 @@ import ( "fmt" "time" - "github.com/onsi/ginkgo/v2" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -33,12 +31,10 @@ import ( var ( timeout = 10 * time.Minute waitTime = 2 * time.Second -) -// SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-scheduling] "+text, body) -} + // SIGDescribe annotates the test with the SIG label. + SIGDescribe = framework.SIGDescribe("scheduling") +) // WaitForStableCluster waits until all existing pods are scheduled and returns their amount. func WaitForStableCluster(c clientset.Interface, workerNodes sets.Set[string]) int { diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 53764bcbc5c..d7dfc4062fe 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -53,8 +53,6 @@ var testDrivers = []func() storageframework.TestDriver{ // This executes testSuites for in-tree volumes. var _ = utils.SIGDescribe("In-tree Volumes", func() { - framework.Logf("Enabling in-tree volume drivers") - gceEnabled := false for _, driver := range framework.TestContext.EnabledVolumeDrivers { switch driver { diff --git a/test/e2e/storage/utils/framework.go b/test/e2e/storage/utils/framework.go index 7bd007044f2..2257e03287c 100644 --- a/test/e2e/storage/utils/framework.go +++ b/test/e2e/storage/utils/framework.go @@ -16,9 +16,7 @@ limitations under the License. package utils -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-storage] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("storage") diff --git a/test/e2e/windows/cpu_limits.go b/test/e2e/windows/cpu_limits.go index 3e7003c4018..bc799af3d04 100644 --- a/test/e2e/windows/cpu_limits.go +++ b/test/e2e/windows/cpu_limits.go @@ -35,7 +35,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Cpu Resources [Serial]", func() { +var _ = sigDescribe("[Feature:Windows] Cpu Resources [Serial]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("cpu-resources-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -100,7 +100,7 @@ var _ = SIGDescribe("[Feature:Windows] Cpu Resources [Serial]", func() { } }) }) -}) +})) // newCPUBurnPods creates a list of pods (specification) with a workload that will consume all available CPU resources up to container limit func newCPUBurnPods(numPods int, image imageutils.Config, cpuLimit string, memoryLimit string) []*v1.Pod { diff --git a/test/e2e/windows/density.go b/test/e2e/windows/density.go index b344b663e34..ceb98cbf5c3 100644 --- a/test/e2e/windows/density.go +++ b/test/e2e/windows/density.go @@ -40,7 +40,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Density [Serial] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] Density [Serial] [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("density-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -72,7 +72,7 @@ var _ = SIGDescribe("[Feature:Windows] Density [Serial] [Slow]", func() { } }) -}) +})) type densityTest struct { // number of pods diff --git a/test/e2e/windows/device_plugin.go b/test/e2e/windows/device_plugin.go index dfcaa0babb1..416e1969a74 100644 --- a/test/e2e/windows/device_plugin.go +++ b/test/e2e/windows/device_plugin.go @@ -39,7 +39,7 @@ const ( testSlowMultiplier = 60 ) -var _ = SIGDescribe("[Feature:GPUDevicePlugin] Device Plugin", func() { +var _ = sigDescribe("[Feature:GPUDevicePlugin] Device Plugin", skipUnlessWindows(func() { f := framework.NewDefaultFramework("device-plugin") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -132,4 +132,4 @@ var _ = SIGDescribe("[Feature:GPUDevicePlugin] Device Plugin", func() { _, envVarDirectxGpuNameErr := e2eoutput.LookForStringInPodExec(defaultNs, windowsPod.Name, envVarCommand, envVarDirectxGpuName, time.Minute) framework.ExpectNoError(envVarDirectxGpuNameErr, "failed: didn't find expected environment variable.") }) -}) +})) diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go index bb2dcaadc3b..a9de7723076 100644 --- a/test/e2e/windows/dns.go +++ b/test/e2e/windows/dns.go @@ -31,7 +31,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] DNS", func() { +var _ = sigDescribe("[Feature:Windows] DNS", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") @@ -136,4 +136,4 @@ var _ = SIGDescribe("[Feature:Windows] DNS", func() { // TODO: Add more test cases for other DNSPolicies. }) -}) +})) diff --git a/test/e2e/windows/framework.go b/test/e2e/windows/framework.go index a0ec0cabcd9..dd9bcc396cb 100644 --- a/test/e2e/windows/framework.go +++ b/test/e2e/windows/framework.go @@ -17,19 +17,29 @@ limitations under the License. package windows import ( + "k8s.io/kubernetes/test/e2e/framework" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "github.com/onsi/ginkgo/v2" ) -// SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-windows] "+text, func() { +// sigDescribe annotates the test with the SIG label. +// Use this together with skipUnlessWindows to define +// tests that only run if the node OS is Windows: +// +// sigDescribe("foo", skipUnlessWindows(func() { ... })) +var sigDescribe = framework.SIGDescribe("windows") + +// skipUnlessWindows wraps some other Ginkgo callback such that +// a BeforeEach runs before tests defined by that callback which +// skips those tests unless the node OS is Windows. +func skipUnlessWindows(cb func()) func() { + return func() { ginkgo.BeforeEach(func() { // all tests in this package are Windows specific e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) - body() - }) + cb() + } } diff --git a/test/e2e/windows/gmsa_full.go b/test/e2e/windows/gmsa_full.go index 100c0dde0d2..2a3f94a98d1 100644 --- a/test/e2e/windows/gmsa_full.go +++ b/test/e2e/windows/gmsa_full.go @@ -90,7 +90,7 @@ const ( gmsaSharedFolder = "write_test" ) -var _ = SIGDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("gmsa-full-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -220,7 +220,7 @@ var _ = SIGDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", func() { }) }) -}) +})) func isValidOutput(output string) bool { return strings.Contains(output, expectedQueryOutput) && diff --git a/test/e2e/windows/gmsa_kubelet.go b/test/e2e/windows/gmsa_kubelet.go index e6fe25c14f6..a7d041fcd97 100644 --- a/test/e2e/windows/gmsa_kubelet.go +++ b/test/e2e/windows/gmsa_kubelet.go @@ -39,7 +39,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] GMSA Kubelet [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] GMSA Kubelet [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("gmsa-kubelet-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -133,7 +133,7 @@ var _ = SIGDescribe("[Feature:Windows] GMSA Kubelet [Slow]", func() { }) }) }) -}) +})) func generateDummyCredSpecs(domain string) *string { shortName := strings.ToUpper(strings.Split(domain, ".")[0]) diff --git a/test/e2e/windows/host_process.go b/test/e2e/windows/host_process.go index 156b5d821b3..d9562ec5619 100644 --- a/test/e2e/windows/host_process.go +++ b/test/e2e/windows/host_process.go @@ -85,7 +85,7 @@ var ( User_NTAuthoritySystem = "NT AUTHORITY\\SYSTEM" ) -var _ = SIGDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers", func() { +var _ = sigDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) @@ -799,7 +799,7 @@ var _ = SIGDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersi gomega.Expect(strings.ToLower(logs)).ShouldNot(gomega.ContainSubstring("nt authority"), "Container runs 'whoami' and logs should not contain 'nt authority'") }) -}) +})) func makeTestPodWithVolumeMounts(name string) *v1.Pod { hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go index 20a3ef6d32a..e2d05a3ec3c 100644 --- a/test/e2e/windows/hybrid_network.go +++ b/test/e2e/windows/hybrid_network.go @@ -44,7 +44,7 @@ var ( linuxBusyBoxImage = imageutils.GetE2EImage(imageutils.Nginx) ) -var _ = SIGDescribe("Hybrid cluster network", func() { +var _ = sigDescribe("Hybrid cluster network", skipUnlessWindows(func() { f := framework.NewDefaultFramework("hybrid-network") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -99,7 +99,7 @@ var _ = SIGDescribe("Hybrid cluster network", func() { }) }) -}) +})) var ( warmUpDuration = "30s" diff --git a/test/e2e/windows/hyperv.go b/test/e2e/windows/hyperv.go index fe188d80b30..edeccc5feac 100644 --- a/test/e2e/windows/hyperv.go +++ b/test/e2e/windows/hyperv.go @@ -35,7 +35,7 @@ var ( WindowsHyperVContainerRuntimeClass = "runhcs-wcow-hypervisor" ) -var _ = SIGDescribe("[Feature:WindowsHyperVContainers] HyperV containers", func() { +var _ = sigDescribe("[Feature:WindowsHyperVContainers] HyperV containers", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) @@ -143,4 +143,4 @@ var _ = SIGDescribe("[Feature:WindowsHyperVContainers] HyperV containers", func( gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded), "pod should have succeeded") }) -}) +})) diff --git a/test/e2e/windows/kubelet_stats.go b/test/e2e/windows/kubelet_stats.go index 7d5826e9eb0..8c93c177fb5 100644 --- a/test/e2e/windows/kubelet_stats.go +++ b/test/e2e/windows/kubelet_stats.go @@ -37,7 +37,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats [Serial]", func() { +var _ = sigDescribe("[Feature:Windows] Kubelet-Stats [Serial]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("kubelet-stats-test-windows-serial") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -113,8 +113,9 @@ var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats [Serial]", func() { }) }) }) -}) -var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats", func() { +})) + +var _ = sigDescribe("[Feature:Windows] Kubelet-Stats", skipUnlessWindows(func() { f := framework.NewDefaultFramework("kubelet-stats-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -204,7 +205,7 @@ var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats", func() { }) }) }) -}) +})) // findWindowsNode finds a Windows node that is Ready and Schedulable func findWindowsNode(ctx context.Context, f *framework.Framework) (v1.Node, error) { diff --git a/test/e2e/windows/memory_limits.go b/test/e2e/windows/memory_limits.go index da96baad3fe..d108e2a495b 100644 --- a/test/e2e/windows/memory_limits.go +++ b/test/e2e/windows/memory_limits.go @@ -39,7 +39,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] Memory Limits [Serial] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] Memory Limits [Serial] [Slow]", skipUnlessWindows(func() { f := framework.NewDefaultFramework("memory-limit-test-windows") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -60,8 +60,7 @@ var _ = SIGDescribe("[Feature:Windows] Memory Limits [Serial] [Slow]", func() { overrideAllocatableMemoryTest(ctx, f, framework.TestContext.CloudConfig.NumNodes) }) }) - -}) +})) type nodeMemory struct { // capacity diff --git a/test/e2e/windows/reboot_node.go b/test/e2e/windows/reboot_node.go index 51226e377de..a1b7790f60f 100644 --- a/test/e2e/windows/reboot_node.go +++ b/test/e2e/windows/reboot_node.go @@ -34,7 +34,7 @@ import ( admissionapi "k8s.io/pod-security-admission/api" ) -var _ = SIGDescribe("[Feature:Windows] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] RebootHost containers [Serial] [Disruptive] [Slow]", func() { +var _ = sigDescribe("[Feature:Windows] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] RebootHost containers [Serial] [Disruptive] [Slow]", skipUnlessWindows(func() { ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessNodeOSDistroIs("windows") }) @@ -254,4 +254,4 @@ var _ = SIGDescribe("[Feature:Windows] [Excluded:WindowsDocker] [MinimumKubeletV framework.ExpectNoError(err, "Error retrieving pod") gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) }) -}) +})) diff --git a/test/e2e/windows/security_context.go b/test/e2e/windows/security_context.go index 2278b80e2f8..f72dd48ef3e 100644 --- a/test/e2e/windows/security_context.go +++ b/test/e2e/windows/security_context.go @@ -40,7 +40,7 @@ import ( const runAsUserNameContainerName = "run-as-username-container" -var _ = SIGDescribe("[Feature:Windows] SecurityContext", func() { +var _ = sigDescribe("[Feature:Windows] SecurityContext", skipUnlessWindows(func() { f := framework.NewDefaultFramework("windows-run-as-username") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -190,7 +190,7 @@ var _ = SIGDescribe("[Feature:Windows] SecurityContext", func() { expectedEventError := "container's runAsUserName (CONTAINERADMINISTRATOR) which will be regarded as root identity and will break non-root policy" gomega.Expect(event.Message).Should(gomega.ContainSubstring(expectedEventError), "Event error should indicate non-root policy caused container to not start") }) -}) +})) func runAsUserNamePod(username *string) *v1.Pod { podName := "run-as-username-" + string(uuid.NewUUID()) diff --git a/test/e2e/windows/service.go b/test/e2e/windows/service.go index 657c2bc516e..e7c431e2598 100644 --- a/test/e2e/windows/service.go +++ b/test/e2e/windows/service.go @@ -36,7 +36,7 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("Services", func() { +var _ = sigDescribe("Services", skipUnlessWindows(func() { f := framework.NewDefaultFramework("services") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -85,5 +85,4 @@ var _ = SIGDescribe("Services", func() { assertConsistentConnectivity(ctx, f, testPod.ObjectMeta.Name, windowsOS, windowsCheck(fmt.Sprintf("http://%s", net.JoinHostPort(nodeIP, strconv.Itoa(nodePort))))) }) - -}) +})) diff --git a/test/e2e/windows/volumes.go b/test/e2e/windows/volumes.go index 1d1c8fe81d7..31fb6918cb4 100644 --- a/test/e2e/windows/volumes.go +++ b/test/e2e/windows/volumes.go @@ -44,7 +44,7 @@ var ( image = imageutils.GetE2EImage(imageutils.Pause) ) -var _ = SIGDescribe("[Feature:Windows] Windows volume mounts", func() { +var _ = sigDescribe("[Feature:Windows] Windows volume mounts", skipUnlessWindows(func() { f := framework.NewDefaultFramework("windows-volumes") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged var ( @@ -86,8 +86,7 @@ var _ = SIGDescribe("[Feature:Windows] Windows volume mounts", func() { }) }) - -}) +})) func doReadOnlyTest(ctx context.Context, f *framework.Framework, source v1.VolumeSource, volumePath string) { var ( diff --git a/test/e2e_node/.import-restrictions b/test/e2e_node/.import-restrictions index a4fbc3e2e2d..8eb0fdbaa49 100644 --- a/test/e2e_node/.import-restrictions +++ b/test/e2e_node/.import-restrictions @@ -4,9 +4,11 @@ rules: allowedPrefixes: - k8s.io/kubernetes/test/e2e/common - k8s.io/kubernetes/test/e2e/dra/test-driver/app + - k8s.io/kubernetes/test/e2e/feature - k8s.io/kubernetes/test/e2e/framework - k8s.io/kubernetes/test/e2e/storage/utils - k8s.io/kubernetes/test/e2e/network/common + - k8s.io/kubernetes/test/e2e/nodefeature - k8s.io/kubernetes/test/e2e/perftype - k8s.io/kubernetes/test/e2e/testing-manifests - k8s.io/kubernetes/test/e2e_node diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 5d7f54af18d..95ae7d670e5 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -53,6 +53,10 @@ import ( e2enodetestingmanifests "k8s.io/kubernetes/test/e2e_node/testing-manifests" system "k8s.io/system-validators/validators" + // define and freeze constants + _ "k8s.io/kubernetes/test/e2e/feature" + _ "k8s.io/kubernetes/test/e2e/nodefeature" + // reconfigure framework _ "k8s.io/kubernetes/test/e2e/framework/debug/init" _ "k8s.io/kubernetes/test/e2e/framework/metrics/init" diff --git a/test/e2e_node/framework.go b/test/e2e_node/framework.go index f2a7c34651d..ad2b27a6110 100644 --- a/test/e2e_node/framework.go +++ b/test/e2e_node/framework.go @@ -16,9 +16,7 @@ limitations under the License. package e2enode -import "github.com/onsi/ginkgo/v2" +import "k8s.io/kubernetes/test/e2e/framework" // SIGDescribe annotates the test with the SIG label. -func SIGDescribe(text string, body func()) bool { - return ginkgo.Describe("[sig-node] "+text, body) -} +var SIGDescribe = framework.SIGDescribe("node")