diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 97715e10ac3..347a03bb987 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -1232,6 +1232,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate DRAConsumableCapacity: { {Version: version.MustParse("1.34"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.36"), Default: true, PreRelease: featuregate.Beta}, }, DRADeviceBindingConditions: { diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go index 95257297d2d..250ac583626 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go @@ -3422,7 +3422,7 @@ func TestAllocatorSelection(t *testing.T) { // is used. "default": { features: "", - expectImplementation: "stable", + expectImplementation: "incubating", }, // Alpha features need the experimental implementation. diff --git a/test/compatibility_lifecycle/reference/feature_list.md b/test/compatibility_lifecycle/reference/feature_list.md index e2d809c0694..4bc50c19e60 100644 --- a/test/compatibility_lifecycle/reference/feature_list.md +++ b/test/compatibility_lifecycle/reference/feature_list.md @@ -51,7 +51,7 @@ | CustomCPUCFSQuotaPeriod | :ballot_box_with_check: 1.36+ | | 1.12–1.35 | | 1.36– | | | [code](https://cs.k8s.io/?q=%5CbCustomCPUCFSQuotaPeriod%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbCustomCPUCFSQuotaPeriod%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | CustomResourceFieldSelectors | :ballot_box_with_check: 1.31+ | :closed_lock_with_key: 1.32+ | 1.30 | 1.31 | 1.32– | | | [code](https://cs.k8s.io/?q=%5CbCustomResourceFieldSelectors%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbCustomResourceFieldSelectors%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRAAdminAccess | :ballot_box_with_check: 1.34+ | | 1.32–1.33 | 1.34– | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAAdminAccess%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAAdminAccess%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | -| DRAConsumableCapacity | | | 1.34– | | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAConsumableCapacity%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAConsumableCapacity%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | +| DRAConsumableCapacity | :ballot_box_with_check: 1.36+ | | 1.34–1.35 | 1.36– | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAConsumableCapacity%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAConsumableCapacity%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRADeviceBindingConditions | | | 1.34– | | | | DRAResourceClaimDeviceStatus
DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRADeviceBindingConditions%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceBindingConditions%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRADeviceTaintRules | | | 1.35– | | | | DRADeviceTaints | [code](https://cs.k8s.io/?q=%5CbDRADeviceTaintRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceTaintRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRADeviceTaints | | | 1.33– | | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRADeviceTaints%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceTaints%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 4fd44966ef0..9f4397fab65 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -511,6 +511,10 @@ lockToDefault: false preRelease: Alpha version: "1.34" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.36" - name: DRADeviceBindingConditions versionedSpecs: - default: false diff --git a/test/integration/dra/dra_test.go b/test/integration/dra/dra_test.go index e8ed25f1687..7b8e49c5e5f 100644 --- a/test/integration/dra/dra_test.go +++ b/test/integration/dra/dra_test.go @@ -147,7 +147,7 @@ func TestDRA(t *testing.T) { tCtx.Run("Pod", func(tCtx ktesting.TContext) { testPod(tCtx, true) }) // Number of devices per slice is chosen so that Filter takes a few seconds: // without a timeout, the test doesn't run too long, but long enough that a short timeout triggers. - tCtx.Run("FilterTimeout", func(tCtx ktesting.TContext) { testFilterTimeout(tCtx, 9) }) + tCtx.Run("FilterTimeout", func(tCtx ktesting.TContext) { testFilterTimeout(tCtx, 20) }) tCtx.Run("UsesAllResources", testUsesAllResources) }, }, diff --git a/test/integration/scheduler_perf/dra/dra_test.go b/test/integration/scheduler_perf/dra/dra_test.go index bf64abb0877..3dbf68ae013 100644 --- a/test/integration/scheduler_perf/dra/dra_test.go +++ b/test/integration/scheduler_perf/dra/dra_test.go @@ -21,12 +21,15 @@ import ( "os" "testing" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" _ "k8s.io/component-base/logs/json/register" "k8s.io/dynamic-resource-allocation/structured" "k8s.io/kubernetes/pkg/features" perf "k8s.io/kubernetes/test/integration/scheduler_perf" + "k8s.io/kubernetes/test/utils/ktesting" ) func TestMain(m *testing.M) { @@ -57,14 +60,26 @@ func TestSchedulerPerf(t *testing.T) { // with two sub tests: // - "ga-only": keep disabling optional features // - "default": don't change features + var options []perf.SchedulerPerfOption if allocatorName == "stable" { - featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ - features.DRAAdminAccess: false, - features.DRAPrioritizedList: false, - }) + options = append(options, perf.WithPreRunFn(func(tCtx ktesting.TContext) error { + overrides := featuregatetesting.FeatureOverrides{ + features.DRAAdminAccess: false, + features.DRAPrioritizedList: false, + } + // If version emulation already caused features to be off, + // then we do not need and maybe even cannot turn them + // off (pre-alpha = feature doesn't event exist). + if utilfeature.DefaultFeatureGate.(featuregate.MutableVersionedFeatureGate). + EmulationVersion().AtLeast(version.MustParse("1.34")) { + overrides[features.DRAConsumableCapacity] = false + } + featuregatetesting.SetFeatureGatesDuringTest(tCtx, utilfeature.DefaultFeatureGate, overrides) + return nil + })) } - perf.RunIntegrationPerfScheduling(t, "performance-config.yaml") + perf.RunIntegrationPerfScheduling(t, "performance-config.yaml", options...) }) } } diff --git a/test/integration/scheduler_perf/options.go b/test/integration/scheduler_perf/options.go index eae85e8e5c4..b5f26e8ed2e 100644 --- a/test/integration/scheduler_perf/options.go +++ b/test/integration/scheduler_perf/options.go @@ -16,21 +16,38 @@ limitations under the License. package benchmark -import "k8s.io/kubernetes/test/utils/ktesting" +import ( + frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" + "k8s.io/kubernetes/test/utils/ktesting" +) type SchedulerPerfOption func(options *schedulerPerfOptions) -// PrepareFn is a function that is called before each workload is run. -type PrepareFn func(tCtx ktesting.TContext) error +// HookFn is a function that is called while going through the test execution. +// The function may record test errors or abort the current test run through tCtx. +// Alternatively, it may also return a non-nil error. +type HookFn func(tCtx ktesting.TContext) error type schedulerPerfOptions struct { - prepareFn PrepareFn + outOfTreePluginRegistry frameworkruntime.Registry + preRunFn HookFn + prepareFn HookFn } // WithPrepareFn is the option to set a function that is called // before each workload is run. (e.g. applying CRDs for custom plugins) -func WithPrepareFn(prepareFn PrepareFn) SchedulerPerfOption { +// Scheduler and etcd are started at that point. +func WithPrepareFn(prepareFn HookFn) SchedulerPerfOption { return func(s *schedulerPerfOptions) { s.prepareFn = prepareFn } } + +// WithPreRunFn is the option to set a function that is called +// after configuring the process (logging, feature gates) and +// before running any code (etcd, scheduler). +func WithPreRunFn(preRunFn HookFn) SchedulerPerfOption { + return func(s *schedulerPerfOptions) { + s.preRunFn = preRunFn + } +} diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 0f4aa87b869..8e6816f2ae6 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -619,7 +619,7 @@ func initTestOutput(tb testing.TB) io.Writer { var specialFilenameChars = regexp.MustCompile(`[^a-zA-Z0-9-_]`) -func setupTestCase(t testing.TB, tc *testCase, featureGates map[featuregate.Feature]bool, outOfTreePluginRegistry frameworkruntime.Registry) (*scheduler.Scheduler, informers.SharedInformerFactory, ktesting.TContext) { +func setupTestCase(t testing.TB, tc *testCase, featureGates map[featuregate.Feature]bool, opts *schedulerPerfOptions) (*scheduler.Scheduler, informers.SharedInformerFactory, ktesting.TContext) { tCtx := ktesting.Init(t, initoption.PerTestOutput(UseTestingLog)) artifacts, doArtifacts := os.LookupEnv("ARTIFACTS") if !UseTestingLog && doArtifacts { @@ -687,15 +687,6 @@ func setupTestCase(t testing.TB, tc *testCase, featureGates map[featuregate.Feat // quit *before* restoring klog settings. framework.GoleakCheck(t) - // Now that we are ready to run, start - // a brand new etcd. - logger := tCtx.Logger() - if !UseTestingLog { - // Associate output going to the global log with the current test. - logger = logger.WithName(tCtx.Name()) - } - framework.StartEtcd(logger, t, true) - // We need to set emulation version for QueueingHints feature gate, which is locked at 1.34. // Only emulate v1.33 when QueueingHints is explicitly disabled. if qhEnabled, exists := featureGates[features.SchedulerQueueingHints]; exists && !qhEnabled { @@ -708,6 +699,21 @@ func setupTestCase(t testing.TB, tc *testCase, featureGates map[featuregate.Feat } featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featureGates) + if opts.preRunFn != nil { + if err := opts.preRunFn(tCtx); err != nil { + t.Fatalf("pre-run: %v", err) + } + } + + // Now that we are ready to run, start + // a brand new etcd. + logger := tCtx.Logger() + if !UseTestingLog { + // Associate output going to the global log with the current test. + logger = logger.WithName(tCtx.Name()) + } + framework.StartEtcd(logger, t, true) + // 30 minutes should be plenty enough even for the 5000-node tests. timeout := 30 * time.Minute tCtx = tCtx.WithTimeout(timeout, fmt.Sprintf("timed out after the %s per-test timeout", timeout)) @@ -719,7 +725,7 @@ func setupTestCase(t testing.TB, tc *testCase, featureGates map[featuregate.Feat }) } - return setupClusterForWorkload(tCtx, tc.SchedulerConfigPath, featureGates, outOfTreePluginRegistry) + return setupClusterForWorkload(tCtx, tc.SchedulerConfigPath, featureGates, opts) } func featureGatesMerge(src map[featuregate.Feature]bool, overrides map[featuregate.Feature]bool) map[featuregate.Feature]bool { @@ -761,7 +767,9 @@ func fixJSONOutput(b *testing.B) { // Also, you may want to put your plugins in PluginNames variable in this package // to collect metrics for them. func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName string, outOfTreePluginRegistry frameworkruntime.Registry, options ...SchedulerPerfOption) { - opts := &schedulerPerfOptions{} + opts := &schedulerPerfOptions{ + outOfTreePluginRegistry: outOfTreePluginRegistry, + } for _, option := range options { option(opts) @@ -803,7 +811,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin fixJSONOutput(b) featureGates := featureGatesMerge(tc.FeatureGates, w.FeatureGates) - scheduler, informerFactory, tCtx := setupTestCase(b, tc, featureGates, outOfTreePluginRegistry) + scheduler, informerFactory, tCtx := setupTestCase(b, tc, featureGates, opts) err := w.isValid(tc.MetricsCollectorConfig) if err != nil { @@ -898,7 +906,12 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin } // RunIntegrationPerfScheduling runs the scheduler performance integration tests. -func RunIntegrationPerfScheduling(t *testing.T, configFile string) { +func RunIntegrationPerfScheduling(t *testing.T, configFile string, options ...SchedulerPerfOption) { + opts := &schedulerPerfOptions{} + for _, option := range options { + option(opts) + } + testCases, err := getTestCases(configFile) if err != nil { t.Fatal(err) @@ -920,7 +933,7 @@ func RunIntegrationPerfScheduling(t *testing.T, configFile string) { t.Skipf("disabled by label filter %q", TestSchedulingLabelFilter) } featureGates := featureGatesMerge(tc.FeatureGates, w.FeatureGates) - scheduler, informerFactory, tCtx := setupTestCase(t, tc, featureGates, nil) + scheduler, informerFactory, tCtx := setupTestCase(t, tc, featureGates, opts) err := w.isValid(tc.MetricsCollectorConfig) if err != nil { t.Fatalf("workload %s is not valid: %v", w.Name, err) @@ -986,7 +999,7 @@ func unrollWorkloadTemplate(tb ktesting.TB, wt []op, w *workload) []op { return unrolled } -func setupClusterForWorkload(tCtx ktesting.TContext, configPath string, featureGates map[featuregate.Feature]bool, outOfTreePluginRegistry frameworkruntime.Registry) (*scheduler.Scheduler, informers.SharedInformerFactory, ktesting.TContext) { +func setupClusterForWorkload(tCtx ktesting.TContext, configPath string, featureGates map[featuregate.Feature]bool, opts *schedulerPerfOptions) (*scheduler.Scheduler, informers.SharedInformerFactory, ktesting.TContext) { var cfg *config.KubeSchedulerConfiguration var err error if configPath != "" { @@ -998,7 +1011,7 @@ func setupClusterForWorkload(tCtx ktesting.TContext, configPath string, featureG tCtx.Fatalf("validate scheduler config file failed: %v", err) } } - return mustSetupCluster(tCtx, cfg, featureGates, outOfTreePluginRegistry) + return mustSetupCluster(tCtx, cfg, featureGates, opts) } func labelsMatch(actualLabels, requiredLabels map[string]string) bool { diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index db270773747..5fcd1619c03 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -54,7 +54,6 @@ import ( "k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" - frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" schedutil "k8s.io/kubernetes/pkg/scheduler/util" "k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/integration/util" @@ -92,7 +91,7 @@ func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) { // remove resources after finished. // Notes on rate limiter: // - client rate limit is set to 5000. -func mustSetupCluster(tCtx ktesting.TContext, config *config.KubeSchedulerConfiguration, enabledFeatures map[featuregate.Feature]bool, outOfTreePluginRegistry frameworkruntime.Registry) (*scheduler.Scheduler, informers.SharedInformerFactory, ktesting.TContext) { +func mustSetupCluster(tCtx ktesting.TContext, config *config.KubeSchedulerConfiguration, enabledFeatures map[featuregate.Feature]bool, opts *schedulerPerfOptions) (*scheduler.Scheduler, informers.SharedInformerFactory, ktesting.TContext) { var runtimeConfig []string if enabledFeatures[features.DynamicResourceAllocation] { runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapi.SchemeGroupVersion)) @@ -136,7 +135,7 @@ func mustSetupCluster(tCtx ktesting.TContext, config *config.KubeSchedulerConfig // Not all config options will be effective but only those mostly related with scheduler performance will // be applied to start a scheduler, most of them are defined in `scheduler.schedulerOptions`. - scheduler, informerFactory := util.StartScheduler(tCtx, config, outOfTreePluginRegistry) + scheduler, informerFactory := util.StartScheduler(tCtx, config, opts.outOfTreePluginRegistry) util.StartFakePVController(tCtx, tCtx.Client(), informerFactory) runGC := util.CreateGCController(tCtx, tCtx, *cfg, informerFactory) runNS := util.CreateNamespaceController(tCtx, tCtx, *cfg, informerFactory)