From 846ebf78146180e70034846beba318eb010178cb Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 27 Apr 2022 09:57:17 -0700 Subject: [PATCH] Cleanup legacy scheduler perf tests --- test/integration/scheduler_perf/README.md | 11 - .../scheduler_perf_legacy_test.go | 625 ------------------ .../scheduler_perf/scheduler_perf_types.go | 31 - .../scheduler_perf/scheduler_test.go | 300 --------- 4 files changed, 967 deletions(-) delete mode 100644 test/integration/scheduler_perf/scheduler_perf_legacy_test.go delete mode 100644 test/integration/scheduler_perf/scheduler_perf_types.go delete mode 100644 test/integration/scheduler_perf/scheduler_test.go diff --git a/test/integration/scheduler_perf/README.md b/test/integration/scheduler_perf/README.md index 5a6ea8d10ef..af7fd5176f5 100644 --- a/test/integration/scheduler_perf/README.md +++ b/test/integration/scheduler_perf/README.md @@ -20,10 +20,6 @@ We want to have a standard way to reproduce scheduling latency metrics result an Currently the test suite has the following: -- density test (by adding a new Go test) - - schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes - - print out scheduling rate every second - - let you learn the rate changes vs number of scheduled pods - benchmark - make use of `go test -bench` and report nanosecond/op. - schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small: 10 - 100. @@ -32,13 +28,6 @@ Currently the test suite has the following: How To Run ------ -## Density tests - -```shell -# In Kubernetes root path -make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-alsologtostderr=true -logtostderr=true -run=." KUBE_TIMEOUT="--timeout=60m" SHORT="--short=false" -``` - ## Benchmark tests ```shell diff --git a/test/integration/scheduler_perf/scheduler_perf_legacy_test.go b/test/integration/scheduler_perf/scheduler_perf_legacy_test.go deleted file mode 100644 index 7a8a2810cce..00000000000 --- a/test/integration/scheduler_perf/scheduler_perf_legacy_test.go +++ /dev/null @@ -1,625 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package benchmark - -import ( - "fmt" - "sync/atomic" - "testing" - "time" - - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/tools/cache" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/csi-translation-lib/plugins" - csilibplugins "k8s.io/csi-translation-lib/plugins" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/kubernetes/test/integration/framework" - testutils "k8s.io/kubernetes/test/utils" -) - -var ( - defaultNodeStrategy = &testutils.TrivialNodePrepareStrategy{} - - testCSIDriver = plugins.AWSEBSDriverName - // From PV controller - annBindCompleted = "pv.kubernetes.io/bind-completed" - - defaultTests = []struct{ nodes, existingPods, minPods int }{ - {nodes: 500, existingPods: 500, minPods: 1000}, - {nodes: 600, existingPods: 10000, minPods: 1000}, - {nodes: 5000, existingPods: 5000, minPods: 1000}, - } -) - -// BenchmarkScheduling benchmarks the scheduling rate when the cluster has -// various quantities of nodes and scheduled pods. -func BenchmarkScheduling(b *testing.B) { - testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1") - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingPodAntiAffinity benchmarks the scheduling rate of pods with -// PodAntiAffinity rules when the cluster has various quantities of nodes and -// scheduled pods. -func BenchmarkSchedulingPodAntiAffinity(b *testing.B) { - // Since the pods has anti affinity to each other, the number of pods to schedule - // can't exceed the number of nodes (the topology used in the test) - tests := []struct{ nodes, existingPods, minPods int }{ - {nodes: 500, existingPods: 100, minPods: 400}, - {nodes: 5000, existingPods: 1000, minPods: 1000}, - } - testBasePod := makeBasePodWithPodAntiAffinity( - map[string]string{"name": "test", "color": "green"}, - map[string]string{"color": "green"}) - // The test strategy creates pods with anti-affinity to each other, each pod ending up in a separate node. - testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - for _, test := range tests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - var nodeStrategies []testutils.CountToStrategy - for i := 0; i < test.nodes; i++ { - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i)) - nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy}) - } - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingSecrets benchmarks the scheduling rate of pods with -// volumes that don't require any special handling, such as Secrets. -// It can be used to compare scheduler efficiency with the other benchmarks -// that use volume scheduling predicates. -func BenchmarkSchedulingSecrets(b *testing.B) { - // The test strategy creates pods with a secret. - testBasePod := makeBasePodWithSecret() - testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingInTreePVs benchmarks the scheduling rate of pods with -// in-tree volumes (used via PV/PVC). Nodes have default hardcoded attach limits -// (39 for AWS EBS). -func BenchmarkSchedulingInTreePVs(b *testing.B) { - // The test strategy creates pods with AWS EBS volume used via PV. - baseClaim := makeBasePersistentVolumeClaim() - basePod := makeBasePod() - testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, awsVolumeFactory, basePod) - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingWaitForFirstConsumerPVs benchmarks the scheduling rate -// of pods with volumes with VolumeBindingMode set to WaitForFirstConsumer. -func BenchmarkSchedulingWaitForFirstConsumerPVs(b *testing.B) { - tests := []struct{ nodes, existingPods, minPods int }{ - {nodes: 500, existingPods: 500, minPods: 1000}, - // default 5000 existingPods is a way too much for now - } - basePod := makeBasePod() - testStrategy := testutils.NewCreatePodWithPersistentVolumeWithFirstConsumerStrategy(gceVolumeFactory, basePod) - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelTopologyZone, "zone1") - for _, test := range tests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingMigratedInTreePVs benchmarks the scheduling rate of pods with -// in-tree volumes (used via PV/PVC) that are migrated to CSI. CSINode instances exist -// for all nodes and have proper annotation that AWS is migrated. -func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) { - // The test strategy creates pods with AWS EBS volume used via PV. - baseClaim := makeBasePersistentVolumeClaim() - basePod := makeBasePod() - testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, awsVolumeFactory, basePod) - - // Each node can use the same amount of CSI volumes as in-tree AWS volume - // plugin, so the results should be comparable with BenchmarkSchedulingInTreePVs. - driverKey := util.GetCSIAttachLimitKey(testCSIDriver) - allocatable := map[v1.ResourceName]string{ - v1.ResourceName(driverKey): fmt.Sprintf("%d", util.DefaultMaxEBSVolumes), - } - var count int32 = util.DefaultMaxEBSVolumes - csiAllocatable := map[string]*storagev1.VolumeNodeResources{ - testCSIDriver: { - Count: &count, - }, - } - nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{csilibplugins.AWSEBSInTreePluginName}) - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)() - defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)() - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// node.status.allocatable. -func BenchmarkSchedulingCSIPVs(b *testing.B) { - // The test strategy creates pods with CSI volume via PV. - baseClaim := makeBasePersistentVolumeClaim() - basePod := makeBasePod() - testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, csiVolumeFactory, basePod) - - // Each node can use the same amount of CSI volumes as in-tree AWS volume - // plugin, so the results should be comparable with BenchmarkSchedulingInTreePVs. - driverKey := util.GetCSIAttachLimitKey(testCSIDriver) - allocatable := map[v1.ResourceName]string{ - v1.ResourceName(driverKey): fmt.Sprintf("%d", util.DefaultMaxEBSVolumes), - } - var count int32 = util.DefaultMaxEBSVolumes - csiAllocatable := map[string]*storagev1.VolumeNodeResources{ - testCSIDriver: { - Count: &count, - }, - } - nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{}) - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingPodAffinity benchmarks the scheduling rate of pods with -// PodAffinity rules when the cluster has various quantities of nodes and -// scheduled pods. -func BenchmarkSchedulingPodAffinity(b *testing.B) { - testBasePod := makeBasePodWithPodAffinity( - map[string]string{"foo": ""}, - map[string]string{"foo": ""}, - ) - // The test strategy creates pods with affinity for each other. - testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelFailureDomainBetaZone, "zone1") - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingPreferredPodAffinity benchmarks the scheduling rate of pods with -// preferred PodAffinity rules when the cluster has various quantities of nodes and -// scheduled pods. -func BenchmarkSchedulingPreferredPodAffinity(b *testing.B) { - testBasePod := makeBasePodWithPreferredPodAffinity( - map[string]string{"foo": ""}, - map[string]string{"foo": ""}, - ) - // The test strategy creates pods with affinity for each other. - testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - var nodeStrategies []testutils.CountToStrategy - for i := 0; i < test.nodes; i++ { - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i)) - nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy}) - } - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingPreferredPodAntiAffinity benchmarks the scheduling rate of pods with -// preferred PodAntiAffinity rules when the cluster has various quantities of nodes and -// scheduled pods. -func BenchmarkSchedulingPreferredPodAntiAffinity(b *testing.B) { - testBasePod := makeBasePodWithPreferredPodAntiAffinity( - map[string]string{"foo": ""}, - map[string]string{"foo": ""}, - ) - // The test strategy creates pods with anti affinity to each other. - testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - var nodeStrategies []testutils.CountToStrategy - for i := 0; i < test.nodes; i++ { - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i)) - nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy}) - } - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// BenchmarkSchedulingNodeAffinity benchmarks the scheduling rate of pods with -// NodeAffinity rules when the cluster has various quantities of nodes and -// scheduled pods. -func BenchmarkSchedulingNodeAffinity(b *testing.B) { - testBasePod := makeBasePodWithNodeAffinity(v1.LabelFailureDomainBetaZone, []string{"zone1", "zone2"}) - // The test strategy creates pods with node-affinity for each other. - testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelFailureDomainBetaZone, "zone1") - for _, test := range defaultTests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) - b.Run(name, func(b *testing.B) { - nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} - benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) - }) - } -} - -// makeBasePodWithPodAntiAffinity creates a Pod object to be used as a template. -// The Pod has a PodAntiAffinity requirement against pods with the given labels. -func makeBasePodWithPodAntiAffinity(podLabels, affinityLabels map[string]string) *v1.Pod { - basePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "anti-affinity-pod-", - Labels: podLabels, - }, - Spec: testutils.MakePodSpec(), - } - basePod.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affinityLabels, - }, - TopologyKey: v1.LabelHostname, - Namespaces: []string{testNamespace, setupNamespace}, - }, - }, - }, - } - return basePod -} - -// makeBasePodWithPreferredPodAntiAffinity creates a Pod object to be used as a template. -// The Pod has a preferred PodAntiAffinity with pods with the given labels. -func makeBasePodWithPreferredPodAntiAffinity(podLabels, affinityLabels map[string]string) *v1.Pod { - basePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "preferred-affinity-pod-", - Labels: podLabels, - }, - Spec: testutils.MakePodSpec(), - } - basePod.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ - { - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affinityLabels, - }, - TopologyKey: v1.LabelHostname, - Namespaces: []string{testNamespace, setupNamespace}, - }, - Weight: 1, - }, - }, - }, - } - return basePod -} - -// makeBasePodWithPreferredPodAffinity creates a Pod object to be used as a template. -// The Pod has a preferred PodAffinity with pods with the given labels. -func makeBasePodWithPreferredPodAffinity(podLabels, affinityLabels map[string]string) *v1.Pod { - basePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "preferred-affinity-pod-", - Labels: podLabels, - }, - Spec: testutils.MakePodSpec(), - } - basePod.Spec.Affinity = &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ - { - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affinityLabels, - }, - TopologyKey: v1.LabelHostname, - Namespaces: []string{testNamespace, setupNamespace}, - }, - Weight: 1, - }, - }, - }, - } - return basePod -} - -// makeBasePodWithPodAffinity creates a Pod object to be used as a template. -// The Pod has a PodAffinity requirement against pods with the given labels. -func makeBasePodWithPodAffinity(podLabels, affinityZoneLabels map[string]string) *v1.Pod { - basePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "affinity-pod-", - Labels: podLabels, - }, - Spec: testutils.MakePodSpec(), - } - basePod.Spec.Affinity = &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affinityZoneLabels, - }, - TopologyKey: v1.LabelFailureDomainBetaZone, - Namespaces: []string{testNamespace, setupNamespace}, - }, - }, - }, - } - return basePod -} - -// makeBasePodWithNodeAffinity creates a Pod object to be used as a template. -// The Pod has a NodeAffinity requirement against nodes with the given expressions. -func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod { - basePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "node-affinity-", - }, - Spec: testutils.MakePodSpec(), - } - basePod.Spec.Affinity = &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: key, - Operator: v1.NodeSelectorOpIn, - Values: vals, - }, - }, - }, - }, - }, - }, - } - return basePod -} - -// benchmarkScheduling benchmarks scheduling rate with specific number of nodes -// and specific number of pods already scheduled. -// This will schedule numExistingPods pods before the benchmark starts, and at -// least minPods pods during the benchmark. -func benchmarkScheduling(numExistingPods, minPods int, - nodeStrategies []testutils.CountToStrategy, - testPodStrategy testutils.TestPodCreateStrategy, - b *testing.B) { - if b.N < minPods { - b.N = minPods //nolint:staticcheck // SA3001 Set a minimum for b.N to get more meaningful results - } - finalFunc, podInformer, clientset, _ := mustSetupScheduler(nil) - defer finalFunc() - - nodePreparer := framework.NewIntegrationTestNodePreparer( - clientset, - nodeStrategies, - "scheduler-perf-") - if err := nodePreparer.PrepareNodes(0); err != nil { - klog.Fatalf("%v", err) - } - defer nodePreparer.CleanupNodes() - - config := testutils.NewTestPodCreatorConfig() - config.AddStrategy(setupNamespace, numExistingPods, testPodStrategy) - podCreator := testutils.NewTestPodCreator(clientset, config) - podCreator.CreatePods() - - for { - scheduled, err := getScheduledPods(podInformer) - if err != nil { - klog.Fatalf("%v", err) - } - if len(scheduled) >= numExistingPods { - break - } - klog.Infof("got %d existing pods, required: %d", len(scheduled), numExistingPods) - time.Sleep(1 * time.Second) - } - - scheduled := int32(0) - completedCh := make(chan struct{}) - podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - UpdateFunc: func(old, cur interface{}) { - curPod := cur.(*v1.Pod) - oldPod := old.(*v1.Pod) - - if len(oldPod.Spec.NodeName) == 0 && len(curPod.Spec.NodeName) > 0 { - if atomic.AddInt32(&scheduled, 1) >= int32(b.N) { - completedCh <- struct{}{} - } - } - }, - }) - - // start benchmark - b.ResetTimer() - config = testutils.NewTestPodCreatorConfig() - config.AddStrategy(testNamespace, b.N, testPodStrategy) - podCreator = testutils.NewTestPodCreator(clientset, config) - podCreator.CreatePods() - - <-completedCh - - // Note: without this line we're taking the overhead of defer() into account. - b.StopTimer() -} - -// makeBasePodWithSecret creates a Pod object to be used as a template. -// The pod uses a single Secrets volume. -func makeBasePodWithSecret() *v1.Pod { - basePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "secret-volume-", - }, - Spec: testutils.MakePodSpec(), - } - - volumes := []v1.Volume{ - { - Name: "secret", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: "secret", - }, - }, - }, - } - basePod.Spec.Volumes = volumes - return basePod -} - -func makeBasePersistentVolumeClaim() *v1.PersistentVolumeClaim { - return &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - // Name is filled in NewCreatePodWithPersistentVolumeStrategy - Annotations: map[string]string{ - annBindCompleted: "true", - }, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), - }, - }, - }, - } -} - -func awsVolumeFactory(id int) *v1.PersistentVolume { - return &v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("vol-%d", id), - }, - Spec: v1.PersistentVolumeSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}, - Capacity: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), - }, - PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain, - PersistentVolumeSource: v1.PersistentVolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ - // VolumeID must be unique for each PV, so every PV is - // counted as a separate volume in MaxPDVolumeCountChecker - // predicate. - VolumeID: fmt.Sprintf("vol-%d", id), - }, - }, - }, - } -} - -func gceVolumeFactory(id int) *v1.PersistentVolume { - return &v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("vol-%d", id), - }, - Spec: v1.PersistentVolumeSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}, - Capacity: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), - }, - PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain, - PersistentVolumeSource: v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - FSType: "ext4", - PDName: fmt.Sprintf("vol-%d-pvc", id), - }, - }, - NodeAffinity: &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelFailureDomainBetaZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"zone1"}, - }, - }, - }, - }, - }, - }, - }, - } -} - -func csiVolumeFactory(id int) *v1.PersistentVolume { - return &v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("vol-%d", id), - }, - Spec: v1.PersistentVolumeSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}, - Capacity: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), - }, - PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain, - PersistentVolumeSource: v1.PersistentVolumeSource{ - CSI: &v1.CSIPersistentVolumeSource{ - // Handle must be unique for each PV, so every PV is - // counted as a separate volume in CSIMaxVolumeLimitChecker - // predicate. - VolumeHandle: fmt.Sprintf("vol-%d", id), - Driver: testCSIDriver, - }, - }, - }, - } -} diff --git a/test/integration/scheduler_perf/scheduler_perf_types.go b/test/integration/scheduler_perf/scheduler_perf_types.go deleted file mode 100644 index 84e9e24107e..00000000000 --- a/test/integration/scheduler_perf/scheduler_perf_types.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package benchmark - -// High Level Configuration for all predicates and priorities. -type schedulerPerfConfig struct { - NodeCount int // The number of nodes which will be seeded with metadata to match predicates and have non-trivial priority rankings. - PodCount int // The number of pods which will be seeded with metadata to match predicates and have non-trivial priority rankings. - NodeAffinity *nodeAffinity - // TODO: Other predicates and priorities to be added here. -} - -// nodeAffinity priority configuration details. -type nodeAffinity struct { - nodeAffinityKey string // Node Selection Key. - LabelCount int // number of labels to be added to each node or pod. -} diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go deleted file mode 100644 index 93f2907b428..00000000000 --- a/test/integration/scheduler_perf/scheduler_test.go +++ /dev/null @@ -1,300 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package benchmark - -import ( - "context" - "fmt" - "math" - "strconv" - "sync/atomic" - "testing" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - coreinformers "k8s.io/client-go/informers/core/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - testutils "k8s.io/kubernetes/test/utils" - - "k8s.io/klog/v2" -) - -const ( - warning3K = 100 - threshold3K = 30 -) - -var ( - basePodTemplate = &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "sched-perf-pod-", - }, - // TODO: this needs to be configurable. - Spec: testutils.MakePodSpec(), - } - baseNodeTemplate = &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "sample-node-", - }, - Status: v1.NodeStatus{ - Capacity: v1.ResourceList{ - v1.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI), - v1.ResourceCPU: resource.MustParse("4"), - v1.ResourceMemory: resource.MustParse("32Gi"), - }, - Phase: v1.NodeRunning, - Conditions: []v1.NodeCondition{ - {Type: v1.NodeReady, Status: v1.ConditionTrue}, - }, - }, - } -) - -// TestSchedule100Node3KPods schedules 3k pods on 100 nodes. -func TestSchedule100Node3KPods(t *testing.T) { - // TODO (#93112) skip test until appropriate timeout established - if testing.Short() || true { - t.Skip("Skipping because we want to run short tests") - } - - config := getBaseConfig(100, 3000) - err := writePodAndNodeTopologyToConfig(config) - if err != nil { - t.Errorf("Misconfiguration happened for nodes/pods chosen to have predicates and priorities") - } - min := schedulePods(config) - if min < threshold3K { - t.Errorf("Failing: Scheduling rate was too low for an interval, we saw rate of %v, which is the allowed minimum of %v ! ", min, threshold3K) - } else if min < warning3K { - fmt.Printf("Warning: pod scheduling throughput for 3k pods was slow for an interval... Saw an interval with very low (%v) scheduling rate!", min) - } else { - fmt.Printf("Minimal observed throughput for 3k pod test: %v\n", min) - } -} - -// TestSchedule2000Node60KPods schedules 60k pods on 2000 nodes. -// This test won't fit in normal 10 minutes time window. -// func TestSchedule2000Node60KPods(t *testing.T) { -// if testing.Short() { -// t.Skip("Skipping because we want to run short tests") -// } -// config := defaultSchedulerBenchmarkConfig(2000, 60000) -// if min := schedulePods(config); min < threshold60K { -// t.Errorf("Too small pod scheduling throughput for 60k pods. Expected %v got %v", threshold60K, min) -// } else { -// fmt.Printf("Minimal observed throughput for 60k pod test: %v\n", min) -// } -// } - -// testConfig contains the some input parameters needed for running test-suite -type testConfig struct { - numPods int - numNodes int - mutatedNodeTemplate *v1.Node - mutatedPodTemplate *v1.Pod - clientset clientset.Interface - podInformer coreinformers.PodInformer - destroyFunc func() -} - -// getBaseConfig returns baseConfig after initializing number of nodes and pods. -func getBaseConfig(nodes int, pods int) *testConfig { - destroyFunc, podInformer, clientset, _ := mustSetupScheduler(nil) - return &testConfig{ - clientset: clientset, - destroyFunc: destroyFunc, - numNodes: nodes, - numPods: pods, - podInformer: podInformer, - } -} - -// schedulePods schedules specific number of pods on specific number of nodes. -// This is used to learn the scheduling throughput on various -// sizes of cluster and changes as more and more pods are scheduled. -// It won't stop until all pods are scheduled. -// It returns the minimum of throughput over whole run. -func schedulePods(config *testConfig) int32 { - defer config.destroyFunc() - prev := int32(0) - // On startup there may be a latent period where NO scheduling occurs (qps = 0). - // We are interested in low scheduling rates (i.e. qps=2), - minQPS := int32(math.MaxInt32) - start := time.Now() - - // Bake in time for the first pod scheduling event. - for { - time.Sleep(50 * time.Millisecond) - scheduled, err := getScheduledPods(config.podInformer) - if err != nil { - klog.Fatalf("%v", err) - } - // 30,000 pods -> wait till @ least 300 are scheduled to start measuring. - // TODO Find out why sometimes there may be scheduling blips in the beginning. - if len(scheduled) > config.numPods/100 { - break - } - } - - scheduled := int32(0) - ctx, cancel := context.WithCancel(context.Background()) - config.podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - UpdateFunc: func(old, cur interface{}) { - curPod := cur.(*v1.Pod) - oldPod := old.(*v1.Pod) - - if len(oldPod.Spec.NodeName) == 0 && len(curPod.Spec.NodeName) > 0 { - if atomic.AddInt32(&scheduled, 1) >= int32(config.numPods) { - cancel() - } - } - }, - }) - - // map minimum QPS entries in a counter, useful for debugging tests. - qpsStats := map[int32]int{} - - ticker := time.NewTicker(1 * time.Second) - go func() { - for { - select { - case <-ticker.C: - scheduled := atomic.LoadInt32(&scheduled) - qps := scheduled - prev - qpsStats[qps]++ - if qps < minQPS { - minQPS = qps - } - fmt.Printf("%ds\trate: %d\ttotal: %d (qps frequency: %v)\n", time.Since(start)/time.Second, qps, scheduled, qpsStats) - prev = scheduled - - case <-ctx.Done(): - return - } - } - }() - - <-ctx.Done() - - ticker.Stop() - - // We will be completed when all pods are done being scheduled. - // return the worst-case-scenario interval that was seen during this time. - // Note this should never be low due to cold-start, so allow bake in sched time if necessary. - consumed := int(time.Since(start) / time.Second) - if consumed <= 0 { - consumed = 1 - } - fmt.Printf("Scheduled %v Pods in %v seconds (%v per second on average). min QPS was %v\n", - config.numPods, consumed, config.numPods/consumed, minQPS) - return minQPS -} - -// mutateNodeTemplate returns the modified node needed for creation of nodes. -func (na nodeAffinity) mutateNodeTemplate(node *v1.Node) { - labels := make(map[string]string) - for i := 0; i < na.LabelCount; i++ { - value := strconv.Itoa(i) - key := na.nodeAffinityKey + value - labels[key] = value - } - node.ObjectMeta.Labels = labels - return -} - -// mutatePodTemplate returns the modified pod template after applying mutations. -func (na nodeAffinity) mutatePodTemplate(pod *v1.Pod) { - var nodeSelectorRequirements []v1.NodeSelectorRequirement - for i := 0; i < na.LabelCount; i++ { - value := strconv.Itoa(i) - key := na.nodeAffinityKey + value - nodeSelector := v1.NodeSelectorRequirement{Key: key, Values: []string{value}, Operator: v1.NodeSelectorOpIn} - nodeSelectorRequirements = append(nodeSelectorRequirements, nodeSelector) - } - pod.Spec.Affinity = &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: nodeSelectorRequirements, - }, - }, - }, - }, - } -} - -// generateNodes generates nodes to be used for scheduling. -func (inputConfig *schedulerPerfConfig) generateNodes(config *testConfig) { - for i := 0; i < inputConfig.NodeCount; i++ { - config.clientset.CoreV1().Nodes().Create(context.TODO(), config.mutatedNodeTemplate, metav1.CreateOptions{}) - - } - for i := 0; i < config.numNodes-inputConfig.NodeCount; i++ { - config.clientset.CoreV1().Nodes().Create(context.TODO(), baseNodeTemplate, metav1.CreateOptions{}) - } -} - -// generatePods generates pods to be used for scheduling. -func (inputConfig *schedulerPerfConfig) generatePods(config *testConfig) { - testutils.CreatePod(config.clientset, "sample", inputConfig.PodCount, config.mutatedPodTemplate) - testutils.CreatePod(config.clientset, "sample", config.numPods-inputConfig.PodCount, basePodTemplate) -} - -// generatePodAndNodeTopology is the wrapper function for modifying both pods and node objects. -func (inputConfig *schedulerPerfConfig) generatePodAndNodeTopology(config *testConfig) error { - if config.numNodes < inputConfig.NodeCount || config.numPods < inputConfig.PodCount { - return fmt.Errorf("NodeCount cannot be greater than numNodes") - } - nodeAffinity := inputConfig.NodeAffinity - // Node template that needs to be mutated. - mutatedNodeTemplate := baseNodeTemplate - // Pod template that needs to be mutated. - mutatedPodTemplate := basePodTemplate - if nodeAffinity != nil { - nodeAffinity.mutateNodeTemplate(mutatedNodeTemplate) - nodeAffinity.mutatePodTemplate(mutatedPodTemplate) - - } // TODO: other predicates/priorities will be processed in subsequent if statements or a switch:). - config.mutatedPodTemplate = mutatedPodTemplate - config.mutatedNodeTemplate = mutatedNodeTemplate - inputConfig.generateNodes(config) - inputConfig.generatePods(config) - return nil -} - -// writePodAndNodeTopologyToConfig reads a configuration and then applies it to a test configuration. -//TODO: As of now, this function is not doing anything except for reading input values to priority structs. -func writePodAndNodeTopologyToConfig(config *testConfig) error { - // High Level structure that should be filled for every predicate or priority. - inputConfig := &schedulerPerfConfig{ - NodeCount: 100, - PodCount: 3000, - NodeAffinity: &nodeAffinity{ - nodeAffinityKey: "kubernetes.io/sched-perf-node-affinity-", - LabelCount: 10, - }, - } - err := inputConfig.generatePodAndNodeTopology(config) - if err != nil { - return err - } - return nil -}