From 17946fe7994d4492fab89db9ebeb10d9dcf96d18 Mon Sep 17 00:00:00 2001 From: sunnylovestiramisu Date: Thu, 30 Jan 2025 21:40:46 +0000 Subject: [PATCH] Add Stress Tests for VolumeAttributesClass --- test/e2e/storage/drivers/csi.go | 3 + test/e2e/storage/framework/testdriver.go | 9 + test/e2e/storage/framework/volume_resource.go | 2 + test/e2e/storage/testsuites/base.go | 1 + .../testsuites/volume_modify_stress.go | 241 ++++++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 test/e2e/storage/testsuites/volume_modify_stress.go diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 75e757b7728..710850bc59b 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -119,6 +119,9 @@ func initHostPathCSIDriver(name string, capabilities map[storageframework.Capabi NumPods: 10, NumSnapshots: 10, }, + VolumeModifyStressTestOptions: &storageframework.VolumeModifyStressTestOptions{ + NumPods: 10, + }, PerformanceTestOptions: &storageframework.PerformanceTestOptions{ ProvisioningOptions: &storageframework.PerformanceTestProvisioningOptions{ VolumeSize: "1Mi", diff --git a/test/e2e/storage/framework/testdriver.go b/test/e2e/storage/framework/testdriver.go index f48d5898911..22e57d6897a 100644 --- a/test/e2e/storage/framework/testdriver.go +++ b/test/e2e/storage/framework/testdriver.go @@ -264,6 +264,8 @@ type DriverInfo struct { StressTestOptions *StressTestOptions // [Optional] Scale parameters for volume snapshot stress tests. VolumeSnapshotStressTestOptions *VolumeSnapshotStressTestOptions + // [Optional] Scale parameters for volume modify stress tests. + VolumeModifyStressTestOptions *VolumeModifyStressTestOptions // [Optional] Parameters for performance tests PerformanceTestOptions *PerformanceTestOptions } @@ -286,6 +288,13 @@ type VolumeSnapshotStressTestOptions struct { NumSnapshots int } +// VolumeModifyStressTestOptions contains parameters used for volume modify stress tests. +type VolumeModifyStressTestOptions struct { + // Number of pods to create in the test. This may also create + // up to 1 volume with volumeAttributesClass per pod. + NumPods int +} + // Metrics to evaluate performance of an operation // TODO: Add metrics like median, mode, standard deviation, percentile type Metrics struct { diff --git a/test/e2e/storage/framework/volume_resource.go b/test/e2e/storage/framework/volume_resource.go index 9208f39a78b..7f01fa1032a 100644 --- a/test/e2e/storage/framework/volume_resource.go +++ b/test/e2e/storage/framework/volume_resource.go @@ -24,6 +24,7 @@ import ( "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,6 +47,7 @@ type VolumeResource struct { Pvc *v1.PersistentVolumeClaim Pv *v1.PersistentVolume Sc *storagev1.StorageClass + Vac *storagev1beta1.VolumeAttributesClass Volume TestVolume } diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index aa4340b394c..04b4ac163d0 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -86,6 +86,7 @@ var CSISuites = append(BaseSuites, InitPvcDeletionPerformanceTestSuite, InitReadWriteOncePodTestSuite, InitVolumeModifyTestSuite, + InitVolumeModifyStressTestSuite, ) func getVolumeOpsFromMetricsForPlugin(ms testutil.Metrics, pluginName string) opCounts { diff --git a/test/e2e/storage/testsuites/volume_modify_stress.go b/test/e2e/storage/testsuites/volume_modify_stress.go new file mode 100644 index 00000000000..86f87cc9645 --- /dev/null +++ b/test/e2e/storage/testsuites/volume_modify_stress.go @@ -0,0 +1,241 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "sync" + + "github.com/onsi/ginkgo/v2" + /*"github.com/onsi/gomega"*/ + v1 "k8s.io/api/core/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +type volumeModifyStressTestSuite struct { + tsInfo storageframework.TestSuiteInfo +} + +type volumeModifyStressTest struct { + config *storageframework.PerTestConfig + + vac *storagev1beta1.VolumeAttributesClass + + volumes []*storageframework.VolumeResource + pods []*v1.Pod + // stop and wait for any async routines + wg sync.WaitGroup + + testOptions storageframework.VolumeModifyStressTestOptions +} + +var _ storageframework.TestSuite = &volumeModifyStressTestSuite{} + +// InitCustomVolumeModifyStressTestSuite returns volumeModifyStressTestSuite that implements TestSuite interface +// using custom test patterns +func InitCustomVolumeModifyStressTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite { + return &volumeModifyStressTestSuite{ + tsInfo: storageframework.TestSuiteInfo{ + Name: "volume-modify-stress", + TestPatterns: patterns, + SupportedSizeRange: e2evolume.SizeRange{ + Min: "1Gi", + }, + TestTags: []interface{}{framework.WithFeatureGate(features.VolumeAttributesClass)}, + }, + } +} + +// InitVolumeModifyStressTestSuite returns volumeModifyStressTestSuite that implements TestSuite interface +// using testsuite default patterns +func InitVolumeModifyStressTestSuite() storageframework.TestSuite { + patterns := []storageframework.TestPattern{ + storageframework.DefaultFsDynamicPV, + storageframework.BlockVolModeDynamicPV, + storageframework.NtfsDynamicPV, + } + return InitCustomVolumeModifyStressTestSuite(patterns) +} + +func (v *volumeModifyStressTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { + return v.tsInfo +} + +func (v *volumeModifyStressTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { + driverInfo := driver.GetDriverInfo() + if driverInfo.VolumeModifyStressTestOptions == nil { + e2eskipper.Skipf("Driver %s doesn't specify volume modify stress test options -- skipping", driverInfo.Name) + } + if driverInfo.VolumeModifyStressTestOptions.NumPods <= 0 { + framework.Failf("NumPods in modify volume stress test options must be a positive integer, received: %d", driverInfo.VolumeModifyStressTestOptions.NumPods) + } + _, ok := driver.(storageframework.VolumeAttributesClassTestDriver) + if !ok { + e2eskipper.Skipf("Driver %q does not support VolumeAttributesClass tests - skipping", driver.GetDriverInfo().Name) + } + // Skip block storage tests if the driver we are testing against does not support block volumes + // TODO: This should be made generic so that it doesn't have to be re-written for every test that uses the BlockVolModeDynamicPV testcase + if !driverInfo.Capabilities[storageframework.CapBlock] && pattern.VolMode == v1.PersistentVolumeBlock { + e2eskipper.Skipf("Driver %q does not support block volume mode - skipping", driver.GetDriverInfo().Name) + } +} + +func (v *volumeModifyStressTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { + + var ( + dInfo = driver.GetDriverInfo() + cs clientset.Interface + l *volumeModifyStressTest + ) + + // Beware that it also registers an AfterEach which renders f unusable. Any code using + // f must run inside an It or Context callback. + f := framework.NewFrameworkWithCustomTimeouts("volume-modify-stress", storageframework.GetDriverTimeouts(driver)) + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + init := func(ctx context.Context) { + cs = f.ClientSet + l = &volumeModifyStressTest{} + + // Now do the more expensive test initialization. + l.config = driver.PrepareTest(ctx, f) + vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver) + l.volumes = []*storageframework.VolumeResource{} + l.pods = []*v1.Pod{} + l.testOptions = *dInfo.VolumeModifyStressTestOptions + l.vac = vacDriver.GetVolumeAttributesClass(ctx, l.config) + + if l.vac == nil { + e2eskipper.Skipf("Driver %q returned nil VolumeAttributesClass - skipping", dInfo.Name) + } + + ginkgo.By("Creating VolumeAttributesClass") + _, err := f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Create(ctx, l.vac, metav1.CreateOptions{}) + framework.ExpectNoError(err, "While creating VolumeAttributesClass") + } + + createPodsAndVolumes := func(ctx context.Context, createVolumeWithVAC bool) { + for i := 0; i < l.testOptions.NumPods; i++ { + framework.Logf("Creating resources for pod %v/%v", i, l.testOptions.NumPods-1) + testVolumeSizeRange := v.GetTestSuiteInfo().SupportedSizeRange + ginkgo.By("Creating volume") + r := &storageframework.VolumeResource{} + if createVolumeWithVAC { + r = storageframework.CreateVolumeResourceWithVAC(ctx, driver, l.config, pattern, testVolumeSizeRange, &l.vac.Name) + } else { + r = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange) + } + l.volumes = append(l.volumes, r) + podConfig := e2epod.Config{ + NS: f.Namespace.Name, + PVCs: []*v1.PersistentVolumeClaim{r.Pvc}, + SeLinuxLabel: e2epv.SELinuxLabel, + } + pod, err := e2epod.MakeSecPod(&podConfig) + framework.ExpectNoError(err) + + l.pods = append(l.pods, pod) + } + } + + modifyVolumes := func(ctx context.Context) { + for i := 0; i < l.testOptions.NumPods; i++ { + ginkgo.By("Modifying PVC via VAC") + volume := l.volumes[i] + volume.Pvc = SetPVCVACName(ctx, volume.Pvc, l.vac.Name, f.ClientSet, setVACWaitPeriod) + + ginkgo.By("Checking PVC status") + err := e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, volume.Pvc, modifyVolumeWaitPeriod) + framework.ExpectNoError(err, "While waiting for PVC to have expected VAC") + } + } + + cleanup := func(ctx context.Context) { + framework.Logf("Stopping and waiting for all test routines to finish") + l.wg.Wait() + + var ( + errs []error + mu sync.Mutex + wg sync.WaitGroup + ) + + wg.Add(len(l.pods)) + for _, pod := range l.pods { + go func(pod *v1.Pod) { + defer ginkgo.GinkgoRecover() + defer wg.Done() + + framework.Logf("Deleting pod %v", pod.Name) + err := e2epod.DeletePodWithWait(ctx, cs, pod) + mu.Lock() + defer mu.Unlock() + errs = append(errs, err) + }(pod) + } + wg.Wait() + + wg.Add(len(l.volumes)) + for _, volume := range l.volumes { + go func(volume *storageframework.VolumeResource) { + defer ginkgo.GinkgoRecover() + defer wg.Done() + + framework.Logf("Deleting volume %s", volume.Pvc.GetName()) + err := volume.CleanupResource(ctx) + mu.Lock() + defer mu.Unlock() + errs = append(errs, err) + }(volume) + } + wg.Wait() + + if l.vac != nil { + ginkgo.By("Deleting VAC") + CleanupVAC(ctx, l.vac, f.ClientSet, vacCleanupWaitPeriod) + l.vac = nil + } + + framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource") + } + + f.It("multiple pods should provision volumes with volumeAttributesClass", f.WithSlow(), func(ctx context.Context) { + init(ctx) + ginkgo.DeferCleanup(cleanup) + createPodsAndVolumes(ctx, true) + l.wg.Wait() + }) + + f.It("multiple pods should modify volumes with a different volumeAttributesClass", f.WithSlow(), func(ctx context.Context) { + init(ctx) + ginkgo.DeferCleanup(cleanup) + createPodsAndVolumes(ctx, false) + modifyVolumes(ctx) + l.wg.Wait() + }) +}