diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index f167bf23592..0471601a22b 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -363,6 +363,13 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) ValidatingAdmissionPolicy = framework.WithFeature(framework.ValidFeatures.Add("ValidatingAdmissionPolicy")) + // Owner: sig-storage + // Tests related to VolumeAttributesClass (https://kep.k8s.io/3751) + // + // TODO: This label only requires the API storage.k8s.io/v1alpha1 and the VolumeAttributesClass feature-gate enabled. + // It should be removed after k/k #124350 is merged. + VolumeAttributesClass = framework.WithFeature(framework.ValidFeatures.Add("VolumeAttributesClass")) + // TODO: document the feature (owning SIG, when to use this feature for a test) Volumes = framework.WithFeature(framework.ValidFeatures.Add("Volumes")) diff --git a/test/e2e/framework/pv/pv.go b/test/e2e/framework/pv/pv.go index c6afc46a2aa..d748b5b0029 100644 --- a/test/e2e/framework/pv/pv.go +++ b/test/e2e/framework/pv/pv.go @@ -127,10 +127,11 @@ type PersistentVolumeClaimConfig struct { // unspecified ClaimSize string // AccessModes defaults to RWO if unspecified - AccessModes []v1.PersistentVolumeAccessMode - Annotations map[string]string - Selector *metav1.LabelSelector - StorageClassName *string + AccessModes []v1.PersistentVolumeAccessMode + Annotations map[string]string + Selector *metav1.LabelSelector + StorageClassName *string + VolumeAttributesClassName *string // VolumeMode defaults to nil if unspecified or specified as the empty // string VolumeMode *v1.PersistentVolumeMode @@ -661,8 +662,9 @@ func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.P v1.ResourceStorage: resource.MustParse(cfg.ClaimSize), }, }, - StorageClassName: cfg.StorageClassName, - VolumeMode: cfg.VolumeMode, + StorageClassName: cfg.StorageClassName, + VolumeAttributesClassName: cfg.VolumeAttributesClassName, + VolumeMode: cfg.VolumeMode, }, } } diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 67a5691d379..96b7a5c8c69 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -54,6 +54,7 @@ import ( v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -85,6 +86,11 @@ const ( // Prefix of the mock driver grpc log grpcCallPrefix = "gRPCCall:" + + // Parameter to use in hostpath CSI driver VolumeAttributesClass + // Must be passed to the driver via --accepted-mutable-parameter-names + hostpathCSIDriverMutableParameterName = "e2eVacTest" + hostpathCSIDriverMutableParameterValue = "test-value" ) // hostpathCSI @@ -209,6 +215,15 @@ func (h *hostpathCSIDriver) GetSnapshotClass(ctx context.Context, config *storag return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns) } +func (h *hostpathCSIDriver) GetVolumeAttributesClass(_ context.Context, config *storageframework.PerTestConfig) *storagev1alpha1.VolumeAttributesClass { + return storageframework.CopyVolumeAttributesClass(&storagev1alpha1.VolumeAttributesClass{ + DriverName: config.GetUniqueDriverName(), + Parameters: map[string]string{ + hostpathCSIDriverMutableParameterName: hostpathCSIDriverMutableParameterValue, + }, + }, config.Framework.Namespace.Name, "e2e-vac-hostpath") +} + func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig { // Create secondary namespace which will be used for creating driver driverNamespace := utils.CreateDriverNamespace(ctx, f) @@ -230,7 +245,9 @@ func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framew DriverNamespace: driverNamespace, } - o := utils.PatchCSIOptions{ + patches := []utils.PatchCSIOptions{} + + patches = append(patches, utils.PatchCSIOptions{ OldDriverName: h.driverInfo.Name, NewDriverName: config.GetUniqueDriverName(), DriverContainerName: "hostpath", @@ -246,11 +263,31 @@ func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framew ProvisionerContainerName: "csi-provisioner", SnapshotterContainerName: "csi-snapshotter", NodeName: node.Name, - } + }) + + // VAC E2E HostPath patch + // Enables ModifyVolume support in the hostpath CSI driver, and adds an enabled parameter name + patches = append(patches, utils.PatchCSIOptions{ + DriverContainerName: "hostpath", + DriverContainerArguments: []string{"--enable-controller-modify-volume=true", "--accepted-mutable-parameter-names=e2eVacTest"}, + }) + + // VAC E2E FeatureGate patches + // TODO: These can be removed after the VolumeAttributesClass feature is default enabled + patches = append(patches, utils.PatchCSIOptions{ + DriverContainerName: "csi-provisioner", + DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"}, + }) + patches = append(patches, utils.PatchCSIOptions{ + DriverContainerName: "csi-resizer", + DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"}, + }) err = utils.CreateFromManifests(ctx, config.Framework, driverNamespace, func(item interface{}) error { - if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { - return err + for _, o := range patches { + if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { + return err + } } // Remove csi-external-health-monitor-agent and diff --git a/test/e2e/storage/external/external.go b/test/e2e/storage/external/external.go index e9f1861f361..0f3446d4691 100644 --- a/test/e2e/storage/external/external.go +++ b/test/e2e/storage/external/external.go @@ -25,6 +25,7 @@ import ( "time" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -79,6 +80,30 @@ type driverDefinition struct { FromExistingClassName string } + // VolumeAttributesClass must be set to enable volume modification tests. + // The default is to not run those tests. + VolumeAttributesClass struct { + // FromName set to true enables the usage of a + // VolumeAttributesClass with DriverInfo.Name as + // provisioner and no parameters. + FromName bool + + // FromFile is used only when FromName is false. It + // loads a storage class from the given .yaml or .json + // file. File names are resolved by the + // framework.testfiles package, which typically means + // that they can be absolute or relative to the test + // suite's --repo-root parameter. + // + // This can be used when the VolumeAttributesClass + // is meant to have additional parameters. + FromFile string + + // FromExistingClassName specifies the name of a pre-installed + // VolumeAttributesClass that will be copied and used for the tests. + FromExistingClassName string + } + // SnapshotClass must be set to enable snapshotting tests. // The default is to not run those tests. SnapshotClass struct { @@ -405,6 +430,43 @@ func (d *driverDefinition) GetSnapshotClass(ctx context.Context, e2econfig *stor return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns) } +func (d *driverDefinition) GetVolumeAttributesClass(ctx context.Context, e2econfig *storageframework.PerTestConfig) *storagev1alpha1.VolumeAttributesClass { + if !d.VolumeAttributesClass.FromName && d.VolumeAttributesClass.FromFile == "" && d.VolumeAttributesClass.FromExistingClassName == "" { + e2eskipper.Skipf("Driver %q has no configured VolumeAttributesClass - skipping", d.DriverInfo.Name) + return nil + } + + var ( + vac *storagev1alpha1.VolumeAttributesClass + err error + ) + + f := e2econfig.Framework + switch { + case d.VolumeAttributesClass.FromName: + vac = &storagev1alpha1.VolumeAttributesClass{DriverName: d.DriverInfo.Name} + case d.VolumeAttributesClass.FromExistingClassName != "": + vac, err = f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Get(ctx, d.VolumeAttributesClass.FromExistingClassName, metav1.GetOptions{}) + framework.ExpectNoError(err, "getting VolumeAttributesClass %s", d.VolumeAttributesClass.FromExistingClassName) + case d.VolumeAttributesClass.FromFile != "": + var ok bool + items, err := utils.LoadFromManifests(d.VolumeAttributesClass.FromFile) + framework.ExpectNoError(err, "load VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile) + gomega.Expect(items).To(gomega.HaveLen(1), "exactly one item from %s", d.VolumeAttributesClass.FromFile) + err = utils.PatchItems(f, f.Namespace, items...) + framework.ExpectNoError(err, "patch VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile) + + vac, ok = items[0].(*storagev1alpha1.VolumeAttributesClass) + if !ok { + framework.Failf("cast VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile) + } + } + + gomega.Expect(vac).ToNot(gomega.BeNil(), "VolumeAttributesClass is unexpectantly nil") + + return storageframework.CopyVolumeAttributesClass(vac, f.Namespace.Name, "e2e-vac") +} + func (d *driverDefinition) GetVolume(e2econfig *storageframework.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) { if len(d.InlineVolumes) == 0 { e2eskipper.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name) diff --git a/test/e2e/storage/external/testdata/example.yaml b/test/e2e/storage/external/testdata/example.yaml index 7d485e0a3aa..038d9054b09 100644 --- a/test/e2e/storage/external/testdata/example.yaml +++ b/test/e2e/storage/external/testdata/example.yaml @@ -1,5 +1,7 @@ StorageClass: FromExistingClassName: example +VolumeAttributesClass: + FromExistingClassName: example-vac DriverInfo: Name: example RequiredAccessModes: diff --git a/test/e2e/storage/framework/driver_operations.go b/test/e2e/storage/framework/driver_operations.go index 8c850245144..5034b300b38 100644 --- a/test/e2e/storage/framework/driver_operations.go +++ b/test/e2e/storage/framework/driver_operations.go @@ -21,6 +21,7 @@ import ( "fmt" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/storage/names" "k8s.io/kubernetes/pkg/volume/util" @@ -92,3 +93,13 @@ func GetStorageClass( VolumeBindingMode: bindingMode, } } + +// CopyVolumeAttributesClass constructs a new VolumeAttributesClass instance +// with a unique name that is based on namespace + suffix +// using the VolumeAttributesClass passed in as a parameter +func CopyVolumeAttributesClass(vac *storagev1alpha1.VolumeAttributesClass, ns string, suffix string) *storagev1alpha1.VolumeAttributesClass { + copy := vac.DeepCopy() + copy.ObjectMeta.Name = names.SimpleNameGenerator.GenerateName(ns + "-" + suffix) + copy.ResourceVersion = "" + return copy +} diff --git a/test/e2e/storage/framework/testdriver.go b/test/e2e/storage/framework/testdriver.go index c1e71179f81..0c6eb73a9ed 100644 --- a/test/e2e/storage/framework/testdriver.go +++ b/test/e2e/storage/framework/testdriver.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" @@ -130,6 +131,15 @@ type SnapshottableTestDriver interface { GetSnapshotClass(ctx context.Context, config *PerTestConfig, parameters map[string]string) *unstructured.Unstructured } +// VolumeAttributesClassTestDriver represents an interface for a TestDriver that supports +// creating and modifying volumes via VolumeAttributesClass objects +type VolumeAttributesClassTestDriver interface { + TestDriver + // GetVolumeAttributesClass returns a VolumeAttributesClass to create/modify PVCs + // It will return nil if the TestDriver does not support VACs + GetVolumeAttributesClass(ctx context.Context, config *PerTestConfig) *storagev1alpha1.VolumeAttributesClass +} + // CustomTimeoutsTestDriver represents an interface fo a TestDriver that supports custom timeouts. type CustomTimeoutsTestDriver interface { TestDriver diff --git a/test/e2e/storage/framework/volume_resource.go b/test/e2e/storage/framework/volume_resource.go index 8f0a428fe1a..36a02e81a31 100644 --- a/test/e2e/storage/framework/volume_resource.go +++ b/test/e2e/storage/framework/volume_resource.go @@ -53,11 +53,19 @@ type VolumeResource struct { // CreateVolumeResource constructs a VolumeResource for the current test. It knows how to deal with // different test pattern volume types. func CreateVolumeResource(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange) *VolumeResource { - return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes) + return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes, nil) +} + +// CreateVolumeResource constructs a VolumeResource for the current test using the specified VAC name. +func CreateVolumeResourceWithVAC(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, vacName *string) *VolumeResource { + if pattern.VolType != DynamicPV { + framework.Failf("Creating volume with VAC only supported on dynamic PV tests") + } + return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes, vacName) } // CreateVolumeResourceWithAccessModes constructs a VolumeResource for the current test with the provided access modes. -func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode) *VolumeResource { +func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode, vacName *string) *VolumeResource { r := VolumeResource{ Config: config, Pattern: pattern, @@ -107,7 +115,7 @@ func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, switch pattern.VolType { case DynamicPV: r.Pv, r.Pvc = createPVCPVFromDynamicProvisionSC( - ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes) + ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes, vacName) r.VolSource = storageutils.CreateVolumeSource(r.Pvc.Name, false /* readOnly */) case GenericEphemeralVolume: driverVolumeSizeRange := dDriver.GetDriverInfo().SupportedSizeRange @@ -287,17 +295,19 @@ func createPVCPVFromDynamicProvisionSC( sc *storagev1.StorageClass, volMode v1.PersistentVolumeMode, accessModes []v1.PersistentVolumeAccessMode, + vacName *string, ) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) { cs := f.ClientSet ns := f.Namespace.Name ginkgo.By("creating a claim") pvcCfg := e2epv.PersistentVolumeClaimConfig{ - NamePrefix: name, - ClaimSize: claimSize, - StorageClassName: &(sc.Name), - AccessModes: accessModes, - VolumeMode: &volMode, + NamePrefix: name, + ClaimSize: claimSize, + StorageClassName: &(sc.Name), + VolumeAttributesClassName: vacName, + AccessModes: accessModes, + VolumeMode: &volMode, } pvc := e2epv.MakePersistentVolumeClaim(pvcCfg, ns) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 453196b4a44..a6efd496512 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -82,6 +82,7 @@ var CSISuites = append(BaseSuites, InitSnapshottableStressTestSuite, InitVolumePerformanceTestSuite, InitReadWriteOncePodTestSuite, + InitVolumeModifyTestSuite, ) func getVolumeOpsFromMetricsForPlugin(ms testutil.Metrics, pluginName string) opCounts { diff --git a/test/e2e/storage/testsuites/disruptive.go b/test/e2e/storage/testsuites/disruptive.go index 9a7d5c80427..28285c8d45a 100644 --- a/test/e2e/storage/testsuites/disruptive.go +++ b/test/e2e/storage/testsuites/disruptive.go @@ -116,7 +116,8 @@ func (s *disruptiveTestSuite) DefineTests(driver storageframework.TestDriver, pa l.config, pattern, testVolumeSizeRange, - accessModes) + accessModes, + nil) } } diff --git a/test/e2e/storage/testsuites/readwriteoncepod.go b/test/e2e/storage/testsuites/readwriteoncepod.go index 64777cc786c..1926ca3d742 100644 --- a/test/e2e/storage/testsuites/readwriteoncepod.go +++ b/test/e2e/storage/testsuites/readwriteoncepod.go @@ -133,7 +133,7 @@ func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriv ginkgo.It("should preempt lower priority pods using ReadWriteOncePod volumes", func(ctx context.Context) { // Create the ReadWriteOncePod PVC. accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod} - l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes) + l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes, nil) l.priorityClass = &schedulingv1.PriorityClass{ ObjectMeta: metav1.ObjectMeta{Name: "e2e-test-read-write-once-pod-" + string(uuid.NewUUID())}, @@ -189,7 +189,7 @@ func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriv ginkgo.It("should block a second pod from using an in-use ReadWriteOncePod volume on the same node", func(ctx context.Context) { // Create the ReadWriteOncePod PVC. accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod} - l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes) + l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes, nil) podConfig := e2epod.Config{ NS: f.Namespace.Name, diff --git a/test/e2e/storage/testsuites/volume_modify.go b/test/e2e/storage/testsuites/volume_modify.go new file mode 100644 index 00000000000..f2104a3aa55 --- /dev/null +++ b/test/e2e/storage/testsuites/volume_modify.go @@ -0,0 +1,294 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "encoding/json" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/errors" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/features" + e2efeature "k8s.io/kubernetes/test/e2e/feature" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +const ( + modifyPollInterval = 2 * time.Second + setVACWaitPeriod = 30 * time.Second + modifyingConditionSyncWaitPeriod = 2 * time.Minute + modifyVolumeWaitPeriod = 10 * time.Minute + vacCleanupWaitPeriod = 30 * time.Second +) + +type volumeModifyTestSuite struct { + tsInfo storageframework.TestSuiteInfo +} + +// InitCustomVolumeModifyTestSuite returns volumeModifyTestSuite that implements TestSuite interface +// using custom test patterns +func InitCustomVolumeModifyTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite { + return &volumeModifyTestSuite{ + tsInfo: storageframework.TestSuiteInfo{ + Name: "volume-modify", + TestPatterns: patterns, + SupportedSizeRange: e2evolume.SizeRange{ + Min: "1Gi", + }, + TestTags: []interface{}{e2efeature.VolumeAttributesClass, framework.WithFeatureGate(features.VolumeAttributesClass)}, + }, + } +} + +// InitVolumeModifyTestSuite returns volumeModifyTestSuite that implements TestSuite interface +// using testsuite default patterns +func InitVolumeModifyTestSuite() storageframework.TestSuite { + patterns := []storageframework.TestPattern{ + storageframework.DefaultFsDynamicPV, + storageframework.BlockVolModeDynamicPV, + storageframework.NtfsDynamicPV, + } + return InitCustomVolumeModifyTestSuite(patterns) +} + +func (v *volumeModifyTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { + return v.tsInfo +} + +func (v *volumeModifyTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { + _, ok := driver.(storageframework.VolumeAttributesClassTestDriver) + if !ok { + e2eskipper.Skipf("Driver %q does not support VolumeAttributesClass tests - skipping", driver.GetDriverInfo().Name) + } + // Skip block storage tests if the driver we are testing against does not support block volumes + // TODO: This should be made generic so that it doesn't have to be re-written for every test that uses the BlockVolModeDynamicPV testcase + if !driver.GetDriverInfo().Capabilities[storageframework.CapBlock] && pattern.VolMode == v1.PersistentVolumeBlock { + e2eskipper.Skipf("Driver %q does not support block volume mode - skipping", driver.GetDriverInfo().Name) + } +} + +func (v *volumeModifyTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { + type local struct { + config *storageframework.PerTestConfig + + resource *storageframework.VolumeResource + vac *storagev1alpha1.VolumeAttributesClass + } + var l local + + // Beware that it also registers an AfterEach which renders f unusable. Any code using + // f must run inside an It or Context callback. + f := framework.NewFrameworkWithCustomTimeouts("volume-modify", storageframework.GetDriverTimeouts(driver)) + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + init := func(ctx context.Context, createVolumeWithVAC bool) { + l = local{} + + l.config = driver.PrepareTest(ctx, f) + vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver) + l.vac = vacDriver.GetVolumeAttributesClass(ctx, l.config) + + if l.vac == nil { + e2eskipper.Skipf("Driver %q returned nil VolumeAttributesClass - skipping", driver.GetDriverInfo().Name) + } + + ginkgo.By("Creating VolumeAttributesClass") + _, err := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Create(ctx, l.vac, metav1.CreateOptions{}) + framework.ExpectNoError(err, "While creating VolumeAttributesClass") + + ginkgo.By("Creating volume") + testVolumeSizeRange := v.GetTestSuiteInfo().SupportedSizeRange + if createVolumeWithVAC { + l.resource = storageframework.CreateVolumeResourceWithVAC(ctx, driver, l.config, pattern, testVolumeSizeRange, &l.vac.Name) + } else { + l.resource = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange) + } + } + + cleanup := func(ctx context.Context) { + var errs []error + if l.resource != nil { + ginkgo.By("Deleting VolumeResource") + errs = append(errs, l.resource.CleanupResource(ctx)) + l.resource = nil + } + + if l.vac != nil { + ginkgo.By("Deleting VAC") + CleanupVAC(ctx, l.vac, f.ClientSet, vacCleanupWaitPeriod) + l.vac = nil + } + + framework.ExpectNoError(errors.NewAggregate(errs), "While cleaning up") + } + + ginkgo.It("should create a volume with VAC", func(ctx context.Context) { + init(ctx, true /* volume created with VAC */) + ginkgo.DeferCleanup(cleanup) + + ginkgo.By("Creating a pod with dynamically provisioned volume") + podConfig := e2epod.Config{ + NS: f.Namespace.Name, + PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, + SeLinuxLabel: e2epod.GetLinuxLabel(), + NodeSelection: l.config.ClientNodeSelection, + ImageID: e2epod.GetDefaultTestImageID(), + } + pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart) + ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod) + framework.ExpectNoError(err, "While creating test pod with VAC") + + createdPVC, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, l.resource.Pvc.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "While getting created PVC") + // Check VAC matches on created PVC, but not current VAC in status + gomega.Expect(vacMatches(createdPVC, l.vac.Name, false)).To(gomega.BeTrueBecause("Created PVC should match expected VAC")) + }) + + ginkgo.It("should modify volume with no VAC", func(ctx context.Context) { + init(ctx, false /* volume created without VAC */) + ginkgo.DeferCleanup(cleanup) + + var err error + ginkgo.By("Creating a pod with dynamically provisioned volume") + podConfig := e2epod.Config{ + NS: f.Namespace.Name, + PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, + SeLinuxLabel: e2epod.GetLinuxLabel(), + NodeSelection: l.config.ClientNodeSelection, + ImageID: e2epod.GetDefaultTestImageID(), + } + pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart) + ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod) + framework.ExpectNoError(err, "While creating pod for modifying") + + ginkgo.By("Modifying PVC via VAC") + newPVC := SetPVCVACName(ctx, l.resource.Pvc, l.vac.Name, f.ClientSet, setVACWaitPeriod) + l.resource.Pvc = newPVC + gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil()) + + ginkgo.By("Waiting for modification to finish") + WaitForVolumeModification(ctx, l.resource.Pvc, f.ClientSet, modifyVolumeWaitPeriod) + + pvcConditions := l.resource.Pvc.Status.Conditions + gomega.Expect(pvcConditions).To(gomega.BeEmpty(), "PVC should not have conditions") + }) + + ginkgo.It("should modify volume that already has a VAC", func(ctx context.Context) { + init(ctx, true /* volume created with VAC */) + ginkgo.DeferCleanup(cleanup) + + vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver) + newVAC := vacDriver.GetVolumeAttributesClass(ctx, l.config) + gomega.Expect(newVAC).NotTo(gomega.BeNil()) + _, err := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Create(ctx, newVAC, metav1.CreateOptions{}) + framework.ExpectNoError(err, "While creating new VolumeAttributesClass") + ginkgo.DeferCleanup(CleanupVAC, newVAC, f.ClientSet, vacCleanupWaitPeriod) + + ginkgo.By("Creating a pod with dynamically provisioned volume") + podConfig := e2epod.Config{ + NS: f.Namespace.Name, + PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, + SeLinuxLabel: e2epod.GetLinuxLabel(), + NodeSelection: l.config.ClientNodeSelection, + ImageID: e2epod.GetDefaultTestImageID(), + } + pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart) + ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod) + framework.ExpectNoError(err, "While creating pod for modifying") + + ginkgo.By("Modifying PVC via VAC") + newPVC := SetPVCVACName(ctx, l.resource.Pvc, newVAC.Name, f.ClientSet, setVACWaitPeriod) + l.resource.Pvc = newPVC + gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil()) + + ginkgo.By("Waiting for modification to finish") + WaitForVolumeModification(ctx, l.resource.Pvc, f.ClientSet, modifyVolumeWaitPeriod) + + pvcConditions := l.resource.Pvc.Status.Conditions + gomega.Expect(pvcConditions).To(gomega.BeEmpty(), "PVC should not have conditions") + }) +} + +// SetPVCVACName sets the VolumeAttributesClassName on a PVC object +func SetPVCVACName(ctx context.Context, origPVC *v1.PersistentVolumeClaim, name string, c clientset.Interface, timeout time.Duration) *v1.PersistentVolumeClaim { + pvcName := origPVC.Name + var patchedPVC *v1.PersistentVolumeClaim + + gomega.Eventually(ctx, func(g gomega.Gomega) { + var err error + patch := []map[string]interface{}{{"op": "replace", "path": "/spec/volumeAttributesClassName", "value": name}} + patchBytes, _ := json.Marshal(patch) + + patchedPVC, err = c.CoreV1().PersistentVolumeClaims(origPVC.Namespace).Patch(ctx, pvcName, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) + framework.ExpectNoError(err, "While patching PVC to add VAC name") + }, timeout, modifyPollInterval).Should(gomega.Succeed()) + + return patchedPVC +} + +// WaitForVolumeModification waits for the volume to be modified +// The input PVC is assumed to have a VolumeAttributesClassName set +func WaitForVolumeModification(ctx context.Context, pvc *v1.PersistentVolumeClaim, c clientset.Interface, timeout time.Duration) { + pvName := pvc.Spec.VolumeName + gomega.Eventually(ctx, func(g gomega.Gomega) { + pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + framework.ExpectNoError(err, "While getting existing PV") + g.Expect(pv.Spec.VolumeAttributesClassName).NotTo(gomega.BeNil()) + newPVC, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "While getting new PVC") + g.Expect(vacMatches(newPVC, *pv.Spec.VolumeAttributesClassName, true)).To(gomega.BeTrueBecause("Modified PVC should match expected VAC")) + }, timeout, modifyPollInterval).Should(gomega.Succeed()) +} + +func CleanupVAC(ctx context.Context, vac *storagev1alpha1.VolumeAttributesClass, c clientset.Interface, timeout time.Duration) { + gomega.Eventually(ctx, func() error { + return c.StorageV1alpha1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{}) + }, timeout, modifyPollInterval).Should(gomega.BeNil()) +} + +func vacMatches(pvc *v1.PersistentVolumeClaim, expectedVac string, checkStatusCurrentVac bool) bool { + // Check the following to ensure the VAC matches and that all pending modifications are complete: + // 1. VAC Name matches Expected + // 2. PVC Modify Volume status is either nil or has an empty status string + // 3. PVC Status Current VAC Matches Expected (only if checkStatusCurrentVac is true) + // (3) is only expected to be true after a VAC is modified, but not when a VAC is used to create a volume + if pvc.Spec.VolumeAttributesClassName == nil || *pvc.Spec.VolumeAttributesClassName != expectedVac { + return false + } + if pvc.Status.ModifyVolumeStatus != nil && (pvc.Status.ModifyVolumeStatus.Status != "" || pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName != expectedVac) { + return false + } + if checkStatusCurrentVac { + if pvc.Status.CurrentVolumeAttributesClassName == nil || *pvc.Status.CurrentVolumeAttributesClassName != expectedVac { + return false + } + } + + return true +} diff --git a/test/e2e/storage/utils/create.go b/test/e2e/storage/utils/create.go index ac570772163..39f527e168d 100644 --- a/test/e2e/storage/utils/create.go +++ b/test/e2e/storage/utils/create.go @@ -29,6 +29,7 @@ import ( v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -267,6 +268,7 @@ var factories = map[What]ItemFactory{ {"StatefulSet"}: &statefulSetFactory{}, {"Deployment"}: &deploymentFactory{}, {"StorageClass"}: &storageClassFactory{}, + {"VolumeAttributesClass"}: &volumeAttributesClassFactory{}, {"CustomResourceDefinition"}: &customResourceDefinitionFactory{}, } @@ -314,6 +316,8 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, PatchName(f, &item.Name) case *storagev1.StorageClass: PatchName(f, &item.Name) + case *storagev1alpha1.VolumeAttributesClass: + PatchName(f, &item.Name) case *storagev1.CSIDriver: PatchName(f, &item.Name) case *v1.ServiceAccount: @@ -618,6 +622,27 @@ func (*storageClassFactory) Create(ctx context.Context, f *framework.Framework, }, nil } +type volumeAttributesClassFactory struct{} + +func (f *volumeAttributesClassFactory) New() runtime.Object { + return &storagev1alpha1.VolumeAttributesClass{} +} + +func (*volumeAttributesClassFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { + item, ok := i.(*storagev1alpha1.VolumeAttributesClass) + if !ok { + return nil, errorItemNotSupported + } + + client := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses() + if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("create VolumeAttributesClass: %w", err) + } + return func(ctx context.Context) error { + return client.Delete(ctx, item.GetName(), metav1.DeleteOptions{}) + }, nil +} + type csiDriverFactory struct{} func (f *csiDriverFactory) New() runtime.Object {