mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-30 23:15:14 +00:00
Add Happy Path VolumeAttributesClass CSI E2E Tests
Signed-off-by: Connor Catlett <conncatl@amazon.com>
This commit is contained in:
Connor Catlett
parent
a2911e06a7
commit
ea58abfd99
@ -363,6 +363,13 @@ var (
|
||||
// TODO: document the feature (owning SIG, when to use this feature for a test)
|
||||
ValidatingAdmissionPolicy = framework.WithFeature(framework.ValidFeatures.Add("ValidatingAdmissionPolicy"))
|
||||
|
||||
// Owner: sig-storage
|
||||
// Tests related to VolumeAttributesClass (https://kep.k8s.io/3751)
|
||||
//
|
||||
// TODO: This label only requires the API storage.k8s.io/v1alpha1 and the VolumeAttributesClass feature-gate enabled.
|
||||
// It should be removed after k/k #124350 is merged.
|
||||
VolumeAttributesClass = framework.WithFeature(framework.ValidFeatures.Add("VolumeAttributesClass"))
|
||||
|
||||
// TODO: document the feature (owning SIG, when to use this feature for a test)
|
||||
Volumes = framework.WithFeature(framework.ValidFeatures.Add("Volumes"))
|
||||
|
||||
|
||||
@ -127,10 +127,11 @@ type PersistentVolumeClaimConfig struct {
|
||||
// unspecified
|
||||
ClaimSize string
|
||||
// AccessModes defaults to RWO if unspecified
|
||||
AccessModes []v1.PersistentVolumeAccessMode
|
||||
Annotations map[string]string
|
||||
Selector *metav1.LabelSelector
|
||||
StorageClassName *string
|
||||
AccessModes []v1.PersistentVolumeAccessMode
|
||||
Annotations map[string]string
|
||||
Selector *metav1.LabelSelector
|
||||
StorageClassName *string
|
||||
VolumeAttributesClassName *string
|
||||
// VolumeMode defaults to nil if unspecified or specified as the empty
|
||||
// string
|
||||
VolumeMode *v1.PersistentVolumeMode
|
||||
@ -661,8 +662,9 @@ func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.P
|
||||
v1.ResourceStorage: resource.MustParse(cfg.ClaimSize),
|
||||
},
|
||||
},
|
||||
StorageClassName: cfg.StorageClassName,
|
||||
VolumeMode: cfg.VolumeMode,
|
||||
StorageClassName: cfg.StorageClassName,
|
||||
VolumeAttributesClassName: cfg.VolumeAttributesClassName,
|
||||
VolumeMode: cfg.VolumeMode,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@ -54,6 +54,7 @@ import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
@ -85,6 +86,11 @@ const (
|
||||
|
||||
// Prefix of the mock driver grpc log
|
||||
grpcCallPrefix = "gRPCCall:"
|
||||
|
||||
// Parameter to use in hostpath CSI driver VolumeAttributesClass
|
||||
// Must be passed to the driver via --accepted-mutable-parameter-names
|
||||
hostpathCSIDriverMutableParameterName = "e2eVacTest"
|
||||
hostpathCSIDriverMutableParameterValue = "test-value"
|
||||
)
|
||||
|
||||
// hostpathCSI
|
||||
@ -209,6 +215,15 @@ func (h *hostpathCSIDriver) GetSnapshotClass(ctx context.Context, config *storag
|
||||
return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
|
||||
}
|
||||
|
||||
func (h *hostpathCSIDriver) GetVolumeAttributesClass(_ context.Context, config *storageframework.PerTestConfig) *storagev1alpha1.VolumeAttributesClass {
|
||||
return storageframework.CopyVolumeAttributesClass(&storagev1alpha1.VolumeAttributesClass{
|
||||
DriverName: config.GetUniqueDriverName(),
|
||||
Parameters: map[string]string{
|
||||
hostpathCSIDriverMutableParameterName: hostpathCSIDriverMutableParameterValue,
|
||||
},
|
||||
}, config.Framework.Namespace.Name, "e2e-vac-hostpath")
|
||||
}
|
||||
|
||||
func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
|
||||
// Create secondary namespace which will be used for creating driver
|
||||
driverNamespace := utils.CreateDriverNamespace(ctx, f)
|
||||
@ -230,7 +245,9 @@ func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framew
|
||||
DriverNamespace: driverNamespace,
|
||||
}
|
||||
|
||||
o := utils.PatchCSIOptions{
|
||||
patches := []utils.PatchCSIOptions{}
|
||||
|
||||
patches = append(patches, utils.PatchCSIOptions{
|
||||
OldDriverName: h.driverInfo.Name,
|
||||
NewDriverName: config.GetUniqueDriverName(),
|
||||
DriverContainerName: "hostpath",
|
||||
@ -246,11 +263,31 @@ func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framew
|
||||
ProvisionerContainerName: "csi-provisioner",
|
||||
SnapshotterContainerName: "csi-snapshotter",
|
||||
NodeName: node.Name,
|
||||
}
|
||||
})
|
||||
|
||||
// VAC E2E HostPath patch
|
||||
// Enables ModifyVolume support in the hostpath CSI driver, and adds an enabled parameter name
|
||||
patches = append(patches, utils.PatchCSIOptions{
|
||||
DriverContainerName: "hostpath",
|
||||
DriverContainerArguments: []string{"--enable-controller-modify-volume=true", "--accepted-mutable-parameter-names=e2eVacTest"},
|
||||
})
|
||||
|
||||
// VAC E2E FeatureGate patches
|
||||
// TODO: These can be removed after the VolumeAttributesClass feature is default enabled
|
||||
patches = append(patches, utils.PatchCSIOptions{
|
||||
DriverContainerName: "csi-provisioner",
|
||||
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
|
||||
})
|
||||
patches = append(patches, utils.PatchCSIOptions{
|
||||
DriverContainerName: "csi-resizer",
|
||||
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
|
||||
})
|
||||
|
||||
err = utils.CreateFromManifests(ctx, config.Framework, driverNamespace, func(item interface{}) error {
|
||||
if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
|
||||
return err
|
||||
for _, o := range patches {
|
||||
if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Remove csi-external-health-monitor-agent and
|
||||
|
||||
62
test/e2e/storage/external/external.go
vendored
62
test/e2e/storage/external/external.go
vendored
@ -25,6 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
@ -79,6 +80,30 @@ type driverDefinition struct {
|
||||
FromExistingClassName string
|
||||
}
|
||||
|
||||
// VolumeAttributesClass must be set to enable volume modification tests.
|
||||
// The default is to not run those tests.
|
||||
VolumeAttributesClass struct {
|
||||
// FromName set to true enables the usage of a
|
||||
// VolumeAttributesClass with DriverInfo.Name as
|
||||
// provisioner and no parameters.
|
||||
FromName bool
|
||||
|
||||
// FromFile is used only when FromName is false. It
|
||||
// loads a storage class from the given .yaml or .json
|
||||
// file. File names are resolved by the
|
||||
// framework.testfiles package, which typically means
|
||||
// that they can be absolute or relative to the test
|
||||
// suite's --repo-root parameter.
|
||||
//
|
||||
// This can be used when the VolumeAttributesClass
|
||||
// is meant to have additional parameters.
|
||||
FromFile string
|
||||
|
||||
// FromExistingClassName specifies the name of a pre-installed
|
||||
// VolumeAttributesClass that will be copied and used for the tests.
|
||||
FromExistingClassName string
|
||||
}
|
||||
|
||||
// SnapshotClass must be set to enable snapshotting tests.
|
||||
// The default is to not run those tests.
|
||||
SnapshotClass struct {
|
||||
@ -405,6 +430,43 @@ func (d *driverDefinition) GetSnapshotClass(ctx context.Context, e2econfig *stor
|
||||
return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
|
||||
}
|
||||
|
||||
func (d *driverDefinition) GetVolumeAttributesClass(ctx context.Context, e2econfig *storageframework.PerTestConfig) *storagev1alpha1.VolumeAttributesClass {
|
||||
if !d.VolumeAttributesClass.FromName && d.VolumeAttributesClass.FromFile == "" && d.VolumeAttributesClass.FromExistingClassName == "" {
|
||||
e2eskipper.Skipf("Driver %q has no configured VolumeAttributesClass - skipping", d.DriverInfo.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
vac *storagev1alpha1.VolumeAttributesClass
|
||||
err error
|
||||
)
|
||||
|
||||
f := e2econfig.Framework
|
||||
switch {
|
||||
case d.VolumeAttributesClass.FromName:
|
||||
vac = &storagev1alpha1.VolumeAttributesClass{DriverName: d.DriverInfo.Name}
|
||||
case d.VolumeAttributesClass.FromExistingClassName != "":
|
||||
vac, err = f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Get(ctx, d.VolumeAttributesClass.FromExistingClassName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "getting VolumeAttributesClass %s", d.VolumeAttributesClass.FromExistingClassName)
|
||||
case d.VolumeAttributesClass.FromFile != "":
|
||||
var ok bool
|
||||
items, err := utils.LoadFromManifests(d.VolumeAttributesClass.FromFile)
|
||||
framework.ExpectNoError(err, "load VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile)
|
||||
gomega.Expect(items).To(gomega.HaveLen(1), "exactly one item from %s", d.VolumeAttributesClass.FromFile)
|
||||
err = utils.PatchItems(f, f.Namespace, items...)
|
||||
framework.ExpectNoError(err, "patch VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile)
|
||||
|
||||
vac, ok = items[0].(*storagev1alpha1.VolumeAttributesClass)
|
||||
if !ok {
|
||||
framework.Failf("cast VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile)
|
||||
}
|
||||
}
|
||||
|
||||
gomega.Expect(vac).ToNot(gomega.BeNil(), "VolumeAttributesClass is unexpectantly nil")
|
||||
|
||||
return storageframework.CopyVolumeAttributesClass(vac, f.Namespace.Name, "e2e-vac")
|
||||
}
|
||||
|
||||
func (d *driverDefinition) GetVolume(e2econfig *storageframework.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
|
||||
if len(d.InlineVolumes) == 0 {
|
||||
e2eskipper.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name)
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
StorageClass:
|
||||
FromExistingClassName: example
|
||||
VolumeAttributesClass:
|
||||
FromExistingClassName: example-vac
|
||||
DriverInfo:
|
||||
Name: example
|
||||
RequiredAccessModes:
|
||||
|
||||
@ -21,6 +21,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apiserver/pkg/storage/names"
|
||||
"k8s.io/kubernetes/pkg/volume/util"
|
||||
@ -92,3 +93,13 @@ func GetStorageClass(
|
||||
VolumeBindingMode: bindingMode,
|
||||
}
|
||||
}
|
||||
|
||||
// CopyVolumeAttributesClass constructs a new VolumeAttributesClass instance
|
||||
// with a unique name that is based on namespace + suffix
|
||||
// using the VolumeAttributesClass passed in as a parameter
|
||||
func CopyVolumeAttributesClass(vac *storagev1alpha1.VolumeAttributesClass, ns string, suffix string) *storagev1alpha1.VolumeAttributesClass {
|
||||
copy := vac.DeepCopy()
|
||||
copy.ObjectMeta.Name = names.SimpleNameGenerator.GenerateName(ns + "-" + suffix)
|
||||
copy.ResourceVersion = ""
|
||||
return copy
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
@ -130,6 +131,15 @@ type SnapshottableTestDriver interface {
|
||||
GetSnapshotClass(ctx context.Context, config *PerTestConfig, parameters map[string]string) *unstructured.Unstructured
|
||||
}
|
||||
|
||||
// VolumeAttributesClassTestDriver represents an interface for a TestDriver that supports
|
||||
// creating and modifying volumes via VolumeAttributesClass objects
|
||||
type VolumeAttributesClassTestDriver interface {
|
||||
TestDriver
|
||||
// GetVolumeAttributesClass returns a VolumeAttributesClass to create/modify PVCs
|
||||
// It will return nil if the TestDriver does not support VACs
|
||||
GetVolumeAttributesClass(ctx context.Context, config *PerTestConfig) *storagev1alpha1.VolumeAttributesClass
|
||||
}
|
||||
|
||||
// CustomTimeoutsTestDriver represents an interface fo a TestDriver that supports custom timeouts.
|
||||
type CustomTimeoutsTestDriver interface {
|
||||
TestDriver
|
||||
|
||||
@ -53,11 +53,19 @@ type VolumeResource struct {
|
||||
// CreateVolumeResource constructs a VolumeResource for the current test. It knows how to deal with
|
||||
// different test pattern volume types.
|
||||
func CreateVolumeResource(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange) *VolumeResource {
|
||||
return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes)
|
||||
return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes, nil)
|
||||
}
|
||||
|
||||
// CreateVolumeResource constructs a VolumeResource for the current test using the specified VAC name.
|
||||
func CreateVolumeResourceWithVAC(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, vacName *string) *VolumeResource {
|
||||
if pattern.VolType != DynamicPV {
|
||||
framework.Failf("Creating volume with VAC only supported on dynamic PV tests")
|
||||
}
|
||||
return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes, vacName)
|
||||
}
|
||||
|
||||
// CreateVolumeResourceWithAccessModes constructs a VolumeResource for the current test with the provided access modes.
|
||||
func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode) *VolumeResource {
|
||||
func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode, vacName *string) *VolumeResource {
|
||||
r := VolumeResource{
|
||||
Config: config,
|
||||
Pattern: pattern,
|
||||
@ -107,7 +115,7 @@ func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver,
|
||||
switch pattern.VolType {
|
||||
case DynamicPV:
|
||||
r.Pv, r.Pvc = createPVCPVFromDynamicProvisionSC(
|
||||
ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes)
|
||||
ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes, vacName)
|
||||
r.VolSource = storageutils.CreateVolumeSource(r.Pvc.Name, false /* readOnly */)
|
||||
case GenericEphemeralVolume:
|
||||
driverVolumeSizeRange := dDriver.GetDriverInfo().SupportedSizeRange
|
||||
@ -287,17 +295,19 @@ func createPVCPVFromDynamicProvisionSC(
|
||||
sc *storagev1.StorageClass,
|
||||
volMode v1.PersistentVolumeMode,
|
||||
accessModes []v1.PersistentVolumeAccessMode,
|
||||
vacName *string,
|
||||
) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) {
|
||||
cs := f.ClientSet
|
||||
ns := f.Namespace.Name
|
||||
|
||||
ginkgo.By("creating a claim")
|
||||
pvcCfg := e2epv.PersistentVolumeClaimConfig{
|
||||
NamePrefix: name,
|
||||
ClaimSize: claimSize,
|
||||
StorageClassName: &(sc.Name),
|
||||
AccessModes: accessModes,
|
||||
VolumeMode: &volMode,
|
||||
NamePrefix: name,
|
||||
ClaimSize: claimSize,
|
||||
StorageClassName: &(sc.Name),
|
||||
VolumeAttributesClassName: vacName,
|
||||
AccessModes: accessModes,
|
||||
VolumeMode: &volMode,
|
||||
}
|
||||
|
||||
pvc := e2epv.MakePersistentVolumeClaim(pvcCfg, ns)
|
||||
|
||||
@ -82,6 +82,7 @@ var CSISuites = append(BaseSuites,
|
||||
InitSnapshottableStressTestSuite,
|
||||
InitVolumePerformanceTestSuite,
|
||||
InitReadWriteOncePodTestSuite,
|
||||
InitVolumeModifyTestSuite,
|
||||
)
|
||||
|
||||
func getVolumeOpsFromMetricsForPlugin(ms testutil.Metrics, pluginName string) opCounts {
|
||||
|
||||
@ -116,7 +116,8 @@ func (s *disruptiveTestSuite) DefineTests(driver storageframework.TestDriver, pa
|
||||
l.config,
|
||||
pattern,
|
||||
testVolumeSizeRange,
|
||||
accessModes)
|
||||
accessModes,
|
||||
nil)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -133,7 +133,7 @@ func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriv
|
||||
ginkgo.It("should preempt lower priority pods using ReadWriteOncePod volumes", func(ctx context.Context) {
|
||||
// Create the ReadWriteOncePod PVC.
|
||||
accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}
|
||||
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes)
|
||||
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes, nil)
|
||||
|
||||
l.priorityClass = &schedulingv1.PriorityClass{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "e2e-test-read-write-once-pod-" + string(uuid.NewUUID())},
|
||||
@ -189,7 +189,7 @@ func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriv
|
||||
ginkgo.It("should block a second pod from using an in-use ReadWriteOncePod volume on the same node", func(ctx context.Context) {
|
||||
// Create the ReadWriteOncePod PVC.
|
||||
accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}
|
||||
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes)
|
||||
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes, nil)
|
||||
|
||||
podConfig := e2epod.Config{
|
||||
NS: f.Namespace.Name,
|
||||
|
||||
294
test/e2e/storage/testsuites/volume_modify.go
Normal file
294
test/e2e/storage/testsuites/volume_modify.go
Normal file
@ -0,0 +1,294 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testsuites
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/gomega"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/errors"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
e2efeature "k8s.io/kubernetes/test/e2e/feature"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
|
||||
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
)
|
||||
|
||||
const (
|
||||
modifyPollInterval = 2 * time.Second
|
||||
setVACWaitPeriod = 30 * time.Second
|
||||
modifyingConditionSyncWaitPeriod = 2 * time.Minute
|
||||
modifyVolumeWaitPeriod = 10 * time.Minute
|
||||
vacCleanupWaitPeriod = 30 * time.Second
|
||||
)
|
||||
|
||||
type volumeModifyTestSuite struct {
|
||||
tsInfo storageframework.TestSuiteInfo
|
||||
}
|
||||
|
||||
// InitCustomVolumeModifyTestSuite returns volumeModifyTestSuite that implements TestSuite interface
|
||||
// using custom test patterns
|
||||
func InitCustomVolumeModifyTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
|
||||
return &volumeModifyTestSuite{
|
||||
tsInfo: storageframework.TestSuiteInfo{
|
||||
Name: "volume-modify",
|
||||
TestPatterns: patterns,
|
||||
SupportedSizeRange: e2evolume.SizeRange{
|
||||
Min: "1Gi",
|
||||
},
|
||||
TestTags: []interface{}{e2efeature.VolumeAttributesClass, framework.WithFeatureGate(features.VolumeAttributesClass)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// InitVolumeModifyTestSuite returns volumeModifyTestSuite that implements TestSuite interface
|
||||
// using testsuite default patterns
|
||||
func InitVolumeModifyTestSuite() storageframework.TestSuite {
|
||||
patterns := []storageframework.TestPattern{
|
||||
storageframework.DefaultFsDynamicPV,
|
||||
storageframework.BlockVolModeDynamicPV,
|
||||
storageframework.NtfsDynamicPV,
|
||||
}
|
||||
return InitCustomVolumeModifyTestSuite(patterns)
|
||||
}
|
||||
|
||||
func (v *volumeModifyTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
|
||||
return v.tsInfo
|
||||
}
|
||||
|
||||
func (v *volumeModifyTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
|
||||
_, ok := driver.(storageframework.VolumeAttributesClassTestDriver)
|
||||
if !ok {
|
||||
e2eskipper.Skipf("Driver %q does not support VolumeAttributesClass tests - skipping", driver.GetDriverInfo().Name)
|
||||
}
|
||||
// Skip block storage tests if the driver we are testing against does not support block volumes
|
||||
// TODO: This should be made generic so that it doesn't have to be re-written for every test that uses the BlockVolModeDynamicPV testcase
|
||||
if !driver.GetDriverInfo().Capabilities[storageframework.CapBlock] && pattern.VolMode == v1.PersistentVolumeBlock {
|
||||
e2eskipper.Skipf("Driver %q does not support block volume mode - skipping", driver.GetDriverInfo().Name)
|
||||
}
|
||||
}
|
||||
|
||||
func (v *volumeModifyTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
|
||||
type local struct {
|
||||
config *storageframework.PerTestConfig
|
||||
|
||||
resource *storageframework.VolumeResource
|
||||
vac *storagev1alpha1.VolumeAttributesClass
|
||||
}
|
||||
var l local
|
||||
|
||||
// Beware that it also registers an AfterEach which renders f unusable. Any code using
|
||||
// f must run inside an It or Context callback.
|
||||
f := framework.NewFrameworkWithCustomTimeouts("volume-modify", storageframework.GetDriverTimeouts(driver))
|
||||
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
|
||||
|
||||
init := func(ctx context.Context, createVolumeWithVAC bool) {
|
||||
l = local{}
|
||||
|
||||
l.config = driver.PrepareTest(ctx, f)
|
||||
vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver)
|
||||
l.vac = vacDriver.GetVolumeAttributesClass(ctx, l.config)
|
||||
|
||||
if l.vac == nil {
|
||||
e2eskipper.Skipf("Driver %q returned nil VolumeAttributesClass - skipping", driver.GetDriverInfo().Name)
|
||||
}
|
||||
|
||||
ginkgo.By("Creating VolumeAttributesClass")
|
||||
_, err := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Create(ctx, l.vac, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "While creating VolumeAttributesClass")
|
||||
|
||||
ginkgo.By("Creating volume")
|
||||
testVolumeSizeRange := v.GetTestSuiteInfo().SupportedSizeRange
|
||||
if createVolumeWithVAC {
|
||||
l.resource = storageframework.CreateVolumeResourceWithVAC(ctx, driver, l.config, pattern, testVolumeSizeRange, &l.vac.Name)
|
||||
} else {
|
||||
l.resource = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange)
|
||||
}
|
||||
}
|
||||
|
||||
cleanup := func(ctx context.Context) {
|
||||
var errs []error
|
||||
if l.resource != nil {
|
||||
ginkgo.By("Deleting VolumeResource")
|
||||
errs = append(errs, l.resource.CleanupResource(ctx))
|
||||
l.resource = nil
|
||||
}
|
||||
|
||||
if l.vac != nil {
|
||||
ginkgo.By("Deleting VAC")
|
||||
CleanupVAC(ctx, l.vac, f.ClientSet, vacCleanupWaitPeriod)
|
||||
l.vac = nil
|
||||
}
|
||||
|
||||
framework.ExpectNoError(errors.NewAggregate(errs), "While cleaning up")
|
||||
}
|
||||
|
||||
ginkgo.It("should create a volume with VAC", func(ctx context.Context) {
|
||||
init(ctx, true /* volume created with VAC */)
|
||||
ginkgo.DeferCleanup(cleanup)
|
||||
|
||||
ginkgo.By("Creating a pod with dynamically provisioned volume")
|
||||
podConfig := e2epod.Config{
|
||||
NS: f.Namespace.Name,
|
||||
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
|
||||
SeLinuxLabel: e2epod.GetLinuxLabel(),
|
||||
NodeSelection: l.config.ClientNodeSelection,
|
||||
ImageID: e2epod.GetDefaultTestImageID(),
|
||||
}
|
||||
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
|
||||
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
|
||||
framework.ExpectNoError(err, "While creating test pod with VAC")
|
||||
|
||||
createdPVC, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, l.resource.Pvc.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "While getting created PVC")
|
||||
// Check VAC matches on created PVC, but not current VAC in status
|
||||
gomega.Expect(vacMatches(createdPVC, l.vac.Name, false)).To(gomega.BeTrueBecause("Created PVC should match expected VAC"))
|
||||
})
|
||||
|
||||
ginkgo.It("should modify volume with no VAC", func(ctx context.Context) {
|
||||
init(ctx, false /* volume created without VAC */)
|
||||
ginkgo.DeferCleanup(cleanup)
|
||||
|
||||
var err error
|
||||
ginkgo.By("Creating a pod with dynamically provisioned volume")
|
||||
podConfig := e2epod.Config{
|
||||
NS: f.Namespace.Name,
|
||||
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
|
||||
SeLinuxLabel: e2epod.GetLinuxLabel(),
|
||||
NodeSelection: l.config.ClientNodeSelection,
|
||||
ImageID: e2epod.GetDefaultTestImageID(),
|
||||
}
|
||||
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
|
||||
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
|
||||
framework.ExpectNoError(err, "While creating pod for modifying")
|
||||
|
||||
ginkgo.By("Modifying PVC via VAC")
|
||||
newPVC := SetPVCVACName(ctx, l.resource.Pvc, l.vac.Name, f.ClientSet, setVACWaitPeriod)
|
||||
l.resource.Pvc = newPVC
|
||||
gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
|
||||
|
||||
ginkgo.By("Waiting for modification to finish")
|
||||
WaitForVolumeModification(ctx, l.resource.Pvc, f.ClientSet, modifyVolumeWaitPeriod)
|
||||
|
||||
pvcConditions := l.resource.Pvc.Status.Conditions
|
||||
gomega.Expect(pvcConditions).To(gomega.BeEmpty(), "PVC should not have conditions")
|
||||
})
|
||||
|
||||
ginkgo.It("should modify volume that already has a VAC", func(ctx context.Context) {
|
||||
init(ctx, true /* volume created with VAC */)
|
||||
ginkgo.DeferCleanup(cleanup)
|
||||
|
||||
vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver)
|
||||
newVAC := vacDriver.GetVolumeAttributesClass(ctx, l.config)
|
||||
gomega.Expect(newVAC).NotTo(gomega.BeNil())
|
||||
_, err := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Create(ctx, newVAC, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "While creating new VolumeAttributesClass")
|
||||
ginkgo.DeferCleanup(CleanupVAC, newVAC, f.ClientSet, vacCleanupWaitPeriod)
|
||||
|
||||
ginkgo.By("Creating a pod with dynamically provisioned volume")
|
||||
podConfig := e2epod.Config{
|
||||
NS: f.Namespace.Name,
|
||||
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
|
||||
SeLinuxLabel: e2epod.GetLinuxLabel(),
|
||||
NodeSelection: l.config.ClientNodeSelection,
|
||||
ImageID: e2epod.GetDefaultTestImageID(),
|
||||
}
|
||||
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
|
||||
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
|
||||
framework.ExpectNoError(err, "While creating pod for modifying")
|
||||
|
||||
ginkgo.By("Modifying PVC via VAC")
|
||||
newPVC := SetPVCVACName(ctx, l.resource.Pvc, newVAC.Name, f.ClientSet, setVACWaitPeriod)
|
||||
l.resource.Pvc = newPVC
|
||||
gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
|
||||
|
||||
ginkgo.By("Waiting for modification to finish")
|
||||
WaitForVolumeModification(ctx, l.resource.Pvc, f.ClientSet, modifyVolumeWaitPeriod)
|
||||
|
||||
pvcConditions := l.resource.Pvc.Status.Conditions
|
||||
gomega.Expect(pvcConditions).To(gomega.BeEmpty(), "PVC should not have conditions")
|
||||
})
|
||||
}
|
||||
|
||||
// SetPVCVACName sets the VolumeAttributesClassName on a PVC object
|
||||
func SetPVCVACName(ctx context.Context, origPVC *v1.PersistentVolumeClaim, name string, c clientset.Interface, timeout time.Duration) *v1.PersistentVolumeClaim {
|
||||
pvcName := origPVC.Name
|
||||
var patchedPVC *v1.PersistentVolumeClaim
|
||||
|
||||
gomega.Eventually(ctx, func(g gomega.Gomega) {
|
||||
var err error
|
||||
patch := []map[string]interface{}{{"op": "replace", "path": "/spec/volumeAttributesClassName", "value": name}}
|
||||
patchBytes, _ := json.Marshal(patch)
|
||||
|
||||
patchedPVC, err = c.CoreV1().PersistentVolumeClaims(origPVC.Namespace).Patch(ctx, pvcName, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
|
||||
framework.ExpectNoError(err, "While patching PVC to add VAC name")
|
||||
}, timeout, modifyPollInterval).Should(gomega.Succeed())
|
||||
|
||||
return patchedPVC
|
||||
}
|
||||
|
||||
// WaitForVolumeModification waits for the volume to be modified
|
||||
// The input PVC is assumed to have a VolumeAttributesClassName set
|
||||
func WaitForVolumeModification(ctx context.Context, pvc *v1.PersistentVolumeClaim, c clientset.Interface, timeout time.Duration) {
|
||||
pvName := pvc.Spec.VolumeName
|
||||
gomega.Eventually(ctx, func(g gomega.Gomega) {
|
||||
pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "While getting existing PV")
|
||||
g.Expect(pv.Spec.VolumeAttributesClassName).NotTo(gomega.BeNil())
|
||||
newPVC, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "While getting new PVC")
|
||||
g.Expect(vacMatches(newPVC, *pv.Spec.VolumeAttributesClassName, true)).To(gomega.BeTrueBecause("Modified PVC should match expected VAC"))
|
||||
}, timeout, modifyPollInterval).Should(gomega.Succeed())
|
||||
}
|
||||
|
||||
func CleanupVAC(ctx context.Context, vac *storagev1alpha1.VolumeAttributesClass, c clientset.Interface, timeout time.Duration) {
|
||||
gomega.Eventually(ctx, func() error {
|
||||
return c.StorageV1alpha1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})
|
||||
}, timeout, modifyPollInterval).Should(gomega.BeNil())
|
||||
}
|
||||
|
||||
func vacMatches(pvc *v1.PersistentVolumeClaim, expectedVac string, checkStatusCurrentVac bool) bool {
|
||||
// Check the following to ensure the VAC matches and that all pending modifications are complete:
|
||||
// 1. VAC Name matches Expected
|
||||
// 2. PVC Modify Volume status is either nil or has an empty status string
|
||||
// 3. PVC Status Current VAC Matches Expected (only if checkStatusCurrentVac is true)
|
||||
// (3) is only expected to be true after a VAC is modified, but not when a VAC is used to create a volume
|
||||
if pvc.Spec.VolumeAttributesClassName == nil || *pvc.Spec.VolumeAttributesClassName != expectedVac {
|
||||
return false
|
||||
}
|
||||
if pvc.Status.ModifyVolumeStatus != nil && (pvc.Status.ModifyVolumeStatus.Status != "" || pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName != expectedVac) {
|
||||
return false
|
||||
}
|
||||
if checkStatusCurrentVac {
|
||||
if pvc.Status.CurrentVolumeAttributesClassName == nil || *pvc.Status.CurrentVolumeAttributesClassName != expectedVac {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
@ -29,6 +29,7 @@ import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
@ -267,6 +268,7 @@ var factories = map[What]ItemFactory{
|
||||
{"StatefulSet"}: &statefulSetFactory{},
|
||||
{"Deployment"}: &deploymentFactory{},
|
||||
{"StorageClass"}: &storageClassFactory{},
|
||||
{"VolumeAttributesClass"}: &volumeAttributesClassFactory{},
|
||||
{"CustomResourceDefinition"}: &customResourceDefinitionFactory{},
|
||||
}
|
||||
|
||||
@ -314,6 +316,8 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace,
|
||||
PatchName(f, &item.Name)
|
||||
case *storagev1.StorageClass:
|
||||
PatchName(f, &item.Name)
|
||||
case *storagev1alpha1.VolumeAttributesClass:
|
||||
PatchName(f, &item.Name)
|
||||
case *storagev1.CSIDriver:
|
||||
PatchName(f, &item.Name)
|
||||
case *v1.ServiceAccount:
|
||||
@ -618,6 +622,27 @@ func (*storageClassFactory) Create(ctx context.Context, f *framework.Framework,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type volumeAttributesClassFactory struct{}
|
||||
|
||||
func (f *volumeAttributesClassFactory) New() runtime.Object {
|
||||
return &storagev1alpha1.VolumeAttributesClass{}
|
||||
}
|
||||
|
||||
func (*volumeAttributesClassFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) {
|
||||
item, ok := i.(*storagev1alpha1.VolumeAttributesClass)
|
||||
if !ok {
|
||||
return nil, errorItemNotSupported
|
||||
}
|
||||
|
||||
client := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses()
|
||||
if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("create VolumeAttributesClass: %w", err)
|
||||
}
|
||||
return func(ctx context.Context) error {
|
||||
return client.Delete(ctx, item.GetName(), metav1.DeleteOptions{})
|
||||
}, nil
|
||||
}
|
||||
|
||||
type csiDriverFactory struct{}
|
||||
|
||||
func (f *csiDriverFactory) New() runtime.Object {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user