github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-02 16:29:21 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #72434 from pohly/storage-volume-testsuites-config

Browse Source 
e2e/storage: delay CreateDriver, simplify APIs and test definition
This commit is contained in:
Kubernetes Prow Robot 2019-02-20 05:37:45 -08:00 committed by GitHub
commit 07cf5fa08e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 1468 additions and 1764 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/e2e/storage/BUILD
View File

@ -67,11 +67,9 @@ go_library(
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
"//test/e2e/framework:go_default_library", "//test/e2e/framework:go_default_library",
"//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/metrics:go_default_library",
"//test/e2e/framework/podlogs:go_default_library",
"//test/e2e/framework/providers/gce:go_default_library", "//test/e2e/framework/providers/gce:go_default_library",
"//test/e2e/framework/testfiles:go_default_library", "//test/e2e/framework/testfiles:go_default_library",
"//test/e2e/storage/drivers:go_default_library", "//test/e2e/storage/drivers:go_default_library",
"//test/e2e/storage/testpatterns:go_default_library",
"//test/e2e/storage/testsuites:go_default_library", "//test/e2e/storage/testsuites:go_default_library",
"//test/e2e/storage/utils:go_default_library", "//test/e2e/storage/utils:go_default_library",
"//test/utils/image:go_default_library", "//test/utils/image:go_default_library",

206
test/e2e/storage/csi_volumes.go
View File

@ -17,10 +17,8 @@ limitations under the License.
package storage package storage
import ( import (
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"regexp"
"strings" "strings"
"time" "time"
@ -32,9 +30,7 @@ import (
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/podlogs"
"k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
"k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils" "k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image" imageutils "k8s.io/kubernetes/test/utils/image"
@ -47,9 +43,9 @@ import (
) )
// List of testDrivers to be executed in below loop // List of testDrivers to be executed in below loop
var csiTestDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{ var csiTestDrivers = []func() testsuites.TestDriver{
drivers.InitHostPathCSIDriver, drivers.InitHostPathCSIDriver,
drivers.InitGcePDCSIDriver, func() testsuites.TestDriver { return drivers.InitGcePDCSIDriver(false /* topology enabled */) },
drivers.InitGcePDExternalCSIDriver, drivers.InitGcePDExternalCSIDriver,
drivers.InitHostPathV0CSIDriver, drivers.InitHostPathV0CSIDriver,
// Don't run tests with mock driver (drivers.InitMockCSIDriver), it does not provide persistent storage. // Don't run tests with mock driver (drivers.InitMockCSIDriver), it does not provide persistent storage.
@ -65,133 +61,58 @@ var csiTestSuites = []func() testsuites.TestSuite{
testsuites.InitSnapshottableTestSuite, testsuites.InitSnapshottableTestSuite,
} }
func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
tunedPatterns := []testpatterns.TestPattern{}
for _, pattern := range patterns {
// Skip inline volume and pre-provsioned PV tests for csi drivers
if pattern.VolType == testpatterns.InlineVolume || pattern.VolType == testpatterns.PreprovisionedPV {
continue
}
tunedPatterns = append(tunedPatterns, pattern)
}
return tunedPatterns
}
// This executes testSuites for csi volumes. // This executes testSuites for csi volumes.
var _ = utils.SIGDescribe("CSI Volumes", func() { var _ = utils.SIGDescribe("CSI Volumes", func() {
f := framework.NewDefaultFramework("csi-volumes")
var (
cancel context.CancelFunc
cs clientset.Interface
csics csiclient.Interface
ns *v1.Namespace
// Common configuration options for each driver.
config = testsuites.TestConfig{
Framework: f,
Prefix: "csi",
}
)
BeforeEach(func() {
ctx, c := context.WithCancel(context.Background())
cancel = c
cs = f.ClientSet
csics = f.CSIClientSet
ns = f.Namespace
// Debugging of the following tests heavily depends on the log output
// of the different containers. Therefore include all of that in log
// files (when using --report-dir, as in the CI) or the output stream
// (otherwise).
to := podlogs.LogOutput{
StatusWriter: GinkgoWriter,
}
if framework.TestContext.ReportDir == "" {
to.LogWriter = GinkgoWriter
} else {
test := CurrentGinkgoTestDescription()
reg := regexp.MustCompile("[^a-zA-Z0-9_-]+")
// We end the prefix with a slash to ensure that all logs
// end up in a directory named after the current test.
to.LogPathPrefix = framework.TestContext.ReportDir + "/" +
reg.ReplaceAllString(test.FullTestText, "_") + "/"
}
podlogs.CopyAllLogs(ctx, cs, ns.Name, to)
// pod events are something that the framework already collects itself
// after a failed test. Logging them live is only useful for interactive
// debugging, not when we collect reports.
if framework.TestContext.ReportDir == "" {
podlogs.WatchPods(ctx, cs, ns.Name, GinkgoWriter)
}
})
AfterEach(func() {
cancel()
})
for _, initDriver := range csiTestDrivers { for _, initDriver := range csiTestDrivers {
curDriver := initDriver(config) curDriver := initDriver()
curConfig := curDriver.GetDriverInfo().Config
Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() {
BeforeEach(func() { testsuites.DefineTestSuite(curDriver, csiTestSuites)
// Reset config. The driver might have modified its copy
// in a previous test.
curDriver.GetDriverInfo().Config = curConfig
// setupDriver
curDriver.CreateDriver()
})
AfterEach(func() {
// Cleanup driver
curDriver.CleanupDriver()
})
testsuites.RunTestSuite(f, curDriver, csiTestSuites, csiTunePattern)
}) })
} }
Context("CSI Topology test using GCE PD driver [Feature:CSINodeInfo]", func() { Context("CSI Topology test using GCE PD driver [Feature:CSINodeInfo]", func() {
newConfig := config f := framework.NewDefaultFramework("csitopology")
newConfig.TopologyEnabled = true driver := drivers.InitGcePDCSIDriver(true /* topology enabled */).(testsuites.DynamicPVTestDriver) // TODO (#71289) eliminate by moving this test to common test suite.
driver := drivers.InitGcePDCSIDriver(newConfig).(testsuites.DynamicPVTestDriver) // TODO (#71289) eliminate by moving this test to common test suite. var (
config *testsuites.PerTestConfig
testCleanup func()
)
BeforeEach(func() { BeforeEach(func() {
driver.CreateDriver() config, testCleanup = driver.PrepareTest(f)
}) })
AfterEach(func() { AfterEach(func() {
driver.CleanupDriver() if testCleanup != nil {
testCleanup()
}
}) })
It("should provision zonal PD with immediate volume binding and AllowedTopologies set and mount the volume to a pod", func() { It("should provision zonal PD with immediate volume binding and AllowedTopologies set and mount the volume to a pod", func() {
suffix := "topology-positive" suffix := "topology-positive"
testTopologyPositive(cs, suffix, ns.GetName(), false /* delayBinding */, true /* allowedTopologies */) testTopologyPositive(config.Framework.ClientSet, suffix, config.Framework.Namespace.GetName(), false /* delayBinding */, true /* allowedTopologies */)
}) })
It("should provision zonal PD with delayed volume binding and mount the volume to a pod", func() { It("should provision zonal PD with delayed volume binding and mount the volume to a pod", func() {
suffix := "delayed" suffix := "delayed"
testTopologyPositive(cs, suffix, ns.GetName(), true /* delayBinding */, false /* allowedTopologies */) testTopologyPositive(config.Framework.ClientSet, suffix, config.Framework.Namespace.GetName(), true /* delayBinding */, false /* allowedTopologies */)
}) })
It("should provision zonal PD with delayed volume binding and AllowedTopologies set and mount the volume to a pod", func() { It("should provision zonal PD with delayed volume binding and AllowedTopologies set and mount the volume to a pod", func() {
suffix := "delayed-topology-positive" suffix := "delayed-topology-positive"
testTopologyPositive(cs, suffix, ns.GetName(), true /* delayBinding */, true /* allowedTopologies */) testTopologyPositive(config.Framework.ClientSet, suffix, config.Framework.Namespace.GetName(), true /* delayBinding */, true /* allowedTopologies */)
}) })
It("should fail to schedule a pod with a zone missing from AllowedTopologies; PD is provisioned with immediate volume binding", func() { It("should fail to schedule a pod with a zone missing from AllowedTopologies; PD is provisioned with immediate volume binding", func() {
framework.SkipUnlessMultizone(cs) framework.SkipUnlessMultizone(config.Framework.ClientSet)
suffix := "topology-negative" suffix := "topology-negative"
testTopologyNegative(cs, suffix, ns.GetName(), false /* delayBinding */) testTopologyNegative(config.Framework.ClientSet, suffix, config.Framework.Namespace.GetName(), false /* delayBinding */)
}) })
It("should fail to schedule a pod with a zone missing from AllowedTopologies; PD is provisioned with delayed volume binding", func() { It("should fail to schedule a pod with a zone missing from AllowedTopologies; PD is provisioned with delayed volume binding", func() {
framework.SkipUnlessMultizone(cs) framework.SkipUnlessMultizone(config.Framework.ClientSet)
suffix := "delayed-topology-negative" suffix := "delayed-topology-negative"
testTopologyNegative(cs, suffix, ns.GetName(), true /* delayBinding */) testTopologyNegative(config.Framework.ClientSet, suffix, config.Framework.Namespace.GetName(), true /* delayBinding */)
}) })
}) })
@ -227,29 +148,30 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
for _, t := range tests { for _, t := range tests {
test := t test := t
It(test.name, func() { f := framework.NewDefaultFramework("csiattach")
By("Deploying mock CSI driver")
config := testsuites.TestConfig{
Framework: f,
Prefix: "csi-attach",
}
driver = drivers.InitMockCSIDriver(config, test.deployDriverCRD, test.driverAttachable, nil) It(test.name, func() {
driver.CreateDriver() cs := f.ClientSet
defer driver.CleanupDriver() csics := f.CSIClientSet
ns := f.Namespace
driver = drivers.InitMockCSIDriver(test.deployDriverCRD, test.driverAttachable, nil)
config, testCleanup := driver.PrepareTest(f)
driverName := config.GetUniqueDriverName()
defer testCleanup()
if test.deployDriverCRD { if test.deployDriverCRD {
err = waitForCSIDriver(csics, driver) err = waitForCSIDriver(csics, driverName)
framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err) framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err)
defer destroyCSIDriver(csics, driver) defer destroyCSIDriver(csics, driverName)
} }
By("Creating pod") By("Creating pod")
var sc *storagev1.StorageClass var sc *storagev1.StorageClass
if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok { if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok {
sc = dDriver.GetDynamicProvisionStorageClass("") sc = dDriver.GetDynamicProvisionStorageClass(config, "")
} }
nodeName := driver.GetDriverInfo().Config.ClientNodeName nodeName := config.ClientNodeName
scTest := testsuites.StorageClassTest{ scTest := testsuites.StorageClassTest{
Name: driver.GetDriverInfo().Name, Name: driver.GetDriverInfo().Name,
Provisioner: sc.Provisioner, Provisioner: sc.Provisioner,
@ -347,29 +269,30 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
} }
for _, t := range tests { for _, t := range tests {
test := t test := t
It(test.name, func() { f := framework.NewDefaultFramework("csiworkload")
By("Deploying mock CSI driver")
config := testsuites.TestConfig{
Framework: f,
Prefix: "csi-workload",
}
driver = drivers.InitMockCSIDriver(config, test.deployDriverCRD, true, test.podInfoOnMountVersion) It(test.name, func() {
driver.CreateDriver() cs := f.ClientSet
defer driver.CleanupDriver() csics := f.CSIClientSet
ns := f.Namespace
driver = drivers.InitMockCSIDriver(test.deployDriverCRD, true, test.podInfoOnMountVersion)
config, testCleanup := driver.PrepareTest(f)
driverName := config.GetUniqueDriverName()
defer testCleanup()
if test.deployDriverCRD { if test.deployDriverCRD {
err = waitForCSIDriver(csics, driver) err = waitForCSIDriver(csics, driverName)
framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err) framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err)
defer destroyCSIDriver(csics, driver) defer destroyCSIDriver(csics, driverName)
} }
By("Creating pod") By("Creating pod")
var sc *storagev1.StorageClass var sc *storagev1.StorageClass
if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok { if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok {
sc = dDriver.GetDynamicProvisionStorageClass("") sc = dDriver.GetDynamicProvisionStorageClass(config, "")
} }
nodeName := driver.GetDriverInfo().Config.ClientNodeName nodeName := config.ClientNodeName
scTest := testsuites.StorageClassTest{ scTest := testsuites.StorageClassTest{
Name: driver.GetDriverInfo().Name, Name: driver.GetDriverInfo().Name,
Parameters: sc.Parameters, Parameters: sc.Parameters,
@ -420,14 +343,16 @@ func testTopologyPositive(cs clientset.Interface, suffix, namespace string, dela
topoZone := getRandomClusterZone(cs) topoZone := getRandomClusterZone(cs)
addSingleCSIZoneAllowedTopologyToStorageClass(cs, class, topoZone) addSingleCSIZoneAllowedTopologyToStorageClass(cs, class, topoZone)
} }
claim := newClaim(test, namespace, suffix) test.Client = cs
claim.Spec.StorageClassName = &class.Name test.Claim = newClaim(test, namespace, suffix)
test.Claim.Spec.StorageClassName = &class.Name
test.Class = class
if delayBinding { if delayBinding {
_, node := testsuites.TestBindingWaitForFirstConsumer(test, cs, claim, class, nil /* node selector */, false /* expect unschedulable */) _, node := test.TestBindingWaitForFirstConsumer(nil /* node selector */, false /* expect unschedulable */)
Expect(node).ToNot(BeNil(), "Unexpected nil node found") Expect(node).ToNot(BeNil(), "Unexpected nil node found")
} else { } else {
testsuites.TestDynamicProvisioning(test, cs, claim, class) test.TestDynamicProvisioning()
} }
} }
@ -447,12 +372,13 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela
test.DelayBinding = delayBinding test.DelayBinding = delayBinding
nodeSelector := map[string]string{v1.LabelZoneFailureDomain: podZone} nodeSelector := map[string]string{v1.LabelZoneFailureDomain: podZone}
class := newStorageClass(test, namespace, suffix) test.Client = cs
addSingleCSIZoneAllowedTopologyToStorageClass(cs, class, pvZone) test.Class = newStorageClass(test, namespace, suffix)
claim := newClaim(test, namespace, suffix) addSingleCSIZoneAllowedTopologyToStorageClass(cs, test.Class, pvZone)
claim.Spec.StorageClassName = &class.Name test.Claim = newClaim(test, namespace, suffix)
test.Claim.Spec.StorageClassName = &test.Class.Name
if delayBinding { if delayBinding {
testsuites.TestBindingWaitForFirstConsumer(test, cs, claim, class, nodeSelector, true /* expect unschedulable */) test.TestBindingWaitForFirstConsumer(nodeSelector, true /* expect unschedulable */)
} else { } else {
test.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) { test.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
// Ensure that a pod cannot be scheduled in an unsuitable zone. // Ensure that a pod cannot be scheduled in an unsuitable zone.
@ -461,13 +387,12 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela
defer testsuites.StopPod(cs, pod) defer testsuites.StopPod(cs, pod)
framework.ExpectNoError(framework.WaitForPodNameUnschedulableInNamespace(cs, pod.Name, pod.Namespace), "pod should be unschedulable") framework.ExpectNoError(framework.WaitForPodNameUnschedulableInNamespace(cs, pod.Name, pod.Namespace), "pod should be unschedulable")
} }
testsuites.TestDynamicProvisioning(test, cs, claim, class) test.TestDynamicProvisioning()
} }
} }
func waitForCSIDriver(csics csiclient.Interface, driver testsuites.TestDriver) error { func waitForCSIDriver(csics csiclient.Interface, driverName string) error {
timeout := 2 * time.Minute timeout := 2 * time.Minute
driverName := testsuites.GetUniqueDriverName(driver)
framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName) framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName)
for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) { for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) {
@ -479,8 +404,7 @@ func waitForCSIDriver(csics csiclient.Interface, driver testsuites.TestDriver) e
return fmt.Errorf("gave up after waiting %v for CSIDriver %q.", timeout, driverName) return fmt.Errorf("gave up after waiting %v for CSIDriver %q.", timeout, driverName)
} }
func destroyCSIDriver(csics csiclient.Interface, driver testsuites.TestDriver) { func destroyCSIDriver(csics csiclient.Interface, driverName string) {
driverName := testsuites.GetUniqueDriverName(driver)
driverGet, err := csics.CsiV1alpha1().CSIDrivers().Get(driverName, metav1.GetOptions{}) driverGet, err := csics.CsiV1alpha1().CSIDrivers().Get(driverName, metav1.GetOptions{})
if err == nil { if err == nil {
framework.Logf("deleting %s.%s: %s", driverGet.TypeMeta.APIVersion, driverGet.TypeMeta.Kind, driverGet.ObjectMeta.Name) framework.Logf("deleting %s.%s: %s", driverGet.TypeMeta.APIVersion, driverGet.TypeMeta.Kind, driverGet.ObjectMeta.Name)

164
test/e2e/storage/drivers/csi.go
View File

@ -56,12 +56,11 @@ const (
// hostpathCSI // hostpathCSI
type hostpathCSIDriver struct { type hostpathCSIDriver struct {
cleanup func()
driverInfo testsuites.DriverInfo driverInfo testsuites.DriverInfo
manifests []string manifests []string
} }
func initHostPathCSIDriver(name string, config testsuites.TestConfig, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver { func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver {
return &hostpathCSIDriver{ return &hostpathCSIDriver{
driverInfo: testsuites.DriverInfo{ driverInfo: testsuites.DriverInfo{
Name: name, Name: name,
@ -71,7 +70,6 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, capabiliti
"", // Default fsType "", // Default fsType
), ),
Capabilities: capabilities, Capabilities: capabilities,
Config: config,
}, },
manifests: manifests, manifests: manifests,
} }
@ -82,8 +80,8 @@ var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{}
var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{} var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{}
// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { func InitHostPathCSIDriver() testsuites.TestDriver {
return initHostPathCSIDriver("csi-hostpath", config, return initHostPathCSIDriver("csi-hostpath",
map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapDataSource: true, testsuites.CapMultiPODs: true}, map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapDataSource: true, testsuites.CapMultiPODs: true},
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
@ -104,19 +102,19 @@ func (h *hostpathCSIDriver) GetDriverInfo() *testsuites.DriverInfo {
func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
} }
func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
provisioner := testsuites.GetUniqueDriverName(h) provisioner := config.GetUniqueDriverName()
parameters := map[string]string{} parameters := map[string]string{}
ns := h.driverInfo.Config.Framework.Namespace.Name ns := config.Framework.Namespace.Name
suffix := fmt.Sprintf("%s-sc", provisioner) suffix := fmt.Sprintf("%s-sc", provisioner)
return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
} }
func (h *hostpathCSIDriver) GetSnapshotClass() *unstructured.Unstructured { func (h *hostpathCSIDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *unstructured.Unstructured {
snapshotter := testsuites.GetUniqueDriverName(h) snapshotter := config.GetUniqueDriverName()
parameters := map[string]string{} parameters := map[string]string{}
ns := h.driverInfo.Config.Framework.Namespace.Name ns := config.Framework.Namespace.Name
suffix := fmt.Sprintf("%s-vsc", snapshotter) suffix := fmt.Sprintf("%s-vsc", snapshotter)
return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix) return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
@ -126,57 +124,60 @@ func (h *hostpathCSIDriver) GetClaimSize() string {
return "5Gi" return "5Gi"
} }
func (h *hostpathCSIDriver) CreateDriver() { func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name)) By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name))
f := h.driverInfo.Config.Framework cancelLogging := testsuites.StartPodLogs(f)
cs := f.ClientSet cs := f.ClientSet
// The hostpath CSI driver only works when everything runs on the same node. // The hostpath CSI driver only works when everything runs on the same node.
nodes := framework.GetReadySchedulableNodesOrDie(cs) nodes := framework.GetReadySchedulableNodesOrDie(cs)
nodeName := nodes.Items[rand.Intn(len(nodes.Items))].Name nodeName := nodes.Items[rand.Intn(len(nodes.Items))].Name
h.driverInfo.Config.ClientNodeName = nodeName config := &testsuites.PerTestConfig{
Driver: h,
Prefix: "hostpath",
Framework: f,
ClientNodeName: nodeName,
}
// TODO (?): the storage.csi.image.version and storage.csi.image.registry // TODO (?): the storage.csi.image.version and storage.csi.image.registry
// settings are ignored for this test. We could patch the image definitions. // settings are ignored for this test. We could patch the image definitions.
o := utils.PatchCSIOptions{ o := utils.PatchCSIOptions{
OldDriverName: h.driverInfo.Name, OldDriverName: h.driverInfo.Name,
NewDriverName: testsuites.GetUniqueDriverName(h), NewDriverName: config.GetUniqueDriverName(),
DriverContainerName: "hostpath", DriverContainerName: "hostpath",
DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName}, DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName()},
ProvisionerContainerName: "csi-provisioner", ProvisionerContainerName: "csi-provisioner",
SnapshotterContainerName: "csi-snapshotter", SnapshotterContainerName: "csi-snapshotter",
NodeName: nodeName, NodeName: nodeName,
} }
cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error { cleanup, err := config.Framework.CreateFromManifests(func(item interface{}) error {
return utils.PatchCSIDeployment(h.driverInfo.Config.Framework, o, item) return utils.PatchCSIDeployment(config.Framework, o, item)
}, },
h.manifests...) h.manifests...)
h.cleanup = cleanup
if err != nil { if err != nil {
framework.Failf("deploying %s driver: %v", h.driverInfo.Name, err) framework.Failf("deploying %s driver: %v", h.driverInfo.Name, err)
} }
}
func (h *hostpathCSIDriver) CleanupDriver() { return config, func() {
if h.cleanup != nil {
By(fmt.Sprintf("uninstalling %s driver", h.driverInfo.Name)) By(fmt.Sprintf("uninstalling %s driver", h.driverInfo.Name))
h.cleanup() cleanup()
cancelLogging()
} }
} }
// mockCSI // mockCSI
type mockCSIDriver struct { type mockCSIDriver struct {
cleanup func()
driverInfo testsuites.DriverInfo driverInfo testsuites.DriverInfo
manifests []string manifests []string
podInfoVersion *string podInfoVersion *string
attachable bool
} }
var _ testsuites.TestDriver = &mockCSIDriver{} var _ testsuites.TestDriver = &mockCSIDriver{}
var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{}
// InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface
func InitMockCSIDriver(config testsuites.TestConfig, registerDriver, driverAttachable bool, podInfoVersion *string) testsuites.TestDriver { func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfoVersion *string) testsuites.TestDriver {
driverManifests := []string{ driverManifests := []string{
"test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
@ -187,16 +188,12 @@ func InitMockCSIDriver(config testsuites.TestConfig, registerDriver, driverAttac
"test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml",
} }
config.ServerConfig = &framework.VolumeTestConfig{}
if registerDriver { if registerDriver {
driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml") driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml")
} }
if driverAttachable { if driverAttachable {
driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml") driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml")
} else {
config.ServerConfig.ServerArgs = append(config.ServerConfig.ServerArgs, "--disable-attach")
} }
return &mockCSIDriver{ return &mockCSIDriver{
@ -212,10 +209,10 @@ func InitMockCSIDriver(config testsuites.TestConfig, registerDriver, driverAttac
testsuites.CapFsGroup: false, testsuites.CapFsGroup: false,
testsuites.CapExec: false, testsuites.CapExec: false,
}, },
Config: config,
}, },
manifests: driverManifests, manifests: driverManifests,
podInfoVersion: podInfoVersion, podInfoVersion: podInfoVersion,
attachable: driverAttachable,
} }
} }
@ -226,10 +223,10 @@ func (m *mockCSIDriver) GetDriverInfo() *testsuites.DriverInfo {
func (m *mockCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { func (m *mockCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
} }
func (m *mockCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { func (m *mockCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
provisioner := testsuites.GetUniqueDriverName(m) provisioner := config.GetUniqueDriverName()
parameters := map[string]string{} parameters := map[string]string{}
ns := m.driverInfo.Config.Framework.Namespace.Name ns := config.Framework.Namespace.Name
suffix := fmt.Sprintf("%s-sc", provisioner) suffix := fmt.Sprintf("%s-sc", provisioner)
return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
@ -239,20 +236,24 @@ func (m *mockCSIDriver) GetClaimSize() string {
return "5Gi" return "5Gi"
} }
func (m *mockCSIDriver) CreateDriver() { func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
By("deploying csi mock driver") By("deploying csi mock driver")
f := m.driverInfo.Config.Framework cancelLogging := testsuites.StartPodLogs(f)
cs := f.ClientSet cs := f.ClientSet
// pods should be scheduled on the node // pods should be scheduled on the node
nodes := framework.GetReadySchedulableNodesOrDie(cs) nodes := framework.GetReadySchedulableNodesOrDie(cs)
node := nodes.Items[rand.Intn(len(nodes.Items))] node := nodes.Items[rand.Intn(len(nodes.Items))]
m.driverInfo.Config.ClientNodeName = node.Name config := &testsuites.PerTestConfig{
Driver: m,
Prefix: "mock",
Framework: f,
ClientNodeName: node.Name,
}
containerArgs := []string{"--name=csi-mock-" + f.UniqueName} containerArgs := []string{"--name=csi-mock-" + f.UniqueName}
if !m.attachable {
if m.driverInfo.Config.ServerConfig != nil && m.driverInfo.Config.ServerConfig.ServerArgs != nil { containerArgs = append(containerArgs, "--disable-attach")
containerArgs = append(containerArgs, m.driverInfo.Config.ServerConfig.ServerArgs...)
} }
// TODO (?): the storage.csi.image.version and storage.csi.image.registry // TODO (?): the storage.csi.image.version and storage.csi.image.registry
@ -264,29 +265,27 @@ func (m *mockCSIDriver) CreateDriver() {
DriverContainerArguments: containerArgs, DriverContainerArguments: containerArgs,
ProvisionerContainerName: "csi-provisioner", ProvisionerContainerName: "csi-provisioner",
ClusterRegistrarContainerName: "csi-cluster-driver-registrar", ClusterRegistrarContainerName: "csi-cluster-driver-registrar",
NodeName: m.driverInfo.Config.ClientNodeName, NodeName: config.ClientNodeName,
PodInfoVersion: m.podInfoVersion, PodInfoVersion: m.podInfoVersion,
} }
cleanup, err := f.CreateFromManifests(func(item interface{}) error { cleanup, err := f.CreateFromManifests(func(item interface{}) error {
return utils.PatchCSIDeployment(f, o, item) return utils.PatchCSIDeployment(f, o, item)
}, },
m.manifests...) m.manifests...)
m.cleanup = cleanup
if err != nil { if err != nil {
framework.Failf("deploying csi mock driver: %v", err) framework.Failf("deploying csi mock driver: %v", err)
} }
}
func (m *mockCSIDriver) CleanupDriver() { return config, func() {
if m.cleanup != nil {
By("uninstalling csi mock driver") By("uninstalling csi mock driver")
m.cleanup() cleanup()
cancelLogging()
} }
} }
// InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests. // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests.
func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver { func InitHostPathV0CSIDriver() testsuites.TestDriver {
return initHostPathCSIDriver("csi-hostpath-v0", config, return initHostPathCSIDriver("csi-hostpath-v0",
map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapMultiPODs: true}, map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapMultiPODs: true},
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
@ -300,16 +299,17 @@ func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver
// gce-pd // gce-pd
type gcePDCSIDriver struct { type gcePDCSIDriver struct {
cleanup func() topologyEnabled bool
driverInfo testsuites.DriverInfo driverInfo testsuites.DriverInfo
} }
var _ testsuites.TestDriver = &gcePDCSIDriver{} var _ testsuites.TestDriver = &gcePDCSIDriver{}
var _ testsuites.DynamicPVTestDriver = &gcePDCSIDriver{} var _ testsuites.DynamicPVTestDriver = &gcePDCSIDriver{}
// InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface // InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface
func InitGcePDCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { func InitGcePDCSIDriver(topologyEnabled bool) testsuites.TestDriver {
return &gcePDCSIDriver{ return &gcePDCSIDriver{
topologyEnabled: topologyEnabled,
driverInfo: testsuites.DriverInfo{ driverInfo: testsuites.DriverInfo{
Name: GCEPDCSIProvisionerName, Name: GCEPDCSIProvisionerName,
FeatureTag: "[Serial]", FeatureTag: "[Serial]",
@ -327,8 +327,6 @@ func InitGcePDCSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
testsuites.CapExec: true, testsuites.CapExec: true,
testsuites.CapMultiPODs: true, testsuites.CapMultiPODs: true,
}, },
Config: config,
}, },
} }
} }
@ -338,21 +336,14 @@ func (g *gcePDCSIDriver) GetDriverInfo() *testsuites.DriverInfo {
} }
func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
f := g.driverInfo.Config.Framework
framework.SkipUnlessProviderIs("gce", "gke") framework.SkipUnlessProviderIs("gce", "gke")
if !g.driverInfo.Config.TopologyEnabled {
// Topology is disabled in external-provisioner, so in a multizone cluster, a pod could be
// scheduled in a different zone from the provisioned volume, causing basic provisioning
// tests to fail.
framework.SkipIfMultizone(f.ClientSet)
}
if pattern.FsType == "xfs" { if pattern.FsType == "xfs" {
framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom")
} }
} }
func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
ns := g.driverInfo.Config.Framework.Namespace.Name ns := config.Framework.Namespace.Name
provisioner := g.driverInfo.Name provisioner := g.driverInfo.Name
suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
@ -368,8 +359,16 @@ func (g *gcePDCSIDriver) GetClaimSize() string {
return "5Gi" return "5Gi"
} }
func (g *gcePDCSIDriver) CreateDriver() { func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
if !g.topologyEnabled {
// Topology is disabled in external-provisioner, so in a multizone cluster, a pod could be
// scheduled in a different zone from the provisioned volume, causing basic provisioning
// tests to fail.
framework.SkipIfMultizone(f.ClientSet)
}
By("deploying csi gce-pd driver") By("deploying csi gce-pd driver")
cancelLogging := testsuites.StartPodLogs(f)
// It would be safer to rename the gcePD driver, but that // It would be safer to rename the gcePD driver, but that
// hasn't been done before either and attempts to do so now led to // hasn't been done before either and attempts to do so now led to
// errors during driver registration, therefore it is disabled // errors during driver registration, therefore it is disabled
@ -382,7 +381,7 @@ func (g *gcePDCSIDriver) CreateDriver() {
// DriverContainerName: "gce-driver", // DriverContainerName: "gce-driver",
// ProvisionerContainerName: "csi-external-provisioner", // ProvisionerContainerName: "csi-external-provisioner",
// } // }
createGCESecrets(g.driverInfo.Config.Framework.ClientSet, g.driverInfo.Config.Framework.Namespace.Name) createGCESecrets(f.ClientSet, f.Namespace.Name)
manifests := []string{ manifests := []string{
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
@ -392,23 +391,25 @@ func (g *gcePDCSIDriver) CreateDriver() {
"test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", "test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml",
} }
if g.driverInfo.Config.TopologyEnabled { if g.topologyEnabled {
manifests = append(manifests, "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml") manifests = append(manifests, "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml")
} else { } else {
manifests = append(manifests, "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml") manifests = append(manifests, "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml")
} }
cleanup, err := g.driverInfo.Config.Framework.CreateFromManifests(nil, manifests...) cleanup, err := f.CreateFromManifests(nil, manifests...)
g.cleanup = cleanup
if err != nil { if err != nil {
framework.Failf("deploying csi gce-pd driver: %v", err) framework.Failf("deploying csi gce-pd driver: %v", err)
} }
}
func (g *gcePDCSIDriver) CleanupDriver() { return &testsuites.PerTestConfig{
By("uninstalling gce-pd driver") Driver: g,
if g.cleanup != nil { Prefix: "gcepd",
g.cleanup() Framework: f,
} }, func() {
By("uninstalling gce-pd driver")
cleanup()
cancelLogging()
}
} }
// gcePd-external // gcePd-external
@ -420,7 +421,7 @@ var _ testsuites.TestDriver = &gcePDExternalCSIDriver{}
var _ testsuites.DynamicPVTestDriver = &gcePDExternalCSIDriver{} var _ testsuites.DynamicPVTestDriver = &gcePDExternalCSIDriver{}
// InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface // InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface
func InitGcePDExternalCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { func InitGcePDExternalCSIDriver() testsuites.TestDriver {
return &gcePDExternalCSIDriver{ return &gcePDExternalCSIDriver{
driverInfo: testsuites.DriverInfo{ driverInfo: testsuites.DriverInfo{
Name: GCEPDCSIProvisionerName, Name: GCEPDCSIProvisionerName,
@ -440,8 +441,6 @@ func InitGcePDExternalCSIDriver(config testsuites.TestConfig) testsuites.TestDri
testsuites.CapExec: true, testsuites.CapExec: true,
testsuites.CapMultiPODs: true, testsuites.CapMultiPODs: true,
}, },
Config: config,
}, },
} }
} }
@ -452,14 +451,13 @@ func (g *gcePDExternalCSIDriver) GetDriverInfo() *testsuites.DriverInfo {
func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
framework.SkipUnlessProviderIs("gce", "gke") framework.SkipUnlessProviderIs("gce", "gke")
framework.SkipIfMultizone(g.driverInfo.Config.Framework.ClientSet)
if pattern.FsType == "xfs" { if pattern.FsType == "xfs" {
framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom")
} }
} }
func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
ns := g.driverInfo.Config.Framework.Namespace.Name ns := config.Framework.Namespace.Name
provisioner := g.driverInfo.Name provisioner := g.driverInfo.Name
suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
@ -475,8 +473,12 @@ func (g *gcePDExternalCSIDriver) GetClaimSize() string {
return "5Gi" return "5Gi"
} }
func (g *gcePDExternalCSIDriver) CreateDriver() { func (g *gcePDExternalCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
} framework.SkipIfMultizone(f.ClientSet)
func (g *gcePDExternalCSIDriver) CleanupDriver() { return &testsuites.PerTestConfig{
Driver: g,
Prefix: "gcepdext",
Framework: f,
}, func() {}
} }

703
test/e2e/storage/drivers/in_tree.go
View File

File diff suppressed because it is too large Load Diff

38
test/e2e/storage/in_tree_volumes.go
View File

@ -18,15 +18,13 @@ package storage
import ( import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
"k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils" "k8s.io/kubernetes/test/e2e/storage/utils"
) )
// List of testDrivers to be executed in below loop // List of testDrivers to be executed in below loop
var testDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{ var testDrivers = []func() testsuites.TestDriver{
drivers.InitNFSDriver, drivers.InitNFSDriver,
drivers.InitGlusterFSDriver, drivers.InitGlusterFSDriver,
drivers.InitISCSIDriver, drivers.InitISCSIDriver,
@ -59,41 +57,13 @@ var testSuites = []func() testsuites.TestSuite{
testsuites.InitProvisioningTestSuite, testsuites.InitProvisioningTestSuite,
} }
func intreeTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
return patterns
}
// This executes testSuites for in-tree volumes. // This executes testSuites for in-tree volumes.
var _ = utils.SIGDescribe("In-tree Volumes", func() { var _ = utils.SIGDescribe("In-tree Volumes", func() {
f := framework.NewDefaultFramework("volumes")
var (
// Common configuration options for all drivers.
config = testsuites.TestConfig{
Framework: f,
Prefix: "in-tree",
}
)
for _, initDriver := range testDrivers { for _, initDriver := range testDrivers {
curDriver := initDriver(config) curDriver := initDriver()
curConfig := curDriver.GetDriverInfo().Config
Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() {
BeforeEach(func() { testsuites.DefineTestSuite(curDriver, testSuites)
// Reset config. The driver might have modified its copy
// in a previous test.
curDriver.GetDriverInfo().Config = curConfig
// setupDriver
curDriver.CreateDriver()
})
AfterEach(func() {
// Cleanup driver
curDriver.CleanupDriver()
})
testsuites.RunTestSuite(f, curDriver, testSuites, intreeTunePattern)
}) })
} }
}) })

37
test/e2e/storage/regional_pd.go
View File

@ -143,10 +143,11 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
} }
for _, test := range tests { for _, test := range tests {
class := newStorageClass(test, ns, "" /* suffix */) test.Client = c
claim := newClaim(test, ns, "" /* suffix */) test.Class = newStorageClass(test, ns, "" /* suffix */)
claim.Spec.StorageClassName = &class.Name test.Claim = newClaim(test, ns, "" /* suffix */)
testsuites.TestDynamicProvisioning(test, c, claim, class) test.Claim.Spec.StorageClassName = &test.Class.Name
test.TestDynamicProvisioning()
} }
} }
@ -301,6 +302,7 @@ func addTaint(c clientset.Interface, ns string, nodes []v1.Node, podZone string)
func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int) { func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int) {
test := testsuites.StorageClassTest{ test := testsuites.StorageClassTest{
Client: c,
Name: "Regional PD storage class with waitForFirstConsumer test on GCE", Name: "Regional PD storage class with waitForFirstConsumer test on GCE",
Provisioner: "kubernetes.io/gce-pd", Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{ Parameters: map[string]string{
@ -312,14 +314,14 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int)
} }
suffix := "delayed-regional" suffix := "delayed-regional"
class := newStorageClass(test, ns, suffix) test.Class = newStorageClass(test, ns, suffix)
var claims []*v1.PersistentVolumeClaim var claims []*v1.PersistentVolumeClaim
for i := 0; i < pvcCount; i++ { for i := 0; i < pvcCount; i++ {
claim := newClaim(test, ns, suffix) claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name claim.Spec.StorageClassName = &test.Class.Name
claims = append(claims, claim) claims = append(claims, claim)
} }
pvs, node := testsuites.TestBindingWaitForFirstConsumerMultiPVC(test, c, claims, class, nil /* node selector */, false /* expect unschedulable */) pvs, node := test.TestBindingWaitForFirstConsumerMultiPVC(claims, nil /* node selector */, false /* expect unschedulable */)
if node == nil { if node == nil {
framework.Failf("unexpected nil node found") framework.Failf("unexpected nil node found")
} }
@ -345,17 +347,20 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
} }
suffix := "topo-regional" suffix := "topo-regional"
class := newStorageClass(test, ns, suffix) test.Client = c
test.Class = newStorageClass(test, ns, suffix)
zones := getTwoRandomZones(c) zones := getTwoRandomZones(c)
addAllowedTopologiesToStorageClass(c, class, zones) addAllowedTopologiesToStorageClass(c, test.Class, zones)
claim := newClaim(test, ns, suffix) test.Claim = newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name test.Claim.Spec.StorageClassName = &test.Class.Name
pv := testsuites.TestDynamicProvisioning(test, c, claim, class)
pv := test.TestDynamicProvisioning()
checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true) checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true)
} }
func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string, pvcCount int) { func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string, pvcCount int) {
test := testsuites.StorageClassTest{ test := testsuites.StorageClassTest{
Client: c,
Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE", Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE",
Provisioner: "kubernetes.io/gce-pd", Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{ Parameters: map[string]string{
@ -367,16 +372,16 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s
} }
suffix := "topo-delayed-regional" suffix := "topo-delayed-regional"
class := newStorageClass(test, ns, suffix) test.Class = newStorageClass(test, ns, suffix)
topoZones := getTwoRandomZones(c) topoZones := getTwoRandomZones(c)
addAllowedTopologiesToStorageClass(c, class, topoZones) addAllowedTopologiesToStorageClass(c, test.Class, topoZones)
var claims []*v1.PersistentVolumeClaim var claims []*v1.PersistentVolumeClaim
for i := 0; i < pvcCount; i++ { for i := 0; i < pvcCount; i++ {
claim := newClaim(test, ns, suffix) claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name claim.Spec.StorageClassName = &test.Class.Name
claims = append(claims, claim) claims = append(claims, claim)
} }
pvs, node := testsuites.TestBindingWaitForFirstConsumerMultiPVC(test, c, claims, class, nil /* node selector */, false /* expect unschedulable */) pvs, node := test.TestBindingWaitForFirstConsumerMultiPVC(claims, nil /* node selector */, false /* expect unschedulable */)
if node == nil { if node == nil {
framework.Failf("unexpected nil node found") framework.Failf("unexpected nil node found")
} }

1
test/e2e/storage/testsuites/BUILD
View File

@ -31,6 +31,7 @@ go_library(
"//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/framework:go_default_library", "//test/e2e/framework:go_default_library",
"//test/e2e/framework/podlogs:go_default_library",
"//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/testpatterns:go_default_library",
"//test/e2e/storage/utils:go_default_library", "//test/e2e/storage/utils:go_default_library",
"//test/utils/image:go_default_library", "//test/utils/image:go_default_library",

123
test/e2e/storage/testsuites/base.go
View File

@ -17,7 +17,9 @@ limitations under the License.
package testsuites package testsuites
import ( import (
"context"
"fmt" "fmt"
"regexp"
"time" "time"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
@ -32,6 +34,7 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors" utilerrors "k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/podlogs"
"k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testpatterns"
) )
@ -39,10 +42,10 @@ import (
type TestSuite interface { type TestSuite interface {
// getTestSuiteInfo returns the TestSuiteInfo for this TestSuite // getTestSuiteInfo returns the TestSuiteInfo for this TestSuite
getTestSuiteInfo() TestSuiteInfo getTestSuiteInfo() TestSuiteInfo
// skipUnsupportedTest skips the test if this TestSuite is not suitable to be tested with the combination of TestPattern and TestDriver // defineTest defines tests of the testpattern for the driver.
skipUnsupportedTest(testpatterns.TestPattern, TestDriver) // Called inside a Ginkgo context that reflects the current driver and test pattern,
// execTest executes test of the testpattern for the driver // so the test suite can define tests directly with ginkgo.It.
execTest(TestDriver, testpatterns.TestPattern) defineTests(TestDriver, testpatterns.TestPattern)
} }
// TestSuiteInfo represents a set of parameters for TestSuite // TestSuiteInfo represents a set of parameters for TestSuite
@ -54,11 +57,8 @@ type TestSuiteInfo struct {
// TestResource represents an interface for resources that is used by TestSuite // TestResource represents an interface for resources that is used by TestSuite
type TestResource interface { type TestResource interface {
// setupResource sets up test resources to be used for the tests with the // cleanupResource cleans up the test resources created when setting up the resource
// combination of TestDriver and TestPattern cleanupResource()
setupResource(TestDriver, testpatterns.TestPattern)
// cleanupResource clean up the test resources created in SetupResource
cleanupResource(TestDriver, testpatterns.TestPattern)
} }
func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string { func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string {
@ -66,27 +66,34 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string {
return fmt.Sprintf("[Testpattern: %s]%s %s%s", pattern.Name, pattern.FeatureTag, tsInfo.name, tsInfo.featureTag) return fmt.Sprintf("[Testpattern: %s]%s %s%s", pattern.Name, pattern.FeatureTag, tsInfo.name, tsInfo.featureTag)
} }
// RunTestSuite runs all testpatterns of all testSuites for a driver // DefineTestSuite defines tests for all testpatterns and all testSuites for a driver
func RunTestSuite(f *framework.Framework, driver TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { func DefineTestSuite(driver TestDriver, tsInits []func() TestSuite) {
for _, testSuiteInit := range tsInits { for _, testSuiteInit := range tsInits {
suite := testSuiteInit() suite := testSuiteInit()
patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns) for _, pattern := range suite.getTestSuiteInfo().testPatterns {
p := pattern
for _, pattern := range patterns { Context(getTestNameStr(suite, p), func() {
suite.execTest(driver, pattern) BeforeEach(func() {
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(driver, p)
})
suite.defineTests(driver, p)
})
} }
} }
} }
// skipUnsupportedTest will skip tests if the combination of driver, testsuite, and testpattern // skipUnsupportedTest will skip tests if the combination of driver, and testpattern
// is not suitable to be tested. // is not suitable to be tested.
// Whether it needs to be skipped is checked by following steps: // Whether it needs to be skipped is checked by following steps:
// 1. Check if Whether SnapshotType is supported by driver from its interface // 1. Check if Whether SnapshotType is supported by driver from its interface
// 2. Check if Whether volType is supported by driver from its interface // 2. Check if Whether volType is supported by driver from its interface
// 3. Check if fsType is supported // 3. Check if fsType is supported
// 4. Check with driver specific logic // 4. Check with driver specific logic
// 5. Check with testSuite specific logic //
func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) { // Test suites can also skip tests inside their own defineTests function or in
// individual tests.
func skipUnsupportedTest(driver TestDriver, pattern testpatterns.TestPattern) {
dInfo := driver.GetDriverInfo() dInfo := driver.GetDriverInfo()
var isSupported bool var isSupported bool
@ -130,9 +137,6 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
// 4. Check with driver specific logic // 4. Check with driver specific logic
driver.SkipUnsupportedTest(pattern) driver.SkipUnsupportedTest(pattern)
// 5. Check with testSuite specific logic
suite.skipUnsupportedTest(pattern, driver)
} }
// genericVolumeTestResource is a generic implementation of TestResource that wil be able to // genericVolumeTestResource is a generic implementation of TestResource that wil be able to
@ -141,40 +145,45 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
// Also, see subpath.go in the same directory for how to extend and use it. // Also, see subpath.go in the same directory for how to extend and use it.
type genericVolumeTestResource struct { type genericVolumeTestResource struct {
driver TestDriver driver TestDriver
config *PerTestConfig
pattern testpatterns.TestPattern
volType string volType string
volSource *v1.VolumeSource volSource *v1.VolumeSource
pvc *v1.PersistentVolumeClaim pvc *v1.PersistentVolumeClaim
pv *v1.PersistentVolume pv *v1.PersistentVolume
sc *storagev1.StorageClass sc *storagev1.StorageClass
driverTestResource interface{} volume TestVolume
} }
var _ TestResource = &genericVolumeTestResource{} var _ TestResource = &genericVolumeTestResource{}
// setupResource sets up genericVolumeTestResource func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, pattern testpatterns.TestPattern) *genericVolumeTestResource {
func (r *genericVolumeTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { r := genericVolumeTestResource{
r.driver = driver driver: driver,
config: config,
pattern: pattern,
}
dInfo := driver.GetDriverInfo() dInfo := driver.GetDriverInfo()
f := dInfo.Config.Framework f := config.Framework
cs := f.ClientSet cs := f.ClientSet
fsType := pattern.FsType fsType := pattern.FsType
volType := pattern.VolType volType := pattern.VolType
// Create volume for pre-provisioned volume tests // Create volume for pre-provisioned volume tests
r.driverTestResource = CreateVolume(driver, volType) r.volume = CreateVolume(driver, config, volType)
switch volType { switch volType {
case testpatterns.InlineVolume: case testpatterns.InlineVolume:
framework.Logf("Creating resource for inline volume") framework.Logf("Creating resource for inline volume")
if iDriver, ok := driver.(InlineVolumeTestDriver); ok { if iDriver, ok := driver.(InlineVolumeTestDriver); ok {
r.volSource = iDriver.GetVolumeSource(false, fsType, r.driverTestResource) r.volSource = iDriver.GetVolumeSource(false, fsType, r.volume)
r.volType = dInfo.Name r.volType = dInfo.Name
} }
case testpatterns.PreprovisionedPV: case testpatterns.PreprovisionedPV:
framework.Logf("Creating resource for pre-provisioned PV") framework.Logf("Creating resource for pre-provisioned PV")
if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok { if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, fsType, r.driverTestResource) pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, fsType, r.volume)
if pvSource != nil { if pvSource != nil {
r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false) r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false)
} }
@ -184,7 +193,7 @@ func (r *genericVolumeTestResource) setupResource(driver TestDriver, pattern tes
framework.Logf("Creating resource for dynamic PV") framework.Logf("Creating resource for dynamic PV")
if dDriver, ok := driver.(DynamicPVTestDriver); ok { if dDriver, ok := driver.(DynamicPVTestDriver); ok {
claimSize := dDriver.GetClaimSize() claimSize := dDriver.GetClaimSize()
r.sc = dDriver.GetDynamicProvisionStorageClass(fsType) r.sc = dDriver.GetDynamicProvisionStorageClass(r.config, fsType)
By("creating a StorageClass " + r.sc.Name) By("creating a StorageClass " + r.sc.Name)
var err error var err error
@ -204,13 +213,14 @@ func (r *genericVolumeTestResource) setupResource(driver TestDriver, pattern tes
if r.volSource == nil { if r.volSource == nil {
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, volType) framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, volType)
} }
return &r
} }
// cleanupResource cleans up genericVolumeTestResource // cleanupResource cleans up genericVolumeTestResource
func (r *genericVolumeTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { func (r *genericVolumeTestResource) cleanupResource() {
dInfo := driver.GetDriverInfo() f := r.config.Framework
f := dInfo.Config.Framework volType := r.pattern.VolType
volType := pattern.VolType
if r.pvc != nil || r.pv != nil { if r.pvc != nil || r.pv != nil {
switch volType { switch volType {
@ -241,7 +251,9 @@ func (r *genericVolumeTestResource) cleanupResource(driver TestDriver, pattern t
} }
// Cleanup volume for pre-provisioned volume tests // Cleanup volume for pre-provisioned volume tests
DeleteVolume(driver, volType, r.driverTestResource) if r.volume != nil {
r.volume.DeleteVolume()
}
} }
func createVolumeSourceWithPVCPV( func createVolumeSourceWithPVCPV(
@ -354,7 +366,7 @@ func deleteStorageClass(cs clientset.Interface, className string) {
// the testsuites package whereas framework.VolumeTestConfig is merely // the testsuites package whereas framework.VolumeTestConfig is merely
// an implementation detail. It contains fields that have no effect, // an implementation detail. It contains fields that have no effect,
// which makes it unsuitable for use in the testsuits public API. // which makes it unsuitable for use in the testsuits public API.
func convertTestConfig(in *TestConfig) framework.VolumeTestConfig { func convertTestConfig(in *PerTestConfig) framework.VolumeTestConfig {
if in.ServerConfig != nil { if in.ServerConfig != nil {
return *in.ServerConfig return *in.ServerConfig
} }
@ -388,3 +400,42 @@ func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.U
return snapshot return snapshot
} }
// StartPodLogs begins capturing log output and events from current
// and future pods running in the namespace of the framework. That
// ends when the returned cleanup function is called.
//
// The output goes to log files (when using --report-dir, as in the
// CI) or the output stream (otherwise).
func StartPodLogs(f *framework.Framework) func() {
ctx, cancel := context.WithCancel(context.Background())
cs := f.ClientSet
ns := f.Namespace
to := podlogs.LogOutput{
StatusWriter: GinkgoWriter,
}
if framework.TestContext.ReportDir == "" {
to.LogWriter = GinkgoWriter
} else {
test := CurrentGinkgoTestDescription()
reg := regexp.MustCompile("[^a-zA-Z0-9_-]+")
// We end the prefix with a slash to ensure that all logs
// end up in a directory named after the current test.
//
// TODO: use a deeper directory hierarchy once gubernator
// supports that (https://github.com/kubernetes/test-infra/issues/10289).
to.LogPathPrefix = framework.TestContext.ReportDir + "/" +
reg.ReplaceAllString(test.FullTestText, "_") + "/"
}
podlogs.CopyAllLogs(ctx, cs, ns.Name, to)
// pod events are something that the framework already collects itself
// after a failed test. Logging them live is only useful for interactive
// debugging, not when we collect reports.
if framework.TestContext.ReportDir == "" {
podlogs.WatchPods(ctx, cs, ns.Name, GinkgoWriter)
}
return cancel
}

25
test/e2e/storage/testsuites/driveroperations.go
View File

@ -37,13 +37,13 @@ func GetDriverNameWithFeatureTags(driver TestDriver) string {
} }
// CreateVolume creates volume for test unless dynamicPV test // CreateVolume creates volume for test unless dynamicPV test
func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface{} { func CreateVolume(driver TestDriver, config *PerTestConfig, volType testpatterns.TestVolType) TestVolume {
switch volType { switch volType {
case testpatterns.InlineVolume: case testpatterns.InlineVolume:
fallthrough fallthrough
case testpatterns.PreprovisionedPV: case testpatterns.PreprovisionedPV:
if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok {
return pDriver.CreateVolume(volType) return pDriver.CreateVolume(config, volType)
} }
case testpatterns.DynamicPV: case testpatterns.DynamicPV:
// No need to create volume // No need to create volume
@ -53,22 +53,6 @@ func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface
return nil return nil
} }
// DeleteVolume deletes volume for test unless dynamicPV test
func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResource interface{}) {
switch volType {
case testpatterns.InlineVolume:
fallthrough
case testpatterns.PreprovisionedPV:
if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok {
pDriver.DeleteVolume(volType, testResource)
}
case testpatterns.DynamicPV:
// No need to delete volume
default:
framework.Failf("Invalid volType specified: %v", volType)
}
}
// GetStorageClass constructs a new StorageClass instance // GetStorageClass constructs a new StorageClass instance
// with a unique name that is based on namespace + suffix. // with a unique name that is based on namespace + suffix.
func GetStorageClass( func GetStorageClass(
@ -119,8 +103,3 @@ func GetSnapshotClass(
return snapshotClass return snapshotClass
} }
// GetUniqueDriverName returns unique driver name that can be used parallelly in tests
func GetUniqueDriverName(driver TestDriver) string {
return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName)
}

298
test/e2e/storage/testsuites/provisioning.go
View File

@ -41,6 +41,9 @@ import (
// StorageClassTest represents parameters to be used by provisioning tests. // StorageClassTest represents parameters to be used by provisioning tests.
// Not all parameters are used by all tests. // Not all parameters are used by all tests.
type StorageClassTest struct { type StorageClassTest struct {
Client clientset.Interface
Claim *v1.PersistentVolumeClaim
Class *storage.StorageClass
Name string Name string
CloudProviders []string CloudProviders []string
Provisioner string Provisioner string
@ -76,183 +79,162 @@ func (p *provisioningTestSuite) getTestSuiteInfo() TestSuiteInfo {
return p.tsInfo return p.tsInfo
} }
func (p *provisioningTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
} type local struct {
config *PerTestConfig
testCleanup func()
func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPattern) (provisioningTestResource, provisioningTestInput) { testCase *StorageClassTest
// Setup test resource for driver and testpattern cs clientset.Interface
resource := provisioningTestResource{} pvc *v1.PersistentVolumeClaim
resource.setupResource(driver, pattern) sc *storage.StorageClass
input := provisioningTestInput{
testCase: StorageClassTest{
ClaimSize: resource.claimSize,
ExpectedSize: resource.claimSize,
},
cs: driver.GetDriverInfo().Config.Framework.ClientSet,
dc: driver.GetDriverInfo().Config.Framework.DynamicClient,
pvc: resource.pvc,
sc: resource.sc,
vsc: resource.vsc,
dInfo: driver.GetDriverInfo(),
nodeName: driver.GetDriverInfo().Config.ClientNodeName,
} }
var (
dInfo = driver.GetDriverInfo()
dDriver DynamicPVTestDriver
l local
)
return resource, input BeforeEach(func() {
} // Check preconditions.
if pattern.VolType != testpatterns.DynamicPV {
func (p *provisioningTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { framework.Skipf("Suite %q does not support %v", p.tsInfo.name, pattern.VolType)
Context(getTestNameStr(p, pattern), func() { }
var ( ok := false
resource provisioningTestResource dDriver, ok = driver.(DynamicPVTestDriver)
input provisioningTestInput if !ok {
needsCleanup bool framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
) }
})
BeforeEach(func() {
needsCleanup = false // This intentionally comes after checking the preconditions because it
// Skip unsupported tests to avoid unnecessary resource initialization // registers its own BeforeEach which creates the namespace. Beware that it
skipUnsupportedTest(p, driver, pattern) // also registers an AfterEach which renders f unusable. Any code using
needsCleanup = true // f must run inside an It or Context callback.
f := framework.NewDefaultFramework("provisioning")
// Create test input
resource, input = createProvisioningTestInput(driver, pattern) init := func() {
}) l = local{}
AfterEach(func() { // Now do the more expensive test initialization.
if needsCleanup { l.config, l.testCleanup = driver.PrepareTest(f)
resource.cleanupResource(driver, pattern) l.cs = l.config.Framework.ClientSet
} claimSize := dDriver.GetClaimSize()
}) l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, "")
if l.sc == nil {
// Ginkgo's "Global Shared Behaviors" require arguments for a shared function framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
// to be a single struct and to be passed as a pointer. }
// Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details. l.pvc = getClaim(claimSize, l.config.Framework.Namespace.Name)
testProvisioning(&input) l.pvc.Spec.StorageClassName = &l.sc.Name
}) framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", l.sc, l.pvc)
} l.testCase = &StorageClassTest{
Client: l.config.Framework.ClientSet,
type provisioningTestResource struct { Claim: l.pvc,
driver TestDriver Class: l.sc,
ClaimSize: claimSize,
claimSize string ExpectedSize: claimSize,
sc *storage.StorageClass
pvc *v1.PersistentVolumeClaim
// follow parameter is used to test provision volume from snapshot
vsc *unstructured.Unstructured
}
var _ TestResource = &provisioningTestResource{}
func (p *provisioningTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) {
// Setup provisioningTest resource
switch pattern.VolType {
case testpatterns.DynamicPV:
if dDriver, ok := driver.(DynamicPVTestDriver); ok {
p.sc = dDriver.GetDynamicProvisionStorageClass("")
if p.sc == nil {
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
}
p.driver = driver
p.claimSize = dDriver.GetClaimSize()
p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
p.pvc.Spec.StorageClassName = &p.sc.Name
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)
if sDriver, ok := driver.(SnapshottableTestDriver); ok {
p.vsc = sDriver.GetSnapshotClass()
}
} }
default:
framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType)
} }
}
func (p *provisioningTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { cleanup := func() {
} if l.testCleanup != nil {
l.testCleanup()
type provisioningTestInput struct { l.testCleanup = nil
testCase StorageClassTest }
cs clientset.Interface
dc dynamic.Interface
pvc *v1.PersistentVolumeClaim
sc *storage.StorageClass
vsc *unstructured.Unstructured
dInfo *DriverInfo
nodeName string
}
func testProvisioning(input *provisioningTestInput) {
// common checker for most of the test cases below
pvcheck := func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
PVWriteReadSingleNodeCheck(input.cs, claim, volume, NodeSelection{Name: input.nodeName})
} }
It("should provision storage with defaults", func() { It("should provision storage with defaults", func() {
input.testCase.PvCheck = pvcheck init()
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) defer cleanup()
l.testCase.TestDynamicProvisioning()
}) })
It("should provision storage with mount options", func() { It("should provision storage with mount options", func() {
if input.dInfo.SupportedMountOption == nil { if dInfo.SupportedMountOption == nil {
framework.Skipf("Driver %q does not define supported mount option - skipping", input.dInfo.Name) framework.Skipf("Driver %q does not define supported mount option - skipping", dInfo.Name)
} }
input.sc.MountOptions = input.dInfo.SupportedMountOption.Union(input.dInfo.RequiredMountOption).List() init()
input.testCase.PvCheck = pvcheck defer cleanup()
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
l.testCase.Class.MountOptions = dInfo.SupportedMountOption.Union(dInfo.RequiredMountOption).List()
l.testCase.TestDynamicProvisioning()
}) })
It("should access volume from different nodes", func() { It("should access volume from different nodes", func() {
init()
defer cleanup()
// The assumption is that if the test hasn't been // The assumption is that if the test hasn't been
// locked onto a single node, then the driver is // locked onto a single node, then the driver is
// usable on all of them *and* supports accessing a volume // usable on all of them *and* supports accessing a volume
// from any node. // from any node.
if input.nodeName != "" { if l.config.ClientNodeName != "" {
framework.Skipf("Driver %q only supports testing on one node - skipping", input.dInfo.Name) framework.Skipf("Driver %q only supports testing on one node - skipping", dInfo.Name)
} }
// Ensure that we actually have more than one node. // Ensure that we actually have more than one node.
nodes := framework.GetReadySchedulableNodesOrDie(input.cs) nodes := framework.GetReadySchedulableNodesOrDie(l.cs)
if len(nodes.Items) <= 1 { if len(nodes.Items) <= 1 {
framework.Skipf("need more than one node - skipping") framework.Skipf("need more than one node - skipping")
} }
input.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) { l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
PVMultiNodeCheck(input.cs, claim, volume, NodeSelection{Name: input.nodeName}) PVMultiNodeCheck(l.cs, claim, volume, NodeSelection{Name: l.config.ClientNodeName})
} }
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) l.testCase.TestDynamicProvisioning()
}) })
It("should create and delete block persistent volumes", func() { It("should create and delete block persistent volumes", func() {
if !input.dInfo.Capabilities[CapBlock] { if !dInfo.Capabilities[CapBlock] {
framework.Skipf("Driver %q does not support BlockVolume - skipping", input.dInfo.Name) framework.Skipf("Driver %q does not support BlockVolume - skipping", dInfo.Name)
} }
init()
defer cleanup()
block := v1.PersistentVolumeBlock block := v1.PersistentVolumeBlock
input.testCase.VolumeMode = &block l.testCase.VolumeMode = &block
input.pvc.Spec.VolumeMode = &block l.pvc.Spec.VolumeMode = &block
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) l.testCase.TestDynamicProvisioning()
}) })
It("should provision storage with snapshot data source [Feature:VolumeSnapshotDataSource]", func() { It("should provision storage with snapshot data source [Feature:VolumeSnapshotDataSource]", func() {
if !input.dInfo.Capabilities[CapDataSource] { if !dInfo.Capabilities[CapDataSource] {
framework.Skipf("Driver %q does not support populate data from snapshot - skipping", input.dInfo.Name) framework.Skipf("Driver %q does not support populate data from snapshot - skipping", dInfo.Name)
} }
dataSource, cleanupFunc := prepareDataSourceForProvisioning(NodeSelection{Name: input.nodeName}, input.cs, input.dc, input.pvc, input.sc, input.vsc) sDriver, ok := driver.(SnapshottableTestDriver)
if !ok {
framework.Failf("Driver %q has CapDataSource but does not implement SnapshottableTestDriver", dInfo.Name)
}
init()
defer cleanup()
dc := l.config.Framework.DynamicClient
vsc := sDriver.GetSnapshotClass(l.config)
dataSource, cleanupFunc := prepareDataSourceForProvisioning(NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
defer cleanupFunc() defer cleanupFunc()
input.pvc.Spec.DataSource = dataSource l.pvc.Spec.DataSource = dataSource
input.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) { l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
By("checking whether the created volume has the pre-populated data") By("checking whether the created volume has the pre-populated data")
command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace) command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
RunInPodWithVolume(input.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, NodeSelection{Name: input.nodeName}) RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, NodeSelection{Name: l.config.ClientNodeName})
} }
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) l.testCase.TestDynamicProvisioning()
}) })
It("should allow concurrent writes on the same node", func() { It("should allow concurrent writes on the same node", func() {
if !input.dInfo.Capabilities[CapMultiPODs] { if !dInfo.Capabilities[CapMultiPODs] {
framework.Skipf("Driver %q does not support multiple concurrent pods - skipping", input.dInfo.Name) framework.Skipf("Driver %q does not support multiple concurrent pods - skipping", dInfo.Name)
} }
input.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
init()
defer cleanup()
l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
// We start two pods concurrently on the same node, // We start two pods concurrently on the same node,
// using the same PVC. Both wait for other to create a // using the same PVC. Both wait for other to create a
// file before returning. The pods are forced onto the // file before returning. The pods are forced onto the
@ -265,7 +247,7 @@ func testProvisioning(input *provisioningTestInput) {
defer GinkgoRecover() defer GinkgoRecover()
defer wg.Done() defer wg.Done()
node := NodeSelection{ node := NodeSelection{
Name: input.nodeName, Name: l.config.ClientNodeName,
} }
if podName == secondPodName { if podName == secondPodName {
node.Affinity = &v1.Affinity{ node.Affinity = &v1.Affinity{
@ -283,18 +265,24 @@ func testProvisioning(input *provisioningTestInput) {
}, },
} }
} }
RunInPodWithVolume(input.cs, claim.Namespace, claim.Name, podName, command, node) RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, podName, command, node)
} }
go run(firstPodName, "touch /mnt/test/first && while ! [ -f /mnt/test/second ]; do sleep 1; done") go run(firstPodName, "touch /mnt/test/first && while ! [ -f /mnt/test/second ]; do sleep 1; done")
go run(secondPodName, "touch /mnt/test/second && while ! [ -f /mnt/test/first ]; do sleep 1; done") go run(secondPodName, "touch /mnt/test/second && while ! [ -f /mnt/test/first ]; do sleep 1; done")
wg.Wait() wg.Wait()
} }
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) l.testCase.TestDynamicProvisioning()
}) })
} }
// TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass // TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest
func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) *v1.PersistentVolume { func (t StorageClassTest) TestDynamicProvisioning() *v1.PersistentVolume {
client := t.Client
Expect(client).NotTo(BeNil(), "StorageClassTest.Client is required")
claim := t.Claim
Expect(claim).NotTo(BeNil(), "StorageClassTest.Claim is required")
class := t.Class
var err error var err error
if class != nil { if class != nil {
Expect(*claim.Spec.StorageClassName).To(Equal(class.Name)) Expect(*claim.Spec.StorageClassName).To(Equal(class.Name))
@ -493,29 +481,29 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai
pod = nil pod = nil
} }
func TestBindingWaitForFirstConsumer(t StorageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass, nodeSelector map[string]string, expectUnschedulable bool) (*v1.PersistentVolume, *v1.Node) { func (t StorageClassTest) TestBindingWaitForFirstConsumer(nodeSelector map[string]string, expectUnschedulable bool) (*v1.PersistentVolume, *v1.Node) {
pvs, node := TestBindingWaitForFirstConsumerMultiPVC(t, client, []*v1.PersistentVolumeClaim{claim}, class, nodeSelector, expectUnschedulable) pvs, node := t.TestBindingWaitForFirstConsumerMultiPVC([]*v1.PersistentVolumeClaim{t.Claim}, nodeSelector, expectUnschedulable)
if pvs == nil { if pvs == nil {
return nil, node return nil, node
} }
return pvs[0], node return pvs[0], node
} }
func TestBindingWaitForFirstConsumerMultiPVC(t StorageClassTest, client clientset.Interface, claims []*v1.PersistentVolumeClaim, class *storage.StorageClass, nodeSelector map[string]string, expectUnschedulable bool) ([]*v1.PersistentVolume, *v1.Node) { func (t StorageClassTest) TestBindingWaitForFirstConsumerMultiPVC(claims []*v1.PersistentVolumeClaim, nodeSelector map[string]string, expectUnschedulable bool) ([]*v1.PersistentVolume, *v1.Node) {
var err error var err error
Expect(len(claims)).ToNot(Equal(0)) Expect(len(claims)).ToNot(Equal(0))
namespace := claims[0].Namespace namespace := claims[0].Namespace
By("creating a storage class " + class.Name) By("creating a storage class " + t.Class.Name)
class, err = client.StorageV1().StorageClasses().Create(class) class, err := t.Client.StorageV1().StorageClasses().Create(t.Class)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer deleteStorageClass(client, class.Name) defer deleteStorageClass(t.Client, class.Name)
By("creating claims") By("creating claims")
var claimNames []string var claimNames []string
var createdClaims []*v1.PersistentVolumeClaim var createdClaims []*v1.PersistentVolumeClaim
for _, claim := range claims { for _, claim := range claims {
c, err := client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) c, err := t.Client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
claimNames = append(claimNames, c.Name) claimNames = append(claimNames, c.Name)
createdClaims = append(createdClaims, c) createdClaims = append(createdClaims, c)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -523,7 +511,7 @@ func TestBindingWaitForFirstConsumerMultiPVC(t StorageClassTest, client clientse
defer func() { defer func() {
var errors map[string]error var errors map[string]error
for _, claim := range createdClaims { for _, claim := range createdClaims {
err := framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace) err := framework.DeletePersistentVolumeClaim(t.Client, claim.Name, claim.Namespace)
if err != nil { if err != nil {
errors[claim.Name] = err errors[claim.Name] = err
} }
@ -537,44 +525,44 @@ func TestBindingWaitForFirstConsumerMultiPVC(t StorageClassTest, client clientse
// Wait for ClaimProvisionTimeout (across all PVCs in parallel) and make sure the phase did not become Bound i.e. the Wait errors out // Wait for ClaimProvisionTimeout (across all PVCs in parallel) and make sure the phase did not become Bound i.e. the Wait errors out
By("checking the claims are in pending state") By("checking the claims are in pending state")
err = framework.WaitForPersistentVolumeClaimsPhase(v1.ClaimBound, client, namespace, claimNames, 2*time.Second /* Poll */, framework.ClaimProvisionShortTimeout, true) err = framework.WaitForPersistentVolumeClaimsPhase(v1.ClaimBound, t.Client, namespace, claimNames, 2*time.Second /* Poll */, framework.ClaimProvisionShortTimeout, true)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
verifyPVCsPending(client, createdClaims) verifyPVCsPending(t.Client, createdClaims)
By("creating a pod referring to the claims") By("creating a pod referring to the claims")
// Create a pod referring to the claim and wait for it to get to running // Create a pod referring to the claim and wait for it to get to running
var pod *v1.Pod var pod *v1.Pod
if expectUnschedulable { if expectUnschedulable {
pod, err = framework.CreateUnschedulablePod(client, namespace, nodeSelector, createdClaims, true /* isPrivileged */, "" /* command */) pod, err = framework.CreateUnschedulablePod(t.Client, namespace, nodeSelector, createdClaims, true /* isPrivileged */, "" /* command */)
} else { } else {
pod, err = framework.CreatePod(client, namespace, nil /* nodeSelector */, createdClaims, true /* isPrivileged */, "" /* command */) pod, err = framework.CreatePod(t.Client, namespace, nil /* nodeSelector */, createdClaims, true /* isPrivileged */, "" /* command */)
} }
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer func() { defer func() {
framework.DeletePodOrFail(client, pod.Namespace, pod.Name) framework.DeletePodOrFail(t.Client, pod.Namespace, pod.Name)
framework.WaitForPodToDisappear(client, pod.Namespace, pod.Name, labels.Everything(), framework.Poll, framework.PodDeleteTimeout) framework.WaitForPodToDisappear(t.Client, pod.Namespace, pod.Name, labels.Everything(), framework.Poll, framework.PodDeleteTimeout)
}() }()
if expectUnschedulable { if expectUnschedulable {
// Verify that no claims are provisioned. // Verify that no claims are provisioned.
verifyPVCsPending(client, createdClaims) verifyPVCsPending(t.Client, createdClaims)
return nil, nil return nil, nil
} }
// collect node details // collect node details
node, err := client.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) node, err := t.Client.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
By("re-checking the claims to see they binded") By("re-checking the claims to see they binded")
var pvs []*v1.PersistentVolume var pvs []*v1.PersistentVolume
for _, claim := range createdClaims { for _, claim := range createdClaims {
// Get new copy of the claim // Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) claim, err = t.Client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// make sure claim did bind // make sure claim did bind
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, t.Client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) pv, err := t.Client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
pvs = append(pvs, pv) pvs = append(pvs, pv)
} }

317
test/e2e/storage/testsuites/snapshottable.go
View File

@ -24,13 +24,10 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testpatterns"
) )
@ -48,13 +45,12 @@ var (
) )
type SnapshotClassTest struct { type SnapshotClassTest struct {
Name string Name string
CloudProviders []string CloudProviders []string
Snapshotter string Snapshotter string
Parameters map[string]string Parameters map[string]string
NodeName string NodeName string
NodeSelector map[string]string // NodeSelector for the pod NodeSelector map[string]string // NodeSelector for the pod
SnapshotContentCheck func(snapshotContent *unstructured.Unstructured) error
} }
type snapshottableTestSuite struct { type snapshottableTestSuite struct {
@ -79,218 +75,129 @@ func (s *snapshottableTestSuite) getTestSuiteInfo() TestSuiteInfo {
return s.tsInfo return s.tsInfo
} }
func (s *snapshottableTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { func (s *snapshottableTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
dInfo := driver.GetDriverInfo() var (
if !dInfo.Capabilities[CapDataSource] { sDriver SnapshottableTestDriver
framework.Skipf("Driver %q does not support snapshots - skipping", dInfo.Name) dDriver DynamicPVTestDriver
} )
}
func createSnapshottableTestInput(driver TestDriver, pattern testpatterns.TestPattern) (snapshottableTestResource, snapshottableTestInput) { BeforeEach(func() {
// Setup test resource for driver and testpattern // Check preconditions.
resource := snapshottableTestResource{} Expect(pattern.SnapshotType).To(Equal(testpatterns.DynamicCreatedSnapshot))
resource.setupResource(driver, pattern) dInfo := driver.GetDriverInfo()
ok := false
dInfo := driver.GetDriverInfo() sDriver, ok = driver.(SnapshottableTestDriver)
input := snapshottableTestInput{ if !dInfo.Capabilities[CapDataSource] || !ok {
testCase: SnapshotClassTest{ framework.Skipf("Driver %q does not support snapshots - skipping", dInfo.Name)
NodeName: dInfo.Config.ClientNodeName, }
}, dDriver, ok = driver.(DynamicPVTestDriver)
cs: dInfo.Config.Framework.ClientSet, if !ok {
dc: dInfo.Config.Framework.DynamicClient, framework.Skipf("Driver %q does not support dynamic provisioning - skipping", driver.GetDriverInfo().Name)
pvc: resource.pvc, }
sc: resource.sc,
vsc: resource.vsc,
dInfo: dInfo,
}
return resource, input
}
func (s *snapshottableTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) {
Context(getTestNameStr(s, pattern), func() {
var (
resource snapshottableTestResource
input snapshottableTestInput
needsCleanup bool
)
BeforeEach(func() {
needsCleanup = false
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(s, driver, pattern)
needsCleanup = true
// Create test input
resource, input = createSnapshottableTestInput(driver, pattern)
})
AfterEach(func() {
if needsCleanup {
resource.cleanupResource(driver, pattern)
}
})
// Ginkgo's "Global Shared Behaviors" require arguments for a shared function
// to be a single struct and to be passed as a pointer.
// Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details.
testSnapshot(&input)
}) })
}
type snapshottableTestResource struct { // This intentionally comes after checking the preconditions because it
driver TestDriver // registers its own BeforeEach which creates the namespace. Beware that it
claimSize string // also registers an AfterEach which renders f unusable. Any code using
// f must run inside an It or Context callback.
f := framework.NewDefaultFramework("snapshotting")
sc *storage.StorageClass It("should create snapshot with defaults [Feature:VolumeSnapshotDataSource]", func() {
pvc *v1.PersistentVolumeClaim cs := f.ClientSet
// volume snapshot class dc := f.DynamicClient
vsc *unstructured.Unstructured
}
var _ TestResource = &snapshottableTestResource{} // Now do the more expensive test initialization.
config, testCleanup := driver.PrepareTest(f)
defer testCleanup()
func (s *snapshottableTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { vsc := sDriver.GetSnapshotClass(config)
// Setup snapshottableTest resource class := dDriver.GetDynamicProvisionStorageClass(config, "")
switch pattern.SnapshotType { if class == nil {
case testpatterns.DynamicCreatedSnapshot: framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
if dDriver, ok := driver.(DynamicPVTestDriver); ok {
s.sc = dDriver.GetDynamicProvisionStorageClass("")
if s.sc == nil {
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
}
s.driver = driver
s.claimSize = dDriver.GetClaimSize()
s.pvc = getClaim(s.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
s.pvc.Spec.StorageClassName = &s.sc.Name
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", s.sc, s.pvc)
if sDriver, ok := driver.(SnapshottableTestDriver); ok {
s.vsc = sDriver.GetSnapshotClass()
}
} }
default: claimSize := dDriver.GetClaimSize()
framework.Failf("Dynamic Snapshot test doesn't support: %s", pattern.SnapshotType) pvc := getClaim(claimSize, config.Framework.Namespace.Name)
} pvc.Spec.StorageClassName = &class.Name
} framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", class, pvc)
func (s *snapshottableTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) {
}
type snapshottableTestInput struct {
testCase SnapshotClassTest
cs clientset.Interface
dc dynamic.Interface
pvc *v1.PersistentVolumeClaim
sc *storage.StorageClass
// volume snapshot class
vsc *unstructured.Unstructured
dInfo *DriverInfo
}
func testSnapshot(input *snapshottableTestInput) {
It("should create snapshot with defaults [Feature:VolumeSnapshotDataSource]", func() {
TestCreateSnapshot(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
})
}
// TestCreateSnapshot tests dynamic creating snapshot with specified SnapshotClassTest and snapshotClass
func TestCreateSnapshot(
t SnapshotClassTest,
client clientset.Interface,
dynamicClient dynamic.Interface,
claim *v1.PersistentVolumeClaim,
class *storage.StorageClass,
snapshotClass *unstructured.Unstructured,
) *unstructured.Unstructured {
var err error
if class != nil {
By("creating a StorageClass " + class.Name) By("creating a StorageClass " + class.Name)
class, err = client.StorageV1().StorageClasses().Create(class) class, err := cs.StorageV1().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer func() { defer func() {
framework.Logf("deleting storage class %s", class.Name) framework.Logf("deleting storage class %s", class.Name)
framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil)) framework.ExpectNoError(cs.StorageV1().StorageClasses().Delete(class.Name, nil))
}() }()
}
By("creating a claim") By("creating a claim")
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) pvc, err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer func() { defer func() {
framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name) framework.Logf("deleting claim %q/%q", pvc.Namespace, pvc.Name)
// typically this claim has already been deleted // typically this claim has already been deleted
err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil) err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)
if err != nil && !apierrs.IsNotFound(err) { if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err) framework.Failf("Error deleting claim %q. Error: %v", pvc.Name, err)
} }
}() }()
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the claim")
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound PV
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
By("creating a SnapshotClass")
snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
}()
By("creating a snapshot")
snapshot := getSnapshot(claim.Name, claim.Namespace, snapshotClass.GetName())
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Create(snapshot, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
// typically this snapshot has already been deleted
err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Delete(snapshot.GetName(), nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting snapshot %q. Error: %v", claim.Name, err)
}
}()
err = WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the snapshot")
// Get new copy of the snapshot
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound snapshotContent
snapshotSpec := snapshot.Object["spec"].(map[string]interface{})
snapshotContentName := snapshotSpec["snapshotContentName"].(string)
snapshotContent, err := dynamicClient.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{})
volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{})
// Check SnapshotContent properties
By("checking the SnapshotContent")
Expect(snapshotContentSpec["snapshotClassName"]).To(Equal(snapshotClass.GetName()))
Expect(volumeSnapshotRef["name"]).To(Equal(snapshot.GetName()))
Expect(volumeSnapshotRef["namespace"]).To(Equal(snapshot.GetNamespace()))
Expect(persistentVolumeRef["name"]).To(Equal(pv.Name))
// Run the checker
if t.SnapshotContentCheck != nil {
err = t.SnapshotContentCheck(snapshotContent)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}
return snapshotContent By("checking the claim")
// Get new copy of the claim
pvc, err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound PV
pv, err := cs.CoreV1().PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
By("creating a SnapshotClass")
vsc, err = dc.Resource(snapshotClassGVR).Create(vsc, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting SnapshotClass %s", vsc.GetName())
framework.ExpectNoError(dc.Resource(snapshotClassGVR).Delete(vsc.GetName(), nil))
}()
By("creating a snapshot")
snapshot := getSnapshot(pvc.Name, pvc.Namespace, vsc.GetName())
snapshot, err = dc.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Create(snapshot, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
// typically this snapshot has already been deleted
err = dc.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Delete(snapshot.GetName(), nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting snapshot %q. Error: %v", pvc.Name, err)
}
}()
err = WaitForSnapshotReady(dc, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the snapshot")
// Get new copy of the snapshot
snapshot, err = dc.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound snapshotContent
snapshotSpec := snapshot.Object["spec"].(map[string]interface{})
snapshotContentName := snapshotSpec["snapshotContentName"].(string)
snapshotContent, err := dc.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{})
volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{})
// Check SnapshotContent properties
By("checking the SnapshotContent")
Expect(snapshotContentSpec["snapshotClassName"]).To(Equal(vsc.GetName()))
Expect(volumeSnapshotRef["name"]).To(Equal(snapshot.GetName()))
Expect(volumeSnapshotRef["namespace"]).To(Equal(snapshot.GetNamespace()))
Expect(persistentVolumeRef["name"]).To(Equal(pv.Name))
})
} }
// WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first. // WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first.

381
test/e2e/storage/testsuites/subpath.go
View File

@ -71,346 +71,351 @@ func (s *subPathTestSuite) getTestSuiteInfo() TestSuiteInfo {
return s.tsInfo return s.tsInfo
} }
func (s *subPathTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
} type local struct {
config *PerTestConfig
testCleanup func()
func createSubPathTestInput(pattern testpatterns.TestPattern, resource subPathTestResource) subPathTestInput { resource *genericVolumeTestResource
driver := resource.driver roVolSource *v1.VolumeSource
dInfo := driver.GetDriverInfo() pod *v1.Pod
f := dInfo.Config.Framework formatPod *v1.Pod
subPath := f.Namespace.Name subPathDir string
subPathDir := filepath.Join(volumePath, subPath) filePathInSubpath string
filePathInVolume string
return subPathTestInput{
f: f,
subPathDir: subPathDir,
filePathInSubpath: filepath.Join(volumePath, fileName),
filePathInVolume: filepath.Join(subPathDir, fileName),
volType: resource.volType,
pod: resource.pod,
formatPod: resource.formatPod,
volSource: resource.genericVolumeTestResource.volSource,
roVol: resource.roVolSource,
} }
} var l local
func (s *subPathTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { // No preconditions to test. Normally they would be in a BeforeEach here.
Context(getTestNameStr(s, pattern), func() {
var (
resource subPathTestResource
input subPathTestInput
needsCleanup bool
)
BeforeEach(func() { // This intentionally comes after checking the preconditions because it
needsCleanup = false // registers its own BeforeEach which creates the namespace. Beware that it
// Skip unsupported tests to avoid unnecessary resource initialization // also registers an AfterEach which renders f unusable. Any code using
skipUnsupportedTest(s, driver, pattern) // f must run inside an It or Context callback.
needsCleanup = true f := framework.NewDefaultFramework("provisioning")
// Setup test resource for driver and testpattern init := func() {
resource = subPathTestResource{} l = local{}
resource.setupResource(driver, pattern)
// Create test input // Now do the more expensive test initialization.
input = createSubPathTestInput(pattern, resource) l.config, l.testCleanup = driver.PrepareTest(f)
}) l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
AfterEach(func() { // Setup subPath test dependent resource
if needsCleanup { volType := pattern.VolType
resource.cleanupResource(driver, pattern) switch volType {
case testpatterns.InlineVolume:
if iDriver, ok := driver.(InlineVolumeTestDriver); ok {
l.roVolSource = iDriver.GetVolumeSource(true, pattern.FsType, l.resource.volume)
} }
}) case testpatterns.PreprovisionedPV:
l.roVolSource = &v1.VolumeSource{
testSubPath(&input) PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
}) ClaimName: l.resource.pvc.Name,
} ReadOnly: true,
},
type subPathTestResource struct { }
genericVolumeTestResource case testpatterns.DynamicPV:
l.roVolSource = &v1.VolumeSource{
roVolSource *v1.VolumeSource PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
pod *v1.Pod ClaimName: l.resource.pvc.Name,
formatPod *v1.Pod ReadOnly: true,
} },
}
var _ TestResource = &subPathTestResource{} default:
framework.Failf("SubPath test doesn't support: %s", volType)
func (s *subPathTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) {
s.driver = driver
dInfo := s.driver.GetDriverInfo()
f := dInfo.Config.Framework
fsType := pattern.FsType
volType := pattern.VolType
// Setup generic test resource
s.genericVolumeTestResource.setupResource(driver, pattern)
// Setup subPath test dependent resource
switch volType {
case testpatterns.InlineVolume:
if iDriver, ok := driver.(InlineVolumeTestDriver); ok {
s.roVolSource = iDriver.GetVolumeSource(true, fsType, s.genericVolumeTestResource.driverTestResource)
} }
case testpatterns.PreprovisionedPV:
s.roVolSource = &v1.VolumeSource{ subPath := f.Namespace.Name
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ l.pod = SubpathTestPod(f, subPath, l.resource.volType, l.resource.volSource, true)
ClaimName: s.genericVolumeTestResource.pvc.Name, l.pod.Spec.NodeName = l.config.ClientNodeName
ReadOnly: true, l.pod.Spec.NodeSelector = l.config.ClientNodeSelector
},
} l.formatPod = volumeFormatPod(f, l.resource.volSource)
case testpatterns.DynamicPV: l.formatPod.Spec.NodeName = l.config.ClientNodeName
s.roVolSource = &v1.VolumeSource{ l.formatPod.Spec.NodeSelector = l.config.ClientNodeSelector
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: s.genericVolumeTestResource.pvc.Name, l.subPathDir = filepath.Join(volumePath, subPath)
ReadOnly: true, l.filePathInSubpath = filepath.Join(volumePath, fileName)
}, l.filePathInVolume = filepath.Join(l.subPathDir, fileName)
}
default:
framework.Failf("SubPath test doesn't support: %s", volType)
} }
subPath := f.Namespace.Name cleanup := func() {
config := dInfo.Config if l.pod != nil {
s.pod = SubpathTestPod(f, subPath, s.volType, s.volSource, true) By("Deleting pod")
s.pod.Spec.NodeName = config.ClientNodeName err := framework.DeletePodWithWait(f, f.ClientSet, l.pod)
s.pod.Spec.NodeSelector = config.ClientNodeSelector Expect(err).ToNot(HaveOccurred(), "while deleting pod")
l.pod = nil
}
s.formatPod = volumeFormatPod(f, s.volSource) if l.resource != nil {
s.formatPod.Spec.NodeName = config.ClientNodeName l.resource.cleanupResource()
s.formatPod.Spec.NodeSelector = config.ClientNodeSelector l.resource = nil
} }
func (s *subPathTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { if l.testCleanup != nil {
dInfo := driver.GetDriverInfo() l.testCleanup()
f := dInfo.Config.Framework l.testCleanup = nil
}
}
// Cleanup subPath test dependent resource
By("Deleting pod")
err := framework.DeletePodWithWait(f, f.ClientSet, s.pod)
Expect(err).ToNot(HaveOccurred(), "while deleting pod")
// Cleanup generic test resource
s.genericVolumeTestResource.cleanupResource(driver, pattern)
}
type subPathTestInput struct {
f *framework.Framework
subPathDir string
filePathInSubpath string
filePathInVolume string
volType string
pod *v1.Pod
formatPod *v1.Pod
volSource *v1.VolumeSource
roVol *v1.VolumeSource
}
func testSubPath(input *subPathTestInput) {
It("should support non-existent path", func() { It("should support non-existent path", func() {
init()
defer cleanup()
// Write the file in the subPath from init container 1 // Write the file in the subPath from init container 1
setWriteCommand(input.filePathInSubpath, &input.pod.Spec.InitContainers[1]) setWriteCommand(l.filePathInSubpath, &l.pod.Spec.InitContainers[1])
// Read it from outside the subPath from container 1 // Read it from outside the subPath from container 1
testReadFile(input.f, input.filePathInVolume, input.pod, 1) testReadFile(f, l.filePathInVolume, l.pod, 1)
}) })
It("should support existing directory", func() { It("should support existing directory", func() {
init()
defer cleanup()
// Create the directory // Create the directory
setInitCommand(input.pod, fmt.Sprintf("mkdir -p %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("mkdir -p %s", l.subPathDir))
// Write the file in the subPath from init container 1 // Write the file in the subPath from init container 1
setWriteCommand(input.filePathInSubpath, &input.pod.Spec.InitContainers[1]) setWriteCommand(l.filePathInSubpath, &l.pod.Spec.InitContainers[1])
// Read it from outside the subPath from container 1 // Read it from outside the subPath from container 1
testReadFile(input.f, input.filePathInVolume, input.pod, 1) testReadFile(f, l.filePathInVolume, l.pod, 1)
}) })
It("should support existing single file", func() { It("should support existing single file", func() {
init()
defer cleanup()
// Create the file in the init container // Create the file in the init container
setInitCommand(input.pod, fmt.Sprintf("mkdir -p %s; echo \"mount-tester new file\" > %s", input.subPathDir, input.filePathInVolume)) setInitCommand(l.pod, fmt.Sprintf("mkdir -p %s; echo \"mount-tester new file\" > %s", l.subPathDir, l.filePathInVolume))
// Read it from inside the subPath from container 0 // Read it from inside the subPath from container 0
testReadFile(input.f, input.filePathInSubpath, input.pod, 0) testReadFile(f, l.filePathInSubpath, l.pod, 0)
}) })
It("should support file as subpath", func() { It("should support file as subpath", func() {
// Create the file in the init container init()
setInitCommand(input.pod, fmt.Sprintf("echo %s > %s", input.f.Namespace.Name, input.subPathDir)) defer cleanup()
TestBasicSubpath(input.f, input.f.Namespace.Name, input.pod) // Create the file in the init container
setInitCommand(l.pod, fmt.Sprintf("echo %s > %s", f.Namespace.Name, l.subPathDir))
TestBasicSubpath(f, f.Namespace.Name, l.pod)
}) })
It("should fail if subpath directory is outside the volume [Slow]", func() { It("should fail if subpath directory is outside the volume [Slow]", func() {
init()
defer cleanup()
// Create the subpath outside the volume // Create the subpath outside the volume
setInitCommand(input.pod, fmt.Sprintf("ln -s /bin %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("ln -s /bin %s", l.subPathDir))
// Pod should fail // Pod should fail
testPodFailSubpath(input.f, input.pod, false) testPodFailSubpath(f, l.pod, false)
}) })
It("should fail if subpath file is outside the volume [Slow]", func() { It("should fail if subpath file is outside the volume [Slow]", func() {
init()
defer cleanup()
// Create the subpath outside the volume // Create the subpath outside the volume
setInitCommand(input.pod, fmt.Sprintf("ln -s /bin/sh %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("ln -s /bin/sh %s", l.subPathDir))
// Pod should fail // Pod should fail
testPodFailSubpath(input.f, input.pod, false) testPodFailSubpath(f, l.pod, false)
}) })
It("should fail if non-existent subpath is outside the volume [Slow]", func() { It("should fail if non-existent subpath is outside the volume [Slow]", func() {
init()
defer cleanup()
// Create the subpath outside the volume // Create the subpath outside the volume
setInitCommand(input.pod, fmt.Sprintf("ln -s /bin/notanexistingpath %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("ln -s /bin/notanexistingpath %s", l.subPathDir))
// Pod should fail // Pod should fail
testPodFailSubpath(input.f, input.pod, false) testPodFailSubpath(f, l.pod, false)
}) })
It("should fail if subpath with backstepping is outside the volume [Slow]", func() { It("should fail if subpath with backstepping is outside the volume [Slow]", func() {
init()
defer cleanup()
// Create the subpath outside the volume // Create the subpath outside the volume
setInitCommand(input.pod, fmt.Sprintf("ln -s ../ %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("ln -s ../ %s", l.subPathDir))
// Pod should fail // Pod should fail
testPodFailSubpath(input.f, input.pod, false) testPodFailSubpath(f, l.pod, false)
}) })
It("should support creating multiple subpath from same volumes [Slow]", func() { It("should support creating multiple subpath from same volumes [Slow]", func() {
init()
defer cleanup()
subpathDir1 := filepath.Join(volumePath, "subpath1") subpathDir1 := filepath.Join(volumePath, "subpath1")
subpathDir2 := filepath.Join(volumePath, "subpath2") subpathDir2 := filepath.Join(volumePath, "subpath2")
filepath1 := filepath.Join("/test-subpath1", fileName) filepath1 := filepath.Join("/test-subpath1", fileName)
filepath2 := filepath.Join("/test-subpath2", fileName) filepath2 := filepath.Join("/test-subpath2", fileName)
setInitCommand(input.pod, fmt.Sprintf("mkdir -p %s; mkdir -p %s", subpathDir1, subpathDir2)) setInitCommand(l.pod, fmt.Sprintf("mkdir -p %s; mkdir -p %s", subpathDir1, subpathDir2))
addSubpathVolumeContainer(&input.pod.Spec.Containers[0], v1.VolumeMount{ addSubpathVolumeContainer(&l.pod.Spec.Containers[0], v1.VolumeMount{
Name: volumeName, Name: volumeName,
MountPath: "/test-subpath1", MountPath: "/test-subpath1",
SubPath: "subpath1", SubPath: "subpath1",
}) })
addSubpathVolumeContainer(&input.pod.Spec.Containers[0], v1.VolumeMount{ addSubpathVolumeContainer(&l.pod.Spec.Containers[0], v1.VolumeMount{
Name: volumeName, Name: volumeName,
MountPath: "/test-subpath2", MountPath: "/test-subpath2",
SubPath: "subpath2", SubPath: "subpath2",
}) })
// Write the files from container 0 and instantly read them back // Write the files from container 0 and instantly read them back
addMultipleWrites(&input.pod.Spec.Containers[0], filepath1, filepath2) addMultipleWrites(&l.pod.Spec.Containers[0], filepath1, filepath2)
testMultipleReads(input.f, input.pod, 0, filepath1, filepath2) testMultipleReads(f, l.pod, 0, filepath1, filepath2)
}) })
It("should support restarting containers using directory as subpath [Slow]", func() { It("should support restarting containers using directory as subpath [Slow]", func() {
// Create the directory init()
setInitCommand(input.pod, fmt.Sprintf("mkdir -p %v; touch %v", input.subPathDir, probeFilePath)) defer cleanup()
testPodContainerRestart(input.f, input.pod) // Create the directory
setInitCommand(l.pod, fmt.Sprintf("mkdir -p %v; touch %v", l.subPathDir, probeFilePath))
testPodContainerRestart(f, l.pod)
}) })
It("should support restarting containers using file as subpath [Slow]", func() { It("should support restarting containers using file as subpath [Slow]", func() {
// Create the file init()
setInitCommand(input.pod, fmt.Sprintf("touch %v; touch %v", input.subPathDir, probeFilePath)) defer cleanup()
testPodContainerRestart(input.f, input.pod) // Create the file
setInitCommand(l.pod, fmt.Sprintf("touch %v; touch %v", l.subPathDir, probeFilePath))
testPodContainerRestart(f, l.pod)
}) })
It("should unmount if pod is gracefully deleted while kubelet is down [Disruptive][Slow]", func() { It("should unmount if pod is gracefully deleted while kubelet is down [Disruptive][Slow]", func() {
testSubpathReconstruction(input.f, input.pod, false) init()
defer cleanup()
testSubpathReconstruction(f, l.pod, false)
}) })
It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() { It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() {
if strings.HasPrefix(input.volType, "hostPath") || strings.HasPrefix(input.volType, "csi-hostpath") { init()
defer cleanup()
if strings.HasPrefix(l.resource.volType, "hostPath") || strings.HasPrefix(l.resource.volType, "csi-hostpath") {
// TODO: This skip should be removed once #61446 is fixed // TODO: This skip should be removed once #61446 is fixed
framework.Skipf("%s volume type does not support reconstruction, skipping", input.volType) framework.Skipf("%s volume type does not support reconstruction, skipping", l.resource.volType)
} }
testSubpathReconstruction(input.f, input.pod, true)
testSubpathReconstruction(f, l.pod, true)
}) })
It("should support readOnly directory specified in the volumeMount", func() { It("should support readOnly directory specified in the volumeMount", func() {
init()
defer cleanup()
// Create the directory // Create the directory
setInitCommand(input.pod, fmt.Sprintf("mkdir -p %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("mkdir -p %s", l.subPathDir))
// Write the file in the volume from init container 2 // Write the file in the volume from init container 2
setWriteCommand(input.filePathInVolume, &input.pod.Spec.InitContainers[2]) setWriteCommand(l.filePathInVolume, &l.pod.Spec.InitContainers[2])
// Read it from inside the subPath from container 0 // Read it from inside the subPath from container 0
input.pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true l.pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true
testReadFile(input.f, input.filePathInSubpath, input.pod, 0) testReadFile(f, l.filePathInSubpath, l.pod, 0)
}) })
It("should support readOnly file specified in the volumeMount", func() { It("should support readOnly file specified in the volumeMount", func() {
init()
defer cleanup()
// Create the file // Create the file
setInitCommand(input.pod, fmt.Sprintf("touch %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("touch %s", l.subPathDir))
// Write the file in the volume from init container 2 // Write the file in the volume from init container 2
setWriteCommand(input.subPathDir, &input.pod.Spec.InitContainers[2]) setWriteCommand(l.subPathDir, &l.pod.Spec.InitContainers[2])
// Read it from inside the subPath from container 0 // Read it from inside the subPath from container 0
input.pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true l.pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true
testReadFile(input.f, volumePath, input.pod, 0) testReadFile(f, volumePath, l.pod, 0)
}) })
It("should support existing directories when readOnly specified in the volumeSource", func() { It("should support existing directories when readOnly specified in the volumeSource", func() {
if input.roVol == nil { init()
framework.Skipf("Volume type %v doesn't support readOnly source", input.volType) defer cleanup()
if l.roVolSource == nil {
framework.Skipf("Volume type %v doesn't support readOnly source", l.resource.volType)
} }
pod := input.pod.DeepCopy() origpod := l.pod.DeepCopy()
// Create the directory // Create the directory
setInitCommand(input.pod, fmt.Sprintf("mkdir -p %s", input.subPathDir)) setInitCommand(l.pod, fmt.Sprintf("mkdir -p %s", l.subPathDir))
// Write the file in the subPath from init container 1 // Write the file in the subPath from init container 1
setWriteCommand(input.filePathInSubpath, &input.pod.Spec.InitContainers[1]) setWriteCommand(l.filePathInSubpath, &l.pod.Spec.InitContainers[1])
// Read it from inside the subPath from container 0 // Read it from inside the subPath from container 0
testReadFile(input.f, input.filePathInSubpath, input.pod, 0) testReadFile(f, l.filePathInSubpath, l.pod, 0)
// Reset the pod // Reset the pod
input.pod = pod l.pod = origpod
// Set volume source to read only // Set volume source to read only
input.pod.Spec.Volumes[0].VolumeSource = *input.roVol l.pod.Spec.Volumes[0].VolumeSource = *l.roVolSource
// Read it from inside the subPath from container 0 // Read it from inside the subPath from container 0
testReadFile(input.f, input.filePathInSubpath, input.pod, 0) testReadFile(f, l.filePathInSubpath, l.pod, 0)
}) })
It("should verify container cannot write to subpath readonly volumes", func() { It("should verify container cannot write to subpath readonly volumes", func() {
if input.roVol == nil { init()
framework.Skipf("Volume type %v doesn't support readOnly source", input.volType) defer cleanup()
if l.roVolSource == nil {
framework.Skipf("Volume type %v doesn't support readOnly source", l.resource.volType)
} }
// Format the volume while it's writable // Format the volume while it's writable
formatVolume(input.f, input.formatPod) formatVolume(f, l.formatPod)
// Set volume source to read only // Set volume source to read only
input.pod.Spec.Volumes[0].VolumeSource = *input.roVol l.pod.Spec.Volumes[0].VolumeSource = *l.roVolSource
// Write the file in the volume from container 0 // Write the file in the volume from container 0
setWriteCommand(input.subPathDir, &input.pod.Spec.Containers[0]) setWriteCommand(l.subPathDir, &l.pod.Spec.Containers[0])
// Pod should fail // Pod should fail
testPodFailSubpath(input.f, input.pod, true) testPodFailSubpath(f, l.pod, true)
}) })
It("should be able to unmount after the subpath directory is deleted", func() { It("should be able to unmount after the subpath directory is deleted", func() {
// Change volume container to busybox so we can exec later init()
input.pod.Spec.Containers[1].Image = imageutils.GetE2EImage(imageutils.BusyBox) defer cleanup()
input.pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", "sleep 100000"}
By(fmt.Sprintf("Creating pod %s", input.pod.Name)) // Change volume container to busybox so we can exec later
removeUnusedContainers(input.pod) l.pod.Spec.Containers[1].Image = imageutils.GetE2EImage(imageutils.BusyBox)
pod, err := input.f.ClientSet.CoreV1().Pods(input.f.Namespace.Name).Create(input.pod) l.pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", "sleep 100000"}
By(fmt.Sprintf("Creating pod %s", l.pod.Name))
removeUnusedContainers(l.pod)
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(l.pod)
Expect(err).ToNot(HaveOccurred(), "while creating pod") Expect(err).ToNot(HaveOccurred(), "while creating pod")
defer func() { defer func() {
By(fmt.Sprintf("Deleting pod %s", pod.Name)) By(fmt.Sprintf("Deleting pod %s", pod.Name))
framework.DeletePodWithWait(input.f, input.f.ClientSet, pod) framework.DeletePodWithWait(f, f.ClientSet, pod)
}() }()
// Wait for pod to be running // Wait for pod to be running
err = framework.WaitForPodRunningInNamespace(input.f.ClientSet, pod) err = framework.WaitForPodRunningInNamespace(f.ClientSet, l.pod)
Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running") Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running")
// Exec into container that mounted the volume, delete subpath directory // Exec into container that mounted the volume, delete subpath directory
rmCmd := fmt.Sprintf("rm -rf %s", input.subPathDir) rmCmd := fmt.Sprintf("rm -rf %s", l.subPathDir)
_, err = podContainerExec(pod, 1, rmCmd) _, err = podContainerExec(l.pod, 1, rmCmd)
Expect(err).ToNot(HaveOccurred(), "while removing subpath directory") Expect(err).ToNot(HaveOccurred(), "while removing subpath directory")
// Delete pod (from defer) and wait for it to be successfully deleted // Delete pod (from defer) and wait for it to be successfully deleted

75
test/e2e/storage/testsuites/testdriver.go
View File

@ -25,35 +25,53 @@ import (
"k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testpatterns"
) )
// TestDriver represents an interface for a driver to be tested in TestSuite // TestDriver represents an interface for a driver to be tested in TestSuite.
// Except for GetDriverInfo, all methods will be called at test runtime and thus
// can use framework.Skipf, framework.Fatal, Gomega assertions, etc.
type TestDriver interface { type TestDriver interface {
// GetDriverInfo returns DriverInfo for the TestDriver // GetDriverInfo returns DriverInfo for the TestDriver. This must be static
// information.
GetDriverInfo() *DriverInfo GetDriverInfo() *DriverInfo
// CreateDriver creates all driver resources that is required for TestDriver method
// except CreateVolume // SkipUnsupportedTest skips test if Testpattern is not
CreateDriver() // suitable to test with the TestDriver. It gets called after
// CreateDriver cleanup all the resources that is created in CreateDriver // parsing parameters of the test suite and before the
CleanupDriver() // framework is initialized. Cheap tests that just check
// SkipUnsupportedTest skips test in Testpattern is not suitable to test with the TestDriver // parameters like the cloud provider can and should be
// done in SkipUnsupportedTest to avoid setting up more
// expensive resources like framework.Framework. Tests that
// depend on a connection to the cluster can be done in
// PrepareTest once the framework is ready.
SkipUnsupportedTest(testpatterns.TestPattern) SkipUnsupportedTest(testpatterns.TestPattern)
// PrepareTest is called at test execution time each time a new test case is about to start.
// It sets up all necessary resources and returns the per-test configuration
// plus a cleanup function that frees all allocated resources.
PrepareTest(f *framework.Framework) (*PerTestConfig, func())
}
// TestVolume is the result of PreprovisionedVolumeTestDriver.CreateVolume.
// The only common functionality is to delete it. Individual driver interfaces
// have additional methods that work with volumes created by them.
type TestVolume interface {
DeleteVolume()
} }
// PreprovisionedVolumeTestDriver represents an interface for a TestDriver that has pre-provisioned volume // PreprovisionedVolumeTestDriver represents an interface for a TestDriver that has pre-provisioned volume
type PreprovisionedVolumeTestDriver interface { type PreprovisionedVolumeTestDriver interface {
TestDriver TestDriver
// CreateVolume creates a pre-provisioned volume. // CreateVolume creates a pre-provisioned volume of the desired volume type.
CreateVolume(testpatterns.TestVolType) interface{} CreateVolume(config *PerTestConfig, volumeType testpatterns.TestVolType) TestVolume
// DeleteVolume deletes a volume that is created in CreateVolume
DeleteVolume(testpatterns.TestVolType, interface{})
} }
// InlineVolumeTestDriver represents an interface for a TestDriver that supports InlineVolume // InlineVolumeTestDriver represents an interface for a TestDriver that supports InlineVolume
type InlineVolumeTestDriver interface { type InlineVolumeTestDriver interface {
PreprovisionedVolumeTestDriver PreprovisionedVolumeTestDriver
// GetVolumeSource returns a volumeSource for inline volume. // GetVolumeSource returns a volumeSource for inline volume.
// It will set readOnly and fsType to the volumeSource, if TestDriver supports both of them. // It will set readOnly and fsType to the volumeSource, if TestDriver supports both of them.
// It will return nil, if the TestDriver doesn't support either of the parameters. // It will return nil, if the TestDriver doesn't support either of the parameters.
GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource GetVolumeSource(readOnly bool, fsType string, testVolume TestVolume) *v1.VolumeSource
} }
// PreprovisionedPVTestDriver represents an interface for a TestDriver that supports PreprovisionedPV // PreprovisionedPVTestDriver represents an interface for a TestDriver that supports PreprovisionedPV
@ -62,8 +80,7 @@ type PreprovisionedPVTestDriver interface {
// GetPersistentVolumeSource returns a PersistentVolumeSource with volume node affinity for pre-provisioned Persistent Volume. // GetPersistentVolumeSource returns a PersistentVolumeSource with volume node affinity for pre-provisioned Persistent Volume.
// It will set readOnly and fsType to the PersistentVolumeSource, if TestDriver supports both of them. // It will set readOnly and fsType to the PersistentVolumeSource, if TestDriver supports both of them.
// It will return nil, if the TestDriver doesn't support either of the parameters. // It will return nil, if the TestDriver doesn't support either of the parameters.
// Volume node affinity is optional, it will be nil for volumes which does not have volume node affinity. GetPersistentVolumeSource(readOnly bool, fsType string, testVolume TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity)
GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity)
} }
// DynamicPVTestDriver represents an interface for a TestDriver that supports DynamicPV // DynamicPVTestDriver represents an interface for a TestDriver that supports DynamicPV
@ -72,7 +89,7 @@ type DynamicPVTestDriver interface {
// GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume. // GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume.
// It will set fsType to the StorageClass, if TestDriver supports it. // It will set fsType to the StorageClass, if TestDriver supports it.
// It will return nil, if the TestDriver doesn't support it. // It will return nil, if the TestDriver doesn't support it.
GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass GetDynamicProvisionStorageClass(config *PerTestConfig, fsType string) *storagev1.StorageClass
// GetClaimSize returns the size of the volume that is to be provisioned ("5Gi", "1Mi"). // GetClaimSize returns the size of the volume that is to be provisioned ("5Gi", "1Mi").
// The size must be chosen so that the resulting volume is large enough for all // The size must be chosen so that the resulting volume is large enough for all
@ -85,7 +102,7 @@ type SnapshottableTestDriver interface {
TestDriver TestDriver
// GetSnapshotClass returns a SnapshotClass to create snapshot. // GetSnapshotClass returns a SnapshotClass to create snapshot.
// It will return nil, if the TestDriver doesn't support it. // It will return nil, if the TestDriver doesn't support it.
GetSnapshotClass() *unstructured.Unstructured GetSnapshotClass(config *PerTestConfig) *unstructured.Unstructured
} }
// Capability represents a feature that a volume plugin supports // Capability represents a feature that a volume plugin supports
@ -106,7 +123,7 @@ const (
CapMultiPODs Capability = "multipods" CapMultiPODs Capability = "multipods"
) )
// DriverInfo represents a combination of parameters to be used in implementation of TestDriver // DriverInfo represents static information about a TestDriver.
type DriverInfo struct { type DriverInfo struct {
Name string // Name of the driver Name string // Name of the driver
FeatureTag string // FeatureTag for the driver FeatureTag string // FeatureTag for the driver
@ -116,14 +133,15 @@ type DriverInfo struct {
SupportedMountOption sets.String // Map of string for supported mount option SupportedMountOption sets.String // Map of string for supported mount option
RequiredMountOption sets.String // Map of string for required mount option (Optional) RequiredMountOption sets.String // Map of string for required mount option (Optional)
Capabilities map[Capability]bool // Map that represents plugin capabilities Capabilities map[Capability]bool // Map that represents plugin capabilities
Config TestConfig // Test configuration for the current test.
} }
// TestConfig represents parameters that control test execution. // PerTestConfig represents parameters that control test execution.
// They can still be modified after defining tests, for example // One instance gets allocated for each test and is then passed
// in a BeforeEach or when creating the driver. // via pointer to functions involved in the test.
type TestConfig struct { type PerTestConfig struct {
// The test driver for the test.
Driver TestDriver
// Some short word that gets inserted into dynamically // Some short word that gets inserted into dynamically
// generated entities (pods, paths) as first part of the name // generated entities (pods, paths) as first part of the name
// to make debugging easier. Can be the same for different // to make debugging easier. Can be the same for different
@ -148,8 +166,9 @@ type TestConfig struct {
// the configuration that then has to be used to run tests. // the configuration that then has to be used to run tests.
// The values above are ignored for such tests. // The values above are ignored for such tests.
ServerConfig *framework.VolumeTestConfig ServerConfig *framework.VolumeTestConfig
}
// TopologyEnabled indicates that the Topology feature gate
// should be enabled in external-provisioner // GetUniqueDriverName returns unique driver name that can be used parallelly in tests
TopologyEnabled bool func (config *PerTestConfig) GetUniqueDriverName() string {
return config.Driver.GetDriverInfo().Name + "-" + config.Framework.UniqueName
} }

124
test/e2e/storage/testsuites/volume_io.go
View File

@ -74,87 +74,65 @@ func (t *volumeIOTestSuite) getTestSuiteInfo() TestSuiteInfo {
return t.tsInfo return t.tsInfo
} }
func (t *volumeIOTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
} type local struct {
config *PerTestConfig
testCleanup func()
func createVolumeIOTestInput(pattern testpatterns.TestPattern, resource genericVolumeTestResource) volumeIOTestInput { resource *genericVolumeTestResource
var fsGroup *int64 }
driver := resource.driver var (
dInfo := driver.GetDriverInfo() dInfo = driver.GetDriverInfo()
f := dInfo.Config.Framework l local
fileSizes := createFileSizes(dInfo.MaxFileSize) )
volSource := resource.volSource
if volSource == nil { // No preconditions to test. Normally they would be in a BeforeEach here.
framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
// This intentionally comes after checking the preconditions because it
// registers its own BeforeEach which creates the namespace. Beware that it
// also registers an AfterEach which renders f unusable. Any code using
// f must run inside an It or Context callback.
f := framework.NewDefaultFramework("volumeio")
init := func() {
l = local{}
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
if l.resource.volSource == nil {
framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
}
} }
if dInfo.Capabilities[CapFsGroup] { cleanup := func() {
fsGroupVal := int64(1234) if l.resource != nil {
fsGroup = &fsGroupVal l.resource.cleanupResource()
l.resource = nil
}
if l.testCleanup != nil {
l.testCleanup()
l.testCleanup = nil
}
} }
return volumeIOTestInput{
f: f,
name: dInfo.Name,
config: &dInfo.Config,
volSource: *volSource,
testFile: fmt.Sprintf("%s_io_test_%s", dInfo.Name, f.Namespace.Name),
podSec: v1.PodSecurityContext{
FSGroup: fsGroup,
},
fileSizes: fileSizes,
}
}
func (t *volumeIOTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) {
Context(getTestNameStr(t, pattern), func() {
var (
resource genericVolumeTestResource
input volumeIOTestInput
needsCleanup bool
)
BeforeEach(func() {
needsCleanup = false
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(t, driver, pattern)
needsCleanup = true
// Setup test resource for driver and testpattern
resource = genericVolumeTestResource{}
resource.setupResource(driver, pattern)
// Create test input
input = createVolumeIOTestInput(pattern, resource)
})
AfterEach(func() {
if needsCleanup {
resource.cleanupResource(driver, pattern)
}
})
execTestVolumeIO(&input)
})
}
type volumeIOTestInput struct {
f *framework.Framework
name string
config *TestConfig
volSource v1.VolumeSource
testFile string
podSec v1.PodSecurityContext
fileSizes []int64
}
func execTestVolumeIO(input *volumeIOTestInput) {
It("should write files of various sizes, verify size, validate content [Slow]", func() { It("should write files of various sizes, verify size, validate content [Slow]", func() {
f := input.f init()
cs := f.ClientSet defer cleanup()
err := testVolumeIO(f, cs, convertTestConfig(input.config), input.volSource, &input.podSec, input.testFile, input.fileSizes) cs := f.ClientSet
fileSizes := createFileSizes(dInfo.MaxFileSize)
testFile := fmt.Sprintf("%s_io_test_%s", dInfo.Name, f.Namespace.Name)
var fsGroup *int64
if dInfo.Capabilities[CapFsGroup] {
fsGroupVal := int64(1234)
fsGroup = &fsGroupVal
}
podSec := v1.PodSecurityContext{
FSGroup: fsGroup,
}
err := testVolumeIO(f, cs, convertTestConfig(l.config), *l.resource.volSource, &podSec, testFile, fileSizes)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
} }

528
test/e2e/storage/testsuites/volumemode.go
View File

@ -26,6 +26,7 @@ import (
storagev1 "k8s.io/api/storage/v1" storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors" utilerrors "k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testpatterns"
"k8s.io/kubernetes/test/e2e/storage/utils" "k8s.io/kubernetes/test/e2e/storage/utils"
@ -61,316 +62,253 @@ func (t *volumeModeTestSuite) getTestSuiteInfo() TestSuiteInfo {
return t.tsInfo return t.tsInfo
} }
func (t *volumeModeTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
} type local struct {
config *PerTestConfig
testCleanup func()
func createVolumeModeTestInput(pattern testpatterns.TestPattern, resource volumeModeTestResource) volumeModeTestInput { cs clientset.Interface
driver := resource.driver ns *v1.Namespace
dInfo := driver.GetDriverInfo() sc *storagev1.StorageClass
f := dInfo.Config.Framework pvc *v1.PersistentVolumeClaim
pv *v1.PersistentVolume
return volumeModeTestInput{ volume TestVolume
f: f,
sc: resource.sc,
pvc: resource.pvc,
pv: resource.pv,
testVolType: pattern.VolType,
nodeName: dInfo.Config.ClientNodeName,
volMode: pattern.VolMode,
isBlockSupported: dInfo.Capabilities[CapBlock],
} }
}
func getVolumeModeTestFunc(pattern testpatterns.TestPattern, driver TestDriver) func(*volumeModeTestInput) {
dInfo := driver.GetDriverInfo()
isBlockSupported := dInfo.Capabilities[CapBlock]
volMode := pattern.VolMode
volType := pattern.VolType
switch volType {
case testpatterns.PreprovisionedPV:
if volMode == v1.PersistentVolumeBlock && !isBlockSupported {
return testVolumeModeFailForPreprovisionedPV
}
return testVolumeModeSuccessForPreprovisionedPV
case testpatterns.DynamicPV:
if volMode == v1.PersistentVolumeBlock && !isBlockSupported {
return testVolumeModeFailForDynamicPV
}
return testVolumeModeSuccessForDynamicPV
default:
framework.Failf("Volume mode test doesn't support volType: %v", volType)
}
return nil
}
func (t *volumeModeTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) {
Context(getTestNameStr(t, pattern), func() {
var (
resource volumeModeTestResource
input volumeModeTestInput
testFunc func(*volumeModeTestInput)
needsCleanup bool
)
testFunc = getVolumeModeTestFunc(pattern, driver)
BeforeEach(func() {
needsCleanup = false
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(t, driver, pattern)
needsCleanup = true
// Setup test resource for driver and testpattern
resource = volumeModeTestResource{}
resource.setupResource(driver, pattern)
// Create test input
input = createVolumeModeTestInput(pattern, resource)
})
AfterEach(func() {
if needsCleanup {
resource.cleanupResource(driver, pattern)
}
})
testFunc(&input)
})
}
type volumeModeTestResource struct {
driver TestDriver
sc *storagev1.StorageClass
pvc *v1.PersistentVolumeClaim
pv *v1.PersistentVolume
driverTestResource interface{}
}
var _ TestResource = &volumeModeTestResource{}
func (s *volumeModeTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) {
s.driver = driver
dInfo := driver.GetDriverInfo()
f := dInfo.Config.Framework
ns := f.Namespace
fsType := pattern.FsType
volBindMode := storagev1.VolumeBindingImmediate
volMode := pattern.VolMode
volType := pattern.VolType
var ( var (
scName string dInfo = driver.GetDriverInfo()
pvSource *v1.PersistentVolumeSource l local
volumeNodeAffinity *v1.VolumeNodeAffinity
) )
// Create volume for pre-provisioned volume tests // No preconditions to test. Normally they would be in a BeforeEach here.
s.driverTestResource = CreateVolume(driver, volType)
switch volType { // This intentionally comes after checking the preconditions because it
case testpatterns.PreprovisionedPV: // registers its own BeforeEach which creates the namespace. Beware that it
if volMode == v1.PersistentVolumeBlock { // also registers an AfterEach which renders f unusable. Any code using
scName = fmt.Sprintf("%s-%s-sc-for-block", ns.Name, dInfo.Name) // f must run inside an It or Context callback.
} else if volMode == v1.PersistentVolumeFilesystem { f := framework.NewDefaultFramework("volumemode")
scName = fmt.Sprintf("%s-%s-sc-for-file", ns.Name, dInfo.Name)
} init := func() {
if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok { l = local{}
pvSource, volumeNodeAffinity = pDriver.GetPersistentVolumeSource(false, fsType, s.driverTestResource) l.ns = f.Namespace
if pvSource == nil { l.cs = f.ClientSet
framework.Skipf("Driver %q does not define PersistentVolumeSource - skipping", dInfo.Name)
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
fsType := pattern.FsType
volBindMode := storagev1.VolumeBindingImmediate
var (
scName string
pvSource *v1.PersistentVolumeSource
volumeNodeAffinity *v1.VolumeNodeAffinity
)
// Create volume for pre-provisioned volume tests
l.volume = CreateVolume(driver, l.config, pattern.VolType)
switch pattern.VolType {
case testpatterns.PreprovisionedPV:
if pattern.VolMode == v1.PersistentVolumeBlock {
scName = fmt.Sprintf("%s-%s-sc-for-block", l.ns.Name, dInfo.Name)
} else if pattern.VolMode == v1.PersistentVolumeFilesystem {
scName = fmt.Sprintf("%s-%s-sc-for-file", l.ns.Name, dInfo.Name)
} }
if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
pvSource, volumeNodeAffinity = pDriver.GetPersistentVolumeSource(false, fsType, l.volume)
if pvSource == nil {
framework.Skipf("Driver %q does not define PersistentVolumeSource - skipping", dInfo.Name)
}
sc, pvConfig, pvcConfig := generateConfigsForPreprovisionedPVTest(scName, volBindMode, volMode, *pvSource, volumeNodeAffinity) storageClass, pvConfig, pvcConfig := generateConfigsForPreprovisionedPVTest(scName, volBindMode, pattern.VolMode, *pvSource, volumeNodeAffinity)
s.sc = sc l.sc = storageClass
s.pv = framework.MakePersistentVolume(pvConfig) l.pv = framework.MakePersistentVolume(pvConfig)
s.pvc = framework.MakePersistentVolumeClaim(pvcConfig, ns.Name) l.pvc = framework.MakePersistentVolumeClaim(pvcConfig, l.ns.Name)
}
case testpatterns.DynamicPV:
if dDriver, ok := driver.(DynamicPVTestDriver); ok {
l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, fsType)
if l.sc == nil {
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
}
l.sc.VolumeBindingMode = &volBindMode
claimSize := dDriver.GetClaimSize()
l.pvc = getClaim(claimSize, l.ns.Name)
l.pvc.Spec.StorageClassName = &l.sc.Name
l.pvc.Spec.VolumeMode = &pattern.VolMode
}
default:
framework.Failf("Volume mode test doesn't support: %s", pattern.VolType)
}
}
cleanup := func() {
if l.pv != nil || l.pvc != nil {
By("Deleting pv and pvc")
errs := framework.PVPVCCleanup(f.ClientSet, f.Namespace.Name, l.pv, l.pvc)
if len(errs) > 0 {
framework.Logf("Failed to delete PV and/or PVC: %v", utilerrors.NewAggregate(errs))
}
l.pv = nil
l.pvc = nil
}
if l.sc != nil {
By("Deleting sc")
deleteStorageClass(f.ClientSet, l.sc.Name)
l.sc = nil
}
if l.volume != nil {
l.volume.DeleteVolume()
l.volume = nil
}
if l.testCleanup != nil {
l.testCleanup()
l.testCleanup = nil
}
}
// We register different tests depending on the drive
isBlockSupported := dInfo.Capabilities[CapBlock]
switch pattern.VolType {
case testpatterns.PreprovisionedPV:
if pattern.VolMode == v1.PersistentVolumeBlock && !isBlockSupported {
It("should fail to create pod by failing to mount volume", func() {
init()
defer cleanup()
var err error
By("Creating sc")
l.sc, err = l.cs.StorageV1().StorageClasses().Create(l.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
l.pv, err = l.cs.CoreV1().PersistentVolumes().Create(l.pv)
Expect(err).NotTo(HaveOccurred())
// Prebind pv
l.pvc.Spec.VolumeName = l.pv.Name
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(l.pvc)
Expect(err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, l.config.ClientNodeName, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
}()
Expect(err).To(HaveOccurred())
})
} else {
It("should create sc, pod, pv, and pvc, read/write to the pv, and delete all created resources", func() {
init()
defer cleanup()
var err error
By("Creating sc")
l.sc, err = l.cs.StorageV1().StorageClasses().Create(l.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
l.pv, err = l.cs.CoreV1().PersistentVolumes().Create(l.pv)
Expect(err).NotTo(HaveOccurred())
// Prebind pv
l.pvc.Spec.VolumeName = l.pv.Name
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(l.pvc)
Expect(err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, l.config.ClientNodeName, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
}()
Expect(err).NotTo(HaveOccurred())
By("Checking if persistent volume exists as expected volume mode")
utils.CheckVolumeModeOfPath(pod, pattern.VolMode, "/mnt/volume1")
By("Checking if read/write to persistent volume works properly")
utils.CheckReadWriteToPath(pod, pattern.VolMode, "/mnt/volume1")
})
// TODO(mkimuram): Add more tests
} }
case testpatterns.DynamicPV: case testpatterns.DynamicPV:
if dDriver, ok := driver.(DynamicPVTestDriver); ok { if pattern.VolMode == v1.PersistentVolumeBlock && !isBlockSupported {
s.sc = dDriver.GetDynamicProvisionStorageClass(fsType) It("should fail in binding dynamic provisioned PV to PVC", func() {
if s.sc == nil { init()
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name) defer cleanup()
}
s.sc.VolumeBindingMode = &volBindMode
claimSize := dDriver.GetClaimSize() var err error
s.pvc = getClaim(claimSize, ns.Name)
s.pvc.Spec.StorageClassName = &s.sc.Name By("Creating sc")
s.pvc.Spec.VolumeMode = &volMode l.sc, err = l.cs.StorageV1().StorageClasses().Create(l.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(l.pvc)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, l.cs, l.pvc.Namespace, l.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).To(HaveOccurred())
})
} else {
It("should create sc, pod, pv, and pvc, read/write to the pv, and delete all created resources", func() {
init()
defer cleanup()
var err error
By("Creating sc")
l.sc, err = l.cs.StorageV1().StorageClasses().Create(l.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(l.pvc)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, l.cs, l.pvc.Namespace, l.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.pvc.Namespace).Get(l.pvc.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
l.pv, err = l.cs.CoreV1().PersistentVolumes().Get(l.pvc.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, l.config.ClientNodeName, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
}()
Expect(err).NotTo(HaveOccurred())
By("Checking if persistent volume exists as expected volume mode")
utils.CheckVolumeModeOfPath(pod, pattern.VolMode, "/mnt/volume1")
By("Checking if read/write to persistent volume works properly")
utils.CheckReadWriteToPath(pod, pattern.VolMode, "/mnt/volume1")
})
// TODO(mkimuram): Add more tests
} }
default: default:
framework.Failf("Volume mode test doesn't support: %s", volType) framework.Failf("Volume mode test doesn't support volType: %v", pattern.VolType)
}
}
func (s *volumeModeTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) {
dInfo := driver.GetDriverInfo()
f := dInfo.Config.Framework
cs := f.ClientSet
ns := f.Namespace
volType := pattern.VolType
By("Deleting pv and pvc")
errs := framework.PVPVCCleanup(cs, ns.Name, s.pv, s.pvc)
if len(errs) > 0 {
framework.Failf("Failed to delete PV and/or PVC: %v", utilerrors.NewAggregate(errs))
}
By("Deleting sc")
if s.sc != nil {
deleteStorageClass(cs, s.sc.Name)
} }
// Cleanup volume for pre-provisioned volume tests
DeleteVolume(driver, volType, s.driverTestResource)
}
type volumeModeTestInput struct {
f *framework.Framework
sc *storagev1.StorageClass
pvc *v1.PersistentVolumeClaim
pv *v1.PersistentVolume
testVolType testpatterns.TestVolType
nodeName string
volMode v1.PersistentVolumeMode
isBlockSupported bool
}
func testVolumeModeFailForPreprovisionedPV(input *volumeModeTestInput) {
It("should fail to create pod by failing to mount volume", func() {
f := input.f
cs := f.ClientSet
ns := f.Namespace
var err error
By("Creating sc")
input.sc, err = cs.StorageV1().StorageClasses().Create(input.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
input.pv, err = cs.CoreV1().PersistentVolumes().Create(input.pv)
Expect(err).NotTo(HaveOccurred())
// Prebind pv
input.pvc.Spec.VolumeName = input.pv.Name
input.pvc, err = cs.CoreV1().PersistentVolumeClaims(ns.Name).Create(input.pvc)
Expect(err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitOnPVandPVC(cs, ns.Name, input.pv, input.pvc))
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(cs, ns.Name, []*v1.PersistentVolumeClaim{input.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, input.nodeName, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, cs, pod))
}()
Expect(err).To(HaveOccurred())
})
}
func testVolumeModeSuccessForPreprovisionedPV(input *volumeModeTestInput) {
It("should create sc, pod, pv, and pvc, read/write to the pv, and delete all created resources", func() {
f := input.f
cs := f.ClientSet
ns := f.Namespace
var err error
By("Creating sc")
input.sc, err = cs.StorageV1().StorageClasses().Create(input.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
input.pv, err = cs.CoreV1().PersistentVolumes().Create(input.pv)
Expect(err).NotTo(HaveOccurred())
// Prebind pv
input.pvc.Spec.VolumeName = input.pv.Name
input.pvc, err = cs.CoreV1().PersistentVolumeClaims(ns.Name).Create(input.pvc)
Expect(err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitOnPVandPVC(cs, ns.Name, input.pv, input.pvc))
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(cs, ns.Name, []*v1.PersistentVolumeClaim{input.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, input.nodeName, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, cs, pod))
}()
Expect(err).NotTo(HaveOccurred())
By("Checking if persistent volume exists as expected volume mode")
utils.CheckVolumeModeOfPath(pod, input.volMode, "/mnt/volume1")
By("Checking if read/write to persistent volume works properly")
utils.CheckReadWriteToPath(pod, input.volMode, "/mnt/volume1")
})
// TODO(mkimuram): Add more tests
}
func testVolumeModeFailForDynamicPV(input *volumeModeTestInput) {
It("should fail in binding dynamic provisioned PV to PVC", func() {
f := input.f
cs := f.ClientSet
ns := f.Namespace
var err error
By("Creating sc")
input.sc, err = cs.StorageV1().StorageClasses().Create(input.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
input.pvc, err = cs.CoreV1().PersistentVolumeClaims(ns.Name).Create(input.pvc)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, input.pvc.Namespace, input.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).To(HaveOccurred())
})
}
func testVolumeModeSuccessForDynamicPV(input *volumeModeTestInput) {
It("should create sc, pod, pv, and pvc, read/write to the pv, and delete all created resources", func() {
f := input.f
cs := f.ClientSet
ns := f.Namespace
var err error
By("Creating sc")
input.sc, err = cs.StorageV1().StorageClasses().Create(input.sc)
Expect(err).NotTo(HaveOccurred())
By("Creating pv and pvc")
input.pvc, err = cs.CoreV1().PersistentVolumeClaims(ns.Name).Create(input.pvc)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, input.pvc.Namespace, input.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
input.pvc, err = cs.CoreV1().PersistentVolumeClaims(input.pvc.Namespace).Get(input.pvc.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
input.pv, err = cs.CoreV1().PersistentVolumes().Get(input.pvc.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(cs, ns.Name, []*v1.PersistentVolumeClaim{input.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, input.nodeName, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, cs, pod))
}()
Expect(err).NotTo(HaveOccurred())
By("Checking if persistent volume exists as expected volume mode")
utils.CheckVolumeModeOfPath(pod, input.volMode, "/mnt/volume1")
By("Checking if read/write to persistent volume works properly")
utils.CheckReadWriteToPath(pod, input.volMode, "/mnt/volume1")
})
// TODO(mkimuram): Add more tests
} }
func generateConfigsForPreprovisionedPVTest(scName string, volBindMode storagev1.VolumeBindingMode, func generateConfigsForPreprovisionedPVTest(scName string, volBindMode storagev1.VolumeBindingMode,

139
test/e2e/storage/testsuites/volumes.go
View File

@ -89,101 +89,80 @@ func skipExecTest(driver TestDriver) {
} }
} }
func createVolumesTestInput(pattern testpatterns.TestPattern, resource genericVolumeTestResource) volumesTestInput { func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
var fsGroup *int64 type local struct {
driver := resource.driver config *PerTestConfig
dInfo := driver.GetDriverInfo() testCleanup func()
f := dInfo.Config.Framework
volSource := resource.volSource
if volSource == nil { resource *genericVolumeTestResource
framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name) }
var dInfo = driver.GetDriverInfo()
var l local
// No preconditions to test. Normally they would be in a BeforeEach here.
// This intentionally comes after checking the preconditions because it
// registers its own BeforeEach which creates the namespace. Beware that it
// also registers an AfterEach which renders f unusable. Any code using
// f must run inside an It or Context callback.
f := framework.NewDefaultFramework("volumeio")
init := func() {
l = local{}
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
if l.resource.volSource == nil {
framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
}
} }
if dInfo.Capabilities[CapFsGroup] { cleanup := func() {
fsGroupVal := int64(1234) if l.resource != nil {
fsGroup = &fsGroupVal l.resource.cleanupResource()
l.resource = nil
}
if l.testCleanup != nil {
l.testCleanup()
l.testCleanup = nil
}
} }
return volumesTestInput{ It("should be mountable", func() {
f: f, skipPersistenceTest(driver)
name: dInfo.Name, init()
config: &dInfo.Config, defer func() {
fsGroup: fsGroup, framework.VolumeTestCleanup(f, convertTestConfig(l.config))
resource: resource, cleanup()
fsType: pattern.FsType, }()
tests: []framework.VolumeTest{
tests := []framework.VolumeTest{
{ {
Volume: *volSource, Volume: *l.resource.volSource,
File: "index.html", File: "index.html",
// Must match content // Must match content
ExpectedContent: fmt.Sprintf("Hello from %s from namespace %s", ExpectedContent: fmt.Sprintf("Hello from %s from namespace %s",
dInfo.Name, f.Namespace.Name), dInfo.Name, f.Namespace.Name),
}, },
}, }
} config := convertTestConfig(l.config)
} framework.InjectHtml(f.ClientSet, config, tests[0].Volume, tests[0].ExpectedContent)
var fsGroup *int64
func (t *volumesTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { if dInfo.Capabilities[CapFsGroup] {
Context(getTestNameStr(t, pattern), func() { fsGroupVal := int64(1234)
var ( fsGroup = &fsGroupVal
resource genericVolumeTestResource }
input volumesTestInput framework.TestVolumeClient(f.ClientSet, config, fsGroup, pattern.FsType, tests)
needsCleanup bool
)
BeforeEach(func() {
needsCleanup = false
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(t, driver, pattern)
needsCleanup = true
// Setup test resource for driver and testpattern
resource = genericVolumeTestResource{}
resource.setupResource(driver, pattern)
// Create test input
input = createVolumesTestInput(pattern, resource)
})
AfterEach(func() {
if needsCleanup {
resource.cleanupResource(driver, pattern)
}
})
testVolumes(&input)
}) })
}
type volumesTestInput struct {
f *framework.Framework
name string
config *TestConfig
fsGroup *int64
fsType string
tests []framework.VolumeTest
resource genericVolumeTestResource
}
func testVolumes(input *volumesTestInput) {
It("should be mountable", func() {
f := input.f
cs := f.ClientSet
defer framework.VolumeTestCleanup(f, convertTestConfig(input.config))
skipPersistenceTest(input.resource.driver)
volumeTest := input.tests
config := convertTestConfig(input.config)
framework.InjectHtml(cs, config, volumeTest[0].Volume, volumeTest[0].ExpectedContent)
framework.TestVolumeClient(cs, config, input.fsGroup, input.fsType, input.tests)
})
It("should allow exec of files on the volume", func() { It("should allow exec of files on the volume", func() {
f := input.f skipExecTest(driver)
skipExecTest(input.resource.driver) init()
defer cleanup()
testScriptInPod(f, input.resource.volType, input.resource.volSource, input.resource.driver.GetDriverInfo().Config.ClientNodeSelector) testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config.ClientNodeSelector)
}) })
} }

71
test/e2e/storage/volume_provisioning.go
View File

@ -212,21 +212,22 @@ func testZonalDelayedBinding(c clientset.Interface, ns string, specifyAllowedTop
action := "creating claims with class with waitForFirstConsumer" action := "creating claims with class with waitForFirstConsumer"
suffix := "delayed" suffix := "delayed"
var topoZone string var topoZone string
class := newStorageClass(test, ns, suffix) test.Client = c
test.Class = newStorageClass(test, ns, suffix)
if specifyAllowedTopology { if specifyAllowedTopology {
action += " and allowedTopologies" action += " and allowedTopologies"
suffix += "-topo" suffix += "-topo"
topoZone = getRandomClusterZone(c) topoZone = getRandomClusterZone(c)
addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) addSingleZoneAllowedTopologyToStorageClass(c, test.Class, topoZone)
} }
By(action) By(action)
var claims []*v1.PersistentVolumeClaim var claims []*v1.PersistentVolumeClaim
for i := 0; i < pvcCount; i++ { for i := 0; i < pvcCount; i++ {
claim := newClaim(test, ns, suffix) claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name claim.Spec.StorageClassName = &test.Class.Name
claims = append(claims, claim) claims = append(claims, claim)
} }
pvs, node := testsuites.TestBindingWaitForFirstConsumerMultiPVC(test, c, claims, class, nil /* node selector */, false /* expect unschedulable */) pvs, node := test.TestBindingWaitForFirstConsumerMultiPVC(claims, nil /* node selector */, false /* expect unschedulable */)
if node == nil { if node == nil {
framework.Failf("unexpected nil node found") framework.Failf("unexpected nil node found")
} }
@ -439,10 +440,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
By("Testing " + test.Name) By("Testing " + test.Name)
suffix := fmt.Sprintf("%d", i) suffix := fmt.Sprintf("%d", i)
class := newStorageClass(test, ns, suffix) test.Client = c
claim := newClaim(test, ns, suffix) test.Class = newStorageClass(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name test.Claim = newClaim(test, ns, suffix)
testsuites.TestDynamicProvisioning(test, c, claim, class) test.Claim.Spec.StorageClassName = &test.Class.Name
test.TestDynamicProvisioning()
} }
// Run the last test with storage.k8s.io/v1beta1 on pvc // Run the last test with storage.k8s.io/v1beta1 on pvc
@ -454,9 +456,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer deleteStorageClass(c, class.Name) defer deleteStorageClass(c, class.Name)
claim := newClaim(*betaTest, ns, "beta") betaTest.Client = c
claim.Spec.StorageClassName = &(class.Name) betaTest.Class = nil
testsuites.TestDynamicProvisioning(*betaTest, c, claim, nil) betaTest.Claim = newClaim(*betaTest, ns, "beta")
betaTest.Claim.Spec.StorageClassName = &(class.Name)
(*betaTest).TestDynamicProvisioning()
} }
}) })
@ -464,6 +468,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
framework.SkipUnlessProviderIs("gce", "gke") framework.SkipUnlessProviderIs("gce", "gke")
test := testsuites.StorageClassTest{ test := testsuites.StorageClassTest{
Client: c,
Name: "HDD PD on GCE/GKE", Name: "HDD PD on GCE/GKE",
CloudProviders: []string{"gce", "gke"}, CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd", Provisioner: "kubernetes.io/gce-pd",
@ -478,12 +483,12 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{}) testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
}, },
} }
class := newStorageClass(test, ns, "reclaimpolicy") test.Class = newStorageClass(test, ns, "reclaimpolicy")
retain := v1.PersistentVolumeReclaimRetain retain := v1.PersistentVolumeReclaimRetain
class.ReclaimPolicy = &retain test.Class.ReclaimPolicy = &retain
claim := newClaim(test, ns, "reclaimpolicy") test.Claim = newClaim(test, ns, "reclaimpolicy")
claim.Spec.StorageClassName = &class.Name test.Claim.Spec.StorageClassName = &test.Class.Name
pv := testsuites.TestDynamicProvisioning(test, c, claim, class) pv := test.TestDynamicProvisioning()
By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased)) By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
framework.ExpectNoError(framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second)) framework.ExpectNoError(framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
@ -717,17 +722,18 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
By("creating a StorageClass") By("creating a StorageClass")
test := testsuites.StorageClassTest{ test := testsuites.StorageClassTest{
Client: c,
Name: "external provisioner test", Name: "external provisioner test",
Provisioner: externalPluginName, Provisioner: externalPluginName,
ClaimSize: "1500Mi", ClaimSize: "1500Mi",
ExpectedSize: "1500Mi", ExpectedSize: "1500Mi",
} }
class := newStorageClass(test, ns, "external") test.Class = newStorageClass(test, ns, "external")
claim := newClaim(test, ns, "external") test.Claim = newClaim(test, ns, "external")
claim.Spec.StorageClassName = &(class.Name) test.Claim.Spec.StorageClassName = &test.Class.Name
By("creating a claim with a external provisioning annotation") By("creating a claim with a external provisioning annotation")
testsuites.TestDynamicProvisioning(test, c, claim, class) test.TestDynamicProvisioning()
}) })
}) })
@ -737,13 +743,14 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
By("creating a claim with no annotation") By("creating a claim with no annotation")
test := testsuites.StorageClassTest{ test := testsuites.StorageClassTest{
Client: c,
Name: "default", Name: "default",
ClaimSize: "2Gi", ClaimSize: "2Gi",
ExpectedSize: "2Gi", ExpectedSize: "2Gi",
} }
claim := newClaim(test, ns, "default") test.Claim = newClaim(test, ns, "default")
testsuites.TestDynamicProvisioning(test, c, claim, nil) test.TestDynamicProvisioning()
}) })
// Modifying the default storage class can be disruptive to other tests that depend on it // Modifying the default storage class can be disruptive to other tests that depend on it
@ -816,6 +823,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
serverUrl := "http://" + pod.Status.PodIP + ":8081" serverUrl := "http://" + pod.Status.PodIP + ":8081"
By("creating a StorageClass") By("creating a StorageClass")
test := testsuites.StorageClassTest{ test := testsuites.StorageClassTest{
Client: c,
Name: "Gluster Dynamic provisioner test", Name: "Gluster Dynamic provisioner test",
Provisioner: "kubernetes.io/glusterfs", Provisioner: "kubernetes.io/glusterfs",
ClaimSize: "2Gi", ClaimSize: "2Gi",
@ -823,13 +831,13 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
Parameters: map[string]string{"resturl": serverUrl}, Parameters: map[string]string{"resturl": serverUrl},
} }
suffix := fmt.Sprintf("glusterdptest") suffix := fmt.Sprintf("glusterdptest")
class := newStorageClass(test, ns, suffix) test.Class = newStorageClass(test, ns, suffix)
By("creating a claim object with a suffix for gluster dynamic provisioner") By("creating a claim object with a suffix for gluster dynamic provisioner")
claim := newClaim(test, ns, suffix) test.Claim = newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name test.Claim.Spec.StorageClassName = &test.Class.Name
testsuites.TestDynamicProvisioning(test, c, claim, class) test.TestDynamicProvisioning()
}) })
}) })
@ -928,12 +936,13 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
} }
By("creating a claim with class with allowedTopologies set") By("creating a claim with class with allowedTopologies set")
suffix := "topology" suffix := "topology"
class := newStorageClass(test, ns, suffix) test.Client = c
test.Class = newStorageClass(test, ns, suffix)
zone := getRandomClusterZone(c) zone := getRandomClusterZone(c)
addSingleZoneAllowedTopologyToStorageClass(c, class, zone) addSingleZoneAllowedTopologyToStorageClass(c, test.Class, zone)
claim := newClaim(test, ns, suffix) test.Claim = newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name test.Claim.Spec.StorageClassName = &test.Class.Name
pv := testsuites.TestDynamicProvisioning(test, c, claim, class) pv := test.TestDynamicProvisioning()
checkZoneFromLabelAndAffinity(pv, zone, true) checkZoneFromLabelAndAffinity(pv, zone, true)
} }
}) })