mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
Merge pull request #69036 from wackxu/snapshottest
add e2e test for snapshot
This commit is contained in:
commit
8f7ccf8d4c
@ -186,6 +186,9 @@ const (
|
|||||||
// restart before test is considered failed.
|
// restart before test is considered failed.
|
||||||
RestartPodReadyAgainTimeout = 5 * time.Minute
|
RestartPodReadyAgainTimeout = 5 * time.Minute
|
||||||
|
|
||||||
|
// How long for snapshot to create snapshotContent
|
||||||
|
SnapshotCreateTimeout = 5 * time.Minute
|
||||||
|
|
||||||
// Number of objects that gc can delete in a second.
|
// Number of objects that gc can delete in a second.
|
||||||
// GC issues 2 requestes for single delete.
|
// GC issues 2 requestes for single delete.
|
||||||
gcThroughput = 10
|
gcThroughput = 10
|
||||||
|
@ -62,6 +62,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
|
|||||||
testsuites.InitVolumeModeTestSuite,
|
testsuites.InitVolumeModeTestSuite,
|
||||||
testsuites.InitSubPathTestSuite,
|
testsuites.InitSubPathTestSuite,
|
||||||
testsuites.InitProvisioningTestSuite,
|
testsuites.InitProvisioningTestSuite,
|
||||||
|
testsuites.InitSnapshottableTestSuite,
|
||||||
}
|
}
|
||||||
|
|
||||||
func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
|
func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
|
||||||
|
@ -15,6 +15,7 @@ go_library(
|
|||||||
"//staging/src/k8s.io/api/storage/v1:go_default_library",
|
"//staging/src/k8s.io/api/storage/v1:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
|
||||||
|
@ -41,6 +41,7 @@ import (
|
|||||||
|
|
||||||
. "github.com/onsi/ginkgo"
|
. "github.com/onsi/ginkgo"
|
||||||
storagev1 "k8s.io/api/storage/v1"
|
storagev1 "k8s.io/api/storage/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
||||||
@ -60,7 +61,7 @@ type hostpathCSIDriver struct {
|
|||||||
manifests []string
|
manifests []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests ...string) testsuites.TestDriver {
|
func initHostPathCSIDriver(name string, config testsuites.TestConfig, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver {
|
||||||
return &hostpathCSIDriver{
|
return &hostpathCSIDriver{
|
||||||
driverInfo: testsuites.DriverInfo{
|
driverInfo: testsuites.DriverInfo{
|
||||||
Name: name,
|
Name: name,
|
||||||
@ -69,10 +70,7 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests
|
|||||||
SupportedFsType: sets.NewString(
|
SupportedFsType: sets.NewString(
|
||||||
"", // Default fsType
|
"", // Default fsType
|
||||||
),
|
),
|
||||||
Capabilities: map[testsuites.Capability]bool{
|
Capabilities: capabilities,
|
||||||
testsuites.CapPersistence: true,
|
|
||||||
},
|
|
||||||
|
|
||||||
Config: config,
|
Config: config,
|
||||||
},
|
},
|
||||||
manifests: manifests,
|
manifests: manifests,
|
||||||
@ -81,15 +79,19 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests
|
|||||||
|
|
||||||
var _ testsuites.TestDriver = &hostpathCSIDriver{}
|
var _ testsuites.TestDriver = &hostpathCSIDriver{}
|
||||||
var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{}
|
var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{}
|
||||||
|
var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{}
|
||||||
|
|
||||||
// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
|
// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
|
||||||
func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
|
func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
|
||||||
return initHostPathCSIDriver("csi-hostpath", config,
|
return initHostPathCSIDriver("csi-hostpath", config,
|
||||||
|
map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapDataSource: true},
|
||||||
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
|
||||||
|
"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
|
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
|
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
|
||||||
|
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
|
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
|
||||||
)
|
)
|
||||||
@ -111,6 +113,15 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *stor
|
|||||||
return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
|
return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *hostpathCSIDriver) GetSnapshotClass() *unstructured.Unstructured {
|
||||||
|
snapshotter := testsuites.GetUniqueDriverName(h)
|
||||||
|
parameters := map[string]string{}
|
||||||
|
ns := h.driverInfo.Config.Framework.Namespace.Name
|
||||||
|
suffix := fmt.Sprintf("%s-vsc", snapshotter)
|
||||||
|
|
||||||
|
return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
|
||||||
|
}
|
||||||
|
|
||||||
func (h *hostpathCSIDriver) GetClaimSize() string {
|
func (h *hostpathCSIDriver) GetClaimSize() string {
|
||||||
return "5Gi"
|
return "5Gi"
|
||||||
}
|
}
|
||||||
@ -133,6 +144,7 @@ func (h *hostpathCSIDriver) CreateDriver() {
|
|||||||
DriverContainerName: "hostpath",
|
DriverContainerName: "hostpath",
|
||||||
DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName},
|
DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName},
|
||||||
ProvisionerContainerName: "csi-provisioner",
|
ProvisionerContainerName: "csi-provisioner",
|
||||||
|
SnapshotterContainerName: "csi-snapshotter",
|
||||||
NodeName: nodeName,
|
NodeName: nodeName,
|
||||||
}
|
}
|
||||||
cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error {
|
cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error {
|
||||||
@ -247,6 +259,7 @@ func (m *mockCSIDriver) CleanupDriver() {
|
|||||||
// InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests.
|
// InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests.
|
||||||
func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
|
func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
|
||||||
return initHostPathCSIDriver("csi-hostpath-v0", config,
|
return initHostPathCSIDriver("csi-hostpath-v0", config,
|
||||||
|
map[testsuites.Capability]bool{testsuites.CapPersistence: true},
|
||||||
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
|
||||||
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
|
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
|
||||||
|
@ -17,7 +17,7 @@ limitations under the License.
|
|||||||
package testpatterns
|
package testpatterns
|
||||||
|
|
||||||
import (
|
import (
|
||||||
v1 "k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -45,6 +45,14 @@ var (
|
|||||||
DynamicPV TestVolType = "DynamicPV"
|
DynamicPV TestVolType = "DynamicPV"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TestSnapshotType represents a snapshot type to be tested in a TestSuite
|
||||||
|
type TestSnapshotType string
|
||||||
|
|
||||||
|
var (
|
||||||
|
// DynamicCreatedSnapshot represents a snapshot type for dynamic created snapshot
|
||||||
|
DynamicCreatedSnapshot TestSnapshotType = "DynamicSnapshot"
|
||||||
|
)
|
||||||
|
|
||||||
// TestPattern represents a combination of parameters to be tested in a TestSuite
|
// TestPattern represents a combination of parameters to be tested in a TestSuite
|
||||||
type TestPattern struct {
|
type TestPattern struct {
|
||||||
Name string // Name of TestPattern
|
Name string // Name of TestPattern
|
||||||
@ -52,6 +60,7 @@ type TestPattern struct {
|
|||||||
VolType TestVolType // Volume type of the volume
|
VolType TestVolType // Volume type of the volume
|
||||||
FsType string // Fstype of the volume
|
FsType string // Fstype of the volume
|
||||||
VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume
|
VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume
|
||||||
|
SnapshotType TestSnapshotType // Snapshot type of the snapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -165,4 +174,12 @@ var (
|
|||||||
VolType: DynamicPV,
|
VolType: DynamicPV,
|
||||||
VolMode: v1.PersistentVolumeBlock,
|
VolMode: v1.PersistentVolumeBlock,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Definitions for snapshot case
|
||||||
|
|
||||||
|
// DynamicSnapshot is TestPattern for "Dynamic snapshot"
|
||||||
|
DynamicSnapshot = TestPattern{
|
||||||
|
Name: "Dynamic Snapshot",
|
||||||
|
SnapshotType: DynamicCreatedSnapshot,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
@ -6,6 +6,7 @@ go_library(
|
|||||||
"base.go",
|
"base.go",
|
||||||
"driveroperations.go",
|
"driveroperations.go",
|
||||||
"provisioning.go",
|
"provisioning.go",
|
||||||
|
"snapshottable.go",
|
||||||
"subpath.go",
|
"subpath.go",
|
||||||
"testdriver.go",
|
"testdriver.go",
|
||||||
"volume_io.go",
|
"volume_io.go",
|
||||||
@ -20,11 +21,14 @@ go_library(
|
|||||||
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
|
||||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||||
"//test/e2e/framework:go_default_library",
|
"//test/e2e/framework:go_default_library",
|
||||||
"//test/e2e/storage/testpatterns:go_default_library",
|
"//test/e2e/storage/testpatterns:go_default_library",
|
||||||
|
@ -22,11 +22,13 @@ import (
|
|||||||
|
|
||||||
. "github.com/onsi/ginkgo"
|
. "github.com/onsi/ginkgo"
|
||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
storagev1 "k8s.io/api/storage/v1"
|
storagev1 "k8s.io/api/storage/v1"
|
||||||
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
@ -79,15 +81,31 @@ func RunTestSuite(f *framework.Framework, driver TestDriver, tsInits []func() Te
|
|||||||
// skipUnsupportedTest will skip tests if the combination of driver, testsuite, and testpattern
|
// skipUnsupportedTest will skip tests if the combination of driver, testsuite, and testpattern
|
||||||
// is not suitable to be tested.
|
// is not suitable to be tested.
|
||||||
// Whether it needs to be skipped is checked by following steps:
|
// Whether it needs to be skipped is checked by following steps:
|
||||||
// 1. Check if Whether volType is supported by driver from its interface
|
// 1. Check if Whether SnapshotType is supported by driver from its interface
|
||||||
// 2. Check if fsType is supported
|
// 2. Check if Whether volType is supported by driver from its interface
|
||||||
// 3. Check with driver specific logic
|
// 3. Check if fsType is supported
|
||||||
// 4. Check with testSuite specific logic
|
// 4. Check with driver specific logic
|
||||||
|
// 5. Check with testSuite specific logic
|
||||||
func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) {
|
func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) {
|
||||||
dInfo := driver.GetDriverInfo()
|
dInfo := driver.GetDriverInfo()
|
||||||
|
|
||||||
// 1. Check if Whether volType is supported by driver from its interface
|
|
||||||
var isSupported bool
|
var isSupported bool
|
||||||
|
|
||||||
|
// 1. Check if Whether SnapshotType is supported by driver from its interface
|
||||||
|
// if isSupported, so it must be a snapshot test case, we just return.
|
||||||
|
if len(pattern.SnapshotType) > 0 {
|
||||||
|
switch pattern.SnapshotType {
|
||||||
|
case testpatterns.DynamicCreatedSnapshot:
|
||||||
|
_, isSupported = driver.(SnapshottableTestDriver)
|
||||||
|
default:
|
||||||
|
isSupported = false
|
||||||
|
}
|
||||||
|
if !isSupported {
|
||||||
|
framework.Skipf("Driver %s doesn't support snapshot type %v -- skipping", dInfo.Name, pattern.SnapshotType)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Check if Whether volType is supported by driver from its interface
|
||||||
switch pattern.VolType {
|
switch pattern.VolType {
|
||||||
case testpatterns.InlineVolume:
|
case testpatterns.InlineVolume:
|
||||||
_, isSupported = driver.(InlineVolumeTestDriver)
|
_, isSupported = driver.(InlineVolumeTestDriver)
|
||||||
@ -103,7 +121,7 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
|
|||||||
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
|
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Check if fsType is supported
|
// 3. Check if fsType is supported
|
||||||
if !dInfo.SupportedFsType.Has(pattern.FsType) {
|
if !dInfo.SupportedFsType.Has(pattern.FsType) {
|
||||||
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.FsType)
|
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.FsType)
|
||||||
}
|
}
|
||||||
@ -111,10 +129,10 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
|
|||||||
framework.Skipf("Distro doesn't support xfs -- skipping")
|
framework.Skipf("Distro doesn't support xfs -- skipping")
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Check with driver specific logic
|
// 4. Check with driver specific logic
|
||||||
driver.SkipUnsupportedTest(pattern)
|
driver.SkipUnsupportedTest(pattern)
|
||||||
|
|
||||||
// 4. Check with testSuite specific logic
|
// 5. Check with testSuite specific logic
|
||||||
suite.skipUnsupportedTest(pattern, driver)
|
suite.skipUnsupportedTest(pattern, driver)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -349,3 +367,25 @@ func convertTestConfig(in *TestConfig) framework.VolumeTestConfig {
|
|||||||
NodeSelector: in.ClientNodeSelector,
|
NodeSelector: in.ClientNodeSelector,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.Unstructured {
|
||||||
|
snapshot := &unstructured.Unstructured{
|
||||||
|
Object: map[string]interface{}{
|
||||||
|
"kind": "VolumeSnapshot",
|
||||||
|
"apiVersion": snapshotAPIVersion,
|
||||||
|
"metadata": map[string]interface{}{
|
||||||
|
"generateName": "snapshot-",
|
||||||
|
"namespace": ns,
|
||||||
|
},
|
||||||
|
"spec": map[string]interface{}{
|
||||||
|
"snapshotClassName": snapshotClassName,
|
||||||
|
"source": map[string]interface{}{
|
||||||
|
"name": claimName,
|
||||||
|
"kind": "PersistentVolumeClaim",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
@ -21,6 +21,7 @@ import (
|
|||||||
|
|
||||||
storagev1 "k8s.io/api/storage/v1"
|
storagev1 "k8s.io/api/storage/v1"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
||||||
)
|
)
|
||||||
@ -95,6 +96,30 @@ func GetStorageClass(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetSnapshotClass constructs a new SnapshotClass instance
|
||||||
|
// with a unique name that is based on namespace + suffix.
|
||||||
|
func GetSnapshotClass(
|
||||||
|
snapshotter string,
|
||||||
|
parameters map[string]string,
|
||||||
|
ns string,
|
||||||
|
suffix string,
|
||||||
|
) *unstructured.Unstructured {
|
||||||
|
snapshotClass := &unstructured.Unstructured{
|
||||||
|
Object: map[string]interface{}{
|
||||||
|
"kind": "VolumeSnapshotClass",
|
||||||
|
"apiVersion": snapshotAPIVersion,
|
||||||
|
"metadata": map[string]interface{}{
|
||||||
|
// Name must be unique, so let's base it on namespace name
|
||||||
|
"name": ns + "-" + suffix,
|
||||||
|
},
|
||||||
|
"snapshotter": snapshotter,
|
||||||
|
"parameters": parameters,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshotClass
|
||||||
|
}
|
||||||
|
|
||||||
// GetUniqueDriverName returns unique driver name that can be used parallelly in tests
|
// GetUniqueDriverName returns unique driver name that can be used parallelly in tests
|
||||||
func GetUniqueDriverName(driver TestDriver) string {
|
func GetUniqueDriverName(driver TestDriver) string {
|
||||||
return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName)
|
return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName)
|
||||||
|
@ -23,13 +23,14 @@ import (
|
|||||||
. "github.com/onsi/ginkgo"
|
. "github.com/onsi/ginkgo"
|
||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
|
|
||||||
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
storage "k8s.io/api/storage/v1"
|
storage "k8s.io/api/storage/v1"
|
||||||
|
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/apimachinery/pkg/labels"
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
|
"k8s.io/client-go/dynamic"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
||||||
@ -90,8 +91,10 @@ func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPat
|
|||||||
ExpectedSize: resource.claimSize,
|
ExpectedSize: resource.claimSize,
|
||||||
},
|
},
|
||||||
cs: driver.GetDriverInfo().Config.Framework.ClientSet,
|
cs: driver.GetDriverInfo().Config.Framework.ClientSet,
|
||||||
|
dc: driver.GetDriverInfo().Config.Framework.DynamicClient,
|
||||||
pvc: resource.pvc,
|
pvc: resource.pvc,
|
||||||
sc: resource.sc,
|
sc: resource.sc,
|
||||||
|
vsc: resource.vsc,
|
||||||
dInfo: driver.GetDriverInfo(),
|
dInfo: driver.GetDriverInfo(),
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -139,6 +142,8 @@ type provisioningTestResource struct {
|
|||||||
claimSize string
|
claimSize string
|
||||||
sc *storage.StorageClass
|
sc *storage.StorageClass
|
||||||
pvc *v1.PersistentVolumeClaim
|
pvc *v1.PersistentVolumeClaim
|
||||||
|
// follow parameter is used to test provision volume from snapshot
|
||||||
|
vsc *unstructured.Unstructured
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ TestResource = &provisioningTestResource{}
|
var _ TestResource = &provisioningTestResource{}
|
||||||
@ -157,6 +162,9 @@ func (p *provisioningTestResource) setupResource(driver TestDriver, pattern test
|
|||||||
p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
|
p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
|
||||||
p.pvc.Spec.StorageClassName = &p.sc.Name
|
p.pvc.Spec.StorageClassName = &p.sc.Name
|
||||||
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)
|
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)
|
||||||
|
if sDriver, ok := driver.(SnapshottableTestDriver); ok {
|
||||||
|
p.vsc = sDriver.GetSnapshotClass()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType)
|
framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType)
|
||||||
@ -169,8 +177,10 @@ func (p *provisioningTestResource) cleanupResource(driver TestDriver, pattern te
|
|||||||
type provisioningTestInput struct {
|
type provisioningTestInput struct {
|
||||||
testCase StorageClassTest
|
testCase StorageClassTest
|
||||||
cs clientset.Interface
|
cs clientset.Interface
|
||||||
|
dc dynamic.Interface
|
||||||
pvc *v1.PersistentVolumeClaim
|
pvc *v1.PersistentVolumeClaim
|
||||||
sc *storage.StorageClass
|
sc *storage.StorageClass
|
||||||
|
vsc *unstructured.Unstructured
|
||||||
dInfo *DriverInfo
|
dInfo *DriverInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -198,6 +208,19 @@ func testProvisioning(input *provisioningTestInput) {
|
|||||||
input.pvc.Spec.VolumeMode = &block
|
input.pvc.Spec.VolumeMode = &block
|
||||||
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
|
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
It("should provision storage with snapshot data source [Feature:VolumeSnapshotDataSource]", func() {
|
||||||
|
if !input.dInfo.Capabilities[CapDataSource] {
|
||||||
|
framework.Skipf("Driver %q does not support populate data from snapshot - skipping", input.dInfo.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
input.testCase.SkipWriteReadCheck = true
|
||||||
|
dataSource, cleanupFunc := prepareDataSourceForProvisioning(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
|
||||||
|
defer cleanupFunc()
|
||||||
|
|
||||||
|
input.pvc.Spec.DataSource = dataSource
|
||||||
|
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass
|
// TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass
|
||||||
@ -205,7 +228,11 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
|
|||||||
var err error
|
var err error
|
||||||
if class != nil {
|
if class != nil {
|
||||||
By("creating a StorageClass " + class.Name)
|
By("creating a StorageClass " + class.Name)
|
||||||
class, err = client.StorageV1().StorageClasses().Create(class)
|
_, err = client.StorageV1().StorageClasses().Create(class)
|
||||||
|
// The "should provision storage with snapshot data source" test already has created the class.
|
||||||
|
// TODO: make class creation optional and remove the IsAlreadyExists exception
|
||||||
|
Expect(err == nil || apierrs.IsAlreadyExists(err)).To(Equal(true))
|
||||||
|
class, err = client.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
|
||||||
Expect(err).NotTo(HaveOccurred())
|
Expect(err).NotTo(HaveOccurred())
|
||||||
defer func() {
|
defer func() {
|
||||||
framework.Logf("deleting storage class %s", class.Name)
|
framework.Logf("deleting storage class %s", class.Name)
|
||||||
@ -268,6 +295,12 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
|
|||||||
Expect(err).NotTo(HaveOccurred())
|
Expect(err).NotTo(HaveOccurred())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if claim.Spec.DataSource != nil {
|
||||||
|
By("checking the created volume whether has the pre-populated data")
|
||||||
|
command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
|
||||||
|
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable)
|
||||||
|
}
|
||||||
|
|
||||||
if !t.SkipWriteReadCheck {
|
if !t.SkipWriteReadCheck {
|
||||||
// We start two pods:
|
// We start two pods:
|
||||||
// - The first writes 'hello word' to the /mnt/test (= the volume).
|
// - The first writes 'hello word' to the /mnt/test (= the volume).
|
||||||
@ -288,6 +321,7 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
|
|||||||
By("checking the created volume is readable and retains data")
|
By("checking the created volume is readable and retains data")
|
||||||
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data", t.NodeSelector, t.ExpectUnschedulable)
|
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data", t.NodeSelector, t.ExpectUnschedulable)
|
||||||
}
|
}
|
||||||
|
|
||||||
By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name))
|
By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name))
|
||||||
framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))
|
framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))
|
||||||
|
|
||||||
@ -466,3 +500,76 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl
|
|||||||
Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
|
Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func prepareDataSourceForProvisioning(
|
||||||
|
t StorageClassTest,
|
||||||
|
client clientset.Interface,
|
||||||
|
dynamicClient dynamic.Interface,
|
||||||
|
initClaim *v1.PersistentVolumeClaim,
|
||||||
|
class *storage.StorageClass,
|
||||||
|
snapshotClass *unstructured.Unstructured,
|
||||||
|
) (*v1.TypedLocalObjectReference, func()) {
|
||||||
|
var err error
|
||||||
|
if class != nil {
|
||||||
|
By("[Initialize dataSource]creating a StorageClass " + class.Name)
|
||||||
|
_, err = client.StorageV1().StorageClasses().Create(class)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
}
|
||||||
|
|
||||||
|
By("[Initialize dataSource]creating a initClaim")
|
||||||
|
updatedClaim, err := client.CoreV1().PersistentVolumeClaims(initClaim.Namespace).Create(initClaim)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, updatedClaim.Namespace, updatedClaim.Name, framework.Poll, framework.ClaimProvisionTimeout)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
By("[Initialize dataSource]checking the initClaim")
|
||||||
|
// Get new copy of the initClaim
|
||||||
|
_, err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Get(updatedClaim.Name, metav1.GetOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
// write namespace to the /mnt/test (= the volume).
|
||||||
|
By("[Initialize dataSource]write data to volume")
|
||||||
|
command := fmt.Sprintf("echo '%s' > /mnt/test/initialData", updatedClaim.GetNamespace())
|
||||||
|
runInPodWithVolume(client, updatedClaim.Namespace, updatedClaim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable)
|
||||||
|
|
||||||
|
By("[Initialize dataSource]creating a SnapshotClass")
|
||||||
|
snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
|
||||||
|
|
||||||
|
By("[Initialize dataSource]creating a snapshot")
|
||||||
|
snapshot := getSnapshot(updatedClaim.Name, updatedClaim.Namespace, snapshotClass.GetName())
|
||||||
|
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Create(snapshot, metav1.CreateOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
By("[Initialize dataSource]checking the snapshot")
|
||||||
|
// Get new copy of the snapshot
|
||||||
|
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
group := "snapshot.storage.k8s.io"
|
||||||
|
dataSourceRef := &v1.TypedLocalObjectReference{
|
||||||
|
APIGroup: &group,
|
||||||
|
Kind: "VolumeSnapshot",
|
||||||
|
Name: snapshot.GetName(),
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupFunc := func() {
|
||||||
|
framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
|
||||||
|
err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Delete(snapshot.GetName(), nil)
|
||||||
|
if err != nil && !apierrs.IsNotFound(err) {
|
||||||
|
framework.Failf("Error deleting snapshot %q. Error: %v", snapshot.GetName(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
framework.Logf("deleting initClaim %q/%q", updatedClaim.Namespace, updatedClaim.Name)
|
||||||
|
err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Delete(updatedClaim.Name, nil)
|
||||||
|
if err != nil && !apierrs.IsNotFound(err) {
|
||||||
|
framework.Failf("Error deleting initClaim %q. Error: %v", updatedClaim.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
|
||||||
|
framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
return dataSourceRef, cleanupFunc
|
||||||
|
}
|
||||||
|
323
test/e2e/storage/testsuites/snapshottable.go
Normal file
323
test/e2e/storage/testsuites/snapshottable.go
Normal file
@ -0,0 +1,323 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package testsuites
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
. "github.com/onsi/ginkgo"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
|
||||||
|
"k8s.io/api/core/v1"
|
||||||
|
storage "k8s.io/api/storage/v1"
|
||||||
|
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
"k8s.io/client-go/dynamic"
|
||||||
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
|
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
||||||
|
)
|
||||||
|
|
||||||
|
// snapshot CRD api group
|
||||||
|
const snapshotGroup = "snapshot.storage.k8s.io"
|
||||||
|
|
||||||
|
// snapshot CRD api version
|
||||||
|
const snapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1"
|
||||||
|
|
||||||
|
var (
|
||||||
|
snapshotGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshots"}
|
||||||
|
snapshotClassGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotclasses"}
|
||||||
|
snapshotContentGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotcontents"}
|
||||||
|
)
|
||||||
|
|
||||||
|
type SnapshotClassTest struct {
|
||||||
|
Name string
|
||||||
|
CloudProviders []string
|
||||||
|
Snapshotter string
|
||||||
|
Parameters map[string]string
|
||||||
|
NodeName string
|
||||||
|
NodeSelector map[string]string // NodeSelector for the pod
|
||||||
|
SnapshotContentCheck func(snapshotContent *unstructured.Unstructured) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type snapshottableTestSuite struct {
|
||||||
|
tsInfo TestSuiteInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ TestSuite = &snapshottableTestSuite{}
|
||||||
|
|
||||||
|
// InitSnapshottableTestSuite returns snapshottableTestSuite that implements TestSuite interface
|
||||||
|
func InitSnapshottableTestSuite() TestSuite {
|
||||||
|
return &snapshottableTestSuite{
|
||||||
|
tsInfo: TestSuiteInfo{
|
||||||
|
name: "snapshottable",
|
||||||
|
testPatterns: []testpatterns.TestPattern{
|
||||||
|
testpatterns.DynamicSnapshot,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *snapshottableTestSuite) getTestSuiteInfo() TestSuiteInfo {
|
||||||
|
return s.tsInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *snapshottableTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) {
|
||||||
|
}
|
||||||
|
|
||||||
|
func createSnapshottableTestInput(driver TestDriver, pattern testpatterns.TestPattern) (snapshottableTestResource, snapshottableTestInput) {
|
||||||
|
// Setup test resource for driver and testpattern
|
||||||
|
resource := snapshottableTestResource{}
|
||||||
|
resource.setupResource(driver, pattern)
|
||||||
|
|
||||||
|
input := snapshottableTestInput{
|
||||||
|
testCase: SnapshotClassTest{},
|
||||||
|
cs: driver.GetDriverInfo().Config.Framework.ClientSet,
|
||||||
|
dc: driver.GetDriverInfo().Config.Framework.DynamicClient,
|
||||||
|
pvc: resource.pvc,
|
||||||
|
sc: resource.sc,
|
||||||
|
vsc: resource.vsc,
|
||||||
|
dInfo: driver.GetDriverInfo(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if driver.GetDriverInfo().Config.ClientNodeName != "" {
|
||||||
|
input.testCase.NodeName = driver.GetDriverInfo().Config.ClientNodeName
|
||||||
|
}
|
||||||
|
|
||||||
|
return resource, input
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *snapshottableTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) {
|
||||||
|
Context(getTestNameStr(s, pattern), func() {
|
||||||
|
var (
|
||||||
|
resource snapshottableTestResource
|
||||||
|
input snapshottableTestInput
|
||||||
|
needsCleanup bool
|
||||||
|
)
|
||||||
|
|
||||||
|
BeforeEach(func() {
|
||||||
|
needsCleanup = false
|
||||||
|
// Skip unsupported tests to avoid unnecessary resource initialization
|
||||||
|
skipUnsupportedTest(s, driver, pattern)
|
||||||
|
needsCleanup = true
|
||||||
|
|
||||||
|
// Create test input
|
||||||
|
resource, input = createSnapshottableTestInput(driver, pattern)
|
||||||
|
})
|
||||||
|
|
||||||
|
AfterEach(func() {
|
||||||
|
if needsCleanup {
|
||||||
|
resource.cleanupResource(driver, pattern)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Ginkgo's "Global Shared Behaviors" require arguments for a shared function
|
||||||
|
// to be a single struct and to be passed as a pointer.
|
||||||
|
// Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details.
|
||||||
|
testSnapshot(&input)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type snapshottableTestResource struct {
|
||||||
|
driver TestDriver
|
||||||
|
claimSize string
|
||||||
|
|
||||||
|
sc *storage.StorageClass
|
||||||
|
pvc *v1.PersistentVolumeClaim
|
||||||
|
// volume snapshot class
|
||||||
|
vsc *unstructured.Unstructured
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ TestResource = &snapshottableTestResource{}
|
||||||
|
|
||||||
|
func (s *snapshottableTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) {
|
||||||
|
// Setup snapshottableTest resource
|
||||||
|
switch pattern.SnapshotType {
|
||||||
|
case testpatterns.DynamicCreatedSnapshot:
|
||||||
|
if dDriver, ok := driver.(DynamicPVTestDriver); ok {
|
||||||
|
s.sc = dDriver.GetDynamicProvisionStorageClass("")
|
||||||
|
if s.sc == nil {
|
||||||
|
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
|
||||||
|
}
|
||||||
|
s.driver = driver
|
||||||
|
s.claimSize = dDriver.GetClaimSize()
|
||||||
|
s.pvc = getClaim(s.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
|
||||||
|
s.pvc.Spec.StorageClassName = &s.sc.Name
|
||||||
|
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", s.sc, s.pvc)
|
||||||
|
|
||||||
|
if sDriver, ok := driver.(SnapshottableTestDriver); ok {
|
||||||
|
s.vsc = sDriver.GetSnapshotClass()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
framework.Failf("Dynamic Snapshot test doesn't support: %s", pattern.SnapshotType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *snapshottableTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) {
|
||||||
|
}
|
||||||
|
|
||||||
|
type snapshottableTestInput struct {
|
||||||
|
testCase SnapshotClassTest
|
||||||
|
cs clientset.Interface
|
||||||
|
dc dynamic.Interface
|
||||||
|
pvc *v1.PersistentVolumeClaim
|
||||||
|
sc *storage.StorageClass
|
||||||
|
// volume snapshot class
|
||||||
|
vsc *unstructured.Unstructured
|
||||||
|
dInfo *DriverInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func testSnapshot(input *snapshottableTestInput) {
|
||||||
|
It("should create snapshot with defaults", func() {
|
||||||
|
if input.dInfo.Name == "csi-hostpath-v0" {
|
||||||
|
framework.Skipf("skip test when using driver csi-hostpath-v0 - skipping")
|
||||||
|
}
|
||||||
|
TestCreateSnapshot(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCreateSnapshot tests dynamic creating snapshot with specified SnapshotClassTest and snapshotClass
|
||||||
|
func TestCreateSnapshot(
|
||||||
|
t SnapshotClassTest,
|
||||||
|
client clientset.Interface,
|
||||||
|
dynamicClient dynamic.Interface,
|
||||||
|
claim *v1.PersistentVolumeClaim,
|
||||||
|
class *storage.StorageClass,
|
||||||
|
snapshotClass *unstructured.Unstructured,
|
||||||
|
) *unstructured.Unstructured {
|
||||||
|
var err error
|
||||||
|
if class != nil {
|
||||||
|
By("creating a StorageClass " + class.Name)
|
||||||
|
class, err = client.StorageV1().StorageClasses().Create(class)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
defer func() {
|
||||||
|
framework.Logf("deleting storage class %s", class.Name)
|
||||||
|
framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil))
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
By("creating a claim")
|
||||||
|
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
defer func() {
|
||||||
|
framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name)
|
||||||
|
// typically this claim has already been deleted
|
||||||
|
err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)
|
||||||
|
if err != nil && !apierrs.IsNotFound(err) {
|
||||||
|
framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
By("checking the claim")
|
||||||
|
// Get new copy of the claim
|
||||||
|
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
// Get the bound PV
|
||||||
|
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
By("creating a SnapshotClass")
|
||||||
|
snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
defer func() {
|
||||||
|
framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
|
||||||
|
framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
|
||||||
|
}()
|
||||||
|
|
||||||
|
By("creating a snapshot")
|
||||||
|
snapshot := getSnapshot(claim.Name, claim.Namespace, snapshotClass.GetName())
|
||||||
|
|
||||||
|
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Create(snapshot, metav1.CreateOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
defer func() {
|
||||||
|
framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
|
||||||
|
// typically this snapshot has already been deleted
|
||||||
|
err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Delete(snapshot.GetName(), nil)
|
||||||
|
if err != nil && !apierrs.IsNotFound(err) {
|
||||||
|
framework.Failf("Error deleting snapshot %q. Error: %v", claim.Name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
err = WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
By("checking the snapshot")
|
||||||
|
// Get new copy of the snapshot
|
||||||
|
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
// Get the bound snapshotContent
|
||||||
|
snapshotSpec := snapshot.Object["spec"].(map[string]interface{})
|
||||||
|
snapshotContentName := snapshotSpec["snapshotContentName"].(string)
|
||||||
|
snapshotContent, err := dynamicClient.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{})
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
|
||||||
|
snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{})
|
||||||
|
volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
|
||||||
|
persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{})
|
||||||
|
|
||||||
|
// Check SnapshotContent properties
|
||||||
|
By("checking the SnapshotContent")
|
||||||
|
Expect(snapshotContentSpec["snapshotClassName"]).To(Equal(snapshotClass.GetName()))
|
||||||
|
Expect(volumeSnapshotRef["name"]).To(Equal(snapshot.GetName()))
|
||||||
|
Expect(volumeSnapshotRef["namespace"]).To(Equal(snapshot.GetNamespace()))
|
||||||
|
Expect(persistentVolumeRef["name"]).To(Equal(pv.Name))
|
||||||
|
|
||||||
|
// Run the checker
|
||||||
|
if t.SnapshotContentCheck != nil {
|
||||||
|
err = t.SnapshotContentCheck(snapshotContent)
|
||||||
|
Expect(err).NotTo(HaveOccurred())
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshotContent
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first.
|
||||||
|
func WaitForSnapshotReady(c dynamic.Interface, ns string, snapshotName string, Poll, timeout time.Duration) error {
|
||||||
|
framework.Logf("Waiting up to %v for VolumeSnapshot %s to become ready", timeout, snapshotName)
|
||||||
|
for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
|
||||||
|
snapshot, err := c.Resource(snapshotGVR).Namespace(ns).Get(snapshotName, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
framework.Logf("Failed to get claim %q, retrying in %v. Error: %v", snapshotName, Poll, err)
|
||||||
|
continue
|
||||||
|
} else {
|
||||||
|
status := snapshot.Object["status"]
|
||||||
|
if status == nil {
|
||||||
|
framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value := status.(map[string]interface{})
|
||||||
|
if value["readyToUse"] == true {
|
||||||
|
framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start))
|
||||||
|
return nil
|
||||||
|
} else if value["ready"] == true {
|
||||||
|
framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start))
|
||||||
|
return nil
|
||||||
|
} else {
|
||||||
|
framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("VolumeSnapshot %s is not ready within %v", snapshotName, timeout)
|
||||||
|
}
|
@ -19,6 +19,7 @@ package testsuites
|
|||||||
import (
|
import (
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
storagev1 "k8s.io/api/storage/v1"
|
storagev1 "k8s.io/api/storage/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
||||||
@ -79,6 +80,14 @@ type DynamicPVTestDriver interface {
|
|||||||
GetClaimSize() string
|
GetClaimSize() string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SnapshottableTestDriver represents an interface for a TestDriver that supports DynamicSnapshot
|
||||||
|
type SnapshottableTestDriver interface {
|
||||||
|
TestDriver
|
||||||
|
// GetSnapshotClass returns a SnapshotClass to create snapshot.
|
||||||
|
// It will return nil, if the TestDriver doesn't support it.
|
||||||
|
GetSnapshotClass() *unstructured.Unstructured
|
||||||
|
}
|
||||||
|
|
||||||
// Capability represents a feature that a volume plugin supports
|
// Capability represents a feature that a volume plugin supports
|
||||||
type Capability string
|
type Capability string
|
||||||
|
|
||||||
@ -87,6 +96,7 @@ const (
|
|||||||
CapBlock Capability = "block" // raw block mode
|
CapBlock Capability = "block" // raw block mode
|
||||||
CapFsGroup Capability = "fsGroup" // volume ownership via fsGroup
|
CapFsGroup Capability = "fsGroup" // volume ownership via fsGroup
|
||||||
CapExec Capability = "exec" // exec a file in the volume
|
CapExec Capability = "exec" // exec a file in the volume
|
||||||
|
CapDataSource Capability = "dataSource" // support populate data from snapshot
|
||||||
)
|
)
|
||||||
|
|
||||||
// DriverInfo represents a combination of parameters to be used in implementation of TestDriver
|
// DriverInfo represents a combination of parameters to be used in implementation of TestDriver
|
||||||
|
@ -93,6 +93,10 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf
|
|||||||
// Driver name is expected to be the same
|
// Driver name is expected to be the same
|
||||||
// as the provisioner here.
|
// as the provisioner here.
|
||||||
container.Args = append(container.Args, "--provisioner="+o.NewDriverName)
|
container.Args = append(container.Args, "--provisioner="+o.NewDriverName)
|
||||||
|
case o.SnapshotterContainerName:
|
||||||
|
// Driver name is expected to be the same
|
||||||
|
// as the snapshotter here.
|
||||||
|
container.Args = append(container.Args, "--snapshotter="+o.NewDriverName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -145,6 +149,10 @@ type PatchCSIOptions struct {
|
|||||||
// If non-empty, --provisioner with new name will be appended
|
// If non-empty, --provisioner with new name will be appended
|
||||||
// to the argument list.
|
// to the argument list.
|
||||||
ProvisionerContainerName string
|
ProvisionerContainerName string
|
||||||
|
// The name of the container which has the snapshotter binary.
|
||||||
|
// If non-empty, --snapshotter with new name will be appended
|
||||||
|
// to the argument list.
|
||||||
|
SnapshotterContainerName string
|
||||||
// If non-empty, all pods are forced to run on this node.
|
// If non-empty, all pods are forced to run on this node.
|
||||||
NodeName string
|
NodeName string
|
||||||
}
|
}
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
# Replaced by individual roles for external-attacher and external-provisioner:
|
# Replaced by individual roles for external-attacher, external-provisioner and external-snapshotter:
|
||||||
# - https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
|
# - https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
|
||||||
# - https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
|
# - https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
|
||||||
|
# - https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml
|
@ -0,0 +1 @@
|
|||||||
|
The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml
|
@ -0,0 +1,65 @@
|
|||||||
|
# Together with the RBAC file for external-provisioner, this YAML file
|
||||||
|
# contains all RBAC objects that are necessary to run external CSI
|
||||||
|
# snapshotter.
|
||||||
|
#
|
||||||
|
# In production, each CSI driver deployment has to be customized:
|
||||||
|
# - to avoid conflicts, use non-default namespace and different names
|
||||||
|
# for non-namespaced entities like the ClusterRole
|
||||||
|
# - optionally rename the non-namespaced ClusterRole if there
|
||||||
|
# are conflicts with other deployments
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: csi-snapshotter
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
# rename if there are conflicts
|
||||||
|
name: external-snapshotter-runner
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["persistentvolumes"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["persistentvolumeclaims"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["storage.k8s.io"]
|
||||||
|
resources: ["storageclasses"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["events"]
|
||||||
|
verbs: ["list", "watch", "create", "update", "patch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["secrets"]
|
||||||
|
verbs: ["get", "list"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotclasses"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotcontents"]
|
||||||
|
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshots"]
|
||||||
|
verbs: ["get", "list", "watch", "update"]
|
||||||
|
- apiGroups: ["apiextensions.k8s.io"]
|
||||||
|
resources: ["customresourcedefinitions"]
|
||||||
|
verbs: ["create", "list", "watch", "delete"]
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: csi-snapshotter-role
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: csi-snapshotter
|
||||||
|
# replace with non-default namespace name
|
||||||
|
namespace: default
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
# change the name also here if the ClusterRole gets renamed
|
||||||
|
name: external-snapshotter-runner
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
@ -1,5 +1,5 @@
|
|||||||
A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example,
|
A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example,
|
||||||
with some modifications:
|
with some modifications:
|
||||||
- serviceAccountName is used instead of the deprecated serviceAccount
|
- serviceAccountName is used instead of the deprecated serviceAccount
|
||||||
- the RBAC roles from driver-registrar, external-attacher and external-provisioner
|
- the RBAC roles from driver-registrar, external-attacher, external-provisioner
|
||||||
are used
|
and external-snapshotter are used
|
||||||
|
@ -0,0 +1,48 @@
|
|||||||
|
kind: Service
|
||||||
|
apiVersion: v1
|
||||||
|
metadata:
|
||||||
|
name: csi-snapshotter
|
||||||
|
labels:
|
||||||
|
app: csi-snapshotter
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: csi-snapshotter
|
||||||
|
ports:
|
||||||
|
- name: dummy
|
||||||
|
port: 12345
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: StatefulSet
|
||||||
|
apiVersion: apps/v1
|
||||||
|
metadata:
|
||||||
|
name: csi-snapshotter
|
||||||
|
spec:
|
||||||
|
serviceName: "csi-snapshotter"
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: csi-snapshotter
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: csi-snapshotter
|
||||||
|
spec:
|
||||||
|
serviceAccount: csi-snapshotter
|
||||||
|
containers:
|
||||||
|
- name: csi-snapshotter
|
||||||
|
image: quay.io/k8scsi/csi-snapshotter:v1.0.1
|
||||||
|
args:
|
||||||
|
- "--csi-address=$(ADDRESS)"
|
||||||
|
- "--connection-timeout=15s"
|
||||||
|
env:
|
||||||
|
- name: ADDRESS
|
||||||
|
value: /csi/csi.sock
|
||||||
|
imagePullPolicy: Always
|
||||||
|
volumeMounts:
|
||||||
|
- name: socket-dir
|
||||||
|
mountPath: /csi
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /var/lib/kubelet/plugins/csi-hostpath
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
name: socket-dir
|
@ -13,6 +13,9 @@ subjects:
|
|||||||
- kind: ServiceAccount
|
- kind: ServiceAccount
|
||||||
name: csi-provisioner
|
name: csi-provisioner
|
||||||
namespace: default
|
namespace: default
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: csi-snapshotter
|
||||||
|
namespace: default
|
||||||
roleRef:
|
roleRef:
|
||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
name: e2e-test-privileged-psp
|
name: e2e-test-privileged-psp
|
||||||
|
Loading…
Reference in New Issue
Block a user