mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 04:54:54 +00:00
test: add preprovisioned vgs test
Signed-off-by: Penghao <pewang@redhat.com>
This commit is contained in:
@@ -62,6 +62,8 @@ var (
|
||||
PreprovisionedCreatedSnapshot TestSnapshotType = "PreprovisionedSnapshot"
|
||||
// VolumeGroupSnapshot represents a group snapshot type for dynamic created volumegroupsnapshot
|
||||
VolumeGroupSnapshot TestSnapshotType = "VolumeGroupSnapshot"
|
||||
// PreprovisionedCreatedVolumeGroupSnapshot represents a group snapshot type for pre-provisioned volumegroupsnapshot
|
||||
PreprovisionedCreatedVolumeGroupSnapshot TestSnapshotType = "PreprovisionedVolumeGroupSnapshot"
|
||||
)
|
||||
|
||||
// TestSnapshotDeletionPolicy represents the deletion policy of the snapshot class
|
||||
@@ -347,6 +349,22 @@ var (
|
||||
VolType: DynamicPV,
|
||||
}
|
||||
|
||||
// PreprovisionedVolumeGroupSnapshotDelete is TestPattern for "Pre-provisioned VolumeGroupSnapshot"
|
||||
PreprovisionedVolumeGroupSnapshotDelete = TestPattern{
|
||||
Name: "Pre-provisioned VolumeGroupSnapshot (delete policy)",
|
||||
SnapshotType: PreprovisionedCreatedVolumeGroupSnapshot,
|
||||
SnapshotDeletionPolicy: DeleteSnapshot,
|
||||
VolType: DynamicPV,
|
||||
}
|
||||
|
||||
// PreprovisionedVolumeGroupSnapshotRetain is TestPattern for "Pre-provisioned VolumeGroupSnapshot"
|
||||
PreprovisionedVolumeGroupSnapshotRetain = TestPattern{
|
||||
Name: "Pre-provisioned VolumeGroupSnapshot (retain policy)",
|
||||
SnapshotType: PreprovisionedCreatedVolumeGroupSnapshot,
|
||||
SnapshotDeletionPolicy: RetainSnapshot,
|
||||
VolType: DynamicPV,
|
||||
}
|
||||
|
||||
// PreprovisionedSnapshotDelete is TestPattern for "Pre-provisioned snapshot"
|
||||
PreprovisionedSnapshotDelete = TestPattern{
|
||||
Name: "Pre-provisioned Snapshot (delete policy)",
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/e2e/storage/utils"
|
||||
@@ -72,9 +73,9 @@ type VolumeGroupSnapshotResource struct {
|
||||
func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VolumeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured, *unstructured.Unstructured) {
|
||||
defer ginkgo.GinkgoRecover()
|
||||
var err error
|
||||
if pattern.SnapshotType != VolumeGroupSnapshot {
|
||||
err = fmt.Errorf("SnapshotType must be set to VolumeGroupSnapshot")
|
||||
framework.ExpectNoError(err, "SnapshotType is set to VolumeGroupSnapshot")
|
||||
if pattern.SnapshotType != VolumeGroupSnapshot && pattern.SnapshotType != PreprovisionedCreatedVolumeGroupSnapshot {
|
||||
err = fmt.Errorf("SnapshotType must be set to either VolumeGroupSnapshot or PreprovisionedCreatedVolumeGroupSnapshot")
|
||||
framework.ExpectNoError(err, "SnapshotType is set to VolumeGroupSnapshot or PreprovisionedCreatedVolumeGroupSnapshot")
|
||||
}
|
||||
dc := config.Framework.DynamicClient
|
||||
|
||||
@@ -318,12 +319,13 @@ func (r *VolumeGroupSnapshotResource) CleanupVGS(ctx context.Context, timeouts *
|
||||
// CleanupVGSClass deletes the VolumeGroupSnapshotClass.
|
||||
func (r *VolumeGroupSnapshotResource) CleanupVGSClass(ctx context.Context, timeouts *framework.TimeoutContext) error {
|
||||
if r.VGSClass == nil {
|
||||
framework.Logf("VGSClass is nil, skipping cleanup")
|
||||
return nil
|
||||
}
|
||||
|
||||
dc := r.Config.Framework.DynamicClient
|
||||
vgsClassName := r.VGSClass.GetName()
|
||||
framework.Logf("deleting groupSnapshotClass %q", vgsClassName)
|
||||
framework.Logf("deleting VolumeGroupSnapshotClass %q", vgsClassName)
|
||||
|
||||
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, vgsClassName, metav1.DeleteOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
@@ -331,10 +333,11 @@ func (r *VolumeGroupSnapshotResource) CleanupVGSClass(ctx context.Context, timeo
|
||||
}
|
||||
|
||||
if err = utils.WaitForGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotClassGVR, vgsClassName, framework.Poll, timeouts.SnapshotDelete); err != nil {
|
||||
return fmt.Errorf("failed waiting for groupSnapshotClass %q deletion: %w", vgsClassName, err)
|
||||
return fmt.Errorf("failed waiting for VolumeGroupSnapshotClass %q deletion: %w", vgsClassName, err)
|
||||
}
|
||||
|
||||
framework.Logf("successfully deleted groupSnapshotClass %q", vgsClassName)
|
||||
framework.Logf("successfully deleted VolumeGroupSnapshotClass %q", vgsClassName)
|
||||
r.VGSClass = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -356,13 +359,288 @@ func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeo
|
||||
|
||||
// CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters.
|
||||
func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VolumeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource {
|
||||
var err error
|
||||
vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
|
||||
vgs := &VolumeGroupSnapshotResource{
|
||||
r := &VolumeGroupSnapshotResource{
|
||||
Config: config,
|
||||
Pattern: pattern,
|
||||
VGS: snapshot,
|
||||
VGSClass: vgsClass,
|
||||
VGSContent: vgsc,
|
||||
}
|
||||
return vgs
|
||||
|
||||
dc := config.Framework.DynamicClient
|
||||
|
||||
if pattern.SnapshotType == PreprovisionedCreatedVolumeGroupSnapshot {
|
||||
// prepare a pre-provisioned VolumeGroupSnapshotContent with certain data
|
||||
// Because this could be run with an external CSI driver, we have no way
|
||||
// to pre-provision the group snapshot as we normally would using their API.
|
||||
// We instead dynamically take a group snapshot (above step), delete the old snapshot,
|
||||
// and create another snapshot using the first snapshot's group snapshot handle.
|
||||
|
||||
// Only update to Retain if the pattern specifies Delete policy
|
||||
if pattern.SnapshotDeletionPolicy == DeleteSnapshot {
|
||||
ginkgo.By("updating the group snapshot content deletion policy to retain")
|
||||
patchData := []byte(`{"spec":{"deletionPolicy":"Retain"}}`)
|
||||
vgsc, err = dc.Resource(utils.VolumeGroupSnapshotContentGVR).Patch(ctx, vgsc.GetName(), types.MergePatchType, patchData, metav1.PatchOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
|
||||
ginkgo.By("recording properties of the pre-provisioned group snapshot")
|
||||
vgscStatus := vgsc.Object["status"].(map[string]interface{})
|
||||
groupSnapshotHandle := vgscStatus["volumeGroupSnapshotHandle"].(string)
|
||||
framework.Logf("Recording group snapshot content handle: %s", groupSnapshotHandle)
|
||||
|
||||
// Extract individual volume snapshot handles from volumeSnapshotInfoList
|
||||
volumeSnapshotInfoList := vgscStatus["volumeSnapshotInfoList"].([]interface{})
|
||||
volumeSnapshotHandles := make([]string, 0, len(volumeSnapshotInfoList))
|
||||
for _, info := range volumeSnapshotInfoList {
|
||||
infoMap := info.(map[string]interface{})
|
||||
snapshotHandle := infoMap["snapshotHandle"].(string)
|
||||
volumeSnapshotHandles = append(volumeSnapshotHandles, snapshotHandle)
|
||||
}
|
||||
framework.Logf("Recording %d volume snapshot handles", len(volumeSnapshotHandles))
|
||||
|
||||
vgscAnnotations := vgsc.GetAnnotations()
|
||||
framework.Logf("Recording group snapshot content annotations: %v", vgscAnnotations)
|
||||
csiDriverName := vgsClass.Object["driver"].(string)
|
||||
framework.Logf("Recording snapshot driver: %s", csiDriverName)
|
||||
|
||||
// Get all VolumeSnapshotContent names owned by this VGS before deletion
|
||||
vgsUID := snapshot.GetUID()
|
||||
contentNamesSet, err := r.getVolumeSnapshotContentNames(ctx, dc, pvcNamespace, vgsUID)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Get annotations from one of the VolumeSnapshotContents
|
||||
// Note: We don't need VolumeSnapshotClass for pre-provisioned VolumeSnapshotContent
|
||||
var snapshotContentAnnotations map[string]string
|
||||
if len(contentNamesSet) > 0 {
|
||||
firstContentName := contentNamesSet.UnsortedList()[0]
|
||||
firstVSC, err := dc.Resource(utils.SnapshotContentGVR).Get(ctx, firstContentName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
snapshotContentAnnotations = firstVSC.GetAnnotations()
|
||||
framework.Logf("Recording VolumeSnapshotContent annotations: %v", snapshotContentAnnotations)
|
||||
}
|
||||
|
||||
// Update all VolumeSnapshotContents to Retain policy BEFORE deleting VGS
|
||||
// This prevents them from being deleted when their VolumeSnapshots are cascade-deleted
|
||||
// Only needed when the pattern specifies Delete policy
|
||||
if pattern.SnapshotDeletionPolicy == DeleteSnapshot {
|
||||
ginkgo.By("updating VolumeSnapshotContents deletion policy to Retain")
|
||||
for _, contentName := range contentNamesSet.UnsortedList() {
|
||||
patchData := []byte(`{"spec":{"deletionPolicy":"Retain"}}`)
|
||||
_, err := dc.Resource(utils.SnapshotContentGVR).Patch(ctx, contentName, types.MergePatchType, patchData, metav1.PatchOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// If the deletion policy is retain on vgsc:
|
||||
// when vgs is deleted vgsc will not be deleted
|
||||
// when the vgsc is manually deleted then the underlying group snapshot resource will not be deleted.
|
||||
// We exploit this to create a group snapshot resource from which we can create a preprovisioned snapshot
|
||||
ginkgo.By("deleting the group snapshot")
|
||||
err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(snapshot.GetNamespace()).Delete(ctx, snapshot.GetName(), metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("checking the VolumeGroupSnapshot has been deleted")
|
||||
err = utils.WaitForNamespacedGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotGVR, snapshot.GetName(), snapshot.GetNamespace(), framework.Poll, timeouts.SnapshotDelete)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Wait for VolumeSnapshots owned by this VGS to be cascade-deleted
|
||||
ginkgo.By("waiting for owned VolumeSnapshots to be deleted")
|
||||
err = utils.WaitForOwnedResourcesDeleted(ctx, dc, utils.SnapshotGVR, pvcNamespace, vgsUID, framework.Poll, timeouts.SnapshotDelete)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Delete the VolumeSnapshotContents (with Retain policy, so physical snapshots remain)
|
||||
ginkgo.By("deleting VolumeSnapshotContents")
|
||||
for _, contentName := range contentNamesSet.UnsortedList() {
|
||||
err := dc.Resource(utils.SnapshotContentGVR).Delete(ctx, contentName, metav1.DeleteOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for VolumeSnapshotContents to be deleted
|
||||
for _, contentName := range contentNamesSet.UnsortedList() {
|
||||
err := utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotContentGVR, contentName, framework.Poll, timeouts.SnapshotDelete)
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
|
||||
// Delete the VolumeGroupSnapshotContent (with Retain policy, so physical group snapshot remains)
|
||||
err = dc.Resource(utils.VolumeGroupSnapshotContentGVR).Delete(ctx, vgsc.GetName(), metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("checking the VolumeGroupSnapshotContent has been deleted")
|
||||
err = utils.WaitForGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotContentGVR, vgsc.GetName(), framework.Poll, timeouts.SnapshotDelete)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Create new pre-provisioned VGSContent with the extracted group snapshot handle
|
||||
ginkgo.By("creating a new pre-provisioned VolumeGroupSnapshotContent with the group snapshot handle")
|
||||
vgsUUID := uuid.NewUUID()
|
||||
|
||||
vgsName := getPreProvisionedVolumeGroupSnapshotName(vgsUUID)
|
||||
vgscName := getPreProvisionedVolumeGroupSnapshotContentName(vgsUUID)
|
||||
|
||||
r.VGSContent = getPreProvisionedVolumeGroupSnapshotContent(vgscName, vgscAnnotations, vgsName, pvcNamespace, groupSnapshotHandle, volumeSnapshotHandles, pattern.SnapshotDeletionPolicy.String(), csiDriverName)
|
||||
r.VGSContent, err = dc.Resource(utils.VolumeGroupSnapshotContentGVR).Create(ctx, r.VGSContent, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("creating a pre-provisioned VGS with that VGSContent")
|
||||
r.VGS = getPreProvisionedVolumeGroupSnapshot(vgsName, pvcNamespace, vgscName)
|
||||
r.VGS, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(r.VGS.GetNamespace()).Create(ctx, r.VGS, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
err = utils.WaitForVolumeGroupSnapshotReady(ctx, dc, r.VGS.GetNamespace(), r.VGS.GetName(), framework.Poll, timeouts.SnapshotCreate*10)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Get the new VGS UID for owner references
|
||||
r.VGS, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(r.VGS.GetNamespace()).Get(ctx, r.VGS.GetName(), metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
newVgsUID := r.VGS.GetUID()
|
||||
vgsAPIVersion := r.VGS.GetAPIVersion()
|
||||
vgsKind := r.VGS.GetKind()
|
||||
|
||||
// Create new pre-provisioned VolumeSnapshotContents and VolumeSnapshots
|
||||
// The VolumeSnapshots need to be owned by the VGS so the controller can discover them
|
||||
ginkgo.By("creating pre-provisioned VolumeSnapshotContents and VolumeSnapshots owned by the VGS")
|
||||
|
||||
for i, snapshotHandle := range volumeSnapshotHandles {
|
||||
// Generate unique names for the pre-provisioned VolumeSnapshotContent and VolumeSnapshot
|
||||
// Using VGS UUID + index to make names predictable and associated with parent VGS
|
||||
snapName := fmt.Sprintf("pre-provisioned-vs-%s-%d", string(vgsUUID), i)
|
||||
snapContentName := fmt.Sprintf("pre-provisioned-vsc-%s-%d", string(vgsUUID), i)
|
||||
|
||||
// Create pre-provisioned VolumeSnapshotContent first
|
||||
// Note: volumeSnapshotClassName is not required for pre-provisioned VolumeSnapshotContent
|
||||
vsc := getPreProvisionedVolumeSnapshotContent(snapContentName, snapshotContentAnnotations, snapName, pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName)
|
||||
_, err := dc.Resource(utils.SnapshotContentGVR).Create(ctx, vsc, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Create pre-provisioned VolumeSnapshot
|
||||
preProvisionedVS := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "VolumeSnapshot",
|
||||
"apiVersion": utils.SnapshotAPIVersion,
|
||||
"metadata": map[string]interface{}{
|
||||
"name": snapName,
|
||||
"namespace": pvcNamespace,
|
||||
"ownerReferences": []interface{}{
|
||||
map[string]interface{}{
|
||||
"apiVersion": vgsAPIVersion,
|
||||
"kind": vgsKind,
|
||||
"name": r.VGS.GetName(),
|
||||
"uid": newVgsUID,
|
||||
},
|
||||
},
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"source": map[string]interface{}{
|
||||
"volumeSnapshotContentName": snapContentName,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err = dc.Resource(utils.SnapshotGVR).Namespace(pvcNamespace).Create(ctx, preProvisionedVS, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Wait for the VolumeSnapshot to be ready
|
||||
err = utils.WaitForSnapshotReady(ctx, dc, pvcNamespace, snapName, framework.Poll, timeouts.SnapshotCreate)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
framework.Logf("Created pre-provisioned VolumeSnapshotContent %s and VolumeSnapshot %s (%d/%d)", snapContentName, snapName, i+1, len(volumeSnapshotHandles))
|
||||
}
|
||||
|
||||
ginkgo.By("getting the group snapshot and group snapshot content")
|
||||
r.VGS, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(r.VGS.GetNamespace()).Get(ctx, r.VGS.GetName(), metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
r.VGSContent, err = dc.Resource(utils.VolumeGroupSnapshotContentGVR).Get(ctx, r.VGSContent.GetName(), metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func getPreProvisionedVolumeGroupSnapshot(vgsName, ns, vgscName string) *unstructured.Unstructured {
|
||||
snapshot := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "VolumeGroupSnapshot",
|
||||
"apiVersion": utils.VolumeGroupSnapshotAPIVersion,
|
||||
"metadata": map[string]interface{}{
|
||||
"name": vgsName,
|
||||
"namespace": ns,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"source": map[string]interface{}{
|
||||
"volumeGroupSnapshotContentName": vgscName,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func getPreProvisionedVolumeGroupSnapshotName(uuid types.UID) string {
|
||||
return fmt.Sprintf("pre-provisioned-vgs-%s", string(uuid))
|
||||
}
|
||||
|
||||
func getPreProvisionedVolumeGroupSnapshotContentName(uuid types.UID) string {
|
||||
return fmt.Sprintf("pre-provisioned-vgsc-%s", string(uuid))
|
||||
}
|
||||
|
||||
func getPreProvisionedVolumeGroupSnapshotContent(vgscName string, vgscAnnotations map[string]string, vgsName, vgsNamespace, groupSnapshotHandle string, volumeSnapshotHandles []string, deletionPolicy, csiDriverName string) *unstructured.Unstructured {
|
||||
snapshotContent := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "VolumeGroupSnapshotContent",
|
||||
"apiVersion": utils.VolumeGroupSnapshotAPIVersion,
|
||||
"metadata": map[string]interface{}{
|
||||
"name": vgscName,
|
||||
"annotations": vgscAnnotations,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"source": map[string]interface{}{
|
||||
"groupSnapshotHandles": map[string]interface{}{
|
||||
"volumeGroupSnapshotHandle": groupSnapshotHandle,
|
||||
"volumeSnapshotHandles": volumeSnapshotHandles,
|
||||
},
|
||||
},
|
||||
"volumeGroupSnapshotRef": map[string]interface{}{
|
||||
"name": vgsName,
|
||||
"namespace": vgsNamespace,
|
||||
},
|
||||
"driver": csiDriverName,
|
||||
"deletionPolicy": deletionPolicy,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return snapshotContent
|
||||
}
|
||||
|
||||
func getPreProvisionedVolumeSnapshotContent(snapcontentName string, snapshotContentAnnotations map[string]string, snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName string) *unstructured.Unstructured {
|
||||
snapshotContent := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "VolumeSnapshotContent",
|
||||
"apiVersion": utils.SnapshotAPIVersion,
|
||||
"metadata": map[string]interface{}{
|
||||
"name": snapcontentName,
|
||||
"annotations": snapshotContentAnnotations,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"source": map[string]interface{}{
|
||||
"snapshotHandle": snapshotHandle,
|
||||
},
|
||||
"volumeSnapshotRef": map[string]interface{}{
|
||||
"name": snapshotName,
|
||||
"namespace": snapshotNamespace,
|
||||
},
|
||||
"driver": csiDriverName,
|
||||
"deletionPolicy": deletionPolicy,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return snapshotContent
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
e2estatefulset "k8s.io/kubernetes/test/e2e/framework/statefulset"
|
||||
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
|
||||
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
|
||||
"k8s.io/kubernetes/test/e2e/storage/utils"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
)
|
||||
|
||||
@@ -61,6 +62,8 @@ func InitVolumeGroupSnapshottableTestSuite() storageframework.TestSuite {
|
||||
patterns := []storageframework.TestPattern{
|
||||
storageframework.VolumeGroupSnapshotDelete,
|
||||
storageframework.VolumeGroupSnapshotRetain,
|
||||
storageframework.PreprovisionedVolumeGroupSnapshotDelete,
|
||||
storageframework.PreprovisionedVolumeGroupSnapshotRetain,
|
||||
}
|
||||
return InitCustomGroupSnapshottableTestSuite(patterns)
|
||||
}
|
||||
@@ -289,62 +292,126 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
|
||||
ginkgo.By("verifying the snapshots in the group are ready to use")
|
||||
status := snapshot.VGS.Object["status"]
|
||||
gomega.Expect(status).ShouldNot(gomega.BeNil(), "failed to get status of group snapshot")
|
||||
volumeListMap := snapshot.VGSContent.Object["status"].(map[string]interface{})
|
||||
gomega.Expect(volumeListMap).ShouldNot(gomega.BeNil(), "failed to get group snapshot list")
|
||||
volumeSnapshotInfoList := volumeListMap["volumeSnapshotInfoList"].([]interface{})
|
||||
gomega.Expect(volumeSnapshotInfoList).ShouldNot(gomega.BeNil(), "failed to get group snapshot handle list")
|
||||
gomega.Expect(volumeSnapshotInfoList).Should(gomega.HaveLen(groupTest.numReplicas), "failed to verify snapshot handle list length")
|
||||
|
||||
var volumeSnapshotInfoList []interface{}
|
||||
// For dynamic snapshots, verify volumeSnapshotInfoList is populated
|
||||
// For pre-provisioned snapshots, this field won't be populated by the controller
|
||||
if pattern.SnapshotType == storageframework.VolumeGroupSnapshot {
|
||||
volumeListMap := snapshot.VGSContent.Object["status"].(map[string]interface{})
|
||||
gomega.Expect(volumeListMap).ShouldNot(gomega.BeNil(), "failed to get group snapshot list")
|
||||
volumeSnapshotInfoList = volumeListMap["volumeSnapshotInfoList"].([]interface{})
|
||||
gomega.Expect(volumeSnapshotInfoList).ShouldNot(gomega.BeNil(), "failed to get group snapshot handle list")
|
||||
gomega.Expect(volumeSnapshotInfoList).Should(gomega.HaveLen(groupTest.numReplicas), "failed to verify snapshot handle list length")
|
||||
}
|
||||
claimSize := s.GetTestSuiteInfo().SupportedSizeRange.Min
|
||||
|
||||
ginkgo.By("creating restored PVCs from snapshots")
|
||||
restoredPVCs := []*v1.PersistentVolumeClaim{}
|
||||
volumeHandleToPVCName := make(map[string]string)
|
||||
|
||||
// First, create mapping from volume handles to original PVC names for StatefulSet
|
||||
for i := 0; i < groupTest.numReplicas; i++ {
|
||||
pvcName := fmt.Sprintf("data-%s-%d", groupTest.statefulSet.Name, i)
|
||||
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, pvcName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "failed to get PVC %s", pvcName)
|
||||
if pattern.SnapshotType == storageframework.VolumeGroupSnapshot {
|
||||
// For dynamic snapshots, use volumeSnapshotInfoList
|
||||
volumeHandleToPVCName := make(map[string]string)
|
||||
|
||||
pv, err := cs.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "failed to get PV for PVC %s", pvcName)
|
||||
volumeHandle := pv.Spec.CSI.VolumeHandle
|
||||
volumeHandleToPVCName[volumeHandle] = pvcName
|
||||
}
|
||||
// First, create mapping from volume handles to original PVC names for StatefulSet
|
||||
for i := 0; i < groupTest.numReplicas; i++ {
|
||||
pvcName := fmt.Sprintf("data-%s-%d", groupTest.statefulSet.Name, i)
|
||||
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, pvcName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "failed to get PVC %s", pvcName)
|
||||
|
||||
for _, info := range volumeSnapshotInfoList {
|
||||
// Create a PVC from the snapshot
|
||||
volumeHandle := info.(map[string]interface{})["volumeHandle"].(string)
|
||||
if volumeHandle == "" {
|
||||
framework.Failf("volumeHandle missing for volume snapshot %v", info)
|
||||
pv, err := cs.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "failed to get PV for PVC %s", pvcName)
|
||||
volumeHandle := pv.Spec.CSI.VolumeHandle
|
||||
volumeHandleToPVCName[volumeHandle] = pvcName
|
||||
}
|
||||
|
||||
uid := snapshot.VGS.Object["metadata"].(map[string]interface{})["uid"].(string)
|
||||
gomega.Expect(uid).NotTo(gomega.BeNil(), "failed to get uuid from content")
|
||||
volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
|
||||
uid+volumeHandle)))
|
||||
for _, info := range volumeSnapshotInfoList {
|
||||
// Create a PVC from the snapshot
|
||||
volumeHandle := info.(map[string]interface{})["volumeHandle"].(string)
|
||||
if volumeHandle == "" {
|
||||
framework.Failf("volumeHandle missing for volume snapshot %v", info)
|
||||
}
|
||||
|
||||
// Use original PVC name as base for restored PVC name
|
||||
originalPVCName := volumeHandleToPVCName[volumeHandle]
|
||||
restoredPVCName := fmt.Sprintf("restored-%s", originalPVCName)
|
||||
uid := snapshot.VGS.Object["metadata"].(map[string]interface{})["uid"].(string)
|
||||
gomega.Expect(uid).NotTo(gomega.BeNil(), "failed to get uuid from content")
|
||||
volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
|
||||
uid+volumeHandle)))
|
||||
|
||||
pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
|
||||
StorageClassName: &groupTest.volumeResources[0].Sc.Name,
|
||||
ClaimSize: claimSize,
|
||||
Name: restoredPVCName,
|
||||
}, f.Namespace.Name)
|
||||
// Use original PVC name as base for restored PVC name
|
||||
originalPVCName := volumeHandleToPVCName[volumeHandle]
|
||||
restoredPVCName := fmt.Sprintf("restored-%s", originalPVCName)
|
||||
|
||||
group := "snapshot.storage.k8s.io"
|
||||
pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
|
||||
StorageClassName: &groupTest.volumeResources[0].Sc.Name,
|
||||
ClaimSize: claimSize,
|
||||
Name: restoredPVCName,
|
||||
}, f.Namespace.Name)
|
||||
|
||||
pvc.Spec.DataSource = &v1.TypedLocalObjectReference{
|
||||
APIGroup: &group,
|
||||
Kind: "VolumeSnapshot",
|
||||
Name: volumeSnapshotName,
|
||||
group := "snapshot.storage.k8s.io"
|
||||
|
||||
pvc.Spec.DataSource = &v1.TypedLocalObjectReference{
|
||||
APIGroup: &group,
|
||||
Kind: "VolumeSnapshot",
|
||||
Name: volumeSnapshotName,
|
||||
}
|
||||
|
||||
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create PVC from snapshot")
|
||||
restoredPVCs = append(restoredPVCs, pvc)
|
||||
}
|
||||
} else {
|
||||
// For pre-provisioned snapshots, query VolumeSnapshots owned by the VGS
|
||||
dc := f.DynamicClient
|
||||
vgsList, err := dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{})
|
||||
framework.ExpectNoError(err, "failed to list VolumeGroupSnapshots")
|
||||
|
||||
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create PVC from snapshot")
|
||||
restoredPVCs = append(restoredPVCs, pvc)
|
||||
var vgsUID string
|
||||
for _, vgs := range vgsList.Items {
|
||||
if vgs.GetName() == snapshot.VGS.GetName() {
|
||||
vgsUID = string(vgs.GetUID())
|
||||
break
|
||||
}
|
||||
}
|
||||
gomega.Expect(vgsUID).NotTo(gomega.BeEmpty(), "failed to get VGS UID")
|
||||
|
||||
// List VolumeSnapshots owned by this VGS
|
||||
vss, err := dc.Resource(utils.SnapshotGVR).Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{})
|
||||
framework.ExpectNoError(err, "failed to list VolumeSnapshots")
|
||||
|
||||
var preProvisionedSnapshots []string
|
||||
for _, vs := range vss.Items {
|
||||
for _, owner := range vs.GetOwnerReferences() {
|
||||
if owner.Kind == "VolumeGroupSnapshot" && string(owner.UID) == vgsUID {
|
||||
preProvisionedSnapshots = append(preProvisionedSnapshots, vs.GetName())
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
gomega.Expect(preProvisionedSnapshots).Should(gomega.HaveLen(groupTest.numReplicas), "failed to verify pre-provisioned snapshot count")
|
||||
|
||||
// Map snapshot index to original PVC name (snapshots are created in the same order as PVCs)
|
||||
for i, vsName := range preProvisionedSnapshots {
|
||||
// Original PVC name for StatefulSet
|
||||
originalPVCName := fmt.Sprintf("data-%s-%d", groupTest.statefulSet.Name, i)
|
||||
restoredPVCName := fmt.Sprintf("restored-%s", originalPVCName)
|
||||
|
||||
pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
|
||||
StorageClassName: &groupTest.volumeResources[0].Sc.Name,
|
||||
ClaimSize: claimSize,
|
||||
Name: restoredPVCName,
|
||||
}, f.Namespace.Name)
|
||||
|
||||
group := "snapshot.storage.k8s.io"
|
||||
|
||||
pvc.Spec.DataSource = &v1.TypedLocalObjectReference{
|
||||
APIGroup: &group,
|
||||
Kind: "VolumeSnapshot",
|
||||
Name: vsName,
|
||||
}
|
||||
|
||||
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create PVC from snapshot")
|
||||
restoredPVCs = append(restoredPVCs, pvc)
|
||||
}
|
||||
}
|
||||
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
|
||||
Reference in New Issue
Block a user