diff --git a/test/e2e/storage/framework/testdriver.go b/test/e2e/storage/framework/testdriver.go index 7c9996f0fc8..259fd1222be 100644 --- a/test/e2e/storage/framework/testdriver.go +++ b/test/e2e/storage/framework/testdriver.go @@ -130,7 +130,8 @@ type SnapshottableTestDriver interface { GetSnapshotClass(ctx context.Context, config *PerTestConfig, parameters map[string]string) *unstructured.Unstructured } -type VoulmeGroupSnapshottableTestDriver interface { +// VolumeGroupSnapshottableTestDriver represents an interface for a TestDriver that supports DynamicGroupSnapshot +type VolumeGroupSnapshottableTestDriver interface { TestDriver // GetVolumeGroupSnapshotClass returns a VolumeGroupSnapshotClass to create group snapshot. GetVolumeGroupSnapshotClass(ctx context.Context, config *PerTestConfig, parameters map[string]string) *unstructured.Unstructured diff --git a/test/e2e/storage/framework/volume_group_snapshot_resource.go b/test/e2e/storage/framework/volume_group_snapshot_resource.go index 842b2da3c68..1a7a180b12b 100644 --- a/test/e2e/storage/framework/volume_group_snapshot_resource.go +++ b/test/e2e/storage/framework/volume_group_snapshot_resource.go @@ -64,7 +64,7 @@ type VolumeGroupSnapshotResource struct { // CreateVolumeGroupSnapshot creates a VolumeGroupSnapshotClass with given SnapshotDeletionPolicy and a VolumeGroupSnapshot // from the VolumeGroupSnapshotClass using a dynamic client. // Returns the unstructured VolumeGroupSnapshotClass and VolumeGroupSnapshot objects. -func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured, *unstructured.Unstructured) { +func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VolumeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured, *unstructured.Unstructured) { defer ginkgo.GinkgoRecover() var err error if pattern.SnapshotType != VolumeGroupSnapshot { @@ -120,7 +120,7 @@ func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeo } // CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters. -func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource { +func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VolumeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource { vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters) vgs := &VolumeGroupSnapshotResource{ Config: config, diff --git a/test/e2e/storage/testsuites/volume_group_snapshottable.go b/test/e2e/storage/testsuites/volume_group_snapshottable.go index e823442a5c6..f603ad6f3c8 100644 --- a/test/e2e/storage/testsuites/volume_group_snapshottable.go +++ b/test/e2e/storage/testsuites/volume_group_snapshottable.go @@ -20,35 +20,42 @@ import ( "context" "crypto/sha256" "fmt" + "strings" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2estatefulset "k8s.io/kubernetes/test/e2e/framework/statefulset" e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" storageframework "k8s.io/kubernetes/test/e2e/storage/framework" admissionapi "k8s.io/pod-security-admission/api" ) type volumeGroupSnapshottableTest struct { - config *storageframework.PerTestConfig - pods []*v1.Pod - volumeGroup [3][]*storageframework.VolumeResource - snapshots []*storageframework.VolumeGroupSnapshotResource - numPods int - numVolumes int + config *storageframework.PerTestConfig + statefulSet *appsv1.StatefulSet + pods []*v1.Pod + volumeResources []*storageframework.VolumeResource + snapshots []*storageframework.VolumeGroupSnapshotResource + numReplicas int } +// VolumeGroupSnapshottableTestSuite represents a test suite for testing volume group snapshot functionality. type VolumeGroupSnapshottableTestSuite struct { tsInfo storageframework.TestSuiteInfo } +// InitVolumeGroupSnapshottableTestSuite initializes the test suite for volume group snapshottable functionality. func InitVolumeGroupSnapshottableTestSuite() storageframework.TestSuite { patterns := []storageframework.TestPattern{ storageframework.VolumeGroupSnapshotDelete, @@ -56,6 +63,8 @@ func InitVolumeGroupSnapshottableTestSuite() storageframework.TestSuite { return InitCustomGroupSnapshottableTestSuite(patterns) } +// InitCustomGroupSnapshottableTestSuite initializes a custom test suite for volume group snapshottable tests +// with the provided test patterns. func InitCustomGroupSnapshottableTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite { return &VolumeGroupSnapshottableTestSuite{ tsInfo: storageframework.TestSuiteInfo{ @@ -69,143 +78,225 @@ func InitCustomGroupSnapshottableTestSuite(patterns []storageframework.TestPatte } } +// SkipUnsupportedTests skips tests if the driver does not support group snapshots. func (s *VolumeGroupSnapshottableTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { // Check preconditions. dInfo := driver.GetDriverInfo() ok := false - _, ok = driver.(storageframework.VoulmeGroupSnapshottableTestDriver) + _, ok = driver.(storageframework.VolumeGroupSnapshottableTestDriver) if !dInfo.Capabilities[storageframework.CapVolumeGroupSnapshot] || !ok { e2eskipper.Skipf("Driver %q does not support group snapshots - skipping", dInfo.Name) } } +// GetTestSuiteInfo returns the test suite information for the VolumeGroupSnapshottableTestSuite. func (s *VolumeGroupSnapshottableTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { return s.tsInfo } +// DefineTests defines the test cases for the VolumeGroupSnapshottableTestSuite. func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { labelKey := "group" labelValue := "test-group" f := framework.NewDefaultFramework("volumegroupsnapshottable") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged ginkgo.Describe("VolumeGroupSnapshottable", func() { + var tk *e2ekubectl.TestKubeconfig + ginkgo.BeforeEach(func() { + tk = e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, f.Namespace.Name) + }) ginkgo.Context("", func() { var ( - snapshottableDriver storageframework.VoulmeGroupSnapshottableTestDriver + snapshottableDriver storageframework.VolumeGroupSnapshottableTestDriver cs clientset.Interface groupTest *volumeGroupSnapshottableTest ) init := func(ctx context.Context) { - snapshottableDriver = driver.(storageframework.VoulmeGroupSnapshottableTestDriver) + snapshottableDriver = driver.(storageframework.VolumeGroupSnapshottableTestDriver) cs = f.ClientSet config := driver.PrepareTest(ctx, f) groupTest = &volumeGroupSnapshottableTest{ - config: config, - volumeGroup: [3][]*storageframework.VolumeResource{}, - snapshots: []*storageframework.VolumeGroupSnapshotResource{}, - pods: []*v1.Pod{}, - numPods: 1, - numVolumes: 3, + config: config, + volumeResources: []*storageframework.VolumeResource{}, + snapshots: []*storageframework.VolumeGroupSnapshotResource{}, + pods: []*v1.Pod{}, + numReplicas: 3, } } - createGroupLabel := func(ctx context.Context, pvc *v1.PersistentVolumeClaim, labelKey, labelValue string) { - if pvc.Labels == nil { - pvc.Labels = map[string]string{} + createStatefulSetAndVolumes := func(ctx context.Context) { + // Create volume resource which includes storage class + volumeResource := storageframework.CreateVolumeResource(ctx, driver, groupTest.config, pattern, s.GetTestSuiteInfo().SupportedSizeRange) + groupTest.volumeResources = append(groupTest.volumeResources, volumeResource) + + // Create StatefulSet with volumeClaimTemplates + statefulSetName := fmt.Sprintf("statefulset-vgs-%s", f.Namespace.Name) + replicas := int32(groupTest.numReplicas) + + statefulSet := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: statefulSetName, + Namespace: f.Namespace.Name, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": statefulSetName, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": statefulSetName, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Image: e2epod.GetDefaultTestImage(), + Command: []string{"sleep", "3600"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + MountPath: "/mnt/data", + }, + }, + }, + }, + }, + }, + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + Labels: map[string]string{ + labelKey: labelValue, + }, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse(s.GetTestSuiteInfo().SupportedSizeRange.Min), + }, + }, + StorageClassName: &volumeResource.Sc.Name, + }, + }, + }, + }, } - pvc.Labels[labelKey] = labelValue - _, err := cs.CoreV1().PersistentVolumeClaims(pvc.GetNamespace()).Update(ctx, pvc, metav1.UpdateOptions{}) - framework.ExpectNoError(err, "failed to update PVC %s", pvc.Name) - } - createPodsAndVolumes := func(ctx context.Context) { - for i := 0; i < groupTest.numPods; i++ { - framework.Logf("Creating resources for pod %d/%d", i, groupTest.numPods-1) - for j := 0; j < groupTest.numVolumes; j++ { - volume := storageframework.CreateVolumeResource(ctx, driver, groupTest.config, pattern, s.GetTestSuiteInfo().SupportedSizeRange) - groupTest.volumeGroup[i] = append(groupTest.volumeGroup[i], volume) - createGroupLabel(ctx, volume.Pvc, labelKey, labelValue) + var err error + groupTest.statefulSet, err = cs.AppsV1().StatefulSets(f.Namespace.Name).Create(ctx, statefulSet, metav1.CreateOptions{}) + framework.ExpectNoError(err, "failed to create StatefulSet") - } - pvcs := []*v1.PersistentVolumeClaim{} - for _, volume := range groupTest.volumeGroup[i] { - pvcs = append(pvcs, volume.Pvc) - } - // Create a pod with multiple volumes - podConfig := e2epod.Config{ - NS: f.Namespace.Name, - PVCs: pvcs, - SeLinuxLabel: e2epv.SELinuxLabel, - } - pod, err := e2epod.MakeSecPod(&podConfig) - framework.ExpectNoError(err, "failed to create pod") + // Wait for StatefulSet to be ready + e2estatefulset.WaitForRunningAndReady(ctx, cs, replicas, groupTest.statefulSet) + + // Get the pods created by StatefulSet + for i := 0; i < groupTest.numReplicas; i++ { + podName := fmt.Sprintf("%s-%d", statefulSetName, i) + pod, err := cs.CoreV1().Pods(f.Namespace.Name).Get(ctx, podName, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get StatefulSet pod %s", podName) groupTest.pods = append(groupTest.pods, pod) } + } + + writeTestDataToVolumes := func(ctx context.Context) map[string]string { + mountPath := "/mnt/data" + writePath := fmt.Sprintf("%s/testfile", mountPath) + originalMntTestData := make(map[string]string) + + ginkgo.By("writing test data to all StatefulSet volumes") for i, pod := range groupTest.pods { - pod, err := cs.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) - if err != nil { - framework.Failf("Failed to create pod-%d [%+v]. Error: %v", i, pod, err) - } - if err = e2epod.WaitForPodRunningInNamespace(ctx, cs, pod); err != nil { - framework.Failf("Failed to wait for pod-%d [%+v] to turn into running status. Error: %v", i, pod, err) - } + // For StatefulSet, each pod has one PVC named "data-{statefulset-name}-{index}" + pvcName := fmt.Sprintf("data-%s-%d", groupTest.statefulSet.Name, i) + testData := fmt.Sprintf("HelloFromStatefulSetPVC%d", i) + originalMntTestData[pvcName] = testData + err := tk.WriteFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, writePath, testData) + framework.ExpectNoError(err, "failed to write test data to StatefulSet pod %s", pod.Name) } + return originalMntTestData } cleanup := func(ctx context.Context) { - for _, pod := range groupTest.pods { - framework.Logf("Deleting pod %s", pod.Name) - err := e2epod.DeletePodWithWait(ctx, cs, pod) - framework.ExpectNoError(err, "failed to delete pod %s", pod.Name) + if groupTest.statefulSet != nil { + framework.Logf("Deleting StatefulSet %s", groupTest.statefulSet.Name) + err := cs.AppsV1().StatefulSets(f.Namespace.Name).Delete(ctx, groupTest.statefulSet.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "failed to delete StatefulSet %s", groupTest.statefulSet.Name) } - for _, group := range groupTest.volumeGroup { - for _, volume := range group { - framework.Logf("Deleting volume %s", volume.Pvc.Name) - err := volume.CleanupResource(ctx) - framework.ExpectNoError(err, "failed to delete volume %s", volume.Pvc.Name) + for _, volumeResource := range groupTest.volumeResources { + if volumeResource != nil { + framework.Logf("Deleting volume resource") + err := volumeResource.CleanupResource(ctx) + framework.ExpectNoError(err, "failed to delete volume resource") } } - } - ginkgo.It("should create snapshots for multiple volumes in a pod", func(ctx context.Context) { + ginkgo.It("should create snapshots for StatefulSet volumes and verify data consistency after restore", func(ctx context.Context) { init(ctx) - createPodsAndVolumes(ctx) + createStatefulSetAndVolumes(ctx) ginkgo.DeferCleanup(cleanup) - snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, groupTest.volumeGroup[0][0].Pvc.GetNamespace(), f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()}) + originalMntTestData := writeTestDataToVolumes(ctx) + + snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, f.Namespace.Name, f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()}) groupTest.snapshots = append(groupTest.snapshots, snapshot) + ginkgo.By("verifying the snapshots in the group are ready to use") status := snapshot.VGS.Object["status"] - err := framework.Gomega().Expect(status).NotTo(gomega.BeNil()) - framework.ExpectNoError(err, "failed to get status of group snapshot") - + gomega.Expect(status).ShouldNot(gomega.BeNil(), "failed to get status of group snapshot") volumeListMap := snapshot.VGSContent.Object["status"].(map[string]interface{}) - err = framework.Gomega().Expect(volumeListMap).NotTo(gomega.BeNil()) - framework.ExpectNoError(err, "failed to get volume snapshot list") + gomega.Expect(volumeListMap).ShouldNot(gomega.BeNil(), "failed to get group snapshot list") volumeSnapshotInfoList := volumeListMap["volumeSnapshotInfoList"].([]interface{}) - err = framework.Gomega().Expect(volumeSnapshotInfoList).NotTo(gomega.BeNil()) - framework.ExpectNoError(err, "failed to get volume snapshot list") - err = framework.Gomega().Expect(len(volumeSnapshotInfoList)).To(gomega.Equal(groupTest.numVolumes)) - framework.ExpectNoError(err, "failed to get volume snapshot list") - claimSize := groupTest.volumeGroup[0][0].Pvc.Spec.Resources.Requests.Storage().String() + gomega.Expect(volumeSnapshotInfoList).ShouldNot(gomega.BeNil(), "failed to get group snapshot handle list") + gomega.Expect(volumeSnapshotInfoList).Should(gomega.HaveLen(groupTest.numReplicas), "failed to verify snapshot handle list length") + claimSize := s.GetTestSuiteInfo().SupportedSizeRange.Min + + ginkgo.By("creating restored PVCs from snapshots") + restoredPVCs := []*v1.PersistentVolumeClaim{} + volumeHandleToPVCName := make(map[string]string) + + // First, create mapping from volume handles to original PVC names for StatefulSet + for i := 0; i < groupTest.numReplicas; i++ { + pvcName := fmt.Sprintf("data-%s-%d", groupTest.statefulSet.Name, i) + pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, pvcName, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get PVC %s", pvcName) + + pv, err := cs.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get PV for PVC %s", pvcName) + volumeHandle := pv.Spec.CSI.VolumeHandle + volumeHandleToPVCName[volumeHandle] = pvcName + } + for _, info := range volumeSnapshotInfoList { // Create a PVC from the snapshot volumeHandle := info.(map[string]interface{})["volumeHandle"].(string) - err = framework.Gomega().Expect(volumeHandle).NotTo(gomega.BeNil()) - framework.ExpectNoError(err, "failed to get volume handle from volume") + if volumeHandle == "" { + framework.Failf("volumeHandle missing for volume snapshot %v", info) + } uid := snapshot.VGS.Object["metadata"].(map[string]interface{})["uid"].(string) - err = framework.Gomega().Expect(uid).NotTo(gomega.BeNil()) - framework.ExpectNoError(err, "failed to get uuid from content") + gomega.Expect(uid).NotTo(gomega.BeNil(), "failed to get uuid from content") volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte( uid+volumeHandle))) + // Use original PVC name as base for restored PVC name + originalPVCName := volumeHandleToPVCName[volumeHandle] + restoredPVCName := fmt.Sprintf("restored-%s", originalPVCName) + pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - StorageClassName: &groupTest.volumeGroup[0][0].Sc.Name, + StorageClassName: &groupTest.volumeResources[0].Sc.Name, ClaimSize: claimSize, + Name: restoredPVCName, }, f.Namespace.Name) group := "snapshot.storage.k8s.io" @@ -216,21 +307,62 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework. Name: volumeSnapshotName, } - volSrc := v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: pvc.Spec, - }, - }, - } - pvc, err = cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{}) + pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create PVC from snapshot") - - pod := StartInPodWithVolumeSource(ctx, cs, volSrc, pvc.Namespace, "snapshot-pod", "sleep 300", groupTest.config.ClientNodeSelection) - ginkgo.DeferCleanup(e2epod.DeletePodWithWait, cs, pod) - framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow), "Pod did not start in expected time") + restoredPVCs = append(restoredPVCs, pvc) } + ginkgo.DeferCleanup(func(ctx context.Context) { + for _, pvc := range restoredPVCs { + framework.Logf("Deleting restored PVC %s", pvc.Name) + err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(ctx, pvc.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "failed to delete restored PVC %s", pvc.Name) + } + }) + + ginkgo.By("creating single pod with all restored volumes") + restoredPodConfig := e2epod.Config{ + NS: f.Namespace.Name, + PVCs: restoredPVCs, + SeLinuxLabel: e2epv.SELinuxLabel, + } + restoredPod, err := e2epod.MakeSecPod(&restoredPodConfig) + framework.ExpectNoError(err, "failed to create restored pod config") + + restoredPod, err = cs.CoreV1().Pods(f.Namespace.Name).Create(ctx, restoredPod, metav1.CreateOptions{}) + framework.ExpectNoError(err, "failed to create restored pod") + ginkgo.DeferCleanup(e2epod.DeletePodWithWait, cs, restoredPod) + framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, restoredPod.Name, restoredPod.Namespace, f.Timeouts.PodStartSlow), "Restored pod did not start in expected time") + + ginkgo.By("verifying data consistency for all restored StatefulSet volumes") + var dataConsistencyCheckErrors []string + for volumeIndex, pvc := range restoredPVCs { + dataPath := "/mnt/volume" + mountPath := fmt.Sprintf("%s%d", dataPath, volumeIndex+1) + readPath := fmt.Sprintf("%s/testfile", mountPath) + + // Extract original PVC name from restored PVC name + const restoredPrefix = "restored-" + if !strings.HasPrefix(pvc.Name, restoredPrefix) { + framework.Failf("unexpected restored PVC name format: %s, expected prefix %s", pvc.Name, restoredPrefix) + } + originalPVCName := pvc.Name[len(restoredPrefix):] + expectedData, ok := originalMntTestData[originalPVCName] + if !ok { + framework.Failf("no test data found for original PVC %s", originalPVCName) + } + + restoredData, err := tk.ReadFileViaContainer(restoredPod.Name, restoredPod.Spec.Containers[0].Name, readPath) + if err != nil || !strings.Contains(restoredData, expectedData) { + dataConsistencyCheckErrors = append(dataConsistencyCheckErrors, fmt.Sprintf("volume %s (from %s), expectedData: %s, restoredData: %s : err: %v", pvc.Name, originalPVCName, expectedData, restoredData, err)) + } else { + framework.Logf("data consistency verified for StatefulSet volume %s (from %s): found expected data '%s'", pvc.Name, originalPVCName, expectedData) + } + } + if len(dataConsistencyCheckErrors) > 0 { + framework.Logf("data verification failed for one or more volumes: %v", dataConsistencyCheckErrors) + } + gomega.Expect(dataConsistencyCheckErrors).Should(gomega.BeEmpty(), "failed to check data consistency") }) }) })