From a6ec62f76d0fda6bef1b41f9eec93ec6377a4988 Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Thu, 7 Jan 2021 21:44:58 +0530 Subject: [PATCH] add e2e tests for DeleteSnapshotsecrets --- test/e2e/storage/csi_mock_volume.go | 160 +++++++++++++++++- test/e2e/storage/drivers/csi.go | 9 +- test/e2e/storage/external/external.go | 3 +- .../storage/framework/snapshot_resource.go | 8 +- test/e2e/storage/framework/testdriver.go | 2 +- test/e2e/storage/testsuites/provisioning.go | 3 +- test/e2e/storage/testsuites/snapshottable.go | 3 +- .../testsuites/snapshottable_stress.go | 3 +- 8 files changed, 174 insertions(+), 17 deletions(-) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 320da7b77c0..28d1c9c09b0 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -1272,7 +1272,8 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { ginkgo.By("Creating snapshot") // TODO: Test VolumeSnapshots with Retain policy - snapshotClass, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, claim.Name, claim.Namespace, f.Timeouts) + parameters := map[string]string{} + snapshotClass, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, claim.Name, claim.Namespace, f.Timeouts, parameters) framework.ExpectNoError(err, "failed to create snapshot") m.vsc[snapshotClass.GetName()] = snapshotClass volumeSnapshotName := snapshot.GetName() @@ -1490,8 +1491,128 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { }) } }) + + ginkgo.Context("CSI Volume Snapshots secrets [Feature:VolumeSnapshotDataSource]", func() { + + var ( + // CSISnapshotterSecretName is the name of the secret to be created + CSISnapshotterSecretName string = "snapshot-secret" + + // CSISnapshotterSecretNameAnnotation is the annotation key for the CSI snapshotter secret name in VolumeSnapshotClass.parameters + CSISnapshotterSecretNameAnnotation string = "csi.storage.k8s.io/snapshotter-secret-name" + + // CSISnapshotterSecretNamespaceAnnotation is the annotation key for the CSI snapshotter secret namespace in VolumeSnapshotClass.parameters + CSISnapshotterSecretNamespaceAnnotation string = "csi.storage.k8s.io/snapshotter-secret-namespace" + + // anotations holds the annotations object + annotations interface{} + ) + + // Global variable in all scripts (called before each test) + globalScript := `counter=0; console.log("globals loaded", OK, DEADLINEEXCEEDED)` + tests := []struct { + name string + createVolumeScript string + createSnapshotScript string + }{ + { + // volume snapshot should be created using secrets successfully even if there is a failure in the first few attempts, + name: "volume snapshot create/delete with secrets", + createVolumeScript: `OK`, + // Fail the first 8 calls to create snapshot and succeed the 9th call. + createSnapshotScript: `console.log("Counter:", ++counter); if (counter < 8) { DEADLINEEXCEEDED; } else { OK; }`, + }, + } + for _, test := range tests { + ginkgo.It(test.name, func() { + scripts := map[string]string{ + "globals": globalScript, + "createVolumeStart": test.createVolumeScript, + "createSnapshotStart": test.createSnapshotScript, + } + init(testParameters{ + disableAttach: true, + registerDriver: true, + enableSnapshot: true, + javascriptHooks: scripts, + }) + + sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) + if !ok { + e2eskipper.Skipf("mock driver does not support snapshots -- skipping") + } + defer cleanup() + + var sc *storagev1.StorageClass + if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok { + sc = dDriver.GetDynamicProvisionStorageClass(m.config, "") + } + ginkgo.By("Creating storage class") + class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create storage class: %v", err) + m.sc[class.Name] = class + pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + Name: "snapshot-test-pvc", + StorageClassName: &(class.Name), + }, f.Namespace.Name) + + ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name)) + pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By("Wait for PVC to be Bound") + _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By("Creating Secret") + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: CSISnapshotterSecretName, + }, + Data: map[string][]byte{ + "secret-data": []byte("secret-value-1"), + }, + } + + if secret, err := m.cs.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { + framework.Failf("unable to create test secret %s: %v", secret.Name, err) + } + + ginkgo.By("Creating snapshot with secrets") + parameters := map[string]string{ + CSISnapshotterSecretNameAnnotation: CSISnapshotterSecretName, + CSISnapshotterSecretNamespaceAnnotation: f.Namespace.Name, + } + + _, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, pvc.Name, pvc.Namespace, f.Timeouts, parameters) + framework.ExpectNoError(err, "failed to create snapshot") + snapshotcontent := utils.GetSnapshotContentFromSnapshot(m.config.Framework.DynamicClient, snapshot) + if annotations, ok = snapshotcontent.Object["metadata"].(map[string]interface{})["annotations"]; !ok { + framework.Failf("Unable to get volume snapshot content annotations") + } + + // checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent. + checkDeleteSnapshotSecrets(m.cs, annotations) + + // delete the snapshot and check if the snapshot is deleted. + deleteSnapshot(m.cs, m.config, snapshot) + }) + } + }) }) +func deleteSnapshot(cs clientset.Interface, config *storageframework.PerTestConfig, snapshot *unstructured.Unstructured) { + // delete the given snapshot + dc := config.Framework.DynamicClient + err := dc.Resource(utils.SnapshotGVR).Namespace(snapshot.GetNamespace()).Delete(context.TODO(), snapshot.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + // check if the snapshot is deleted + _, err = dc.Resource(utils.SnapshotGVR).Get(context.TODO(), snapshot.GetName(), metav1.GetOptions{}) + framework.ExpectError(err) +} + // A lot of this code was copied from e2e/framework. It would be nicer // if it could be reused - see https://github.com/kubernetes/kubernetes/issues/92754 func podRunning(ctx context.Context, c clientset.Interface, podName, namespace string) wait.ConditionFunc { @@ -1988,3 +2109,40 @@ func getVolumeLimitFromCSINode(csiNode *storagev1.CSINode, driverName string) in } return 0 } + +// checkDeleteSnapshotSecrets checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent. +func checkDeleteSnapshotSecrets(cs clientset.Interface, annotations interface{}) error { + ginkgo.By("checking if delete snapshot secrets annotation is applied to the VolumeSnapshotContent") + + var ( + annDeletionSecretName string + annDeletionSecretNamespace string + ok bool + err error + + // CSISnapshotterDeleteSecretNameAnnotation is the annotation key for the CSI snapshotter delete secret name in VolumeSnapshotClass.parameters + CSISnapshotterDeleteSecretNameAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-name" + + // CSISnapshotterDeleteSecretNamespaceAnnotation is the annotation key for the CSI snapshotter delete secret namespace in VolumeSnapshotClass.parameters + CSISnapshotterDeleteSecretNamespaceAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-namespace" + ) + + annotationsObj, ok := annotations.(map[string]interface{}) + if !ok { + framework.Failf("failed to get annotations from annotations object") + } + + if annDeletionSecretName, ok = annotationsObj[CSISnapshotterDeleteSecretNameAnnotation].(string); !ok { + framework.Failf("unable to get secret annotation name") + } + if annDeletionSecretNamespace, ok = annotationsObj[CSISnapshotterDeleteSecretNamespaceAnnotation].(string); !ok { + framework.Failf("unable to get secret annotation namespace") + } + + // verify if secrets exists + if _, err = cs.CoreV1().Secrets(annDeletionSecretNamespace).Get(context.TODO(), annDeletionSecretName, metav1.GetOptions{}); err != nil { + framework.Failf("unable to get test secret %s: %v", annDeletionSecretName, err) + } + + return err +} diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 83490a43129..aea6c18beda 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -166,9 +166,8 @@ func (h *hostpathCSIDriver) GetCSIDriverName(config *storageframework.PerTestCon return config.GetUniqueDriverName() } -func (h *hostpathCSIDriver) GetSnapshotClass(config *storageframework.PerTestConfig) *unstructured.Unstructured { +func (h *hostpathCSIDriver) GetSnapshotClass(config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured { snapshotter := config.GetUniqueDriverName() - parameters := map[string]string{} ns := config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-vsc", snapshotter) @@ -349,8 +348,7 @@ func (m *mockCSIDriver) GetDynamicProvisionStorageClass(config *storageframework return storageframework.GetStorageClass(provisioner, parameters, nil, ns) } -func (m *mockCSIDriver) GetSnapshotClass(config *storageframework.PerTestConfig) *unstructured.Unstructured { - parameters := map[string]string{} +func (m *mockCSIDriver) GetSnapshotClass(config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured { snapshotter := m.driverInfo.Name + "-" + config.Framework.UniqueName ns := config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-vsc", snapshotter) @@ -563,8 +561,7 @@ func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(config *storageframewor return storageframework.GetStorageClass(provisioner, parameters, &delayedBinding, ns) } -func (g *gcePDCSIDriver) GetSnapshotClass(config *storageframework.PerTestConfig) *unstructured.Unstructured { - parameters := map[string]string{} +func (g *gcePDCSIDriver) GetSnapshotClass(config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured { snapshotter := g.driverInfo.Name ns := config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-vsc", snapshotter) diff --git a/test/e2e/storage/external/external.go b/test/e2e/storage/external/external.go index 0a1180b8cec..1fe7377176a 100644 --- a/test/e2e/storage/external/external.go +++ b/test/e2e/storage/external/external.go @@ -348,14 +348,13 @@ func loadSnapshotClass(filename string) (*unstructured.Unstructured, error) { return snapshotClass, nil } -func (d *driverDefinition) GetSnapshotClass(e2econfig *storageframework.PerTestConfig) *unstructured.Unstructured { +func (d *driverDefinition) GetSnapshotClass(e2econfig *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured { if !d.SnapshotClass.FromName && d.SnapshotClass.FromFile == "" && d.SnapshotClass.FromExistingClassName == "" { e2eskipper.Skipf("Driver %q does not support snapshotting - skipping", d.DriverInfo.Name) } f := e2econfig.Framework snapshotter := d.DriverInfo.Name - parameters := map[string]string{} ns := e2econfig.Framework.Namespace.Name suffix := "vsc" diff --git a/test/e2e/storage/framework/snapshot_resource.go b/test/e2e/storage/framework/snapshot_resource.go index b8d01b0440d..026dea2e0a4 100644 --- a/test/e2e/storage/framework/snapshot_resource.go +++ b/test/e2e/storage/framework/snapshot_resource.go @@ -45,7 +45,7 @@ type SnapshotResource struct { // CreateSnapshot creates a VolumeSnapshotClass with given SnapshotDeletionPolicy and a VolumeSnapshot // from the VolumeSnapshotClass using a dynamic client. // Returns the unstructured VolumeSnapshotClass and VolumeSnapshot objects. -func CreateSnapshot(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext) (*unstructured.Unstructured, *unstructured.Unstructured) { +func CreateSnapshot(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured) { defer ginkgo.GinkgoRecover() var err error if pattern.SnapshotType != DynamicCreatedSnapshot && pattern.SnapshotType != PreprovisionedCreatedSnapshot { @@ -55,7 +55,7 @@ func CreateSnapshot(sDriver SnapshottableTestDriver, config *PerTestConfig, patt dc := config.Framework.DynamicClient ginkgo.By("creating a SnapshotClass") - sclass := sDriver.GetSnapshotClass(config) + sclass := sDriver.GetSnapshotClass(config, parameters) if sclass == nil { framework.Failf("Failed to get snapshot class based on test config") } @@ -79,13 +79,13 @@ func CreateSnapshot(sDriver SnapshottableTestDriver, config *PerTestConfig, patt // CreateSnapshotResource creates a snapshot resource for the current test. It knows how to deal with // different test pattern snapshot provisioning and deletion policy -func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext) *SnapshotResource { +func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *SnapshotResource { var err error r := SnapshotResource{ Config: config, Pattern: pattern, } - r.Vsclass, r.Vs = CreateSnapshot(sDriver, config, pattern, pvcName, pvcNamespace, timeouts) + r.Vsclass, r.Vs = CreateSnapshot(sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters) dc := r.Config.Framework.DynamicClient diff --git a/test/e2e/storage/framework/testdriver.go b/test/e2e/storage/framework/testdriver.go index 4e80b3e97ae..861d992faee 100644 --- a/test/e2e/storage/framework/testdriver.go +++ b/test/e2e/storage/framework/testdriver.go @@ -124,7 +124,7 @@ type SnapshottableTestDriver interface { TestDriver // GetSnapshotClass returns a SnapshotClass to create snapshot. // It will return nil, if the TestDriver doesn't support it. - GetSnapshotClass(config *PerTestConfig) *unstructured.Unstructured + GetSnapshotClass(config *PerTestConfig, parameters map[string]string) *unstructured.Unstructured } // CustomTimeoutsTestDriver represents an interface fo a TestDriver that supports custom timeouts. diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index e46f4e5fb84..804aebb17d6 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -817,7 +817,8 @@ func prepareSnapshotDataSourceForProvisioning( } e2evolume.InjectContent(f, config, nil, "", tests) - snapshotResource := storageframework.CreateSnapshotResource(sDriver, perTestConfig, pattern, updatedClaim.GetName(), updatedClaim.GetNamespace(), f.Timeouts) + parameters := map[string]string{} + snapshotResource := storageframework.CreateSnapshotResource(sDriver, perTestConfig, pattern, updatedClaim.GetName(), updatedClaim.GetNamespace(), f.Timeouts, parameters) group := "snapshot.storage.k8s.io" dataSourceRef := &v1.TypedLocalObjectReference{ diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 43732c6f6bf..00685621cfb 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -192,7 +192,8 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver, cleanupSteps = append(cleanupSteps, func() { framework.ExpectNoError(sr.CleanupResource(f.Timeouts)) }) - sr = storageframework.CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace(), f.Timeouts) + parameters := map[string]string{} + sr = storageframework.CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace(), f.Timeouts, parameters) vs = sr.Vs vscontent = sr.Vscontent vsc = sr.Vsclass diff --git a/test/e2e/storage/testsuites/snapshottable_stress.go b/test/e2e/storage/testsuites/snapshottable_stress.go index f006ac12f7a..db81666d677 100644 --- a/test/e2e/storage/testsuites/snapshottable_stress.go +++ b/test/e2e/storage/testsuites/snapshottable_stress.go @@ -275,7 +275,8 @@ func (t *snapshottableStressTestSuite) DefineTests(driver storageframework.TestD return default: framework.Logf("Pod-%d [%s], Iteration %d/%d", podIndex, pod.Name, snapshotIndex, stressTest.testOptions.NumSnapshots-1) - snapshot := storageframework.CreateSnapshotResource(snapshottableDriver, stressTest.config, pattern, volume.Pvc.GetName(), volume.Pvc.GetNamespace(), f.Timeouts) + parameters := map[string]string{} + snapshot := storageframework.CreateSnapshotResource(snapshottableDriver, stressTest.config, pattern, volume.Pvc.GetName(), volume.Pvc.GetNamespace(), f.Timeouts, parameters) stressTest.snapshotsMutex.Lock() defer stressTest.snapshotsMutex.Unlock() stressTest.snapshots = append(stressTest.snapshots, snapshot)