diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/csi_objects.go
index 61f401f21ae..dbb0eb30b92 100644
--- a/test/e2e/storage/csi_objects.go
+++ b/test/e2e/storage/csi_objects.go
@@ -535,21 +535,17 @@ func createCSICRDs(c apiextensionsclient.Interface) {
 	}
 
 	for _, crd := range crds {
-		_, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+		_, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+		if err == nil {
+			continue
+		} else if !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to check for existing of CSI CRD %q: %v", crd.Name, err)
+		}
+		_, err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
 		framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err)
 	}
 }
 
-func deleteCSICRDs(c apiextensionsclient.Interface) {
-	By("Deleting CSI CRDs")
-	csiDriverCRDName := csicrd.CSIDriverCRD().Name
-	csiNodeInfoCRDName := csicrd.CSINodeInfoCRD().Name
-	err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiDriverCRDName, &metav1.DeleteOptions{})
-	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiDriverCRDName, err)
-	err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiNodeInfoCRDName, &metav1.DeleteOptions{})
-	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiNodeInfoCRDName, err)
-}
-
 func shredFile(filePath string) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		framework.Logf("File %v was not found, skipping shredding", filePath)
diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go
index 720660efc44..e84c6b59b9c 100644
--- a/test/e2e/storage/csi_volumes.go
+++ b/test/e2e/storage/csi_volumes.go
@@ -58,7 +58,7 @@ var csiTestDrivers = map[string]func(f *framework.Framework, config framework.Vo
 	"gcePD":    initCSIgcePD,
 }
 
-var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() {
+var _ = utils.SIGDescribe("CSI Volumes", func() {
 	f := framework.NewDefaultFramework("csi-mock-plugin")
 
 	var (
@@ -88,15 +88,11 @@ var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() {
 		createCSICRDs(crdclient)
 	})
 
-	AfterEach(func() {
-		deleteCSICRDs(crdclient)
-	})
-
 	for driverName, initCSIDriver := range csiTestDrivers {
 		curDriverName := driverName
 		curInitCSIDriver := initCSIDriver
 
-		Context(fmt.Sprintf("CSI plugin test using CSI driver: %s [Serial]", curDriverName), func() {
+		Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() {
 			var (
 				driver csiTestDriver
 			)
@@ -406,6 +402,8 @@ func (g *gcePDCSIDriver) createCSIDriver() {
 	g.nodeServiceAccount = csiServiceAccount(cs, config, "gce-node", false /* teardown */)
 	csiClusterRoleBindings(cs, config, false /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles)
 	csiClusterRoleBindings(cs, config, false /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles)
+	utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace, false, /* teardown */
+		[]string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name})
 	role := csiControllerRole(cs, config, false)
 	csiControllerRoleBinding(cs, config, false, role, g.controllerServiceAccount)
 	deployGCEPDCSIDriver(cs, config, false /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount)
@@ -419,6 +417,8 @@ func (g *gcePDCSIDriver) cleanupCSIDriver() {
 	deployGCEPDCSIDriver(cs, config, true /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount)
 	csiClusterRoleBindings(cs, config, true /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles)
 	csiClusterRoleBindings(cs, config, true /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles)
+	utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace, true, /* teardown */
+		[]string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name})
 	role := csiControllerRole(cs, config, true)
 	csiControllerRoleBinding(cs, config, true, role, g.controllerServiceAccount)
 	csiServiceAccount(cs, config, "gce-controller", true /* teardown */)