Merge pull request #70362 from ddebroy/ddebroy-multi-pvc

Add multi PVC versions of e2e tests for WaitForFirstConsumer

test/e2e/framework/util.go

@@ -1016,22 +1016,40 @@ func WaitForPersistentVolumeDeleted(c clientset.Interface, pvName string, Poll,
 
 // WaitForPersistentVolumeClaimPhase waits for a PersistentVolumeClaim to be in a specific phase or until timeout occurs, whichever comes first.
 func WaitForPersistentVolumeClaimPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error {
-	Logf("Waiting up to %v for PersistentVolumeClaim %s to have phase %s", timeout, pvcName, phase)
+	return WaitForPersistentVolumeClaimsPhase(phase, c, ns, []string{pvcName}, Poll, timeout, true)
+}
+
+// WaitForPersistentVolumeClaimPhase waits for any (if matchAny is true) or all (if matchAny is false) PersistentVolumeClaims
+// to be in a specific phase or until timeout occurs, whichever comes first.
+func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcNames []string, Poll, timeout time.Duration, matchAny bool) error {
+	if len(pvcNames) == 0 {
+		return fmt.Errorf("Incorrect parameter: Need at least one PVC to track. Found 0.")
+	}
+	Logf("Waiting up to %v for PersistentVolumeClaims %v to have phase %s", timeout, pvcNames, phase)
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
-		pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{})
+		phaseFoundInAllClaims := true
-		if err != nil {
+		for _, pvcName := range pvcNames {
-			Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err)
+			pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{})
-			continue
+			if err != nil {
-		} else {
+				Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err)
-			if pvc.Status.Phase == phase {
+				continue
-				Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start))
-				return nil
 			} else {
-				Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase)
+				if pvc.Status.Phase == phase {
+					Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start))
+					if matchAny {
+						return nil
+					}
+				} else {
+					Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase)
+					phaseFoundInAllClaims = false
+				}
 			}
 		}
+		if phaseFoundInAllClaims {
+			return nil
+		}
 	}
-	return fmt.Errorf("PersistentVolumeClaim %s not in phase %s within %v", pvcName, phase, timeout)
+	return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout)
 }
 
 // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name.

test/e2e/storage/regional_pd.go

@@ -74,7 +74,8 @@ var _ = utils.SIGDescribe("Regional PD", func() {
 		})
 
 		It("should provision storage with delayed binding [Slow]", func() {
-			testRegionalDelayedBinding(c, ns)
+			testRegionalDelayedBinding(c, ns, 1 /* pvcCount */)
+			testRegionalDelayedBinding(c, ns, 3 /* pvcCount */)
 		})
 
 		It("should provision storage in the allowedTopologies [Slow]", func() {
@@ -82,7 +83,8 @@ var _ = utils.SIGDescribe("Regional PD", func() {
 		})
 
 		It("should provision storage in the allowedTopologies with delayed binding [Slow]", func() {
-			testRegionalAllowedTopologiesWithDelayedBinding(c, ns)
+			testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 1 /* pvcCount */)
+			testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 3 /* pvcCount */)
 		})
 
 		It("should failover to a different zone when all nodes in one zone become unreachable [Slow] [Disruptive]", func() {
@@ -297,7 +299,7 @@ func addTaint(c clientset.Interface, ns string, nodes []v1.Node, podZone string)
 	}
 }
 
-func testRegionalDelayedBinding(c clientset.Interface, ns string) {
+func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int) {
 	test := testsuites.StorageClassTest{
 		Name:        "Regional PD storage class with waitForFirstConsumer test on GCE",
 		Provisioner: "kubernetes.io/gce-pd",
@@ -311,9 +313,13 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) {
 
 	suffix := "delayed-regional"
 	class := newStorageClass(test, ns, suffix)
-	claim := newClaim(test, ns, suffix)
+	var claims []*v1.PersistentVolumeClaim
-	claim.Spec.StorageClassName = &class.Name
+	for i := 0; i < pvcCount; i++ {
-	pv, node := testBindingWaitForFirstConsumer(c, claim, class)
+		claim := newClaim(test, ns, suffix)
+		claim.Spec.StorageClassName = &class.Name
+		claims = append(claims, claim)
+	}
+	pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class)
 	if node == nil {
 		framework.Failf("unexpected nil node found")
 	}
@@ -321,7 +327,9 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) {
 	if !ok {
 		framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
 	}
-	checkZoneFromLabelAndAffinity(pv, zone, false)
+	for _, pv := range pvs {
+		checkZoneFromLabelAndAffinity(pv, zone, false)
+	}
 }
 
 func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
@@ -346,7 +354,7 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
 	checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true)
 }
 
-func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) {
+func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string, pvcCount int) {
 	test := testsuites.StorageClassTest{
 		Name:        "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE",
 		Provisioner: "kubernetes.io/gce-pd",
@@ -362,9 +370,13 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s
 	class := newStorageClass(test, ns, suffix)
 	topoZones := getTwoRandomZones(c)
 	addAllowedTopologiesToStorageClass(c, class, topoZones)
-	claim := newClaim(test, ns, suffix)
+	var claims []*v1.PersistentVolumeClaim
-	claim.Spec.StorageClassName = &class.Name
+	for i := 0; i < pvcCount; i++ {
-	pv, node := testBindingWaitForFirstConsumer(c, claim, class)
+		claim := newClaim(test, ns, suffix)
+		claim.Spec.StorageClassName = &class.Name
+		claims = append(claims, claim)
+	}
+	pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class)
 	if node == nil {
 		framework.Failf("unexpected nil node found")
 	}
@@ -382,7 +394,9 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s
 	if !zoneFound {
 		framework.Failf("zones specified in AllowedTopologies: %v does not contain zone of node where PV got provisioned: %s", topoZones, nodeZone)
 	}
-	checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true)
+	for _, pv := range pvs {
+		checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true)
+	}
 }
 
 func getPVC(c clientset.Interface, ns string, pvcLabels map[string]string) *v1.PersistentVolumeClaim {

test/e2e/storage/volume_provisioning.go

@@ -54,57 +54,87 @@ import (
 const (
 	// Plugin name of the external provisioner
 	externalPluginName = "example.com/nfs"
+	// Number of PVCs for multi PVC tests
+	multiPVCcount = 3
 )
 
 func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) (*v1.PersistentVolume, *v1.Node) {
+	pvs, node := testBindingWaitForFirstConsumerMultiPVC(client, []*v1.PersistentVolumeClaim{claim}, class)
+	return pvs[0], node
+}
+
+func testBindingWaitForFirstConsumerMultiPVC(client clientset.Interface, claims []*v1.PersistentVolumeClaim, class *storage.StorageClass) ([]*v1.PersistentVolume, *v1.Node) {
 	var err error
+	Expect(len(claims)).ToNot(Equal(0))
+	namespace := claims[0].Namespace
 
 	By("creating a storage class " + class.Name)
 	class, err = client.StorageV1().StorageClasses().Create(class)
 	Expect(err).NotTo(HaveOccurred())
 	defer deleteStorageClass(client, class.Name)
 
-	By("creating a claim")
+	By("creating claims")
-	claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
+	var claimNames []string
-	Expect(err).NotTo(HaveOccurred())
+	var createdClaims []*v1.PersistentVolumeClaim
+	for _, claim := range claims {
+		c, err := client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
+		claimNames = append(claimNames, c.Name)
+		createdClaims = append(createdClaims, c)
+		Expect(err).NotTo(HaveOccurred())
+	}
 	defer func() {
-		framework.ExpectNoError(framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace), "Failed to delete PVC ", claim.Name)
+		var errors map[string]error
+		for _, claim := range createdClaims {
+			err := framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace)
+			if err != nil {
+				errors[claim.Name] = err
+			}
+		}
+		if len(errors) > 0 {
+			for claimName, err := range errors {
+				framework.Logf("Failed to delete PVC: %s due to error: %v", claimName, err)
+			}
+		}
 	}()
 
-	// Wait for ClaimProvisionTimeout and make sure the phase did not become Bound i.e. the Wait errors out
+	// Wait for ClaimProvisionTimeout (across all PVCs in parallel) and make sure the phase did not become Bound i.e. the Wait errors out
-	err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
+	By("checking the claims are in pending state")
+	err = framework.WaitForPersistentVolumeClaimsPhase(v1.ClaimBound, client, namespace, claimNames, 2*time.Second, framework.ClaimProvisionShortTimeout, true)
 	Expect(err).To(HaveOccurred())
+	for _, claim := range createdClaims {
+		// Get new copy of the claim
+		claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
+	}
 
-	By("checking the claim is in pending state")
+	By("creating a pod referring to the claims")
-	// Get new copy of the claim
-	claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
-	Expect(err).NotTo(HaveOccurred())
-	Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
-
-	By("creating a pod referring to the claim")
 	// Create a pod referring to the claim and wait for it to get to running
-	pod, err := framework.CreateClientPod(client, claim.Namespace, claim)
+	pod, err := framework.CreatePod(client, namespace, nil /* nodeSelector */, createdClaims, true /* isPrivileged */, "" /* command */)
 	Expect(err).NotTo(HaveOccurred())
 	defer func() {
 		framework.DeletePodOrFail(client, pod.Namespace, pod.Name)
 	}()
-
+	// collect node details
-	By("re-checking the claim to see it binded")
-	// Get new copy of the claim
-	claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
-	Expect(err).NotTo(HaveOccurred())
-	// make sure claim did bind
-	err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
-	Expect(err).NotTo(HaveOccurred())
-
-	// collect node and pv details
 	node, err := client.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
 	Expect(err).NotTo(HaveOccurred())
 
-	pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+	By("re-checking the claims to see they binded")
-	Expect(err).NotTo(HaveOccurred())
+	var pvs []*v1.PersistentVolume
+	for _, claim := range createdClaims {
+		// Get new copy of the claim
+		claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		// make sure claim did bind
+		err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+		Expect(err).NotTo(HaveOccurred())
 
-	return pv, node
+		pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		pvs = append(pvs, pv)
+	}
+	Expect(len(pvs)).ToNot(Equal(0))
+	return pvs, node
 }
 
 func checkZoneFromLabelAndAffinity(pv *v1.PersistentVolume, zone string, matchZone bool) {
@@ -231,6 +261,67 @@ func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
 	return nil
 }
 
+func testZonalDelayedBinding(c clientset.Interface, ns string, specifyAllowedTopology bool, pvcCount int) {
+	storageClassTestNameFmt := "Delayed binding %s storage class test %s"
+	storageClassTestNameSuffix := ""
+	if specifyAllowedTopology {
+		storageClassTestNameSuffix += " with AllowedTopologies"
+	}
+	tests := []testsuites.StorageClassTest{
+		{
+			Name:           fmt.Sprintf(storageClassTestNameFmt, "EBS", storageClassTestNameSuffix),
+			CloudProviders: []string{"aws"},
+			Provisioner:    "kubernetes.io/aws-ebs",
+			ClaimSize:      "2Gi",
+			DelayBinding:   true,
+		},
+		{
+			Name:           fmt.Sprintf(storageClassTestNameFmt, "GCE PD", storageClassTestNameSuffix),
+			CloudProviders: []string{"gce", "gke"},
+			Provisioner:    "kubernetes.io/gce-pd",
+			ClaimSize:      "2Gi",
+			DelayBinding:   true,
+		},
+	}
+	for _, test := range tests {
+		if !framework.ProviderIs(test.CloudProviders...) {
+			framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
+			continue
+		}
+		action := "creating claims with class with waitForFirstConsumer"
+		suffix := "delayed"
+		var topoZone string
+		class := newStorageClass(test, ns, suffix)
+		if specifyAllowedTopology {
+			action += " and allowedTopologies"
+			suffix += "-topo"
+			topoZone = getRandomCloudZone(c)
+			addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone)
+		}
+		By(action)
+		var claims []*v1.PersistentVolumeClaim
+		for i := 0; i < pvcCount; i++ {
+			claim := newClaim(test, ns, suffix)
+			claim.Spec.StorageClassName = &class.Name
+			claims = append(claims, claim)
+		}
+		pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class)
+		if node == nil {
+			framework.Failf("unexpected nil node found")
+		}
+		zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]
+		if !ok {
+			framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
+		}
+		if specifyAllowedTopology && topoZone != zone {
+			framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, zone)
+		}
+		for _, pv := range pvs {
+			checkZoneFromLabelAndAffinity(pv, zone, true)
+		}
+	}
+}
+
 var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 	f := framework.NewDefaultFramework("volume-provisioning")
 
@@ -846,43 +937,9 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 		})
 	})
 	Describe("DynamicProvisioner delayed binding [Slow]", func() {
-		It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() {
+		It("should create persistent volumes in the same zone as node after a pod mounting the claims is started", func() {
-			tests := []testsuites.StorageClassTest{
+			testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 1 /*pvcCount*/)
-				{
+			testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 3 /*pvcCount*/)
-					Name:           "Delayed binding EBS storage class test",
-					CloudProviders: []string{"aws"},
-					Provisioner:    "kubernetes.io/aws-ebs",
-					ClaimSize:      "2Gi",
-					DelayBinding:   true,
-				},
-				{
-					Name:           "Delayed binding GCE PD storage class test",
-					CloudProviders: []string{"gce", "gke"},
-					Provisioner:    "kubernetes.io/gce-pd",
-					ClaimSize:      "2Gi",
-					DelayBinding:   true,
-				},
-			}
-			for _, test := range tests {
-				if !framework.ProviderIs(test.CloudProviders...) {
-					framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
-					continue
-				}
-				By("creating a claim with class with waitForFirstConsumer")
-				suffix := "delayed"
-				class := newStorageClass(test, ns, suffix)
-				claim := newClaim(test, ns, suffix)
-				claim.Spec.StorageClassName = &class.Name
-				pv, node := testBindingWaitForFirstConsumer(c, claim, class)
-				if node == nil {
-					framework.Failf("unexpected nil node found")
-				}
-				zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]
-				if !ok {
-					framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
-				}
-				checkZoneFromLabelAndAffinity(pv, zone, true)
-			}
 		})
 	})
 	Describe("DynamicProvisioner allowedTopologies", func() {
@@ -921,51 +978,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 		})
 	})
 	Describe("DynamicProvisioner delayed binding with allowedTopologies [Slow]", func() {
-		It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() {
+		It("should create persistent volumes in the same zone as specified in allowedTopologies after a pod mounting the claims is started", func() {
-			tests := []testsuites.StorageClassTest{
+			testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 1 /*pvcCount*/)
-				{
+			testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 3 /*pvcCount*/)
-					Name:           "AllowedTopologies and delayed binding EBS storage class test",
-					CloudProviders: []string{"aws"},
-					Provisioner:    "kubernetes.io/aws-ebs",
-					ClaimSize:      "2Gi",
-					DelayBinding:   true,
-				},
-				{
-					Name:           "AllowedTopologies and delayed binding GCE PD storage class test",
-					CloudProviders: []string{"gce", "gke"},
-					Provisioner:    "kubernetes.io/gce-pd",
-					ClaimSize:      "2Gi",
-					DelayBinding:   true,
-				},
-			}
-			for _, test := range tests {
-				if !framework.ProviderIs(test.CloudProviders...) {
-					framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
-					continue
-				}
-				By("creating a claim with class with WaitForFirstConsumer and allowedTopologies")
-				suffix := "delayed-topo"
-				class := newStorageClass(test, ns, suffix)
-				topoZone := getRandomCloudZone(c)
-				addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone)
-				claim := newClaim(test, ns, suffix)
-				claim.Spec.StorageClassName = &class.Name
-				pv, node := testBindingWaitForFirstConsumer(c, claim, class)
-				if node == nil {
-					framework.Failf("unexpected nil node found")
-				}
-				nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]
-				if !ok {
-					framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
-				}
-				if topoZone != nodeZone {
-					framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, nodeZone)
-				}
-				checkZoneFromLabelAndAffinity(pv, topoZone, true)
-			}
 		})
 	})
-
 })
 
 func getDefaultStorageClassName(c clientset.Interface) string {