diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index a6e3aa207e9..1d0b080987f 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -1016,22 +1016,40 @@ func WaitForPersistentVolumeDeleted(c clientset.Interface, pvName string, Poll, // WaitForPersistentVolumeClaimPhase waits for a PersistentVolumeClaim to be in a specific phase or until timeout occurs, whichever comes first. func WaitForPersistentVolumeClaimPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error { - Logf("Waiting up to %v for PersistentVolumeClaim %s to have phase %s", timeout, pvcName, phase) + return WaitForPersistentVolumeClaimsPhase(phase, c, ns, []string{pvcName}, Poll, timeout, true) +} + +// WaitForPersistentVolumeClaimPhase waits for any (if matchAny is true) or all (if matchAny is false) PersistentVolumeClaims +// to be in a specific phase or until timeout occurs, whichever comes first. +func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcNames []string, Poll, timeout time.Duration, matchAny bool) error { + if len(pvcNames) == 0 { + return fmt.Errorf("Incorrect parameter: Need at least one PVC to track. Found 0.") + } + Logf("Waiting up to %v for PersistentVolumeClaims %v to have phase %s", timeout, pvcNames, phase) for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) { - pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{}) - if err != nil { - Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err) - continue - } else { - if pvc.Status.Phase == phase { - Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start)) - return nil + phaseFoundInAllClaims := true + for _, pvcName := range pvcNames { + pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{}) + if err != nil { + Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err) + continue } else { - Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase) + if pvc.Status.Phase == phase { + Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start)) + if matchAny { + return nil + } + } else { + Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase) + phaseFoundInAllClaims = false + } } } + if phaseFoundInAllClaims { + return nil + } } - return fmt.Errorf("PersistentVolumeClaim %s not in phase %s within %v", pvcName, phase, timeout) + return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout) } // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name. diff --git a/test/e2e/storage/regional_pd.go b/test/e2e/storage/regional_pd.go index bbd196b1051..428ef8556da 100644 --- a/test/e2e/storage/regional_pd.go +++ b/test/e2e/storage/regional_pd.go @@ -74,7 +74,8 @@ var _ = utils.SIGDescribe("Regional PD", func() { }) It("should provision storage with delayed binding [Slow]", func() { - testRegionalDelayedBinding(c, ns) + testRegionalDelayedBinding(c, ns, 1 /* pvcCount */) + testRegionalDelayedBinding(c, ns, 3 /* pvcCount */) }) It("should provision storage in the allowedTopologies [Slow]", func() { @@ -82,7 +83,8 @@ var _ = utils.SIGDescribe("Regional PD", func() { }) It("should provision storage in the allowedTopologies with delayed binding [Slow]", func() { - testRegionalAllowedTopologiesWithDelayedBinding(c, ns) + testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 1 /* pvcCount */) + testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 3 /* pvcCount */) }) It("should failover to a different zone when all nodes in one zone become unreachable [Slow] [Disruptive]", func() { @@ -297,7 +299,7 @@ func addTaint(c clientset.Interface, ns string, nodes []v1.Node, podZone string) } } -func testRegionalDelayedBinding(c clientset.Interface, ns string) { +func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int) { test := testsuites.StorageClassTest{ Name: "Regional PD storage class with waitForFirstConsumer test on GCE", Provisioner: "kubernetes.io/gce-pd", @@ -311,9 +313,13 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) { suffix := "delayed-regional" class := newStorageClass(test, ns, suffix) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) + var claims []*v1.PersistentVolumeClaim + for i := 0; i < pvcCount; i++ { + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + claims = append(claims, claim) + } + pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class) if node == nil { framework.Failf("unexpected nil node found") } @@ -321,7 +327,9 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) { if !ok { framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) } - checkZoneFromLabelAndAffinity(pv, zone, false) + for _, pv := range pvs { + checkZoneFromLabelAndAffinity(pv, zone, false) + } } func testRegionalAllowedTopologies(c clientset.Interface, ns string) { @@ -346,7 +354,7 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) { checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true) } -func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) { +func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string, pvcCount int) { test := testsuites.StorageClassTest{ Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE", Provisioner: "kubernetes.io/gce-pd", @@ -362,9 +370,13 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s class := newStorageClass(test, ns, suffix) topoZones := getTwoRandomZones(c) addAllowedTopologiesToStorageClass(c, class, topoZones) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) + var claims []*v1.PersistentVolumeClaim + for i := 0; i < pvcCount; i++ { + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + claims = append(claims, claim) + } + pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class) if node == nil { framework.Failf("unexpected nil node found") } @@ -382,7 +394,9 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s if !zoneFound { framework.Failf("zones specified in AllowedTopologies: %v does not contain zone of node where PV got provisioned: %s", topoZones, nodeZone) } - checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true) + for _, pv := range pvs { + checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true) + } } func getPVC(c clientset.Interface, ns string, pvcLabels map[string]string) *v1.PersistentVolumeClaim { diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index cde6712dc5c..db0e357595d 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -54,57 +54,87 @@ import ( const ( // Plugin name of the external provisioner externalPluginName = "example.com/nfs" + // Number of PVCs for multi PVC tests + multiPVCcount = 3 ) func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) (*v1.PersistentVolume, *v1.Node) { + pvs, node := testBindingWaitForFirstConsumerMultiPVC(client, []*v1.PersistentVolumeClaim{claim}, class) + return pvs[0], node +} + +func testBindingWaitForFirstConsumerMultiPVC(client clientset.Interface, claims []*v1.PersistentVolumeClaim, class *storage.StorageClass) ([]*v1.PersistentVolume, *v1.Node) { var err error + Expect(len(claims)).ToNot(Equal(0)) + namespace := claims[0].Namespace By("creating a storage class " + class.Name) class, err = client.StorageV1().StorageClasses().Create(class) Expect(err).NotTo(HaveOccurred()) defer deleteStorageClass(client, class.Name) - By("creating a claim") - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) - Expect(err).NotTo(HaveOccurred()) + By("creating claims") + var claimNames []string + var createdClaims []*v1.PersistentVolumeClaim + for _, claim := range claims { + c, err := client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) + claimNames = append(claimNames, c.Name) + createdClaims = append(createdClaims, c) + Expect(err).NotTo(HaveOccurred()) + } defer func() { - framework.ExpectNoError(framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace), "Failed to delete PVC ", claim.Name) + var errors map[string]error + for _, claim := range createdClaims { + err := framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace) + if err != nil { + errors[claim.Name] = err + } + } + if len(errors) > 0 { + for claimName, err := range errors { + framework.Logf("Failed to delete PVC: %s due to error: %v", claimName, err) + } + } }() - // Wait for ClaimProvisionTimeout and make sure the phase did not become Bound i.e. the Wait errors out - err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, 2*time.Second, framework.ClaimProvisionShortTimeout) + // Wait for ClaimProvisionTimeout (across all PVCs in parallel) and make sure the phase did not become Bound i.e. the Wait errors out + By("checking the claims are in pending state") + err = framework.WaitForPersistentVolumeClaimsPhase(v1.ClaimBound, client, namespace, claimNames, 2*time.Second, framework.ClaimProvisionShortTimeout, true) Expect(err).To(HaveOccurred()) + for _, claim := range createdClaims { + // Get new copy of the claim + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(claim.Status.Phase).To(Equal(v1.ClaimPending)) + } - By("checking the claim is in pending state") - // Get new copy of the claim - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(claim.Status.Phase).To(Equal(v1.ClaimPending)) - - By("creating a pod referring to the claim") + By("creating a pod referring to the claims") // Create a pod referring to the claim and wait for it to get to running - pod, err := framework.CreateClientPod(client, claim.Namespace, claim) + pod, err := framework.CreatePod(client, namespace, nil /* nodeSelector */, createdClaims, true /* isPrivileged */, "" /* command */) Expect(err).NotTo(HaveOccurred()) defer func() { framework.DeletePodOrFail(client, pod.Namespace, pod.Name) }() - - By("re-checking the claim to see it binded") - // Get new copy of the claim - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - // make sure claim did bind - err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) - Expect(err).NotTo(HaveOccurred()) - - // collect node and pv details + // collect node details node, err := client.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) + By("re-checking the claims to see they binded") + var pvs []*v1.PersistentVolume + for _, claim := range createdClaims { + // Get new copy of the claim + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + // make sure claim did bind + err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred()) - return pv, node + pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + pvs = append(pvs, pv) + } + Expect(len(pvs)).ToNot(Equal(0)) + return pvs, node } func checkZoneFromLabelAndAffinity(pv *v1.PersistentVolume, zone string, matchZone bool) { @@ -231,6 +261,67 @@ func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error { return nil } +func testZonalDelayedBinding(c clientset.Interface, ns string, specifyAllowedTopology bool, pvcCount int) { + storageClassTestNameFmt := "Delayed binding %s storage class test %s" + storageClassTestNameSuffix := "" + if specifyAllowedTopology { + storageClassTestNameSuffix += " with AllowedTopologies" + } + tests := []testsuites.StorageClassTest{ + { + Name: fmt.Sprintf(storageClassTestNameFmt, "EBS", storageClassTestNameSuffix), + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + ClaimSize: "2Gi", + DelayBinding: true, + }, + { + Name: fmt.Sprintf(storageClassTestNameFmt, "GCE PD", storageClassTestNameSuffix), + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + ClaimSize: "2Gi", + DelayBinding: true, + }, + } + for _, test := range tests { + if !framework.ProviderIs(test.CloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) + continue + } + action := "creating claims with class with waitForFirstConsumer" + suffix := "delayed" + var topoZone string + class := newStorageClass(test, ns, suffix) + if specifyAllowedTopology { + action += " and allowedTopologies" + suffix += "-topo" + topoZone = getRandomCloudZone(c) + addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) + } + By(action) + var claims []*v1.PersistentVolumeClaim + for i := 0; i < pvcCount; i++ { + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + claims = append(claims, claim) + } + pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class) + if node == nil { + framework.Failf("unexpected nil node found") + } + zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] + if !ok { + framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) + } + if specifyAllowedTopology && topoZone != zone { + framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, zone) + } + for _, pv := range pvs { + checkZoneFromLabelAndAffinity(pv, zone, true) + } + } +} + var _ = utils.SIGDescribe("Dynamic Provisioning", func() { f := framework.NewDefaultFramework("volume-provisioning") @@ -846,43 +937,9 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) }) Describe("DynamicProvisioner delayed binding [Slow]", func() { - It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() { - tests := []testsuites.StorageClassTest{ - { - Name: "Delayed binding EBS storage class test", - CloudProviders: []string{"aws"}, - Provisioner: "kubernetes.io/aws-ebs", - ClaimSize: "2Gi", - DelayBinding: true, - }, - { - Name: "Delayed binding GCE PD storage class test", - CloudProviders: []string{"gce", "gke"}, - Provisioner: "kubernetes.io/gce-pd", - ClaimSize: "2Gi", - DelayBinding: true, - }, - } - for _, test := range tests { - if !framework.ProviderIs(test.CloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) - continue - } - By("creating a claim with class with waitForFirstConsumer") - suffix := "delayed" - class := newStorageClass(test, ns, suffix) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) - if node == nil { - framework.Failf("unexpected nil node found") - } - zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] - if !ok { - framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) - } - checkZoneFromLabelAndAffinity(pv, zone, true) - } + It("should create persistent volumes in the same zone as node after a pod mounting the claims is started", func() { + testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 1 /*pvcCount*/) + testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 3 /*pvcCount*/) }) }) Describe("DynamicProvisioner allowedTopologies", func() { @@ -921,51 +978,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) }) Describe("DynamicProvisioner delayed binding with allowedTopologies [Slow]", func() { - It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() { - tests := []testsuites.StorageClassTest{ - { - Name: "AllowedTopologies and delayed binding EBS storage class test", - CloudProviders: []string{"aws"}, - Provisioner: "kubernetes.io/aws-ebs", - ClaimSize: "2Gi", - DelayBinding: true, - }, - { - Name: "AllowedTopologies and delayed binding GCE PD storage class test", - CloudProviders: []string{"gce", "gke"}, - Provisioner: "kubernetes.io/gce-pd", - ClaimSize: "2Gi", - DelayBinding: true, - }, - } - for _, test := range tests { - if !framework.ProviderIs(test.CloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) - continue - } - By("creating a claim with class with WaitForFirstConsumer and allowedTopologies") - suffix := "delayed-topo" - class := newStorageClass(test, ns, suffix) - topoZone := getRandomCloudZone(c) - addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) - if node == nil { - framework.Failf("unexpected nil node found") - } - nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] - if !ok { - framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) - } - if topoZone != nodeZone { - framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, nodeZone) - } - checkZoneFromLabelAndAffinity(pv, topoZone, true) - } + It("should create persistent volumes in the same zone as specified in allowedTopologies after a pod mounting the claims is started", func() { + testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 1 /*pvcCount*/) + testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 3 /*pvcCount*/) }) }) - }) func getDefaultStorageClassName(c clientset.Interface) string {