diff --git a/test/e2e/framework/pv_util.go b/test/e2e/framework/pv_util.go index c81e024b603..3b4784f848c 100644 --- a/test/e2e/framework/pv_util.go +++ b/test/e2e/framework/pv_util.go @@ -926,6 +926,26 @@ func CreateClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeC return CreatePod(c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "") } +// CreateUnschedulablePod with given claims based on node selector +func CreateUnschedulablePod(client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) (*v1.Pod, error) { + pod := MakePod(namespace, nodeSelector, pvclaims, isPrivileged, command) + pod, err := client.CoreV1().Pods(namespace).Create(pod) + if err != nil { + return nil, fmt.Errorf("pod Create API error: %v", err) + } + // Waiting for pod to become Unschedulable + err = WaitForPodNameUnschedulableInNamespace(client, pod.Name, namespace) + if err != nil { + return pod, fmt.Errorf("pod %q is not Unschedulable: %v", pod.Name, err) + } + // get fresh pod info + pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + if err != nil { + return pod, fmt.Errorf("pod Get API error: %v", err) + } + return pod, nil +} + // wait until all pvcs phase set to bound func WaitForPVClaimBoundPhase(client clientset.Interface, pvclaims []*v1.PersistentVolumeClaim, timeout time.Duration) ([]*v1.PersistentVolume, error) { persistentvolumes := make([]*v1.PersistentVolume, len(pvclaims)) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index fdb5ad2dd4f..6a5566d842c 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -1547,6 +1547,28 @@ func WaitForPodToDisappear(c clientset.Interface, ns, podName string, label labe }) } +// WaitForPodNameUnschedulableInNamespace returns an error if it takes too long for the pod to become Pending +// and have condition Status equal to Unschedulable, +// if the pod Get api returns an error (IsNotFound or other), or if the pod failed with an unexpected reason. +// Typically called to test that the passed-in pod is Pending and Unschedulable. +func WaitForPodNameUnschedulableInNamespace(c clientset.Interface, podName, namespace string) error { + return WaitForPodCondition(c, namespace, podName, "Unschedulable", PodStartTimeout, func(pod *v1.Pod) (bool, error) { + // Only consider Failed pods. Successful pods will be deleted and detected in + // waitForPodCondition's Get call returning `IsNotFound` + if pod.Status.Phase == v1.PodPending { + for _, cond := range pod.Status.Conditions { + if cond.Type == v1.PodScheduled && cond.Status == v1.ConditionFalse && cond.Reason == "Unschedulable" { + return true, nil + } + } + } + if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { + return true, fmt.Errorf("Expected pod %q in namespace %q to be in phase Pending, but got phase: %v", podName, namespace, pod.Status.Phase) + } + return false, nil + }) +} + // WaitForService waits until the service appears (exist == true), or disappears (exist == false) func WaitForService(c clientset.Interface, namespace, name string, exist bool, interval, timeout time.Duration) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { @@ -5119,8 +5141,8 @@ func waitForServerPreferredNamespacedResources(d discovery.DiscoveryInterface, t return resources, nil } -// WaitForPersistentVolumeClaimBeRemoved waits for a PersistentVolumeClaim to be removed from the system until timeout occurs, whichever comes first. -func WaitForPersistentVolumeClaimBeRemoved(c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error { +// WaitForPersistentVolumeClaimDeleted waits for a PersistentVolumeClaim to be removed from the system until timeout occurs, whichever comes first. +func WaitForPersistentVolumeClaimDeleted(c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error { Logf("Waiting up to %v for PersistentVolumeClaim %s to be removed", timeout, pvcName) for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) { _, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{}) diff --git a/test/e2e/storage/pvc_protection.go b/test/e2e/storage/pvc_protection.go index 8eaac20439c..16f2aa0e57e 100644 --- a/test/e2e/storage/pvc_protection.go +++ b/test/e2e/storage/pvc_protection.go @@ -76,7 +76,7 @@ var _ = utils.SIGDescribe("PVC Protection [Feature:PVCProtection]", func() { By("Deleting the PVC") err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, metav1.NewDeleteOptions(0)) Expect(err).NotTo(HaveOccurred(), "Error deleting PVC") - framework.WaitForPersistentVolumeClaimBeRemoved(client, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) + framework.WaitForPersistentVolumeClaimDeleted(client, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) pvcCreatedAndNotDeleted = false }) @@ -100,7 +100,44 @@ var _ = utils.SIGDescribe("PVC Protection [Feature:PVCProtection]", func() { Expect(err).NotTo(HaveOccurred(), "Error terminating and deleting pod") By("Checking that the PVC is automatically removed from the system because it's no longer in active use by a pod") - framework.WaitForPersistentVolumeClaimBeRemoved(client, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) + framework.WaitForPersistentVolumeClaimDeleted(client, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) + pvcCreatedAndNotDeleted = false + }) + + It("Verify that scheduling of a pod that uses PVC that is being deleted fails and the pod becomes Unschedulable", func() { + By("Creating first Pod that becomes Running and therefore is actively using the PVC") + pvcClaims := []*v1.PersistentVolumeClaim{pvc} + firstPod, err := framework.CreatePod(client, nameSpace, nil, pvcClaims, false, "") + Expect(err).NotTo(HaveOccurred(), "While creating pod that uses the PVC or waiting for the Pod to become Running") + + By("Deleting the PVC, however, the PVC must not be removed from the system as it's in active use by a pod") + err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, metav1.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred(), "Error deleting PVC") + + By("Checking that the PVC status is Terminating") + pvc, err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "While checking PVC status") + Expect(pvc.ObjectMeta.DeletionTimestamp).NotTo(Equal(nil)) + + By("Creating second Pod whose scheduling fails because it uses a PVC that is being deleted") + secondPod, err2 := framework.CreateUnschedulablePod(client, nameSpace, nil, pvcClaims, false, "") + Expect(err2).NotTo(HaveOccurred(), "While creating second pod that uses a PVC that is being deleted and that is Unschedulable") + + By("Deleting the second pod that uses the PVC that is being deleted") + err = framework.DeletePodWithWait(f, client, secondPod) + Expect(err).NotTo(HaveOccurred(), "Error terminating and deleting pod") + + By("Checking again that the PVC status is Terminating") + pvc, err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "While checking PVC status") + Expect(pvc.ObjectMeta.DeletionTimestamp).NotTo(Equal(nil)) + + By("Deleting the first pod that uses the PVC") + err = framework.DeletePodWithWait(f, client, firstPod) + Expect(err).NotTo(HaveOccurred(), "Error terminating and deleting pod") + + By("Checking that the PVC is automatically removed from the system because it's no longer in active use by a pod") + framework.WaitForPersistentVolumeClaimDeleted(client, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) pvcCreatedAndNotDeleted = false }) })