diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index e27929635a7..db209cf9cba 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -1349,7 +1349,7 @@ var _ = SIGDescribe("StatefulSet", func() { }) }) - ginkgo.It("PVC should be recreated when pod is pending due to missing PVC [Feature:AutomaticPVCRecreation]", func() { + ginkgo.It("PVC should be recreated when pod is pending due to missing PVC [Feature:AutomaticPVCRecreation][Disruptive][Serial]", func() { // This test must be run in an environment that will allow the test pod to be scheduled on a node with local volumes ssName := "test-ss" headlessSvcName := "test" @@ -1394,36 +1394,37 @@ var _ = SIGDescribe("StatefulSet", func() { ginkgo.By("Cordoning Node") _, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) framework.ExpectNoError(err) + cordoned := true - // wait for the node to be patched - time.Sleep(5 * time.Second) - node, err = c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - framework.ExpectNoError(err) - framework.ExpectEqual(node.Spec.Unschedulable, true) + defer func() { + if cordoned { + uncordonNode(c, oldData, newData, nodeName) + } + }() + + // wait for the node to be unschedulable + e2enode.WaitForNodeSchedulable(c, nodeName, 10*time.Second, false) ginkgo.By("Deleting Pod") err = c.CoreV1().Pods(ns).Delete(context.TODO(), podName, metav1.DeleteOptions{}) framework.ExpectNoError(err) // wait for the pod to be recreated - time.Sleep(10 * time.Second) + e2estatefulset.WaitForStatusCurrentReplicas(c, ss, 1) _, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) framework.ExpectNoError(err) pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) framework.ExpectNoError(err) + framework.ExpectEqual(len(pvcList.Items), 1) pvcName := pvcList.Items[0].Name ginkgo.By("Deleting PVC") err = c.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvcName, metav1.DeleteOptions{}) framework.ExpectNoError(err) - ginkgo.By("Uncordoning Node") - // uncordon node, by reverting patch - revertPatchBytes, err := strategicpatch.CreateTwoWayMergePatch(newData, oldData, v1.Node{}) - framework.ExpectNoError(err) - _, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, revertPatchBytes, metav1.PatchOptions{}) - framework.ExpectNoError(err) + uncordonNode(c, oldData, newData, nodeName) + cordoned = false ginkgo.By("Confirming PVC recreated") err = verifyStatefulSetPVCsExist(c, ss, []int{0}) @@ -1436,6 +1437,15 @@ var _ = SIGDescribe("StatefulSet", func() { }) }) +func uncordonNode(c clientset.Interface, oldData, newData []byte, nodeName string) { + ginkgo.By("Uncordoning Node") + // uncordon node, by reverting patch + revertPatchBytes, err := strategicpatch.CreateTwoWayMergePatch(newData, oldData, v1.Node{}) + framework.ExpectNoError(err) + _, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, revertPatchBytes, metav1.PatchOptions{}) + framework.ExpectNoError(err) +} + func kubectlExecWithRetries(ns string, args ...string) (out string) { var err error for i := 0; i < 3; i++ { diff --git a/test/e2e/framework/node/wait.go b/test/e2e/framework/node/wait.go index 60b79cd6477..ffda416b693 100644 --- a/test/e2e/framework/node/wait.go +++ b/test/e2e/framework/node/wait.go @@ -143,6 +143,23 @@ func WaitForNodeToBeReady(ctx context.Context, c clientset.Interface, name strin return WaitConditionToBe(ctx, c, name, v1.NodeReady, true, timeout) } +func WaitForNodeSchedulable(c clientset.Interface, name string, timeout time.Duration, wantSchedulable bool) bool { + framework.Logf("Waiting up to %v for node %s to be schedulable: %t", timeout, name, wantSchedulable) + for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { + node, err := c.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + framework.Logf("Couldn't get node %s", name) + continue + } + + if IsNodeSchedulable(node) == wantSchedulable { + return true + } + } + framework.Logf("Node %s didn't reach desired schedulable status (%t) within %v", name, wantSchedulable, timeout) + return false +} + // CheckReady waits up to timeout for cluster to has desired size and // there is no not-ready nodes in it. By cluster size we mean number of schedulable Nodes. func CheckReady(ctx context.Context, c clientset.Interface, size int, timeout time.Duration) ([]v1.Node, error) { diff --git a/test/e2e/framework/statefulset/wait.go b/test/e2e/framework/statefulset/wait.go index f05c50dc353..7e4d7053788 100644 --- a/test/e2e/framework/statefulset/wait.go +++ b/test/e2e/framework/statefulset/wait.go @@ -171,6 +171,31 @@ func WaitForStatusReplicas(ctx context.Context, c clientset.Interface, ss *appsv } } +// WaitForStatusCurrentReplicas waits for the ss.Status.CurrentReplicas to be equal to expectedReplicas +func WaitForStatusCurrentReplicas(c clientset.Interface, ss *appsv1.StatefulSet, expectedReplicas int32) { + framework.Logf("Waiting for statefulset status.currentReplicas updated to %d", expectedReplicas) + + ns, name := ss.Namespace, ss.Name + pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout, + func() (bool, error) { + ssGet, err := c.AppsV1().StatefulSets(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if ssGet.Status.ObservedGeneration < ss.Generation { + return false, nil + } + if ssGet.Status.CurrentReplicas != expectedReplicas { + framework.Logf("Waiting for stateful set status.currentReplicas to become %d, currently %d", expectedReplicas, ssGet.Status.CurrentReplicas) + return false, nil + } + return true, nil + }) + if pollErr != nil { + framework.Failf("Failed waiting for stateful set status.currentReplicas updated to %d: %v", expectedReplicas, pollErr) + } +} + // Saturate waits for all Pods in ss to become Running and Ready. func Saturate(ctx context.Context, c clientset.Interface, ss *appsv1.StatefulSet) { var i int32