mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-20 18:31:15 +00:00
Cleanup statefulset after e2e test
This commit is contained in:
Rahul Rangith
parent
392cd5ce8c
commit
8924b58e2c
@ -1349,99 +1349,113 @@ var _ = SIGDescribe("StatefulSet", func() {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
ginkgo.It("PVC should be recreated when pod is pending due to missing PVC [Disruptive][Serial]", func() {
|
ginkgo.Describe("Automatically recreate PVC for pending pod when PVC is missing", func() {
|
||||||
ssName := "test-ss"
|
ssName := "ss"
|
||||||
headlessSvcName := "test"
|
labels := map[string]string{
|
||||||
// Define StatefulSet Labels
|
"foo": "bar",
|
||||||
ssPodLabels := map[string]string{
|
"baz": "blah",
|
||||||
"name": "sample-pod",
|
|
||||||
"pod": WebserverImageName,
|
|
||||||
}
|
}
|
||||||
|
headlessSvcName := "test"
|
||||||
|
var statefulPodMounts []v1.VolumeMount
|
||||||
|
var ss *appsv1.StatefulSet
|
||||||
|
|
||||||
readyNode, err := e2enode.GetReadySchedulableWorkerNode(c)
|
ginkgo.BeforeEach(func(ctx context.Context) {
|
||||||
framework.ExpectNoError(err)
|
statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}}
|
||||||
hostLabel := "kubernetes.io/hostname"
|
ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, statefulPodMounts, nil, labels)
|
||||||
hostLabelVal := readyNode.Labels[hostLabel]
|
})
|
||||||
|
|
||||||
statefulPodMounts := []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}}
|
ginkgo.AfterEach(func(ctx context.Context) {
|
||||||
ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, statefulPodMounts, nil, ssPodLabels)
|
if ginkgo.CurrentSpecReport().Failed() {
|
||||||
ss.Spec.Template.Spec.NodeSelector = map[string]string{hostLabel: hostLabelVal} // force the pod on a specific node
|
e2eoutput.DumpDebugInfo(ctx, c, ns)
|
||||||
|
|
||||||
e2epv.SkipIfNoDefaultStorageClass(c)
|
|
||||||
ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
|
|
||||||
_, err = c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
ginkgo.By("Confirming PVC exists")
|
|
||||||
err = verifyStatefulSetPVCsExist(c, ss, []int{0})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
ginkgo.By("Confirming Pod is ready")
|
|
||||||
e2estatefulset.WaitForStatusReadyReplicas(c, ss, 1)
|
|
||||||
podName := getStatefulSetPodNameAtIndex(0, ss)
|
|
||||||
pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
nodeName := pod.Spec.NodeName
|
|
||||||
framework.ExpectEqual(nodeName, readyNode.Name)
|
|
||||||
node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
oldData, err := json.Marshal(node)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
node.Spec.Unschedulable = true
|
|
||||||
|
|
||||||
newData, err := json.Marshal(node)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
// cordon node, to make sure pod does not get scheduled to the node until the pvc is deleted
|
|
||||||
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
ginkgo.By("Cordoning Node")
|
|
||||||
_, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
cordoned := true
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if cordoned {
|
|
||||||
uncordonNode(c, oldData, newData, nodeName)
|
|
||||||
}
|
}
|
||||||
}()
|
framework.Logf("Deleting all statefulset in ns %v", ns)
|
||||||
|
e2estatefulset.DeleteAllStatefulSets(ctx, c, ns)
|
||||||
|
})
|
||||||
|
|
||||||
// wait for the node to be unschedulable
|
ginkgo.It("PVC should be recreated when pod is pending due to missing PVC [Disruptive][Serial]", func(ctx context.Context) {
|
||||||
e2enode.WaitForNodeSchedulable(c, nodeName, 10*time.Second, false)
|
e2epv.SkipIfNoDefaultStorageClass(ctx, c)
|
||||||
|
|
||||||
ginkgo.By("Deleting Pod")
|
readyNode, err := e2enode.GetRandomReadySchedulableNode(ctx, c)
|
||||||
err = c.CoreV1().Pods(ns).Delete(context.TODO(), podName, metav1.DeleteOptions{})
|
framework.ExpectNoError(err)
|
||||||
framework.ExpectNoError(err)
|
hostLabel := "kubernetes.io/hostname"
|
||||||
|
hostLabelVal := readyNode.Labels[hostLabel]
|
||||||
|
|
||||||
// wait for the pod to be recreated
|
ss.Spec.Template.Spec.NodeSelector = map[string]string{hostLabel: hostLabelVal} // force the pod on a specific node
|
||||||
e2estatefulset.WaitForStatusCurrentReplicas(c, ss, 1)
|
ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
|
||||||
_, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
|
_, err = c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{})
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()})
|
ginkgo.By("Confirming PVC exists")
|
||||||
framework.ExpectNoError(err)
|
err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0})
|
||||||
framework.ExpectEqual(len(pvcList.Items), 1)
|
framework.ExpectNoError(err)
|
||||||
pvcName := pvcList.Items[0].Name
|
|
||||||
|
|
||||||
ginkgo.By("Deleting PVC")
|
ginkgo.By("Confirming Pod is ready")
|
||||||
err = c.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvcName, metav1.DeleteOptions{})
|
e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 1)
|
||||||
framework.ExpectNoError(err)
|
podName := getStatefulSetPodNameAtIndex(0, ss)
|
||||||
|
pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
uncordonNode(c, oldData, newData, nodeName)
|
nodeName := pod.Spec.NodeName
|
||||||
cordoned = false
|
framework.ExpectEqual(nodeName, readyNode.Name)
|
||||||
|
node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
ginkgo.By("Confirming PVC recreated")
|
oldData, err := json.Marshal(node)
|
||||||
err = verifyStatefulSetPVCsExist(c, ss, []int{0})
|
framework.ExpectNoError(err)
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
ginkgo.By("Confirming Pod is ready after being recreated")
|
node.Spec.Unschedulable = true
|
||||||
e2estatefulset.WaitForStatusReadyReplicas(c, ss, 1)
|
|
||||||
pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
|
newData, err := json.Marshal(node)
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
framework.ExpectEqual(pod.Spec.NodeName, readyNode.Name) // confirm the pod was scheduled back to the original node
|
|
||||||
|
// cordon node, to make sure pod does not get scheduled to the node until the pvc is deleted
|
||||||
|
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
ginkgo.By("Cordoning Node")
|
||||||
|
_, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
cordoned := true
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if cordoned {
|
||||||
|
uncordonNode(c, oldData, newData, nodeName)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// wait for the node to be unschedulable
|
||||||
|
e2enode.WaitForNodeSchedulable(c, nodeName, 10*time.Second, false)
|
||||||
|
|
||||||
|
ginkgo.By("Deleting Pod")
|
||||||
|
err = c.CoreV1().Pods(ns).Delete(context.TODO(), podName, metav1.DeleteOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
|
// wait for the pod to be recreated
|
||||||
|
e2estatefulset.WaitForStatusCurrentReplicas(c, ss, 1)
|
||||||
|
_, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
|
pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
framework.ExpectEqual(len(pvcList.Items), 1)
|
||||||
|
pvcName := pvcList.Items[0].Name
|
||||||
|
|
||||||
|
ginkgo.By("Deleting PVC")
|
||||||
|
err = c.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvcName, metav1.DeleteOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
|
uncordonNode(c, oldData, newData, nodeName)
|
||||||
|
cordoned = false
|
||||||
|
|
||||||
|
ginkgo.By("Confirming PVC recreated")
|
||||||
|
err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
|
ginkgo.By("Confirming Pod is ready after being recreated")
|
||||||
|
e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 1)
|
||||||
|
pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
framework.ExpectEqual(pod.Spec.NodeName, readyNode.Name) // confirm the pod was scheduled back to the original node
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user