mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-01 15:58:37 +00:00
Merge pull request #120731 from Nordix/sts_issue/adil
Fixing CurrentReplicas and CurrentRevision in completeRollingUpdate
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
568aee16e8
@ -582,8 +582,9 @@ func inconsistentStatus(set *apps.StatefulSet, status *apps.StatefulSetStatus) b
|
||||
// are set to 0.
|
||||
func completeRollingUpdate(set *apps.StatefulSet, status *apps.StatefulSetStatus) {
|
||||
if set.Spec.UpdateStrategy.Type == apps.RollingUpdateStatefulSetStrategyType &&
|
||||
status.UpdatedReplicas == status.Replicas &&
|
||||
status.ReadyReplicas == status.Replicas {
|
||||
status.UpdatedReplicas == *set.Spec.Replicas &&
|
||||
status.ReadyReplicas == *set.Spec.Replicas &&
|
||||
status.Replicas == *set.Spec.Replicas {
|
||||
status.CurrentReplicas = status.UpdatedReplicas
|
||||
status.CurrentRevision = status.UpdateRevision
|
||||
}
|
||||
|
||||
@ -77,6 +77,8 @@ const (
|
||||
statefulSetTimeout = 10 * time.Minute
|
||||
// statefulPodTimeout is a timeout for stateful pods to change state
|
||||
statefulPodTimeout = 5 * time.Minute
|
||||
|
||||
testFinalizer = "example.com/test-finalizer"
|
||||
)
|
||||
|
||||
var httpProbe = &v1.Probe{
|
||||
@ -510,6 +512,51 @@ var _ = SIGDescribe("StatefulSet", func() {
|
||||
|
||||
})
|
||||
|
||||
ginkgo.It("should perform canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 without failing container", func(ctx context.Context) {
|
||||
ginkgo.By("Creating a new StatefulSet without failing container")
|
||||
ss := e2estatefulset.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
|
||||
deletingPodForRollingUpdatePartitionTest(ctx, f, c, ns, ss)
|
||||
})
|
||||
|
||||
ginkgo.It("should perform canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 with failing container", func(ctx context.Context) {
|
||||
ginkgo.By("Creating a new StatefulSet with failing container")
|
||||
ss := e2estatefulset.NewStatefulSet("ss3", ns, headlessSvcName, 3, nil, nil, labels)
|
||||
ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, v1.Container{
|
||||
Name: "sleep-exit-with-1",
|
||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
||||
Command: []string{"sh", "-c"},
|
||||
Args: []string{`
|
||||
echo "Running in pod $POD_NAME"
|
||||
_term(){
|
||||
echo "Received SIGTERM signal"
|
||||
if [ "${POD_NAME}" = "ss3-0" ]; then
|
||||
exit 1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
trap _term SIGTERM
|
||||
while true; do
|
||||
echo "Running in infinite loop in $POD_NAME"
|
||||
sleep 1
|
||||
done
|
||||
`,
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "POD_NAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
APIVersion: "v1",
|
||||
FieldPath: "metadata.name",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
deletingPodForRollingUpdatePartitionTest(ctx, f, c, ns, ss)
|
||||
})
|
||||
|
||||
// Do not mark this as Conformance.
|
||||
// The legacy OnDelete strategy only exists for backward compatibility with pre-v1 APIs.
|
||||
ginkgo.It("should implement legacy replacement when the update strategy is OnDelete", func(ctx context.Context) {
|
||||
@ -1945,6 +1992,144 @@ func rollbackTest(ctx context.Context, c clientset.Interface, ns string, ss *app
|
||||
}
|
||||
}
|
||||
|
||||
// This function is used canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0
|
||||
func deletingPodForRollingUpdatePartitionTest(ctx context.Context, f *framework.Framework, c clientset.Interface, ns string, ss *appsv1.StatefulSet) {
|
||||
setHTTPProbe(ss)
|
||||
ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
|
||||
Type: appsv1.RollingUpdateStatefulSetStrategyType,
|
||||
RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
|
||||
return &appsv1.RollingUpdateStatefulSetStrategy{
|
||||
Partition: pointer.Int32(1),
|
||||
}
|
||||
}(),
|
||||
}
|
||||
ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
|
||||
ss = waitForStatus(ctx, c, ss)
|
||||
currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision
|
||||
gomega.Expect(currentRevision).To(gomega.Equal(updateRevision), fmt.Sprintf("StatefulSet %s/%s created with update revision %s not equal to current revision %s",
|
||||
ss.Namespace, ss.Name, updateRevision, currentRevision))
|
||||
pods := e2estatefulset.GetPodList(ctx, c, ss)
|
||||
for i := range pods.Items {
|
||||
gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s revision %s is not equal to currentRevision %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
|
||||
currentRevision))
|
||||
}
|
||||
|
||||
ginkgo.By("Adding finalizer for pod-0")
|
||||
pod0name := getStatefulSetPodNameAtIndex(0, ss)
|
||||
pod0, err := c.CoreV1().Pods(ns).Get(ctx, pod0name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
pod0.Finalizers = append(pod0.Finalizers, testFinalizer)
|
||||
pod0, err = c.CoreV1().Pods(ss.Namespace).Update(ctx, pod0, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
pods.Items[0] = *pod0
|
||||
defer e2epod.NewPodClient(f).RemoveFinalizer(ctx, pod0.Name, testFinalizer)
|
||||
|
||||
ginkgo.By("Updating image on StatefulSet")
|
||||
newImage := NewWebserverImage
|
||||
oldImage := ss.Spec.Template.Spec.Containers[0].Image
|
||||
ginkgo.By(fmt.Sprintf("Updating stateful set template: update image from %s to %s", oldImage, newImage))
|
||||
gomega.Expect(oldImage).ToNot(gomega.Equal(newImage), "Incorrect test setup: should update to a different image")
|
||||
ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
|
||||
update.Spec.Template.Spec.Containers[0].Image = newImage
|
||||
})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("Creating a new revision")
|
||||
ss = waitForStatus(ctx, c, ss)
|
||||
currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision
|
||||
gomega.Expect(currentRevision).ToNot(gomega.Equal(updateRevision), "Current revision should not equal update revision during rolling update")
|
||||
|
||||
ginkgo.By("Await for all replicas running, all are updated but pod-0")
|
||||
e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
|
||||
ss = set2
|
||||
pods = pods2
|
||||
if ss.Status.UpdatedReplicas == *ss.Spec.Replicas-1 && ss.Status.Replicas == *ss.Spec.Replicas && ss.Status.ReadyReplicas == *ss.Spec.Replicas {
|
||||
// rolling updated is not completed, because replica 0 isn't ready
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
ginkgo.By("Verify pod images before pod-0 deletion and recreation")
|
||||
for i := range pods.Items {
|
||||
if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
|
||||
gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to oldimage image %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Spec.Containers[0].Image,
|
||||
oldImage))
|
||||
gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to current revision %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
|
||||
currentRevision))
|
||||
} else {
|
||||
gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), fmt.Sprintf("Pod %s/%s has image %s not equal to new image %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Spec.Containers[0].Image,
|
||||
newImage))
|
||||
gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(updateRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to new revision %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
|
||||
updateRevision))
|
||||
}
|
||||
}
|
||||
|
||||
ginkgo.By("Deleting the pod-0 so that kubelet terminates it and StatefulSet controller recreates it")
|
||||
deleteStatefulPodAtIndex(ctx, c, 0, ss)
|
||||
ginkgo.By("Await for two replicas to be updated, while the pod-0 is not running")
|
||||
e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
|
||||
ss = set2
|
||||
pods = pods2
|
||||
return ss.Status.ReadyReplicas == *ss.Spec.Replicas-1, nil
|
||||
})
|
||||
|
||||
ginkgo.By(fmt.Sprintf("Removing finalizer from pod-0 (%v/%v) to allow recreation", pod0.Namespace, pod0.Name))
|
||||
e2epod.NewPodClient(f).RemoveFinalizer(ctx, pod0.Name, testFinalizer)
|
||||
|
||||
ginkgo.By("Await for recreation of pod-0, so that all replicas are running")
|
||||
e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
|
||||
ss = set2
|
||||
pods = pods2
|
||||
return ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil
|
||||
})
|
||||
|
||||
ginkgo.By("Verify pod images after pod-0 deletion and recreation")
|
||||
pods = e2estatefulset.GetPodList(ctx, c, ss)
|
||||
for i := range pods.Items {
|
||||
if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
|
||||
gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to current image %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Spec.Containers[0].Image,
|
||||
oldImage))
|
||||
gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to current revision %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
|
||||
currentRevision))
|
||||
} else {
|
||||
gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), fmt.Sprintf("Pod %s/%s has image %s not equal to new image %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Spec.Containers[0].Image,
|
||||
newImage))
|
||||
gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(updateRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to new revision %s",
|
||||
pods.Items[i].Namespace,
|
||||
pods.Items[i].Name,
|
||||
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
|
||||
updateRevision))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// confirmStatefulPodCount asserts that the current number of Pods in ss is count, waiting up to timeout for ss to
|
||||
// scale to count.
|
||||
func confirmStatefulPodCount(ctx context.Context, c clientset.Interface, count int, ss *appsv1.StatefulSet, timeout time.Duration, hard bool) {
|
||||
|
||||
@ -43,6 +43,7 @@ import (
|
||||
"k8s.io/kubernetes/pkg/controlplane"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/test/integration/framework"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -489,6 +490,146 @@ func TestAutodeleteOwnerRefs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletingPodForRollingUpdatePartition(t *testing.T) {
|
||||
_, ctx := ktesting.NewTestContext(t)
|
||||
closeFn, rm, informers, c := scSetup(ctx, t)
|
||||
defer closeFn()
|
||||
ns := framework.CreateNamespaceOrDie(c, "test-deleting-pod-for-rolling-update-partition", t)
|
||||
defer framework.DeleteNamespaceOrDie(c, ns, t)
|
||||
cancel := runControllerAndInformers(rm, informers)
|
||||
defer cancel()
|
||||
|
||||
labelMap := labelMap()
|
||||
sts := newSTS("sts", ns.Name, 2)
|
||||
sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
|
||||
Type: appsv1.RollingUpdateStatefulSetStrategyType,
|
||||
RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
|
||||
return &appsv1.RollingUpdateStatefulSetStrategy{
|
||||
Partition: ptr.To[int32](1),
|
||||
}
|
||||
}(),
|
||||
}
|
||||
stss, _ := createSTSsPods(t, c, []*appsv1.StatefulSet{sts}, []*v1.Pod{})
|
||||
sts = stss[0]
|
||||
waitSTSStable(t, c, sts)
|
||||
|
||||
// Verify STS creates 2 pods
|
||||
podClient := c.CoreV1().Pods(ns.Name)
|
||||
pods := getPods(t, podClient, labelMap)
|
||||
if len(pods.Items) != 2 {
|
||||
t.Fatalf("len(pods) = %d, want 2", len(pods.Items))
|
||||
}
|
||||
// Setting all pods in Running, Ready, and Available
|
||||
setPodsReadyCondition(t, c, &v1.PodList{Items: pods.Items}, v1.ConditionTrue, time.Now())
|
||||
|
||||
// 1. Roll out a new image.
|
||||
oldImage := sts.Spec.Template.Spec.Containers[0].Image
|
||||
newImage := "new-image"
|
||||
if oldImage == newImage {
|
||||
t.Fatalf("bad test setup, statefulSet %s roll out with the same image", sts.Name)
|
||||
}
|
||||
// Set finalizers for the pod-0 to trigger pod recreation failure while the status UpdateRevision is bumped
|
||||
pod0 := &pods.Items[0]
|
||||
updatePod(t, podClient, pod0.Name, func(pod *v1.Pod) {
|
||||
pod.Finalizers = []string{"fake.example.com/blockDeletion"}
|
||||
})
|
||||
|
||||
stsClient := c.AppsV1().StatefulSets(ns.Name)
|
||||
_ = updateSTS(t, stsClient, sts.Name, func(sts *appsv1.StatefulSet) {
|
||||
sts.Spec.Template.Spec.Containers[0].Image = newImage
|
||||
})
|
||||
|
||||
// Await for the pod-1 to be recreated, while pod-0 remains running
|
||||
if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
|
||||
ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
pods := getPods(t, podClient, labelMap)
|
||||
recreatedPods := v1.PodList{}
|
||||
for _, pod := range pods.Items {
|
||||
if pod.Status.Phase == v1.PodPending {
|
||||
recreatedPods.Items = append(recreatedPods.Items, pod)
|
||||
}
|
||||
}
|
||||
setPodsReadyCondition(t, c, &v1.PodList{Items: recreatedPods.Items}, v1.ConditionTrue, time.Now())
|
||||
return ss.Status.UpdatedReplicas == *ss.Spec.Replicas-*sts.Spec.UpdateStrategy.RollingUpdate.Partition && ss.Status.Replicas == *ss.Spec.Replicas && ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("failed to await for pod-1 to be recreated by sts %s: %v", sts.Name, err)
|
||||
}
|
||||
|
||||
// Mark pod-0 as terminal and not ready
|
||||
updatePodStatus(t, podClient, pod0.Name, func(pod *v1.Pod) {
|
||||
pod.Status.Phase = v1.PodFailed
|
||||
})
|
||||
|
||||
// Make sure pod-0 gets deletion timestamp so that it is recreated
|
||||
if err := c.CoreV1().Pods(ns.Name).Delete(context.TODO(), pod0.Name, metav1.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("error deleting pod %s: %v", pod0.Name, err)
|
||||
}
|
||||
|
||||
// Await for pod-0 to be not ready
|
||||
if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
|
||||
ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ss.Status.ReadyReplicas == *ss.Spec.Replicas-1, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("failed to await for pod-0 to be not counted as ready in status of sts %s: %v", sts.Name, err)
|
||||
}
|
||||
|
||||
// Remove the finalizer to allow recreation
|
||||
updatePod(t, podClient, pod0.Name, func(pod *v1.Pod) {
|
||||
pod.Finalizers = []string{}
|
||||
})
|
||||
|
||||
// Await for pod-0 to be recreated and make it running
|
||||
if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
|
||||
pods := getPods(t, podClient, labelMap)
|
||||
recreatedPods := v1.PodList{}
|
||||
for _, pod := range pods.Items {
|
||||
if pod.Status.Phase == v1.PodPending {
|
||||
recreatedPods.Items = append(recreatedPods.Items, pod)
|
||||
}
|
||||
}
|
||||
setPodsReadyCondition(t, c, &v1.PodList{Items: recreatedPods.Items}, v1.ConditionTrue, time.Now().Add(-120*time.Minute))
|
||||
return len(recreatedPods.Items) > 0, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("failed to await for pod-0 to be recreated by sts %s: %v", sts.Name, err)
|
||||
}
|
||||
|
||||
// Await for all stateful set status to record all replicas as ready
|
||||
if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
|
||||
ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("failed to verify .Spec.Template.Spec.Containers[0].Image is updated for sts %s: %v", sts.Name, err)
|
||||
}
|
||||
|
||||
// Verify 3 pods exist
|
||||
pods = getPods(t, podClient, labelMap)
|
||||
if len(pods.Items) != int(*sts.Spec.Replicas) {
|
||||
t.Fatalf("Unexpected number of pods")
|
||||
}
|
||||
|
||||
// Verify pod images
|
||||
for i := range pods.Items {
|
||||
if i < int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition) {
|
||||
if pods.Items[i].Spec.Containers[0].Image != oldImage {
|
||||
t.Fatalf("Pod %s has image %s not equal to old image %s", pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, oldImage)
|
||||
}
|
||||
} else {
|
||||
if pods.Items[i].Spec.Containers[0].Image != newImage {
|
||||
t.Fatalf("Pod %s has image %s not equal to new image %s", pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, newImage)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetStartOrdinal(t *testing.T) {
|
||||
tests := []struct {
|
||||
ordinals *appsv1.StatefulSetOrdinals
|
||||
|
||||
Loading…
Reference in New Issue
Block a user