From 2956e294c446721e57ed4b47111a0368373bc984 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Filip=20K=C5=99epinsk=C3=BD?= <atiratree@gmail.com>
Date: Mon, 11 Mar 2024 17:02:59 +0100
Subject: [PATCH] fix conflict race in statefulset rest update

statefulset controller does less requests per sync now and thus can
reconcile status faster, thus resulting in a higher chance for conflicts
---
 test/e2e/framework/statefulset/rest.go | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/test/e2e/framework/statefulset/rest.go b/test/e2e/framework/statefulset/rest.go
index 5047d86c6a1..2f18d6e333e 100644
--- a/test/e2e/framework/statefulset/rest.go
+++ b/test/e2e/framework/statefulset/rest.go
@@ -25,12 +25,12 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/kubectl/pkg/util/podutils"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest"
@@ -247,21 +247,23 @@ func ExecInStatefulPods(ctx context.Context, c clientset.Interface, ss *appsv1.S
 }
 
 // update updates a statefulset, and it is only used within rest.go
-func update(ctx context.Context, c clientset.Interface, ns, name string, replicas int32) *appsv1.StatefulSet {
-	for i := 0; i < 3; i++ {
-		ss, err := c.AppsV1().StatefulSets(ns).Get(ctx, name, metav1.GetOptions{})
+func update(ctx context.Context, c clientset.Interface, ns, name string, replicas int32) (ss *appsv1.StatefulSet) {
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		var err error
+		ss, err = c.AppsV1().StatefulSets(ns).Get(ctx, name, metav1.GetOptions{})
 		if err != nil {
 			framework.Failf("failed to get statefulset %q: %v", name, err)
 		}
+		if *(ss.Spec.Replicas) == replicas {
+			return nil
+		}
 		*(ss.Spec.Replicas) = replicas
 		ss, err = c.AppsV1().StatefulSets(ns).Update(ctx, ss, metav1.UpdateOptions{})
-		if err == nil {
-			return ss
-		}
-		if !apierrors.IsConflict(err) && !apierrors.IsServerTimeout(err) {
-			framework.Failf("failed to update statefulset %q: %v", name, err)
-		}
+		return err
+	})
+	if err == nil {
+		return ss
 	}
-	framework.Failf("too many retries draining statefulset %q", name)
+	framework.Failf("failed to update statefulset  %q: %v", name, err)
 	return nil
 }