From 9d80516abb4244ac1df4ceb84a32d3204fe5b308 Mon Sep 17 00:00:00 2001 From: Iceber Gu Date: Tue, 10 Jan 2023 13:09:07 +0800 Subject: [PATCH 1/2] client-go: fix the wait time for trying to acquire the leader lease Signed-off-by: Iceber Gu --- .../tools/leaderelection/leaderelection.go | 2 +- .../leaderelection/leaderelection_test.go | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go index c64ba9b26b0..f6e5ebc710a 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go @@ -347,7 +347,7 @@ func (le *LeaderElector) tryAcquireOrRenew(ctx context.Context) bool { le.observedRawRecord = oldLeaderElectionRawRecord } if len(oldLeaderElectionRecord.HolderIdentity) > 0 && - le.observedTime.Add(le.config.LeaseDuration).After(now.Time) && + le.observedTime.Add(time.Second*time.Duration(oldLeaderElectionRecord.LeaseDurationSeconds)).After(now.Time) && !le.IsLeader() { klog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity) return false diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go index 6b5b079050d..1f9718caddb 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go @@ -80,6 +80,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { name string observedRecord rl.LeaderElectionRecord observedTime time.Time + retryAfter time.Duration reactors []Reactor expectSuccess bool @@ -125,6 +126,33 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { transitionLeader: true, outHolder: "baz", }, + { + name: "acquire from led object with the lease duration seconds", + reactors: []Reactor{ + { + verb: "get", + reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { + return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil + }, + }, + { + verb: "get", + reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { + return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil + }, + }, + { + verb: "update", + reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { + return true, action.(fakeclient.CreateAction).GetObject(), nil + }, + }, + }, + retryAfter: 3 * time.Second, + expectSuccess: true, + transitionLeader: true, + outHolder: "baz", + }, { name: "acquire from unled object", reactors: []Reactor{ @@ -283,7 +311,14 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { clock: clock.RealClock{}, } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { - t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) + if test.retryAfter != 0 { + time.Sleep(test.retryAfter) + if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { + t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) + } + } else { + t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) + } } le.observedRecord.AcquireTime = metav1.Time{} From 9c31f14a7cfea7cc793a9a3f98bdd117cb438316 Mon Sep 17 00:00:00 2001 From: Iceber Gu Date: Tue, 7 Feb 2023 13:00:55 +0800 Subject: [PATCH 2/2] leaderelection: using le.clock for the time calculations Signed-off-by: Iceber Gu --- .../tools/leaderelection/leaderelection.go | 4 ++-- .../tools/leaderelection/leaderelection_test.go | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go index f6e5ebc710a..f3bf93dace6 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go @@ -295,7 +295,7 @@ func (le *LeaderElector) release() bool { if !le.IsLeader() { return true } - now := metav1.Now() + now := metav1.NewTime(le.clock.Now()) leaderElectionRecord := rl.LeaderElectionRecord{ LeaderTransitions: le.observedRecord.LeaderTransitions, LeaseDurationSeconds: 1, @@ -315,7 +315,7 @@ func (le *LeaderElector) release() bool { // else it tries to renew the lease if it has already been acquired. Returns true // on success else returns false. func (le *LeaderElector) tryAcquireOrRenew(ctx context.Context) bool { - now := metav1.Now() + now := metav1.NewTime(le.clock.Now()) leaderElectionRecord := rl.LeaderElectionRecord{ HolderIdentity: le.config.Lock.Identity(), LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second), diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go index 1f9718caddb..fbb408ae606 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go @@ -73,8 +73,9 @@ type Reactor struct { } func testTryAcquireOrRenew(t *testing.T, objectType string) { - future := time.Now().Add(1000 * time.Hour) - past := time.Now().Add(-1000 * time.Hour) + clock := clock.RealClock{} + future := clock.Now().Add(1000 * time.Hour) + past := clock.Now().Add(-1000 * time.Hour) tests := []struct { name string @@ -308,7 +309,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { observedRecord: test.observedRecord, observedRawRecord: observedRawRecord, observedTime: test.observedTime, - clock: clock.RealClock{}, + clock: clock, } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { if test.retryAfter != 0 { @@ -409,8 +410,9 @@ func GetRawRecordOrDie(t *testing.T, objectType string, ler rl.LeaderElectionRec } func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) { - future := time.Now().Add(1000 * time.Hour) - past := time.Now().Add(-1000 * time.Hour) + clock := clock.RealClock{} + future := clock.Now().Add(1000 * time.Hour) + past := clock.Now().Add(-1000 * time.Hour) primaryType, secondaryType := multiLockType(t, objectType) tests := []struct { name string @@ -867,7 +869,7 @@ func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) { observedRecord: test.observedRecord, observedRawRecord: test.observedRawRecord, observedTime: test.observedTime, - clock: clock.RealClock{}, + clock: clock, } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess)