From b4217e149b42e3494fad9477e9c272e470de44e0 Mon Sep 17 00:00:00 2001 From: Iceber Gu Date: Tue, 10 Jan 2023 13:09:07 +0800 Subject: [PATCH 1/2] client-go: fix the wait time for trying to acquire the leader lease Signed-off-by: Iceber Gu Kubernetes-commit: 9d80516abb4244ac1df4ceb84a32d3204fe5b308 --- tools/leaderelection/leaderelection.go | 2 +- tools/leaderelection/leaderelection_test.go | 37 ++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tools/leaderelection/leaderelection.go b/tools/leaderelection/leaderelection.go index eb72b251..1a229196 100644 --- a/tools/leaderelection/leaderelection.go +++ b/tools/leaderelection/leaderelection.go @@ -344,7 +344,7 @@ func (le *LeaderElector) tryAcquireOrRenew(ctx context.Context) bool { le.observedRawRecord = oldLeaderElectionRawRecord } if len(oldLeaderElectionRecord.HolderIdentity) > 0 && - le.observedTime.Add(le.config.LeaseDuration).After(now.Time) && + le.observedTime.Add(time.Second*time.Duration(oldLeaderElectionRecord.LeaseDurationSeconds)).After(now.Time) && !le.IsLeader() { klog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity) return false diff --git a/tools/leaderelection/leaderelection_test.go b/tools/leaderelection/leaderelection_test.go index ea047658..fbc0426b 100644 --- a/tools/leaderelection/leaderelection_test.go +++ b/tools/leaderelection/leaderelection_test.go @@ -81,6 +81,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { name string observedRecord rl.LeaderElectionRecord observedTime time.Time + retryAfter time.Duration reactors []Reactor expectedEvents []string @@ -127,6 +128,33 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { transitionLeader: true, outHolder: "baz", }, + { + name: "acquire from led object with the lease duration seconds", + reactors: []Reactor{ + { + verb: "get", + reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { + return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil + }, + }, + { + verb: "get", + reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { + return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil + }, + }, + { + verb: "update", + reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { + return true, action.(fakeclient.CreateAction).GetObject(), nil + }, + }, + }, + retryAfter: 3 * time.Second, + expectSuccess: true, + transitionLeader: true, + outHolder: "baz", + }, { name: "acquire from unled object", reactors: []Reactor{ @@ -286,7 +314,14 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { clock: clock.RealClock{}, } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { - t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) + if test.retryAfter != 0 { + time.Sleep(test.retryAfter) + if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { + t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) + } + } else { + t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) + } } le.observedRecord.AcquireTime = metav1.Time{} From 899a10901781d2bf0c6e348e2bf55140c50cfd06 Mon Sep 17 00:00:00 2001 From: Iceber Gu Date: Tue, 7 Feb 2023 13:00:55 +0800 Subject: [PATCH 2/2] leaderelection: using le.clock for the time calculations Signed-off-by: Iceber Gu Kubernetes-commit: 9c31f14a7cfea7cc793a9a3f98bdd117cb438316 --- tools/leaderelection/leaderelection.go | 4 ++-- tools/leaderelection/leaderelection_test.go | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/leaderelection/leaderelection.go b/tools/leaderelection/leaderelection.go index 1a229196..940e7161 100644 --- a/tools/leaderelection/leaderelection.go +++ b/tools/leaderelection/leaderelection.go @@ -292,7 +292,7 @@ func (le *LeaderElector) release() bool { if !le.IsLeader() { return true } - now := metav1.Now() + now := metav1.NewTime(le.clock.Now()) leaderElectionRecord := rl.LeaderElectionRecord{ LeaderTransitions: le.observedRecord.LeaderTransitions, LeaseDurationSeconds: 1, @@ -312,7 +312,7 @@ func (le *LeaderElector) release() bool { // else it tries to renew the lease if it has already been acquired. Returns true // on success else returns false. func (le *LeaderElector) tryAcquireOrRenew(ctx context.Context) bool { - now := metav1.Now() + now := metav1.NewTime(le.clock.Now()) leaderElectionRecord := rl.LeaderElectionRecord{ HolderIdentity: le.config.Lock.Identity(), LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second), diff --git a/tools/leaderelection/leaderelection_test.go b/tools/leaderelection/leaderelection_test.go index fbc0426b..aa0bd61b 100644 --- a/tools/leaderelection/leaderelection_test.go +++ b/tools/leaderelection/leaderelection_test.go @@ -74,8 +74,9 @@ type Reactor struct { } func testTryAcquireOrRenew(t *testing.T, objectType string) { - future := time.Now().Add(1000 * time.Hour) - past := time.Now().Add(-1000 * time.Hour) + clock := clock.RealClock{} + future := clock.Now().Add(1000 * time.Hour) + past := clock.Now().Add(-1000 * time.Hour) tests := []struct { name string @@ -311,7 +312,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { observedRecord: test.observedRecord, observedRawRecord: observedRawRecord, observedTime: test.observedTime, - clock: clock.RealClock{}, + clock: clock, } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { if test.retryAfter != 0 { @@ -413,8 +414,9 @@ func GetRawRecordOrDie(t *testing.T, objectType string, ler rl.LeaderElectionRec } func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) { - future := time.Now().Add(1000 * time.Hour) - past := time.Now().Add(-1000 * time.Hour) + clock := clock.RealClock{} + future := clock.Now().Add(1000 * time.Hour) + past := clock.Now().Add(-1000 * time.Hour) primaryType, secondaryType := multiLockType(t, objectType) tests := []struct { name string @@ -873,7 +875,7 @@ func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) { observedRecord: test.observedRecord, observedRawRecord: test.observedRawRecord, observedTime: test.observedTime, - clock: clock.RealClock{}, + clock: clock, } if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess)