Merge pull request #31044 from caesarxuchao/fix-ratelimiter-gc

Automatic merge from submit-queue

[GarbageCollector] Make Rate Limiter registration more efficient in GC

<!--  Thanks for sending a pull request!  Here are some tips for you:
1. If this is your first time, read our contributor guidelines https://github.com/kubernetes/kubernetes/blob/master/CONTRIBUTING.md and developer guide https://github.com/kubernetes/kubernetes/blob/master/docs/devel/development.md
2. If you want *faster* PR reviews, read how: https://github.com/kubernetes/kubernetes/blob/master/docs/devel/faster_reviews.md
3. Follow the instructions for writing a release note: https://github.com/kubernetes/kubernetes/blob/master/docs/devel/pull-requests.md#release-notes
-->

**What this PR does / why we need it**:
Decrease the CPU consumption of the garbage collector

**Which issue this PR fixes** 
#30759

**Special notes for your reviewer**:
I observed dramatic improvement (dropped from 0.8cpu to 0.3cpu) in load test.

**Release note**:
<!--  Steps to write your release note:
1. Use the release-note-* labels to set the release note state (if you have access) 
2. Enter your extended release note in the below block; leaving it blank means using the PR title as the release note. If no release note is required, just write `NONE`. 
-->
```release-note
```



@wojtek-t @lavalamp @gmarek

pkg/controller/garbagecollector/garbagecollector.go

@@ -541,8 +541,8 @@ func NewGarbageCollector(metaOnlyClientPool dynamic.ClientPool, clientPool dynam
 		clock:                            clock.RealClock{},
 		dirtyQueue:                       workqueue.NewTimedWorkQueue(),
 		orphanQueue:                      workqueue.NewTimedWorkQueue(),
-		registeredRateLimiter:            NewRegisteredRateLimiter(),
-		registeredRateLimiterForMonitors: NewRegisteredRateLimiter(),
+		registeredRateLimiter:            NewRegisteredRateLimiter(resources),
+		registeredRateLimiterForMonitors: NewRegisteredRateLimiter(resources),
 	}
 	gc.propagator = &Propagator{
 		eventQueue: workqueue.NewTimedWorkQueue(),

pkg/controller/garbagecollector/rate_limiter_helper.go

@@ -29,31 +29,32 @@ import (
 // RegisteredRateLimiter records the registered RateLimters to avoid
 // duplication.
 type RegisteredRateLimiter struct {
-	rateLimiters map[unversioned.GroupVersion]struct{}
-	lock         sync.RWMutex
+	rateLimiters map[unversioned.GroupVersion]*sync.Once
 }
 
 // NewRegisteredRateLimiter returns a new RegisteredRateLimiater.
-func NewRegisteredRateLimiter() *RegisteredRateLimiter {
-	return &RegisteredRateLimiter{
-		rateLimiters: make(map[unversioned.GroupVersion]struct{}),
+// TODO: NewRegisteredRateLimiter is not dynamic. We need to find a better way
+// when GC dynamically change the resources it monitors.
+func NewRegisteredRateLimiter(resources []unversioned.GroupVersionResource) *RegisteredRateLimiter {
+	rateLimiters := make(map[unversioned.GroupVersion]*sync.Once)
+	for _, resource := range resources {
+		gv := resource.GroupVersion()
+		if _, found := rateLimiters[gv]; !found {
+			rateLimiters[gv] = &sync.Once{}
+		}
 	}
+	return &RegisteredRateLimiter{rateLimiters: rateLimiters}
 }
 
 func (r *RegisteredRateLimiter) registerIfNotPresent(gv unversioned.GroupVersion, client *dynamic.Client, prefix string) {
-	r.lock.RLock()
-	_, ok := r.rateLimiters[gv]
-	r.lock.RUnlock()
-	if ok {
+	once, found := r.rateLimiters[gv]
+	if !found {
 		return
 	}
-	r.lock.Lock()
-	defer r.lock.Unlock()
-	if _, ok := r.rateLimiters[gv]; !ok {
+	once.Do(func() {
 		if rateLimiter := client.GetRateLimiter(); rateLimiter != nil {
 			group := strings.Replace(gv.Group, ".", ":", -1)
 			metrics.RegisterMetricAndTrackRateLimiterUsage(fmt.Sprintf("%s_%s_%s", prefix, group, gv.Version), rateLimiter)
 		}
-		r.rateLimiters[gv] = struct{}{}
-	}
+	})
 }