github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 15:05:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #47796 from shyamjvs/ep-controller-retries

Browse Source 
Automatic merge from submit-queue

Retry service syncs with exponential backoff in endpoints-controller

This should make it avoid missing any syncs.

/cc @smarterclayton @gmarek @seh
This commit is contained in:
Kubernetes Submit Queue 2017-06-26 06:48:58 -07:00 committed by GitHub
commit ee8b663d22
1 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pkg/controller/endpoint/endpoints_controller.go
View File

@ -44,6 +44,13 @@ import (
) )
const ( const (
// maxRetries is the number of times a service will be retried before it is dropped out of the queue.
// With the current rate-limiter in use (5ms*2^(maxRetries-1)) the following numbers represent the
// sequence of delays between successive queuings of a service.
//
// 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s
maxRetries = 15
// An annotation on the Service denoting if the endpoints controller should // An annotation on the Service denoting if the endpoints controller should
// go ahead and create endpoints for unready pods. This annotation is // go ahead and create endpoints for unready pods. This annotation is
// currently only used by StatefulSets, where we need the pod to be DNS // currently only used by StatefulSets, where we need the pod to be DNS
@ -265,17 +272,28 @@ func (e *EndpointController) processNextWorkItem() bool {
defer e.queue.Done(eKey) defer e.queue.Done(eKey)
err := e.syncService(eKey.(string)) err := e.syncService(eKey.(string))
if err == nil { e.handleErr(err, eKey)
e.queue.Forget(eKey)
return true
}
utilruntime.HandleError(fmt.Errorf("Sync %v failed with %v", eKey, err))
e.queue.AddRateLimited(eKey)
return true return true
} }
func (e *EndpointController) handleErr(err error, key interface{}) {
if err == nil {
e.queue.Forget(key)
return
}
if e.queue.NumRequeues(key) < maxRetries {
glog.V(2).Infof("Error syncing endpoints for service %q: %v", key, err)
e.queue.AddRateLimited(key)
return
}
glog.Warningf("Dropping service %q out of the queue: %v", key, err)
e.queue.Forget(key)
utilruntime.HandleError(err)
}
func (e *EndpointController) syncService(key string) error { func (e *EndpointController) syncService(key string) error {
startTime := time.Now() startTime := time.Now()
defer func() { defer func() {