diff --git a/pkg/apiserver/handlers.go b/pkg/apiserver/handlers.go index e7374f18df0..34163acc923 100644 --- a/pkg/apiserver/handlers.go +++ b/pkg/apiserver/handlers.go @@ -72,6 +72,7 @@ func RateLimit(rl util.RateLimiter, handler http.Handler) http.Handler { return } w.WriteHeader(http.StatusServiceUnavailable) + w.Header().Set("Retry-After", "1") fmt.Fprintf(w, "Rate limit exceeded.") }) } diff --git a/pkg/client/request.go b/pkg/client/request.go index ca0fb9aadc9..6c6c60add69 100644 --- a/pkg/client/request.go +++ b/pkg/client/request.go @@ -35,6 +35,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/watch" watchjson "github.com/GoogleCloudPlatform/kubernetes/pkg/watch/json" + "github.com/golang/glog" ) // specialParams lists parameters that are handled specially and which users of Request @@ -457,6 +458,10 @@ func (r *Request) Do() Result { client = http.DefaultClient } + // Right now we make about ten retry attempts if we get a Retry-After response. + // TODO: Change to a timeout based approach. + retries := 0 + for { if r.err != nil { return Result{err: &RequestConstructionError{r.err}} @@ -478,6 +483,19 @@ func (r *Request) Do() Result { continue } + if resp.StatusCode == http.StatusServiceUnavailable { + if retries < 10 { + retries++ + if waitFor := resp.Header.Get("Retry-After"); waitFor != "" { + delay, err := strconv.Atoi(waitFor) + if err == nil { + glog.V(4).Infof("Got a Retry-After %s response for attempt %d to %v", waitFor, retries, r.finalURL()) + time.Sleep(time.Duration(delay) * time.Second) + continue + } + } + } + } return Result{respBody, created, err, r.codec} } }