Merge pull request #40497 from lavalamp/log2much

Automatic merge from submit-queue

Make HandleError prevent hot-loops

Add an error "handler" that just sleeps for a bit if errors happen more
often than 500ms. Manually tested against #39816. This doesn't fix #39816 but it does keep it from crippling a cluster.

```release-note
Prevent hotloops on error conditions, which could fill up the disk faster than log rotation can free space.
```
This commit is contained in:
Kubernetes Submit Queue 2017-01-27 17:26:03 -08:00 committed by GitHub
commit fe2829c16d

View File

@ -19,6 +19,8 @@ package runtime
import (
"fmt"
"runtime"
"sync"
"time"
"github.com/golang/glog"
)
@ -79,7 +81,18 @@ func getCallers(r interface{}) string {
// ErrorHandlers is a list of functions which will be invoked when an unreturnable
// error occurs.
var ErrorHandlers = []func(error){logError}
// TODO(lavalamp): for testability, this and the below HandleError function
// should be packaged up into a testable and reusable object.
var ErrorHandlers = []func(error){
logError,
(&rudimentaryErrorBackoff{
lastErrorTime: time.Now(),
// 1ms was the number folks were able to stomach as a global rate limit.
// If you need to log errors more than 1000 times a second you
// should probably consider fixing your code instead. :)
minPeriod: time.Millisecond,
}).OnError,
}
// HandlerError is a method to invoke when a non-user facing piece of code cannot
// return an error and needs to indicate it has been ignored. Invoking this method
@ -101,6 +114,26 @@ func logError(err error) {
glog.ErrorDepth(2, err)
}
type rudimentaryErrorBackoff struct {
minPeriod time.Duration // immutable
// TODO(lavalamp): use the clock for testability. Need to move that
// package for that to be accessible here.
lastErrorTimeLock sync.Mutex
lastErrorTime time.Time
}
// OnError will block if it is called more often than the embedded period time.
// This will prevent overly tight hot error loops.
func (r *rudimentaryErrorBackoff) OnError(error) {
r.lastErrorTimeLock.Lock()
defer r.lastErrorTimeLock.Unlock()
d := time.Since(r.lastErrorTime)
if d < r.minPeriod {
time.Sleep(r.minPeriod - d)
}
r.lastErrorTime = time.Now()
}
// GetCaller returns the caller of the function that calls it.
func GetCaller() string {
var pc [1]uintptr