mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #40497 from lavalamp/log2much
Automatic merge from submit-queue Make HandleError prevent hot-loops Add an error "handler" that just sleeps for a bit if errors happen more often than 500ms. Manually tested against #39816. This doesn't fix #39816 but it does keep it from crippling a cluster. ```release-note Prevent hotloops on error conditions, which could fill up the disk faster than log rotation can free space. ```
This commit is contained in:
commit
fe2829c16d
@ -19,6 +19,8 @@ package runtime
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
@ -79,7 +81,18 @@ func getCallers(r interface{}) string {
|
||||
|
||||
// ErrorHandlers is a list of functions which will be invoked when an unreturnable
|
||||
// error occurs.
|
||||
var ErrorHandlers = []func(error){logError}
|
||||
// TODO(lavalamp): for testability, this and the below HandleError function
|
||||
// should be packaged up into a testable and reusable object.
|
||||
var ErrorHandlers = []func(error){
|
||||
logError,
|
||||
(&rudimentaryErrorBackoff{
|
||||
lastErrorTime: time.Now(),
|
||||
// 1ms was the number folks were able to stomach as a global rate limit.
|
||||
// If you need to log errors more than 1000 times a second you
|
||||
// should probably consider fixing your code instead. :)
|
||||
minPeriod: time.Millisecond,
|
||||
}).OnError,
|
||||
}
|
||||
|
||||
// HandlerError is a method to invoke when a non-user facing piece of code cannot
|
||||
// return an error and needs to indicate it has been ignored. Invoking this method
|
||||
@ -101,6 +114,26 @@ func logError(err error) {
|
||||
glog.ErrorDepth(2, err)
|
||||
}
|
||||
|
||||
type rudimentaryErrorBackoff struct {
|
||||
minPeriod time.Duration // immutable
|
||||
// TODO(lavalamp): use the clock for testability. Need to move that
|
||||
// package for that to be accessible here.
|
||||
lastErrorTimeLock sync.Mutex
|
||||
lastErrorTime time.Time
|
||||
}
|
||||
|
||||
// OnError will block if it is called more often than the embedded period time.
|
||||
// This will prevent overly tight hot error loops.
|
||||
func (r *rudimentaryErrorBackoff) OnError(error) {
|
||||
r.lastErrorTimeLock.Lock()
|
||||
defer r.lastErrorTimeLock.Unlock()
|
||||
d := time.Since(r.lastErrorTime)
|
||||
if d < r.minPeriod {
|
||||
time.Sleep(r.minPeriod - d)
|
||||
}
|
||||
r.lastErrorTime = time.Now()
|
||||
}
|
||||
|
||||
// GetCaller returns the caller of the function that calls it.
|
||||
func GetCaller() string {
|
||||
var pc [1]uintptr
|
||||
|
Loading…
Reference in New Issue
Block a user