From b03b5de5af5d5641398ea4dd6c740e9a63d71c77 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Wed, 25 Jan 2017 15:57:44 -0800 Subject: [PATCH 1/2] Make HandleError prevent hot-loops Add an error "handler" that just sleeps for a bit if errors happen more often than 500ms. Manually tested against #39816. --- .../apimachinery/pkg/util/runtime/runtime.go | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go index 976de49d582..c1c9bdabfbc 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go @@ -19,6 +19,8 @@ package runtime import ( "fmt" "runtime" + "sync" + "time" "github.com/golang/glog" ) @@ -79,7 +81,15 @@ func getCallers(r interface{}) string { // ErrorHandlers is a list of functions which will be invoked when an unreturnable // error occurs. -var ErrorHandlers = []func(error){logError} +// TODO(lavalamp): for testability, this and the below HandleError function +// should be packaged up into a testable and reusable object. +var ErrorHandlers = []func(error){ + logError, + (&rudimentaryErrorBackoff{ + lastErrorTime: time.Now(), + minPeriod: 500 * time.Millisecond, + }).OnError, +} // HandlerError is a method to invoke when a non-user facing piece of code cannot // return an error and needs to indicate it has been ignored. Invoking this method @@ -101,6 +111,26 @@ func logError(err error) { glog.ErrorDepth(2, err) } +type rudimentaryErrorBackoff struct { + minPeriod time.Duration // immutable + // TODO(lavalamp): use the clock for testability. Need to move that + // package for that to be accessible here. + lastErrorTimeLock sync.Mutex + lastErrorTime time.Time +} + +// OnError will block if it is called more often than the embedded period time. +// This will prevent overly tight hot error loops. +func (r *rudimentaryErrorBackoff) OnError(error) { + r.lastErrorTimeLock.Lock() + defer r.lastErrorTimeLock.Unlock() + d := time.Since(r.lastErrorTime) + if d < r.minPeriod { + time.Sleep(r.minPeriod - d) + } + r.lastErrorTime = time.Now() +} + // GetCaller returns the caller of the function that calls it. func GetCaller() string { var pc [1]uintptr From 16b7bee56db55c0d5a3101b17bfb7f99dcef3ff6 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Thu, 26 Jan 2017 12:54:25 -0800 Subject: [PATCH 2/2] Adjust global log limit to 1ms --- staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go index c1c9bdabfbc..748174e1919 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go @@ -87,7 +87,10 @@ var ErrorHandlers = []func(error){ logError, (&rudimentaryErrorBackoff{ lastErrorTime: time.Now(), - minPeriod: 500 * time.Millisecond, + // 1ms was the number folks were able to stomach as a global rate limit. + // If you need to log errors more than 1000 times a second you + // should probably consider fixing your code instead. :) + minPeriod: time.Millisecond, }).OnError, }