Add healthz check to ensure logging is not blocked

This commit is contained in:
Jordan Liggitt 2018-06-09 17:32:14 -04:00
parent f29c1301a3
commit b7b4b84afe
No known key found for this signature in database
GPG Key ID: 39928704103C7229
5 changed files with 40 additions and 2 deletions

View File

@ -256,6 +256,7 @@ func (s *Server) InstallAuthFilter() {
func (s *Server) InstallDefaultHandlers() {
healthz.InstallHandler(s.restfulCont,
healthz.PingHealthz,
healthz.LogHealthz,
healthz.NamedCheck("syncloop", s.syncLoopHealthCheck),
)
ws := new(restful.WebService)

View File

@ -255,7 +255,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
HandlerChainWaitGroup: new(utilwaitgroup.SafeWaitGroup),
LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix),
DisabledPostStartHooks: sets.NewString(),
HealthzChecks: []healthz.HealthzChecker{healthz.PingHealthz},
HealthzChecks: []healthz.HealthzChecker{healthz.PingHealthz, healthz.LogHealthz},
EnableIndex: true,
EnableDiscovery: true,
EnableProfiling: true,

View File

@ -101,6 +101,7 @@ func TestNewWithDelegate(t *testing.T) {
"/foo",
"/healthz",
"/healthz/delegate-health",
"/healthz/log",
"/healthz/ping",
"/healthz/poststarthook/delegate-post-start-hook",
"/healthz/poststarthook/generic-apiserver-start-informers",
@ -111,6 +112,7 @@ func TestNewWithDelegate(t *testing.T) {
]
}`, t)
checkPath(server.URL+"/healthz", http.StatusInternalServerError, `[+]ping ok
[+]log ok
[-]wrapping-health failed: reason withheld
[-]delegate-health failed: reason withheld
[+]poststarthook/generic-apiserver-start-informers ok

View File

@ -20,7 +20,10 @@ go_library(
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/healthz",
importpath = "k8s.io/apiserver/pkg/server/healthz",
deps = ["//vendor/github.com/golang/glog:go_default_library"],
deps = [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
],
)
filegroup(

View File

@ -22,8 +22,12 @@ import (
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/wait"
)
// HealthzChecker is a named healthz checker.
@ -56,6 +60,34 @@ func (ping) Check(_ *http.Request) error {
return nil
}
// LogHealthz returns true if logging is not blocked
var LogHealthz HealthzChecker = &log{}
type log struct {
startOnce sync.Once
lastVerified atomic.Value
}
func (l *log) Name() string {
return "log"
}
func (l *log) Check(_ *http.Request) error {
l.startOnce.Do(func() {
l.lastVerified.Store(time.Now())
go wait.Forever(func() {
glog.Flush()
l.lastVerified.Store(time.Now())
}, time.Minute)
})
lastVerified := l.lastVerified.Load().(time.Time)
if time.Since(lastVerified) < (2 * time.Minute) {
return nil
}
return fmt.Errorf("logging blocked")
}
// NamedCheck returns a healthz checker for the given name and function.
func NamedCheck(name string, check func(r *http.Request) error) HealthzChecker {
return &healthzCheck{name, check}