Merge pull request #81969 from logicalhan/livez

add `/livez` endpoint for liveness probing on the kube-apiserver

cmd/kube-apiserver/app/aggregator.go

@@ -153,7 +153,7 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega
 		return nil, err
 	}
 
-	err = aggregatorServer.GenericAPIServer.AddHealthChecks(
+	err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks(
 		makeAPIServiceAvailableHealthCheck(
 			"autoregister-completion",
 			apiServices,

cmd/kube-apiserver/app/testing/BUILD

@@ -17,7 +17,6 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",

cmd/kube-apiserver/app/testing/testserver.go

@@ -31,7 +31,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/registry/generic/registry"
-	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
@@ -46,8 +45,6 @@ type TearDownFunc func()
 type TestServerInstanceOptions struct {
 	// DisableStorageCleanup Disable the automatic storage cleanup
 	DisableStorageCleanup bool
-	// Injected health
-	InjectedHealthChecker healthz.HealthChecker
 }
 
 // TestServer return values supplied by kube-test-ApiServer
@@ -150,13 +147,6 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
 		return result, fmt.Errorf("failed to create server chain: %v", err)
 	}
 
-	if instanceOptions.InjectedHealthChecker != nil {
-		t.Logf("Adding health check with delay %v %v", s.GenericServerRunOptions.MaxStartupSequenceDuration, instanceOptions.InjectedHealthChecker.Name())
-		if err := server.GenericAPIServer.AddDelayedHealthzChecks(s.GenericServerRunOptions.MaxStartupSequenceDuration, instanceOptions.InjectedHealthChecker); err != nil {
-			return result, err
-		}
-	}
-
 	errCh := make(chan error)
 	go func(stopCh <-chan struct{}) {
 		prepared, err := server.PrepareRun()

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -198,7 +198,8 @@ func ClusterRoles() []rbacv1.ClusterRole {
 			ObjectMeta: metav1.ObjectMeta{Name: "system:discovery"},
 			Rules: []rbacv1.PolicyRule{
 				rbacv1helpers.NewRule("get").URLs(
-					"/readyz", "/healthz", "/version", "/version/",
+					"/livez", "/readyz", "/healthz",
+					"/version", "/version/",
 					"/openapi", "/openapi/*",
 					"/api", "/api/*",
 					"/apis", "/apis/*",
@@ -218,7 +219,7 @@ func ClusterRoles() []rbacv1.ClusterRole {
 			ObjectMeta: metav1.ObjectMeta{Name: "system:public-info-viewer"},
 			Rules: []rbacv1.PolicyRule{
 				rbacv1helpers.NewRule("get").URLs(
-					"/readyz", "/healthz", "/version", "/version/",
+					"/livez", "/readyz", "/healthz", "/version", "/version/",
 				).RuleOrDie(),
 			},
 		},

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml

@@ -548,6 +548,7 @@ items:
     - /apis
     - /apis/*
     - /healthz
+    - /livez
     - /openapi
     - /openapi/*
     - /readyz
@@ -1185,6 +1186,7 @@ items:
   rules:
   - nonResourceURLs:
     - /healthz
+    - /livez
     - /readyz
     - /version
     - /version/

staging/src/k8s.io/apiserver/pkg/server/BUILD

@@ -22,6 +22,7 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -139,6 +139,8 @@ type Config struct {
 	DiscoveryAddresses discovery.Addresses
 	// The default set of healthz checks. There might be more added via AddHealthChecks dynamically.
 	HealthzChecks []healthz.HealthChecker
+	// The default set of livez checks. There might be more added via AddHealthChecks dynamically.
+	LivezChecks []healthz.HealthChecker
 	// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
 	ReadyzChecks []healthz.HealthChecker
 	// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
@@ -165,9 +167,9 @@ type Config struct {
 
 	// This represents the maximum amount of time it should take for apiserver to complete its startup
 	// sequence and become healthy. From apiserver's start time to when this amount of time has
-	// elapsed, /healthz will assume that unfinished post-start hooks will complete successfully and
+	// elapsed, /livez will assume that unfinished post-start hooks will complete successfully and
 	// therefore return true.
-	MaxStartupSequenceDuration time.Duration
+	LivezGracePeriod time.Duration
 	// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
 	// have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
 	// but /readyz will return failure.
@@ -281,6 +283,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 		DisabledPostStartHooks:      sets.NewString(),
 		HealthzChecks:               append([]healthz.HealthChecker{}, defaultHealthChecks...),
 		ReadyzChecks:                append([]healthz.HealthChecker{}, defaultHealthChecks...),
+		LivezChecks:                 append([]healthz.HealthChecker{}, defaultHealthChecks...),
 		EnableIndex:                 true,
 		EnableDiscovery:             true,
 		EnableProfiling:             true,
@@ -289,7 +292,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 		MaxMutatingRequestsInFlight: 200,
 		RequestTimeout:              time.Duration(60) * time.Second,
 		MinRequestTimeout:           1800,
-		MaxStartupSequenceDuration:  time.Duration(0),
+		LivezGracePeriod:            time.Duration(0),
 		ShutdownDelayDuration:       time.Duration(0),
 		// 10MB is the recommended maximum client request size in bytes
 		// the etcd server should accept. See
@@ -512,15 +515,16 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		preShutdownHooks:       map[string]preShutdownHookEntry{},
 		disabledPostStartHooks: c.DisabledPostStartHooks,
 
-		healthzChecks:              c.HealthzChecks,
+		healthzChecks:    c.HealthzChecks,
-		readyzChecks:               c.ReadyzChecks,
+		livezChecks:      c.LivezChecks,
-		readinessStopCh:            make(chan struct{}),
+		readyzChecks:     c.ReadyzChecks,
-		maxStartupSequenceDuration: c.MaxStartupSequenceDuration,
+		readinessStopCh:  make(chan struct{}),
+		livezGracePeriod: c.LivezGracePeriod,
 
 		DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
 
 		maxRequestBodyBytes: c.MaxRequestBodyBytes,
-		healthzClock:        clock.RealClock{},
+		livezClock:          clock.RealClock{},
 	}
 
 	for {
@@ -566,9 +570,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		if skip {
 			continue
 		}
-
+		s.AddHealthChecks(delegateCheck)
-		s.healthzChecks = append(s.healthzChecks, delegateCheck)
-		s.readyzChecks = append(s.readyzChecks, delegateCheck)
 	}
 
 	s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}

staging/src/k8s.io/apiserver/pkg/server/config_test.go

@@ -26,6 +26,7 @@ import (
 	"reflect"
 	"testing"
 
+	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/informers"
@@ -135,31 +136,36 @@ func TestNewWithDelegate(t *testing.T) {
 	// Wait for the hooks to finish before checking the response
 	<-delegatePostStartHookChan
 	<-wrappingPostStartHookChan
-
+	expectedPaths := []string{
-	checkPath(server.URL, http.StatusOK, `{
+		"/apis",
-  "paths": [
+		"/bar",
-    "/apis",
+		"/foo",
-    "/bar",
+		"/healthz",
-    "/foo",
+		"/healthz/delegate-health",
-    "/healthz",
+		"/healthz/log",
-    "/healthz/delegate-health",
+		"/healthz/ping",
-    "/healthz/log",
+		"/healthz/poststarthook/delegate-post-start-hook",
-    "/healthz/ping",
+		"/healthz/poststarthook/generic-apiserver-start-informers",
-    "/healthz/poststarthook/delegate-post-start-hook",
+		"/healthz/poststarthook/wrapping-post-start-hook",
-    "/healthz/poststarthook/generic-apiserver-start-informers",
+		"/healthz/wrapping-health",
-    "/healthz/poststarthook/wrapping-post-start-hook",
+		"/livez",
-    "/healthz/wrapping-health",
+		"/livez/delegate-health",
-    "/metrics",
+		"/livez/log",
-    "/readyz",
+		"/livez/ping",
-    "/readyz/delegate-health",
+		"/livez/poststarthook/delegate-post-start-hook",
-    "/readyz/log",
+		"/livez/poststarthook/generic-apiserver-start-informers",
-    "/readyz/ping",
+		"/livez/poststarthook/wrapping-post-start-hook",
-    "/readyz/poststarthook/delegate-post-start-hook",
+		"/metrics",
-    "/readyz/poststarthook/generic-apiserver-start-informers",
+		"/readyz",
-    "/readyz/poststarthook/wrapping-post-start-hook",
+		"/readyz/delegate-health",
-    "/readyz/shutdown"
+		"/readyz/log",
-  ]
+		"/readyz/ping",
-}`, t)
+		"/readyz/poststarthook/delegate-post-start-hook",
+		"/readyz/poststarthook/generic-apiserver-start-informers",
+		"/readyz/poststarthook/wrapping-post-start-hook",
+		"/readyz/shutdown",
+	}
+	checkExpectedPathsAtRoot(server.URL, expectedPaths, t)
 	checkPath(server.URL+"/healthz", http.StatusInternalServerError, `[+]ping ok
 [+]log ok
 [-]wrapping-health failed: reason withheld
@@ -181,22 +187,57 @@ healthz check failed
 }
 
 func checkPath(url string, expectedStatusCode int, expectedBody string, t *testing.T) {
-	resp, err := http.Get(url)
+	t.Run(url, func(t *testing.T) {
-	if err != nil {
+		resp, err := http.Get(url)
-		t.Fatal(err)
+		if err != nil {
-	}
+			t.Fatal(err)
-	dump, _ := httputil.DumpResponse(resp, true)
+		}
-	t.Log(string(dump))
+		dump, _ := httputil.DumpResponse(resp, true)
+		t.Log(string(dump))
 
-	body, err := ioutil.ReadAll(resp.Body)
+		body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
+		if err != nil {
-		t.Fatal(err)
+			t.Fatal(err)
-	}
+		}
 
-	if e, a := expectedBody, string(body); e != a {
+		if e, a := expectedBody, string(body); e != a {
-		t.Errorf("%q expected %v, got %v", url, e, a)
+			t.Errorf("%q expected %v, got %v", url, e, a)
-	}
+		}
-	if e, a := expectedStatusCode, resp.StatusCode; e != a {
+		if e, a := expectedStatusCode, resp.StatusCode; e != a {
-		t.Errorf("%q expected %v, got %v", url, e, a)
+			t.Errorf("%q expected %v, got %v", url, e, a)
-	}
+		}
+	})
+}
+
+func checkExpectedPathsAtRoot(url string, expectedPaths []string, t *testing.T) {
+	t.Run(url, func(t *testing.T) {
+		resp, err := http.Get(url)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dump, _ := httputil.DumpResponse(resp, true)
+		t.Log(string(dump))
+
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var result map[string]interface{}
+		json.Unmarshal(body, &result)
+		paths, ok := result["paths"].([]interface{})
+		if !ok {
+			t.Errorf("paths not found")
+		}
+		pathset := sets.NewString()
+		for _, p := range paths {
+			pathset.Insert(p.(string))
+		}
+		expectedset := sets.NewString(expectedPaths...)
+		for _, p := range pathset.Difference(expectedset) {
+			t.Errorf("Got %v path, which we did not expect", p)
+		}
+		for _, p := range expectedset.Difference(pathset) {
+			t.Errorf(" Expected %v path which we did not get", p)
+		}
+	})
 }

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -146,14 +146,19 @@ type GenericAPIServer struct {
 	preShutdownHooksCalled bool
 
 	// healthz checks
-	healthzLock                sync.Mutex
+	healthzLock            sync.Mutex
-	healthzChecks              []healthz.HealthChecker
+	healthzChecks          []healthz.HealthChecker
-	healthzChecksInstalled     bool
+	healthzChecksInstalled bool
-	readyzLock                 sync.Mutex
+	// livez checks
-	readyzChecks               []healthz.HealthChecker
+	livezLock            sync.Mutex
-	readyzChecksInstalled      bool
+	livezChecks          []healthz.HealthChecker
-	maxStartupSequenceDuration time.Duration
+	livezChecksInstalled bool
-	healthzClock               clock.Clock
+	// readyz checks
+	readyzLock            sync.Mutex
+	readyzChecks          []healthz.HealthChecker
+	readyzChecksInstalled bool
+	livezGracePeriod      time.Duration
+	livezClock            clock.Clock
 	// the readiness stop channel is used to signal that the apiserver has initiated a shutdown sequence, this
 	// will cause readyz to return unhealthy.
 	readinessStopCh chan struct{}
@@ -281,7 +286,12 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 	}
 
 	s.installHealthz()
-	s.installReadyz(s.readinessStopCh)
+	s.installLivez()
+	err := s.addReadyzShutdownCheck(s.readinessStopCh)
+	if err != nil {
+		klog.Errorf("Failed to install readyz shutdown check %s", err)
+	}
+	s.installReadyz()
 
 	// Register audit backend preShutdownHook.
 	if s.AuditBackend != nil {

staging/src/k8s.io/apiserver/pkg/server/healthz.go

@@ -25,51 +25,93 @@ import (
 	"k8s.io/apiserver/pkg/server/healthz"
 )
 
-// AddHealthChecks adds HealthzCheck(s) to both healthz and readyz. All healthz checks
+// AddHealthChecks adds HealthCheck(s) to health endpoints (healthz, livez, readyz) but
-// are automatically added to readyz, since we want to avoid the situation where the
+// configures the liveness grace period to be zero, which means we expect this health check
-// apiserver is ready but not live.
+// to immediately indicate that the apiserver is unhealthy.
 func (s *GenericAPIServer) AddHealthChecks(checks ...healthz.HealthChecker) error {
-	return s.AddDelayedHealthzChecks(0, checks...)
+	// we opt for a delay of zero here, because this entrypoint adds generic health checks
+	// and not health checks which are specifically related to kube-apiserver boot-sequences.
+	return s.addHealthChecks(0, checks...)
 }
 
-// AddReadyzChecks allows you to add a HealthzCheck to readyz.
+// AddBootSequenceHealthChecks adds health checks to the old healthz endpoint (for backwards compatibility reasons)
-func (s *GenericAPIServer) AddReadyzChecks(checks ...healthz.HealthChecker) error {
+// as well as livez and readyz. The livez grace period is defined by the value of the
+// command-line flag --livez-grace-period; before the grace period elapses, the livez health checks
+// will default to healthy. One may want to set a grace period in order to prevent the kubelet from restarting
+// the kube-apiserver due to long-ish boot sequences. Readyz health checks, on the other hand, have no grace period,
+// since readyz should fail until boot fully completes.
+func (s *GenericAPIServer) AddBootSequenceHealthChecks(checks ...healthz.HealthChecker) error {
+	return s.addHealthChecks(s.livezGracePeriod, checks...)
+}
+
+// addHealthChecks adds health checks to healthz, livez, and readyz. The delay passed in will set
+// a corresponding grace period on livez.
+func (s *GenericAPIServer) addHealthChecks(livezGracePeriod time.Duration, checks ...healthz.HealthChecker) error {
+	s.healthzLock.Lock()
+	defer s.healthzLock.Unlock()
+	if s.healthzChecksInstalled {
+		return fmt.Errorf("unable to add because the healthz endpoint has already been created")
+	}
+	s.healthzChecks = append(s.healthzChecks, checks...)
+	return s.addLivezChecks(livezGracePeriod, checks...)
+}
+
+// addReadyzChecks allows you to add a HealthCheck to readyz.
+func (s *GenericAPIServer) addReadyzChecks(checks ...healthz.HealthChecker) error {
 	s.readyzLock.Lock()
 	defer s.readyzLock.Unlock()
-	return s.addReadyzChecks(checks...)
-}
-
-// addReadyzChecks allows you to add a HealthzCheck to readyz.
-// premise: readyzLock has been obtained
-func (s *GenericAPIServer) addReadyzChecks(checks ...healthz.HealthChecker) error {
 	if s.readyzChecksInstalled {
 		return fmt.Errorf("unable to add because the readyz endpoint has already been created")
 	}
-
 	s.readyzChecks = append(s.readyzChecks, checks...)
 	return nil
 }
 
+// addLivezChecks allows you to add a HealthCheck to livez. It will also automatically add a check to readyz,
+// since we want to avoid being ready when we are not live.
+func (s *GenericAPIServer) addLivezChecks(delay time.Duration, checks ...healthz.HealthChecker) error {
+	s.livezLock.Lock()
+	defer s.livezLock.Unlock()
+	if s.livezChecksInstalled {
+		return fmt.Errorf("unable to add because the livez endpoint has already been created")
+	}
+	for _, check := range checks {
+		s.livezChecks = append(s.livezChecks, delayedHealthCheck(check, s.livezClock, delay))
+	}
+	return s.addReadyzChecks(checks...)
+}
+
+// addReadyzShutdownCheck is a convenience function for adding a readyz shutdown check, so
+// that we can register that the api-server is no longer ready while we attempt to gracefully
+// shutdown.
+func (s *GenericAPIServer) addReadyzShutdownCheck(stopCh <-chan struct{}) error {
+	return s.addReadyzChecks(shutdownCheck{stopCh})
+}
+
 // installHealthz creates the healthz endpoint for this server
 func (s *GenericAPIServer) installHealthz() {
 	s.healthzLock.Lock()
 	defer s.healthzLock.Unlock()
 	s.healthzChecksInstalled = true
-
 	healthz.InstallHandler(s.Handler.NonGoRestfulMux, s.healthzChecks...)
 }
 
 // installReadyz creates the readyz endpoint for this server.
-func (s *GenericAPIServer) installReadyz(stopCh <-chan struct{}) {
+func (s *GenericAPIServer) installReadyz() {
 	s.readyzLock.Lock()
 	defer s.readyzLock.Unlock()
-	s.addReadyzChecks(shutdownCheck{stopCh})
-
 	s.readyzChecksInstalled = true
-
 	healthz.InstallReadyzHandler(s.Handler.NonGoRestfulMux, s.readyzChecks...)
 }
 
+// installLivez creates the livez endpoint for this server.
+func (s *GenericAPIServer) installLivez() {
+	s.livezLock.Lock()
+	defer s.livezLock.Unlock()
+	s.livezChecksInstalled = true
+	healthz.InstallLivezHandler(s.Handler.NonGoRestfulMux, s.livezChecks...)
+}
+
 // shutdownCheck fails if the embedded channel is closed. This is intended to allow for graceful shutdown sequences
 // for the apiserver.
 type shutdownCheck struct {
@@ -89,46 +131,27 @@ func (c shutdownCheck) Check(req *http.Request) error {
 	return nil
 }
 
-// AddDelayedHealthzChecks adds a health check to both healthz and readyz. The delay parameter
+// delayedHealthCheck wraps a health check which will not fail until the explicitly defined delay has elapsed. This
-// allows you to set the grace period for healthz checks, which will return healthy while
+// is intended for use primarily for livez health checks.
-// grace period has not yet elapsed. One may want to set a grace period in order to prevent
-// the kubelet from restarting the kube-apiserver due to long-ish boot sequences. Readyz health
-// checks have no grace period, since we want readyz to fail while boot has not completed.
-func (s *GenericAPIServer) AddDelayedHealthzChecks(delay time.Duration, checks ...healthz.HealthChecker) error {
-	s.healthzLock.Lock()
-	defer s.healthzLock.Unlock()
-	if s.healthzChecksInstalled {
-		return fmt.Errorf("unable to add because the healthz endpoint has already been created")
-	}
-	for _, check := range checks {
-		s.healthzChecks = append(s.healthzChecks, delayedHealthCheck(check, s.healthzClock, s.maxStartupSequenceDuration))
-	}
-
-	s.readyzLock.Lock()
-	defer s.readyzLock.Unlock()
-	return s.addReadyzChecks(checks...)
-}
-
-// delayedHealthCheck wraps a health check which will not fail until the explicitly defined delay has elapsed.
 func delayedHealthCheck(check healthz.HealthChecker, clock clock.Clock, delay time.Duration) healthz.HealthChecker {
-	return delayedHealthzCheck{
+	return delayedLivezCheck{
 		check,
 		clock.Now().Add(delay),
 		clock,
 	}
 }
 
-type delayedHealthzCheck struct {
+type delayedLivezCheck struct {
 	check      healthz.HealthChecker
 	startCheck time.Time
 	clock      clock.Clock
 }
 
-func (c delayedHealthzCheck) Name() string {
+func (c delayedLivezCheck) Name() string {
 	return c.check.Name()
 }
 
-func (c delayedHealthzCheck) Check(req *http.Request) error {
+func (c delayedLivezCheck) Check(req *http.Request) error {
 	if c.clock.Now().After(c.startCheck) {
 		return c.check.Check(req)
 	}

staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -101,6 +101,14 @@ func InstallReadyzHandler(mux mux, checks ...HealthChecker) {
 	InstallPathHandler(mux, "/readyz", checks...)
 }
 
+// InstallLivezHandler registers handlers for liveness checking on the path
+// "/livez" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallLivezHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/livez", checks...)
+}
+
 // InstallPathHandler registers handlers for health checking on
 // a specific path to mux. *All handlers* for the path must be
 // specified in exactly one call to InstallPathHandler. Calling

staging/src/k8s.io/apiserver/pkg/server/hooks.go

@@ -22,12 +22,11 @@ import (
 	"net/http"
 	"runtime/debug"
 
-	"k8s.io/klog"
-
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/server/healthz"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog"
 )
 
 // PostStartHookFunc is a function that is called after the server has started.
@@ -97,7 +96,7 @@ func (s *GenericAPIServer) AddPostStartHook(name string, hook PostStartHookFunc)
 	// done is closed when the poststarthook is finished.  This is used by the health check to be able to indicate
 	// that the poststarthook is finished
 	done := make(chan struct{})
-	if err := s.AddDelayedHealthzChecks(s.maxStartupSequenceDuration, postStartHookHealthz{name: "poststarthook/" + name, done: done}); err != nil {
+	if err := s.AddBootSequenceHealthChecks(postStartHookHealthz{name: "poststarthook/" + name, done: done}); err != nil {
 		return err
 	}
 	s.postStartHooks[name] = postStartHookEntry{hook: hook, originatingStack: string(debug.Stack()), done: done}

staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -41,7 +41,7 @@ type ServerRunOptions struct {
 	MaxRequestsInFlight         int
 	MaxMutatingRequestsInFlight int
 	RequestTimeout              time.Duration
-	MaxStartupSequenceDuration  time.Duration
+	LivezGracePeriod            time.Duration
 	MinRequestTimeout           int
 	ShutdownDelayDuration       time.Duration
 	// We intentionally did not add a flag for this option. Users of the
@@ -62,7 +62,7 @@ func NewServerRunOptions() *ServerRunOptions {
 		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
 		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
 		RequestTimeout:              defaults.RequestTimeout,
-		MaxStartupSequenceDuration:  defaults.MaxStartupSequenceDuration,
+		LivezGracePeriod:            defaults.LivezGracePeriod,
 		MinRequestTimeout:           defaults.MinRequestTimeout,
 		ShutdownDelayDuration:       defaults.ShutdownDelayDuration,
 		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
@@ -76,7 +76,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.ExternalAddress = s.ExternalHost
 	c.MaxRequestsInFlight = s.MaxRequestsInFlight
 	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
-	c.MaxStartupSequenceDuration = s.MaxStartupSequenceDuration
+	c.LivezGracePeriod = s.LivezGracePeriod
 	c.RequestTimeout = s.RequestTimeout
 	c.MinRequestTimeout = s.MinRequestTimeout
 	c.ShutdownDelayDuration = s.ShutdownDelayDuration
@@ -112,8 +112,8 @@ func (s *ServerRunOptions) Validate() []error {
 		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
 	}
 
-	if s.MaxStartupSequenceDuration < 0 {
+	if s.LivezGracePeriod < 0 {
-		errors = append(errors, fmt.Errorf("--maximum-startup-sequence-duration can not be a negative value"))
+		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
 	}
 
 	if s.EnableInfightQuotaHandler {
@@ -199,9 +199,9 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
 		"--min-request-timeout for specific types of requests.")
 
-	fs.DurationVar(&s.MaxStartupSequenceDuration, "maximum-startup-sequence-duration", s.MaxStartupSequenceDuration, ""+
+	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
 		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
-		"and become healthy. From apiserver's start time to when this amount of time has elapsed, /healthz will assume "+
+		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
 		"that unfinished post-start hooks will complete successfully and therefore return true.")
 
 	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+

staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go

@@ -137,7 +137,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
 			expectErr: "--max-resource-write-bytes can not be negative value",
 		},
 		{
-			name: "Test when MaxStartupSequenceDuration is negative value",
+			name: "Test when LivezGracePeriod is negative value",
 			testOptions: &ServerRunOptions{
 				AdvertiseAddress:            net.ParseIP("192.168.10.10"),
 				CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
@@ -148,9 +148,9 @@ func TestServerRunOptionsValidate(t *testing.T) {
 				JSONPatchMaxCopyBytes:       10 * 1024 * 1024,
 				MaxRequestBodyBytes:         10 * 1024 * 1024,
 				TargetRAMMB:                 65536,
-				MaxStartupSequenceDuration:  -time.Second,
+				LivezGracePeriod:            -time.Second,
 			},
-			expectErr: "--maximum-startup-sequence-duration can not be a negative value",
+			expectErr: "--livez-grace-period can not be a negative value",
 		},
 		{
 			name: "Test when MinimalShutdownDuration is negative value",

test/integration/master/kube_apiserver_test.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"reflect"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -104,35 +103,16 @@ func endpointReturnsStatusOK(client *kubernetes.Clientset, path string) bool {
 	return status == http.StatusOK
 }
 
-func TestStartupSequenceHealthzAndReadyz(t *testing.T) {
+func TestLivezAndReadyz(t *testing.T) {
-	hc := &delayedCheck{}
+	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--livez-grace-period", "0s"}, framework.SharedEtcd())
-	instanceOptions := &kubeapiservertesting.TestServerInstanceOptions{
-		InjectedHealthChecker: hc,
-	}
-	server := kubeapiservertesting.StartTestServerOrDie(t, instanceOptions, []string{"--maximum-startup-sequence-duration", "15s"}, framework.SharedEtcd())
 	defer server.TearDownFn()
 
 	client, err := kubernetes.NewForConfig(server.ClientConfig)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-
+	if !endpointReturnsStatusOK(client, "/livez") {
-	if endpointReturnsStatusOK(client, "/readyz") {
+		t.Fatalf("livez should be healthy")
-		t.Fatalf("readyz should start unready")
-	}
-	// we need to wait longer than our grace period
-	err = wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		return !endpointReturnsStatusOK(client, "/healthz"), nil
-	})
-	if err != nil {
-		t.Fatalf("healthz should have become unhealthy: %v", err)
-	}
-	hc.makeHealthy()
-	err = wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		return endpointReturnsStatusOK(client, "/healthz"), nil
-	})
-	if err != nil {
-		t.Fatalf("healthz should have become healthy again: %v", err)
 	}
 	if !endpointReturnsStatusOK(client, "/readyz") {
 		t.Fatalf("readyz should be healthy")
@@ -505,27 +485,3 @@ func TestReconcilerMasterLeaseMultiMoreMasters(t *testing.T) {
 func TestReconcilerMasterLeaseMultiCombined(t *testing.T) {
 	testReconcilersMasterLease(t, 3, 3)
 }
-
-type delayedCheck struct {
-	healthLock sync.Mutex
-	isHealthy  bool
-}
-
-func (h *delayedCheck) Name() string {
-	return "delayed-check"
-}
-
-func (h *delayedCheck) Check(req *http.Request) error {
-	h.healthLock.Lock()
-	defer h.healthLock.Unlock()
-	if h.isHealthy {
-		return nil
-	}
-	return fmt.Errorf("isn't healthy")
-}
-
-func (h *delayedCheck) makeHealthy() {
-	h.healthLock.Lock()
-	defer h.healthLock.Unlock()
-	h.isHealthy = true
-}