From b1f7b60515798934b011d27bfeba1cee4fcd26b7 Mon Sep 17 00:00:00 2001
From: Abu Kashem <akashem@redhat.com>
Date: Tue, 17 May 2022 12:00:04 -0400
Subject: [PATCH] apiserver: add lifecycle signal for preshutdown hook

---
 .../k8s.io/apiserver/pkg/server/genericapiserver.go  | 12 ++++++++----
 .../k8s.io/apiserver/pkg/server/lifecycle_signals.go |  5 +++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
index 2377fb8fec0..5326b5fc222 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
@@ -431,7 +431,7 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 //           |                                                       |
 // (ShutdownDelayDuration)                                    (PreShutdownHooks)
 //           |                                                       |
-//  AfterShutdownDelayDuration (delayedStopCh)           preShutdownHooksHasStoppedCh
+//  AfterShutdownDelayDuration (delayedStopCh)   PreShutdownHooksStopped (preShutdownHooksHasStoppedCh)
 //           |                                                       |
 //           |----------------------------------                     |
 //           |                                  |                    |
@@ -504,12 +504,13 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 	}
 
 	// pre-shutdown hooks need to finish before we stop the http server
-	preShutdownHooksHasStoppedCh, stopHttpServerCh := make(chan struct{}), make(chan struct{})
+	preShutdownHooksHasStoppedCh := s.lifecycleSignals.PreShutdownHooksStopped
+	stopHttpServerCh := make(chan struct{})
 	go func() {
 		defer close(stopHttpServerCh)
 
 		<-delayedStopOrDrainedCh
-		<-preShutdownHooksHasStoppedCh
+		<-preShutdownHooksHasStoppedCh.Signaled()
 	}()
 
 	stoppedCh, listenerStoppedCh, err := s.NonBlockingRun(stopHttpServerCh, shutdownTimeout)
@@ -540,7 +541,10 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 	// run shutdown hooks directly. This includes deregistering from
 	// the kubernetes endpoint in case of kube-apiserver.
 	func() {
-		defer close(preShutdownHooksHasStoppedCh)
+		defer func() {
+			preShutdownHooksHasStoppedCh.Signal()
+			klog.V(1).InfoS("[graceful-termination] pre-shutdown hooks completed", "name", preShutdownHooksHasStoppedCh.Name())
+		}()
 		err = s.RunPreShutdownHooks()
 	}()
 	if err != nil {
diff --git a/staging/src/k8s.io/apiserver/pkg/server/lifecycle_signals.go b/staging/src/k8s.io/apiserver/pkg/server/lifecycle_signals.go
index 6b406072b61..fd1c94db2db 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/lifecycle_signals.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/lifecycle_signals.go
@@ -120,6 +120,10 @@ type lifecycleSignals struct {
 	// ShutdownDelayDuration allows the apiserver to delay shutdown for some time.
 	AfterShutdownDelayDuration lifecycleSignal
 
+	// PreShutdownHooksStopped event is signaled when all registered
+	// preshutdown hook(s) have finished running.
+	PreShutdownHooksStopped lifecycleSignal
+
 	// InFlightRequestsDrained event is signaled when the existing requests
 	// in flight have completed. This is used as signal to shut down the audit backends
 	InFlightRequestsDrained lifecycleSignal
@@ -143,6 +147,7 @@ func newLifecycleSignals() lifecycleSignals {
 	return lifecycleSignals{
 		ShutdownInitiated:          newNamedChannelWrapper("ShutdownInitiated"),
 		AfterShutdownDelayDuration: newNamedChannelWrapper("AfterShutdownDelayDuration"),
+		PreShutdownHooksStopped:    newNamedChannelWrapper("PreShutdownHooksStopped"),
 		InFlightRequestsDrained:    newNamedChannelWrapper("InFlightRequestsDrained"),
 		HTTPServerStoppedListening: newNamedChannelWrapper("HTTPServerStoppedListening"),
 		HasBeenReady:               newNamedChannelWrapper("HasBeenReady"),