Merge pull request #110061 from wojtek-t/shutdown_apiextensions

Cleanup CRD storage on shutdown

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -211,6 +211,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 	}
 	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler)
 	s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler)
+	s.GenericAPIServer.RegisterDestroyFunc(crdHandler.destroy)
 
 	discoveryController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler)
 	namingController := status.NewNamingConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go

@@ -583,6 +583,23 @@ func (r *crdHandler) tearDown(oldInfo *crdInfo) {
 	}
 }
 
+// Destroy shuts down storage layer for all registered CRDs.
+// It should be called as a last step of the shutdown sequence.
+func (r *crdHandler) destroy() {
+	r.customStorageLock.Lock()
+	defer r.customStorageLock.Unlock()
+
+	storageMap := r.customStorage.Load().(crdStorageMap)
+	for _, crdInfo := range storageMap {
+		for _, storage := range crdInfo.storages {
+			// DestroyFunc have to be implemented in idempotent way,
+			// so the potential race with r.tearDown() (being called
+			// from a goroutine) is safe.
+			storage.CustomResource.DestroyFunc()
+		}
+	}
+}
+
 // GetCustomResourceListerCollectionDeleter returns the ListerCollectionDeleter of
 // the given crd.
 func (r *crdHandler) GetCustomResourceListerCollectionDeleter(crd *apiextensionsv1.CustomResourceDefinition) (finalizer.ListerCollectionDeleter, error) {

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -317,6 +317,12 @@ func (s *GenericAPIServer) MuxAndDiscoveryCompleteSignals() map[string]<-chan st
 	return s.muxAndDiscoveryCompleteSignals
 }
 
+// RegisterDestroyFunc registers a function that will be called during Destroy().
+// The function have to be idempotent and prepared to be called more than once.
+func (s *GenericAPIServer) RegisterDestroyFunc(destroyFn func()) {
+	s.destroyFns = append(s.destroyFns, destroyFn)
+}
+
 // Destroy cleans up all its and its delegation target resources on shutdown.
 // It starts with destroying its own resources and later proceeds with
 // its delegation target.
@@ -644,7 +650,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
 		resourceInfos = append(resourceInfos, r...)
 	}
 
-	s.destroyFns = append(s.destroyFns, apiGroupInfo.destroyStorage)
+	s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
 		utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {