Clean shutdown of apiserver integration tests

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -222,11 +222,6 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		crdClient.ApiextensionsV1(),
 		crdHandler,
 	)
-	openapiController := openapicontroller.NewController(s.Informers.Apiextensions().V1().CustomResourceDefinitions())
-	var openapiv3Controller *openapiv3controller.Controller
-	if utilfeature.DefaultFeatureGate.Enabled(features.OpenAPIV3) {
-		openapiv3Controller = openapiv3controller.NewController(s.Informers.Apiextensions().V1().CustomResourceDefinitions())
-	}
 
 	s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
 		s.Informers.Start(context.StopCh)
@@ -239,10 +234,12 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		// and StaticOpenAPISpec are both null. In that case we don't run the CRD OpenAPI controller.
 		if s.GenericAPIServer.StaticOpenAPISpec != nil {
 			if s.GenericAPIServer.OpenAPIVersionedService != nil {
+				openapiController := openapicontroller.NewController(s.Informers.Apiextensions().V1().CustomResourceDefinitions())
 				go openapiController.Run(s.GenericAPIServer.StaticOpenAPISpec, s.GenericAPIServer.OpenAPIVersionedService, context.StopCh)
 			}
 
 			if s.GenericAPIServer.OpenAPIV3VersionedService != nil && utilfeature.DefaultFeatureGate.Enabled(features.OpenAPIV3) {
+				openapiv3Controller := openapiv3controller.NewController(s.Informers.Apiextensions().V1().CustomResourceDefinitions())
 				go openapiv3Controller.Run(s.GenericAPIServer.OpenAPIV3VersionedService, context.StopCh)
 			}
 		}

staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go

@@ -34,6 +34,8 @@ import (
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+
+	"k8s.io/klog/v2"
 )
 
 // TearDownFunc is to be called to tear down a test server.
@@ -71,11 +73,22 @@ func NewDefaultTestServerOptions() *TestServerInstanceOptions {
 // 		 enough time to remove temporary files.
 func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
 	stopCh := make(chan struct{})
+	var errCh chan error
 	tearDown := func() {
 		// Closing stopCh is stopping apiextensions apiserver and its
 		// delegates, which itself is cleaning up after itself,
 		// including shutting down its storage layer.
 		close(stopCh)
+
+		// If the apiextensions apiserver was started, let's wait for
+		// it to shutdown clearly.
+		if errCh != nil {
+			err, ok := <-errCh
+			if ok && err != nil {
+				klog.Errorf("Failed to shutdown test server clearly: %v", err)
+			}
+		}
+
 		if len(result.TmpDir) != 0 {
 			os.RemoveAll(result.TmpDir)
 		}
@@ -135,8 +148,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
 		return result, fmt.Errorf("failed to create server: %v", err)
 	}
 
-	errCh := make(chan error)
+	errCh = make(chan error)
 	go func(stopCh <-chan struct{}) {
+		defer close(errCh)
+
 		if err := server.GenericAPIServer.PrepareRun().Run(stopCh); err != nil {
 			errCh <- err
 		}

test/integration/apiserver/apiserver_test.go

@@ -401,10 +401,14 @@ func TestListOptions(t *testing.T) {
 			}
 
 			// compact some of the revision history in etcd so we can test "too old" resource versions
-			_, kvClient, err := integration.GetEtcdClients(*storageTransport)
+			rawClient, kvClient, err := integration.GetEtcdClients(*storageTransport)
 			if err != nil {
 				t.Fatal(err)
 			}
+			// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
+			// close the client (which we can do by closing rawClient).
+			defer rawClient.Close()
+
 			revision, err := strconv.Atoi(oldestUncompactedRv)
 			if err != nil {
 				t.Fatal(err)

test/integration/controlplane/transformation/kms_transformation_test.go

@@ -120,7 +120,6 @@ resources:
        cachesize: 1000
        endpoint: unix:///@kms-provider.sock
 `
-
 	providerName := "kms-provider"
 	pluginMock, err := mock.NewBase64Plugin("@kms-provider.sock")
 	if err != nil {

test/integration/controlplane/transformation/transformation_testcase.go

@@ -236,10 +236,14 @@ func (e *transformTest) createSecret(name, namespace string) (*corev1.Secret, er
 }
 
 func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetResponse, error) {
-	_, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
+	rawClient, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create etcd client: %v", err)
 	}
+	// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
+	// close the client (which we can do by closing rawClient).
+	defer rawClient.Close()
+
 	response, err := etcdClient.Get(context.Background(), path, clientv3.WithPrefix())
 	if err != nil {
 		return nil, fmt.Errorf("failed to retrieve secret from etcd %v", err)
@@ -249,10 +253,14 @@ func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetRespons
 }
 
 func (e *transformTest) writeRawRecordToETCD(path string, data []byte) (*clientv3.PutResponse, error) {
-	_, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
+	rawClient, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create etcd client: %v", err)
 	}
+	// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
+	// close the client (which we can do by closing rawClient).
+	defer rawClient.Close()
+
 	response, err := etcdClient.Put(context.Background(), path, string(data))
 	if err != nil {
 		return nil, fmt.Errorf("failed to write secret to etcd %v", err)