diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go
index b6b17f56592..4cbd940d4cc 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go
@@ -23,6 +23,8 @@ import (
 	"reflect"
 	"testing"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/apis/apiserver"
 )
@@ -281,7 +283,7 @@ spec:
 				if err != nil {
 					t.Fatal(err)
 				}
-				defer os.Remove(f.Name())
+				defer utiltesting.CloseAndRemove(t, f)
 				if err := ioutil.WriteFile(f.Name(), []byte(tc.contents), os.FileMode(0755)); err != nil {
 					t.Fatal(err)
 				}
diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go
index 0857cc36cb8..cd0f6d69a09 100644
--- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go
@@ -704,6 +704,7 @@ func bootstrapTestDir(t *testing.T) string {
 	// Write the certificate files to disk or fail
 	for fileName, fileData := range files {
 		if err := ioutil.WriteFile(filepath.Join(dir, fileName), fileData, 0400); err != nil {
+			os.RemoveAll(dir)
 			t.Fatal(err)
 		}
 	}
@@ -713,6 +714,10 @@ func bootstrapTestDir(t *testing.T) string {
 
 func newKubeConfigFile(config v1.Config) (string, error) {
 	configFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", err
+	}
+	defer configFile.Close()
 
 	if err != nil {
 		return "", fmt.Errorf("unable to create the Kubernetes client config file: %v", err)
diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go
index 6d21e40a2a1..52fa60b9e8d 100644
--- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go
+++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go
@@ -34,6 +34,8 @@ import (
 	"text/template"
 	"time"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	"github.com/google/go-cmp/cmp"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -185,7 +187,7 @@ current-context: default
 				return err
 			}
 			p := tempfile.Name()
-			defer os.Remove(p)
+			defer utiltesting.CloseAndRemove(t, tempfile)
 
 			tmpl, err := template.New("test").Parse(tt.configTmpl)
 			if err != nil {
diff --git a/staging/src/k8s.io/controller-manager/pkg/leadermigration/config/config_test.go b/staging/src/k8s.io/controller-manager/pkg/leadermigration/config/config_test.go
index 07b3c430478..de209698126 100644
--- a/staging/src/k8s.io/controller-manager/pkg/leadermigration/config/config_test.go
+++ b/staging/src/k8s.io/controller-manager/pkg/leadermigration/config/config_test.go
@@ -21,6 +21,8 @@ import (
 	"reflect"
 	"testing"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	internal "k8s.io/controller-manager/config"
 )
@@ -117,7 +119,7 @@ controllerLeaders:
 			if err != nil {
 				t.Fatal(err)
 			}
-			defer os.Remove(configFile.Name())
+			defer utiltesting.CloseAndRemove(t, configFile)
 			err = os.WriteFile(configFile.Name(), []byte(tc.content), os.FileMode(0755))
 			if err != nil {
 				t.Fatal(err)
diff --git a/staging/src/k8s.io/controller-manager/pkg/leadermigration/options/options_test.go b/staging/src/k8s.io/controller-manager/pkg/leadermigration/options/options_test.go
index f78be12adc8..ebd57a5b402 100644
--- a/staging/src/k8s.io/controller-manager/pkg/leadermigration/options/options_test.go
+++ b/staging/src/k8s.io/controller-manager/pkg/leadermigration/options/options_test.go
@@ -21,6 +21,8 @@ import (
 	"reflect"
 	"testing"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	"github.com/spf13/pflag"
 
 	"k8s.io/controller-manager/config"
@@ -188,7 +190,7 @@ controllerLeaders:
 				if err != nil {
 					t.Fatal(err)
 				}
-				defer os.Remove(configFile.Name())
+				defer utiltesting.CloseAndRemove(t, configFile)
 				err = os.WriteFile(configFile.Name(), []byte(tc.configContent), os.FileMode(0755))
 				if err != nil {
 					t.Fatal(err)