diff --git a/test/e2e/apimachinery/certs.go b/test/e2e/apimachinery/certs.go
index 4f18333a82a..cc3db706686 100644
--- a/test/e2e/apimachinery/certs.go
+++ b/test/e2e/apimachinery/certs.go
@@ -19,11 +19,14 @@ package apimachinery
 import (
 	"crypto/x509"
 	"os"
+	"testing"
+
+	utiltesting "k8s.io/client-go/util/testing"
+	"k8s.io/kubernetes/test/utils"
 
 	"k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
 	"k8s.io/kubernetes/test/e2e/framework"
-	"k8s.io/kubernetes/test/utils"
 )
 
 type certContext struct {
@@ -52,6 +55,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 	if err != nil {
 		framework.Failf("Failed to create a temp file for ca cert generation %v", err)
 	}
+	defer utiltesting.CloseAndRemove(&testing.T{}, caCertFile)
 	if err := os.WriteFile(caCertFile.Name(), utils.EncodeCertPEM(signingCert), 0644); err != nil {
 		framework.Failf("Failed to write CA cert %v", err)
 	}
@@ -74,6 +78,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 	if err != nil {
 		framework.Failf("Failed to create a temp file for cert generation %v", err)
 	}
+	defer utiltesting.CloseAndRemove(&testing.T{}, certFile)
 	keyFile, err := os.CreateTemp(certDir, "server.key")
 	if err != nil {
 		framework.Failf("Failed to create a temp file for key generation %v", err)
@@ -88,6 +93,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 	if err = os.WriteFile(keyFile.Name(), privateKeyPEM, 0644); err != nil {
 		framework.Failf("Failed to write key file %v", err)
 	}
+	defer utiltesting.CloseAndRemove(&testing.T{}, keyFile)
 	return &certContext{
 		cert:        utils.EncodeCertPEM(signedCert),
 		key:         privateKeyPEM,
diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go
index 98d5ce9f52d..957f594df21 100644
--- a/test/e2e_node/services/apiserver.go
+++ b/test/e2e_node/services/apiserver.go
@@ -19,6 +19,9 @@ package services
 import (
 	"fmt"
 	"os"
+	"testing"
+
+	utiltesting "k8s.io/client-go/util/testing"
 
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	netutils "k8s.io/utils/net"
@@ -79,7 +82,7 @@ func (a *APIServer) Start() error {
 	if err != nil {
 		return fmt.Errorf("create temp file failed: %w", err)
 	}
-	defer os.RemoveAll(saSigningKeyFile.Name())
+	defer utiltesting.CloseAndRemove(&testing.T{}, saSigningKeyFile)
 	if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
 		return fmt.Errorf("write file %s failed: %w", saSigningKeyFile.Name(), err)
 	}
diff --git a/test/integration/apiserver/admissionwebhook/client_auth_test.go b/test/integration/apiserver/admissionwebhook/client_auth_test.go
index 816a373eab3..8e0315a9d3a 100644
--- a/test/integration/apiserver/admissionwebhook/client_auth_test.go
+++ b/test/integration/apiserver/admissionwebhook/client_auth_test.go
@@ -31,6 +31,8 @@ import (
 	"testing"
 	"time"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	"k8s.io/api/admission/v1beta1"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -87,7 +89,7 @@ func testWebhookClientAuth(t *testing.T, enableAggregatorRouting bool) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.Remove(kubeConfigFile.Name())
+	defer utiltesting.CloseAndRemove(t, kubeConfigFile)
 
 	if err := os.WriteFile(kubeConfigFile.Name(), []byte(`
 apiVersion: v1
@@ -113,7 +115,7 @@ users:
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.Remove(admissionConfigFile.Name())
+	defer utiltesting.CloseAndRemove(t, admissionConfigFile)
 
 	if err := os.WriteFile(admissionConfigFile.Name(), []byte(`
 apiVersion: apiserver.k8s.io/v1alpha1
diff --git a/test/integration/apiserver/admissionwebhook/reinvocation_test.go b/test/integration/apiserver/admissionwebhook/reinvocation_test.go
index 61051f1101c..9d8f6ca163c 100644
--- a/test/integration/apiserver/admissionwebhook/reinvocation_test.go
+++ b/test/integration/apiserver/admissionwebhook/reinvocation_test.go
@@ -44,6 +44,7 @@ import (
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	utiltesting "k8s.io/client-go/util/testing"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration/framework"
 	"k8s.io/kubernetes/test/utils"
@@ -281,7 +282,7 @@ func testWebhookReinvocationPolicy(t *testing.T, watchCache bool) {
 	if err != nil {
 		t.Fatalf("Failed to create audit log file: %v", err)
 	}
-	defer os.Remove(logFile.Name())
+	defer utiltesting.CloseAndRemove(t, logFile)
 
 	s := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{
 		"--disable-admission-plugins=ServiceAccount",
diff --git a/test/integration/apiserver/tracing/tracing_test.go b/test/integration/apiserver/tracing/tracing_test.go
index d284de19857..23594843a8f 100644
--- a/test/integration/apiserver/tracing/tracing_test.go
+++ b/test/integration/apiserver/tracing/tracing_test.go
@@ -38,6 +38,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	client "k8s.io/client-go/kubernetes"
+	utiltesting "k8s.io/client-go/util/testing"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration/framework"
 )
@@ -73,7 +74,7 @@ egressSelections:
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.Remove(tracingConfigFile.Name())
+	defer utiltesting.CloseAndRemove(t, tracingConfigFile)
 
 	if err := os.WriteFile(tracingConfigFile.Name(), []byte(fmt.Sprintf(`
 apiVersion: apiserver.config.k8s.io/v1beta1
diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go
index 674f98dffd7..7412aa84b01 100644
--- a/test/integration/auth/auth_test.go
+++ b/test/integration/auth/auth_test.go
@@ -41,6 +41,8 @@ import (
 	"testing"
 	"time"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	"github.com/google/go-cmp/cmp"
 
 	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
@@ -86,7 +88,7 @@ func getTestWebhookTokenAuth(serverURL string, customDial utilnet.DialFunc) (aut
 	if err != nil {
 		return nil, err
 	}
-	defer os.Remove(kubecfgFile.Name())
+	defer utiltesting.CloseAndRemove(&testing.T{}, kubecfgFile)
 	config := v1.Config{
 		Clusters: []v1.NamedCluster{
 			{
diff --git a/test/integration/auth/dynamic_client_test.go b/test/integration/auth/dynamic_client_test.go
index be862869d0b..90c5454105a 100644
--- a/test/integration/auth/dynamic_client_test.go
+++ b/test/integration/auth/dynamic_client_test.go
@@ -22,6 +22,8 @@ import (
 	"testing"
 	"time"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	clientset "k8s.io/client-go/kubernetes"
@@ -38,7 +40,7 @@ func TestDynamicClientBuilder(t *testing.T) {
 	if err != nil {
 		t.Fatalf("create temp file failed: %v", err)
 	}
-	defer os.RemoveAll(tmpfile.Name())
+	defer utiltesting.CloseAndRemove(t, tmpfile)
 
 	if err = os.WriteFile(tmpfile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
 		t.Fatalf("write file %s failed: %v", tmpfile.Name(), err)
diff --git a/test/integration/controlplane/audit/audit_test.go b/test/integration/controlplane/audit/audit_test.go
index b934a9ec444..316f152a6c9 100644
--- a/test/integration/controlplane/audit/audit_test.go
+++ b/test/integration/controlplane/audit/audit_test.go
@@ -41,6 +41,7 @@ import (
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
 	clientset "k8s.io/client-go/kubernetes"
+	utiltesting "k8s.io/client-go/util/testing"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration/framework"
 	"k8s.io/kubernetes/test/utils"
@@ -245,7 +246,7 @@ func runTestWithVersion(t *testing.T, version string) {
 	if err != nil {
 		t.Fatalf("Failed to create audit log file: %v", err)
 	}
-	defer os.Remove(logFile.Name())
+	defer utiltesting.CloseAndRemove(t, logFile)
 
 	// start api server
 	result := kubeapiservertesting.StartTestServerOrDie(t, nil,
diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go
index f8c9455eb10..34bab086199 100644
--- a/test/integration/etcd/server.go
+++ b/test/integration/etcd/server.go
@@ -26,6 +26,8 @@ import (
 	"testing"
 	"time"
 
+	utiltesting "k8s.io/client-go/util/testing"
+
 	clientv3 "go.etcd.io/etcd/client/v3"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -81,7 +83,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
 	if err != nil {
 		t.Fatalf("create temp file failed: %v", err)
 	}
-	defer os.RemoveAll(saSigningKeyFile.Name())
+	defer utiltesting.CloseAndRemove(t, saSigningKeyFile)
 	if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
 		t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
 	}
diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go
index 0ce4944f65e..6adfd47ec36 100644
--- a/test/integration/examples/apiserver_test.go
+++ b/test/integration/examples/apiserver_test.go
@@ -522,6 +522,7 @@ func writeKubeConfigForWardleServerToKASConnection(t *testing.T, kubeClientConfi
 		t.Fatal(err)
 	}
 
+	defer wardleToKASKubeConfigFile.Close()
 	return wardleToKASKubeConfigFile.Name()
 }
 
diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go
index ce1599ea544..931c5a4a573 100644
--- a/test/integration/framework/test_server.go
+++ b/test/integration/framework/test_server.go
@@ -99,6 +99,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
 	if err := os.WriteFile(proxyCACertFile.Name(), utils.EncodeCertPEM(proxySigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
+	defer proxyCACertFile.Close()
 	clientSigningKey, err := utils.NewPrivateKey()
 	if err != nil {
 		t.Fatal(err)
@@ -111,7 +112,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
 	if err := os.WriteFile(clientCACertFile.Name(), utils.EncodeCertPEM(clientSigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
-
+	defer clientCACertFile.Close()
 	listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
 	if err != nil {
 		t.Fatal(err)
@@ -121,7 +122,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
 	if err != nil {
 		t.Fatalf("create temp file failed: %v", err)
 	}
-	defer os.RemoveAll(saSigningKeyFile.Name())
+	defer saSigningKeyFile.Close()
 	if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
 		t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
 	}