diff --git a/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go b/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go
index 9a7c856bdb4..e7145c59bfc 100644
--- a/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go
+++ b/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go
@@ -84,13 +84,10 @@ type tearDown func()
 
 func setupFakeGRPCServer(service, addr string) (tearDown, error) {
 	ctx, cancel := context.WithCancel(context.Background())
-	teardown := func() {
-		cancel()
-	}
 
 	listener, err := net.Listen("unix", addr)
 	if err != nil {
-		teardown()
+		cancel()
 		return nil, err
 	}
 
@@ -110,20 +107,31 @@ func setupFakeGRPCServer(service, addr string) (tearDown, error) {
 			drapbv1beta1.RegisterDRAPluginServer(s, drapbv1beta1.V1ServerWrapper{DRAPluginServer: fakeGRPCServer})
 			drahealthv1alpha1.RegisterDRAResourceHealthServer(s, fakeGRPCServer)
 		} else {
+			cancel()
 			return nil, err
 		}
 	}
 
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+		if err := s.Serve(listener); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
+			panic(err)
+		}
+	}()
+
 	go func() {
-		go func() {
-			if err := s.Serve(listener); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
-				panic(err)
-			}
-		}()
 		<-ctx.Done()
 		s.GracefulStop()
 	}()
 
+	teardown := func() {
+		cancel()
+		wg.Wait() // wait for Serve() to finish
+	}
+
 	return teardown, nil
 }