diff --git a/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go b/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go
index e948fcf7024..f2ffb49c196 100644
--- a/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go
+++ b/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go
@@ -18,6 +18,7 @@ package plugin
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"path"
@@ -80,7 +81,7 @@ func setupFakeGRPCServer(service, addr string) (tearDown, error) {
 
 	go func() {
 		go func() {
-			if err := s.Serve(listener); err != nil {
+			if err := s.Serve(listener); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
 				panic(err)
 			}
 		}()