diff --git a/staging/src/k8s.io/cri-streaming/pkg/streaming/server.go b/staging/src/k8s.io/cri-streaming/pkg/streaming/server.go
index 97ecb028287..0a7ea51998c 100644
--- a/staging/src/k8s.io/cri-streaming/pkg/streaming/server.go
+++ b/staging/src/k8s.io/cri-streaming/pkg/streaming/server.go
@@ -238,8 +238,10 @@ func (s *server) Start(stayUp bool) error {
 	if err != nil {
 		return err
 	}
-	// Use the actual address as baseURL host. This handles the "0" port case.
-	s.config.BaseURL.Host = listener.Addr().String()
+	if _, port, err := net.SplitHostPort(s.config.Addr); err != nil || port == "0" {
+		// Use the actual address as baseURL host. This handles the "0" port case.
+		s.config.BaseURL.Host = listener.Addr().String()
+	}
 	if s.config.TLSConfig != nil {
 		return s.server.ServeTLS(listener, "", "") // Use certs from TLSConfig.
 	}