diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go
index d9357d2c6d8..a705bb9585d 100644
--- a/pkg/proxy/ipvs/graceful_termination.go
+++ b/pkg/proxy/ipvs/graceful_termination.go
@@ -75,10 +75,10 @@ func (q *graceTerminateRSList) remove(rs *listItem) bool {
 
 	uniqueRS := rs.String()
 	if _, ok := q.list[uniqueRS]; ok {
-		return false
+		delete(q.list, uniqueRS)
+		return true
 	}
-	delete(q.list, uniqueRS)
-	return true
+	return false
 }
 
 func (q *graceTerminateRSList) flushList(handler func(rsToDelete *listItem) (bool, error)) bool {
@@ -164,7 +164,10 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e
 	}
 	for _, rs := range rss {
 		if rsToDelete.RealServer.Equal(rs) {
-			if rs.ActiveConn != 0 {
+			// Delete RS with no connections
+			// For UDP, ActiveConn is always 0
+			// For TCP, InactiveConn are connections not in ESTABLISHED state
+			if rs.ActiveConn+rs.InactiveConn != 0 {
 				return false, nil
 			}
 			klog.Infof("Deleting rs: %s", rsToDelete.String())
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 242a6025ca6..20fd8acca5e 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -1602,7 +1602,7 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
 			Port:    uint16(portNum),
 		}
 
-		klog.V(5).Infof("Using graceful delete to delete: %v", delDest)
+		klog.V(5).Infof("Using graceful delete to delete: %v", uniqueRS)
 		err = proxier.gracefuldeleteManager.GracefulDeleteRS(appliedVirtualServer, delDest)
 		if err != nil {
 			klog.Errorf("Failed to delete destination: %v, error: %v", delDest, err)