diff --git a/pkg/proxy/ipvs/netlink_linux.go b/pkg/proxy/ipvs/netlink_linux.go
index 0f9fe617f49..8277ace35ec 100644
--- a/pkg/proxy/ipvs/netlink_linux.go
+++ b/pkg/proxy/ipvs/netlink_linux.go
@@ -92,7 +92,11 @@ func (h *netlinkHandle) EnsureDummyDevice(devName string) (bool, error) {
 func (h *netlinkHandle) DeleteDummyDevice(devName string) error {
 	link, err := h.LinkByName(devName)
 	if err != nil {
-		return fmt.Errorf("error deleting a non-exist dummy device: %s", devName)
+		_, ok := err.(netlink.LinkNotFoundError)
+		if ok {
+			return nil
+		}
+		return fmt.Errorf("error deleting a non-exist dummy device: %s, %v", devName, err)
 	}
 	dummy, ok := link.(*netlink.Dummy)
 	if !ok {
diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go
index c5ecdc45d33..3615caefecb 100644
--- a/pkg/util/ipset/ipset.go
+++ b/pkg/util/ipset/ipset.go
@@ -356,8 +356,8 @@ func (runner *runner) FlushSet(set string) error {
 
 // DestroySet is used to destroy a named set.
 func (runner *runner) DestroySet(set string) error {
-	if _, err := runner.exec.Command(IPSetCmd, "destroy", set).CombinedOutput(); err != nil {
-		return fmt.Errorf("error destroying set %s:, error: %v", set, err)
+	if out, err := runner.exec.Command(IPSetCmd, "destroy", set).CombinedOutput(); err != nil {
+		return fmt.Errorf("error destroying set %s, error: %v(%s)", set, err, out)
 	}
 	return nil
 }