diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go
index 0a01f0bacc6..d61992125ae 100644
--- a/pkg/proxy/ipvs/ipset.go
+++ b/pkg/proxy/ipvs/ipset.go
@@ -114,7 +114,9 @@ func (set *IPSet) syncIPSetEntries() {
 		// Clean legacy entries
 		for _, entry := range currentIPSetEntries.Difference(set.activeEntries).List() {
 			if err := set.handle.DelEntry(entry, set.Name); err != nil {
-				glog.Errorf("Failed to delete ip set entry: %s from ip set: %s, error: %v", entry, set.Name, err)
+				if !utilipset.IsNotFoundError(err) {
+					glog.Errorf("Failed to delete ip set entry: %s from ip set: %s, error: %v", entry, set.Name, err)
+				}
 			} else {
 				glog.V(3).Infof("Successfully delete legacy ip set entry: %s from ip set: %s", entry, set.Name)
 			}
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 17faadc1814..e7cf693e6fe 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -852,6 +852,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 		encounteredError = false
 		err := ipvs.Flush()
 		if err != nil {
+			glog.Errorf("Error flushing IPVS rules: %v", err)
 			encounteredError = true
 		}
 	}
@@ -859,6 +860,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 	nl := NewNetLinkHandle()
 	err := nl.DeleteDummyDevice(DefaultDummyDevice)
 	if err != nil {
+		glog.Errorf("Error deleting dummy device %s created by IPVS proxier: %v", DefaultDummyDevice, err)
 		encounteredError = true
 	}
 	// Clear iptables created by ipvs Proxier.
@@ -870,7 +872,10 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 	for _, set := range ipSetsToDestroy {
 		err = ipset.DestroySet(set)
 		if err != nil {
-			encounteredError = true
+			if !utilipset.IsNotFoundError(err) {
+				glog.Errorf("Error removing ipset %s, error: %v", set, err)
+				encounteredError = true
+			}
 		}
 	}
 	return encounteredError
diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go
index ba92bff74bf..7ae28050097 100644
--- a/pkg/util/ipset/ipset.go
+++ b/pkg/util/ipset/ipset.go
@@ -322,4 +322,23 @@ func validatePortRange(portRange string) bool {
 	return true
 }
 
+// IsNotFoundError returns true if the error indicates "not found".  It parses
+// the error string looking for known values, which is imperfect but works in
+// practice.
+func IsNotFoundError(err error) bool {
+	es := err.Error()
+	if strings.Contains(es, "does not exist") {
+		// set with the same name already exists
+		// xref: https://github.com/Olipro/ipset/blob/master/lib/errcode.c#L32-L33
+		return true
+	}
+	if strings.Contains(es, "element is missing") {
+		// entry is missing from the set
+		// xref: https://github.com/Olipro/ipset/blob/master/lib/parse.c#L1904
+		// https://github.com/Olipro/ipset/blob/master/lib/parse.c#L1925
+		return true
+	}
+	return false
+}
+
 var _ = Interface(&runner{})