From e6b9b5e0c3633d1486ff2ac9af280fbdc8117b2f Mon Sep 17 00:00:00 2001
From: m1093782566 <dujun5@huawei.com>
Date: Mon, 18 Dec 2017 17:22:26 +0800
Subject: [PATCH 1/3] add not found error for ipset set and entry delete

---
 pkg/proxy/ipvs/ipset.go   |  4 +++-
 pkg/proxy/ipvs/proxier.go |  5 ++++-
 pkg/util/ipset/ipset.go   | 16 ++++++++++++++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go
index 0a01f0bacc6..d61992125ae 100644
--- a/pkg/proxy/ipvs/ipset.go
+++ b/pkg/proxy/ipvs/ipset.go
@@ -114,7 +114,9 @@ func (set *IPSet) syncIPSetEntries() {
 		// Clean legacy entries
 		for _, entry := range currentIPSetEntries.Difference(set.activeEntries).List() {
 			if err := set.handle.DelEntry(entry, set.Name); err != nil {
-				glog.Errorf("Failed to delete ip set entry: %s from ip set: %s, error: %v", entry, set.Name, err)
+				if !utilipset.IsNotFoundError(err) {
+					glog.Errorf("Failed to delete ip set entry: %s from ip set: %s, error: %v", entry, set.Name, err)
+				}
 			} else {
 				glog.V(3).Infof("Successfully delete legacy ip set entry: %s from ip set: %s", entry, set.Name)
 			}
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 5f5a09d2447..104a5e9a353 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -824,7 +824,10 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 	for _, set := range ipSetsToDestroy {
 		err = ipset.DestroySet(set)
 		if err != nil {
-			encounteredError = true
+			if !utilipset.IsNotFoundError(err) {
+				glog.Errorf("Error removing ipset %s, error: %v", set, err)
+				encounteredError = true
+			}
 		}
 	}
 	return encounteredError
diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go
index ba92bff74bf..56993c26305 100644
--- a/pkg/util/ipset/ipset.go
+++ b/pkg/util/ipset/ipset.go
@@ -322,4 +322,20 @@ func validatePortRange(portRange string) bool {
 	return true
 }
 
+// IsNotFoundError returns true if the error indicates "not found".  It parses
+// the error string looking for known values, which is imperfect but works in
+// practice.
+func IsNotFoundError(err error) bool {
+	es := err.Error()
+	if strings.Contains(es, "does not exist") {
+		// set with the same name already exists
+		return true
+	}
+	if strings.Contains(es, "element is missing") {
+		// entry is missing from the set
+		return true
+	}
+	return false
+}
+
 var _ = Interface(&runner{})

From 89e70760d77d2117a44fa2d1c8fd09a2615efc34 Mon Sep 17 00:00:00 2001
From: m1093782566 <dujun5@huawei.com>
Date: Tue, 19 Dec 2017 16:05:32 +0800
Subject: [PATCH 2/3] log error when error occur in CleanupLeftovers()

---
 pkg/proxy/ipvs/proxier.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 104a5e9a353..9a3725f6f3f 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -806,6 +806,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 		encounteredError = false
 		err := ipvs.Flush()
 		if err != nil {
+			glog.Errorf("Error flushing IPVS rules: %v", err)
 			encounteredError = true
 		}
 	}
@@ -813,6 +814,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 	nl := NewNetLinkHandle()
 	err := nl.DeleteDummyDevice(DefaultDummyDevice)
 	if err != nil {
+		glog.Errorf("Error deleting dummy device %s created by IPVS proxier: %v", DefaultDummyDevice, err)
 		encounteredError = true
 	}
 	// Clear iptables created by ipvs Proxier.

From 5052f3aed0d8d975bf7745f73141ef96e86862a0 Mon Sep 17 00:00:00 2001
From: m1093782566 <dujun5@huawei.com>
Date: Fri, 22 Dec 2017 13:49:47 +0800
Subject: [PATCH 3/3] add error string reference

---
 pkg/util/ipset/ipset.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go
index 56993c26305..7ae28050097 100644
--- a/pkg/util/ipset/ipset.go
+++ b/pkg/util/ipset/ipset.go
@@ -329,10 +329,13 @@ func IsNotFoundError(err error) bool {
 	es := err.Error()
 	if strings.Contains(es, "does not exist") {
 		// set with the same name already exists
+		// xref: https://github.com/Olipro/ipset/blob/master/lib/errcode.c#L32-L33
 		return true
 	}
 	if strings.Contains(es, "element is missing") {
 		// entry is missing from the set
+		// xref: https://github.com/Olipro/ipset/blob/master/lib/parse.c#L1904
+		// https://github.com/Olipro/ipset/blob/master/lib/parse.c#L1925
 		return true
 	}
 	return false