Merge pull request #56036 from m1093782566/flush-ipvs

Automatic merge from submit-queue (batch tested with PRs 55103, 56036, 56186). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add cleanup-ipvs flag for kube-proxy 

**What this PR does / why we need it**:

There is no way to tell if a given ipvs rule is created by ipvs proxier or not, and some people have complained that iptables/userspace proxier will clean up their ipvs rules when start up - both iptables and userspace proxiers need to clean up legacy proxy rules created by ipvs proxier.

This PR adds a new `--cleanup-ipvs` flag for kube-proxy for the sake of providing users a way to decide if clean up IPVS rules or not when start iptables or userspace proxier.

**Which issue(s) this PR fixes**:
Fixes #55857 

**Special notes for your reviewer**:

**Release note**:

```release-note
Add cleanup-ipvs flag for kube-proxy 
```

/sig network
/area ipvs
/king bug

cmd/kube-proxy/app/server.go

@@ -95,7 +95,8 @@ type Options struct {
 	WriteConfigTo string
 	// CleanupAndExit, when true, makes the proxy server clean up iptables rules, then exit.
 	CleanupAndExit bool
-
+	// CleanupIPVS, when true, makes the proxy server clean up ipvs rules before running.
+	CleanupIPVS bool
 	// config is the proxy server's configuration object.
 	config *kubeproxyconfig.KubeProxyConfiguration
 
@@ -120,6 +121,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) {
 	fs.BoolVar(&options.CleanupAndExit, "cleanup-iptables", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
 	fs.MarkDeprecated("cleanup-iptables", "This flag is replaced by --cleanup.")
 	fs.BoolVar(&options.CleanupAndExit, "cleanup", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
+	fs.BoolVar(&options.CleanupIPVS, "cleanup-ipvs", options.CleanupIPVS, "If true make kube-proxy cleanup ipvs rules before running.  Default is true")
 
 	// All flags below here are deprecated and will eventually be removed.
 
@@ -173,6 +175,7 @@ func NewOptions() *Options {
 		healthzPort: ports.ProxyHealthzPort,
 		scheme:      scheme.Scheme,
 		codecs:      scheme.Codecs,
+		CleanupIPVS: true,
 	}
 }
 
@@ -215,7 +218,7 @@ func (o *Options) Run() error {
 		return o.writeConfigFile()
 	}
 
-	proxyServer, err := NewProxyServer(o.config, o.CleanupAndExit, o.scheme, o.master)
+	proxyServer, err := NewProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master)
 	if err != nil {
 		return err
 	}
@@ -367,6 +370,7 @@ type ProxyServer struct {
 	ProxyMode              string
 	NodeRef                *v1.ObjectReference
 	CleanupAndExit         bool
+	CleanupIPVS            bool
 	MetricsBindAddress     string
 	EnableProfiling        bool
 	OOMScoreAdj            *int32
@@ -424,7 +428,7 @@ func (s *ProxyServer) Run() error {
 	if s.CleanupAndExit {
 		encounteredError := userspace.CleanupLeftovers(s.IptInterface)
 		encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError
-		encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface) || encounteredError
+		encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface, s.CleanupIPVS) || encounteredError
 		if encounteredError {
 			return errors.New("encountered an error while tearing down rules.")
 		}

cmd/kube-proxy/app/server_others.go

@@ -54,7 +54,7 @@ import (
 )
 
 // NewProxyServer returns a new ProxyServer.
-func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) {
+func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, cleanupIPVS bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) {
 	if config == nil {
 		return nil, errors.New("config is required")
 	}
@@ -161,9 +161,11 @@ func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 		glog.V(0).Info("Tearing down inactive rules.")
 		// TODO this has side effects that should only happen when Run() is invoked.
 		userspace.CleanupLeftovers(iptInterface)
-		// IPVS Proxier will generate some iptables rules,
-		// need to clean them before switching to other proxy mode.
-		ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface)
+		// IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.
+		// Besides, ipvs proxier will create some ipvs rules as well.  Because there is no way to tell if a given
+		// ipvs rule is created by IPVS proxier or not.  Users should explicitly specify `--clean-ipvs=true` to flush
+		// all ipvs rules when kube-proxy start up.  Users do this operation should be with caution.
+		ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
 	} else if proxyMode == proxyModeIPVS {
 		glog.V(0).Info("Using ipvs Proxier.")
 		proxierIPVS, err := ipvs.NewProxier(
@@ -223,9 +225,11 @@ func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 		glog.V(0).Info("Tearing down inactive rules.")
 		// TODO this has side effects that should only happen when Run() is invoked.
 		iptables.CleanupLeftovers(iptInterface)
-		// IPVS Proxier will generate some iptables rules,
-		// need to clean them before switching to other proxy mode.
-		ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface)
+		// IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.
+		// Besides, ipvs proxier will create some ipvs rules as well.  Because there is no way to tell if a given
+		// ipvs rule is created by IPVS proxier or not.  Users should explicitly specify `--clean-ipvs=true` to flush
+		// all ipvs rules when kube-proxy start up.  Users do this operation should be with caution.
+		ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
 	}
 
 	iptInterface.AddReloadFunc(proxier.Sync)

cmd/kube-proxy/app/server_test.go

@@ -162,7 +162,7 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) {
 		}
 		options.CleanupAndExit = true
 
-		proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master)
+		proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.CleanupIPVS, options.scheme, options.master)
 
 		assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
 		assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)

pkg/proxy/ipvs/proxier.go

@@ -797,21 +797,21 @@ func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool
 }
 
 // CleanupLeftovers clean up all ipvs and iptables rules created by ipvs Proxier.
-func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset utilipset.Interface) (encounteredError bool) {
-	// Return immediately when ipvs interface is nil - Probably initialization failed in somewhere.
-	if ipvs == nil {
-		return true
-	}
-	encounteredError = false
-	// Currently we assume only ipvs proxier will create ipvs rules, ipvs proxier will flush all ipvs rules when clean up.
-	// Users do this operation should be with caution.
-	err := ipvs.Flush()
-	if err != nil {
-		encounteredError = true
+func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset utilipset.Interface, cleanupIPVS bool) (encounteredError bool) {
+	if cleanupIPVS {
+		// Return immediately when ipvs interface is nil - Probably initialization failed in somewhere.
+		if ipvs == nil {
+			return true
+		}
+		encounteredError = false
+		err := ipvs.Flush()
+		if err != nil {
+			encounteredError = true
+		}
 	}
 	// Delete dummy interface created by ipvs Proxier.
 	nl := NewNetLinkHandle()
-	err = nl.DeleteDummyDevice(DefaultDummyDevice)
+	err := nl.DeleteDummyDevice(DefaultDummyDevice)
 	if err != nil {
 		encounteredError = true
 	}