From f3e744b661ca44fdbbc24a15ad54986d6400fa35 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Mon, 20 Nov 2017 15:02:18 +0800 Subject: [PATCH] add cleanup-ipvs flag --- cmd/kube-proxy/app/server.go | 10 +++++++--- cmd/kube-proxy/app/server_others.go | 18 +++++++++++------- cmd/kube-proxy/app/server_test.go | 2 +- pkg/proxy/ipvs/proxier.go | 24 ++++++++++++------------ 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 654531dd426..52acff8e762 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -95,7 +95,8 @@ type Options struct { WriteConfigTo string // CleanupAndExit, when true, makes the proxy server clean up iptables rules, then exit. CleanupAndExit bool - + // CleanupIPVS, when true, makes the proxy server clean up ipvs rules before running. + CleanupIPVS bool // config is the proxy server's configuration object. config *kubeproxyconfig.KubeProxyConfiguration @@ -120,6 +121,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) { fs.BoolVar(&options.CleanupAndExit, "cleanup-iptables", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.") fs.MarkDeprecated("cleanup-iptables", "This flag is replaced by --cleanup.") fs.BoolVar(&options.CleanupAndExit, "cleanup", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.") + fs.BoolVar(&options.CleanupIPVS, "cleanup-ipvs", options.CleanupIPVS, "If true make kube-proxy cleanup ipvs rules before running. Default is true") // All flags below here are deprecated and will eventually be removed. @@ -173,6 +175,7 @@ func NewOptions() *Options { healthzPort: ports.ProxyHealthzPort, scheme: scheme.Scheme, codecs: scheme.Codecs, + CleanupIPVS: true, } } @@ -215,7 +218,7 @@ func (o *Options) Run() error { return o.writeConfigFile() } - proxyServer, err := NewProxyServer(o.config, o.CleanupAndExit, o.scheme, o.master) + proxyServer, err := NewProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master) if err != nil { return err } @@ -367,6 +370,7 @@ type ProxyServer struct { ProxyMode string NodeRef *v1.ObjectReference CleanupAndExit bool + CleanupIPVS bool MetricsBindAddress string EnableProfiling bool OOMScoreAdj *int32 @@ -424,7 +428,7 @@ func (s *ProxyServer) Run() error { if s.CleanupAndExit { encounteredError := userspace.CleanupLeftovers(s.IptInterface) encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError - encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface) || encounteredError + encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface, s.CleanupIPVS) || encounteredError if encounteredError { return errors.New("encountered an error while tearing down rules.") } diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index ee68a48540d..fa76f6130b9 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -54,7 +54,7 @@ import ( ) // NewProxyServer returns a new ProxyServer. -func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { +func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, cleanupIPVS bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { if config == nil { return nil, errors.New("config is required") } @@ -161,9 +161,11 @@ func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi glog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. userspace.CleanupLeftovers(iptInterface) - // IPVS Proxier will generate some iptables rules, - // need to clean them before switching to other proxy mode. - ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface) + // IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode. + // Besides, ipvs proxier will create some ipvs rules as well. Because there is no way to tell if a given + // ipvs rule is created by IPVS proxier or not. Users should explicitly specify `--clean-ipvs=true` to flush + // all ipvs rules when kube-proxy start up. Users do this operation should be with caution. + ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS) } else if proxyMode == proxyModeIPVS { glog.V(0).Info("Using ipvs Proxier.") proxierIPVS, err := ipvs.NewProxier( @@ -223,9 +225,11 @@ func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi glog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. iptables.CleanupLeftovers(iptInterface) - // IPVS Proxier will generate some iptables rules, - // need to clean them before switching to other proxy mode. - ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface) + // IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode. + // Besides, ipvs proxier will create some ipvs rules as well. Because there is no way to tell if a given + // ipvs rule is created by IPVS proxier or not. Users should explicitly specify `--clean-ipvs=true` to flush + // all ipvs rules when kube-proxy start up. Users do this operation should be with caution. + ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS) } iptInterface.AddReloadFunc(proxier.Sync) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index f9c70a4dce1..b36904caea1 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -162,7 +162,7 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) { } options.CleanupAndExit = true - proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master) + proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.CleanupIPVS, options.scheme, options.master) assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr) assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 9c81de66e4e..5f5a09d2447 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -797,21 +797,21 @@ func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool } // CleanupLeftovers clean up all ipvs and iptables rules created by ipvs Proxier. -func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset utilipset.Interface) (encounteredError bool) { - // Return immediately when ipvs interface is nil - Probably initialization failed in somewhere. - if ipvs == nil { - return true - } - encounteredError = false - // Currently we assume only ipvs proxier will create ipvs rules, ipvs proxier will flush all ipvs rules when clean up. - // Users do this operation should be with caution. - err := ipvs.Flush() - if err != nil { - encounteredError = true +func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset utilipset.Interface, cleanupIPVS bool) (encounteredError bool) { + if cleanupIPVS { + // Return immediately when ipvs interface is nil - Probably initialization failed in somewhere. + if ipvs == nil { + return true + } + encounteredError = false + err := ipvs.Flush() + if err != nil { + encounteredError = true + } } // Delete dummy interface created by ipvs Proxier. nl := NewNetLinkHandle() - err = nl.DeleteDummyDevice(DefaultDummyDevice) + err := nl.DeleteDummyDevice(DefaultDummyDevice) if err != nil { encounteredError = true }