Move some code in the proxiers

For no real reason, the core Proxier definitions weren't at the start
of the files.

(This just moves code around. It doesn't change anything.)

pkg/proxy/iptables/proxier.go

@@ -95,47 +95,42 @@ const (
 const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
 const sysctlNFConntrackTCPBeLiberal = "net/netfilter/nf_conntrack_tcp_be_liberal"
 
-// internal struct for string service information
+// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies.
-type servicePortInfo struct {
+func NewDualStackProxier(
-	*proxy.BaseServicePortInfo
+	ipt [2]utiliptables.Interface,
-	// The following fields are computed and stored for performance reasons.
+	sysctl utilsysctl.Interface,
-	nameString             string
+	exec utilexec.Interface,
-	clusterPolicyChainName utiliptables.Chain
+	syncPeriod time.Duration,
-	localPolicyChainName   utiliptables.Chain
+	minSyncPeriod time.Duration,
-	firewallChainName      utiliptables.Chain
+	masqueradeAll bool,
-	externalChainName      utiliptables.Chain
+	localhostNodePorts bool,
-}
+	masqueradeBit int,
-
+	localDetectors [2]proxyutiliptables.LocalTrafficDetector,
-// returns a new proxy.ServicePort which abstracts a serviceInfo
+	hostname string,
-func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort {
+	nodeIPs map[v1.IPFamily]net.IP,
-	svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo}
+	recorder events.EventRecorder,
-
+	healthzServer *healthcheck.ProxierHealthServer,
-	// Store the following for performance reasons.
+	nodePortAddresses []string,
-	svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	initOnly bool,
-	svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name}
+) (proxy.Provider, error) {
-	protocol := strings.ToLower(string(svcPort.Protocol()))
+	// Create an ipv4 instance of the single-stack proxier
-	svcPort.nameString = svcPortName.String()
+	ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], sysctl,
-	svcPort.clusterPolicyChainName = servicePortPolicyClusterChain(svcPort.nameString, protocol)
+		exec, syncPeriod, minSyncPeriod, masqueradeAll, localhostNodePorts, masqueradeBit, localDetectors[0], hostname,
-	svcPort.localPolicyChainName = servicePortPolicyLocalChainName(svcPort.nameString, protocol)
+		nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
-	svcPort.firewallChainName = serviceFirewallChainName(svcPort.nameString, protocol)
+	if err != nil {
-	svcPort.externalChainName = serviceExternalChainName(svcPort.nameString, protocol)
+		return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err)
-
-	return svcPort
-}
-
-// internal struct for endpoints information
-type endpointInfo struct {
-	*proxy.BaseEndpointInfo
-
-	ChainName utiliptables.Chain
-}
-
-// returns a new proxy.Endpoint which abstracts a endpointInfo
-func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint {
-	return &endpointInfo{
-		BaseEndpointInfo: baseInfo,
-		ChainName:        servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(svcPortName.Protocol)), baseInfo.String()),
 	}
+
+	ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], sysctl,
+		exec, syncPeriod, minSyncPeriod, masqueradeAll, false, masqueradeBit, localDetectors[1], hostname,
+		nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err)
+	}
+	if initOnly {
+		return nil, nil
+	}
+	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
 }
 
 // Proxier is an iptables based proxy for connections between a localhost:lport
@@ -322,42 +317,47 @@ func NewProxier(ipFamily v1.IPFamily,
 	return proxier, nil
 }
 
-// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies.
+// internal struct for string service information
-func NewDualStackProxier(
+type servicePortInfo struct {
-	ipt [2]utiliptables.Interface,
+	*proxy.BaseServicePortInfo
-	sysctl utilsysctl.Interface,
+	// The following fields are computed and stored for performance reasons.
-	exec utilexec.Interface,
+	nameString             string
-	syncPeriod time.Duration,
+	clusterPolicyChainName utiliptables.Chain
-	minSyncPeriod time.Duration,
+	localPolicyChainName   utiliptables.Chain
-	masqueradeAll bool,
+	firewallChainName      utiliptables.Chain
-	localhostNodePorts bool,
+	externalChainName      utiliptables.Chain
-	masqueradeBit int,
+}
-	localDetectors [2]proxyutiliptables.LocalTrafficDetector,
-	hostname string,
-	nodeIPs map[v1.IPFamily]net.IP,
-	recorder events.EventRecorder,
-	healthzServer *healthcheck.ProxierHealthServer,
-	nodePortAddresses []string,
-	initOnly bool,
-) (proxy.Provider, error) {
-	// Create an ipv4 instance of the single-stack proxier
-	ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], sysctl,
-		exec, syncPeriod, minSyncPeriod, masqueradeAll, localhostNodePorts, masqueradeBit, localDetectors[0], hostname,
-		nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err)
-	}
 
-	ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], sysctl,
+// returns a new proxy.ServicePort which abstracts a serviceInfo
-		exec, syncPeriod, minSyncPeriod, masqueradeAll, false, masqueradeBit, localDetectors[1], hostname,
+func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort {
-		nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
+	svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo}
-	if err != nil {
+
-		return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err)
+	// Store the following for performance reasons.
+	svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name}
+	protocol := strings.ToLower(string(svcPort.Protocol()))
+	svcPort.nameString = svcPortName.String()
+	svcPort.clusterPolicyChainName = servicePortPolicyClusterChain(svcPort.nameString, protocol)
+	svcPort.localPolicyChainName = servicePortPolicyLocalChainName(svcPort.nameString, protocol)
+	svcPort.firewallChainName = serviceFirewallChainName(svcPort.nameString, protocol)
+	svcPort.externalChainName = serviceExternalChainName(svcPort.nameString, protocol)
+
+	return svcPort
+}
+
+// internal struct for endpoints information
+type endpointInfo struct {
+	*proxy.BaseEndpointInfo
+
+	ChainName utiliptables.Chain
+}
+
+// returns a new proxy.Endpoint which abstracts a endpointInfo
+func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint {
+	return &endpointInfo{
+		BaseEndpointInfo: baseInfo,
+		ChainName:        servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(svcPortName.Protocol)), baseInfo.String()),
 	}
-	if initOnly {
-		return nil, nil
-	}
-	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
 }
 
 type iptablesJumpChain struct {

pkg/proxy/ipvs/proxier.go

@@ -98,112 +98,6 @@ const (
 	defaultDummyDevice = "kube-ipvs0"
 )
 
-// iptablesJumpChain is tables of iptables chains that ipvs proxier used to install iptables or cleanup iptables.
-// `to` is the iptables chain we want to operate.
-// `from` is the source iptables chain
-var iptablesJumpChain = []struct {
-	table   utiliptables.Table
-	from    utiliptables.Chain
-	to      utiliptables.Chain
-	comment string
-}{
-	{utiliptables.TableNAT, utiliptables.ChainOutput, kubeServicesChain, "kubernetes service portals"},
-	{utiliptables.TableNAT, utiliptables.ChainPrerouting, kubeServicesChain, "kubernetes service portals"},
-	{utiliptables.TableNAT, utiliptables.ChainPostrouting, kubePostroutingChain, "kubernetes postrouting rules"},
-	{utiliptables.TableFilter, utiliptables.ChainForward, kubeForwardChain, "kubernetes forwarding rules"},
-	{utiliptables.TableFilter, utiliptables.ChainInput, kubeNodePortChain, "kubernetes health check rules"},
-	{utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"},
-	{utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"},
-	{utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"},
-	{utiliptables.TableFilter, utiliptables.ChainOutput, kubeIPVSOutFilterChain, "kubernetes ipvs access filter"},
-}
-
-var iptablesChains = []struct {
-	table utiliptables.Table
-	chain utiliptables.Chain
-}{
-	{utiliptables.TableNAT, kubeServicesChain},
-	{utiliptables.TableNAT, kubePostroutingChain},
-	{utiliptables.TableNAT, kubeNodePortChain},
-	{utiliptables.TableNAT, kubeLoadBalancerChain},
-	{utiliptables.TableNAT, kubeMarkMasqChain},
-	{utiliptables.TableFilter, kubeForwardChain},
-	{utiliptables.TableFilter, kubeNodePortChain},
-	{utiliptables.TableFilter, kubeProxyFirewallChain},
-	{utiliptables.TableFilter, kubeSourceRangesFirewallChain},
-	{utiliptables.TableFilter, kubeIPVSFilterChain},
-	{utiliptables.TableFilter, kubeIPVSOutFilterChain},
-}
-
-var iptablesCleanupChains = []struct {
-	table utiliptables.Table
-	chain utiliptables.Chain
-}{
-	{utiliptables.TableNAT, kubeServicesChain},
-	{utiliptables.TableNAT, kubePostroutingChain},
-	{utiliptables.TableNAT, kubeNodePortChain},
-	{utiliptables.TableNAT, kubeLoadBalancerChain},
-	{utiliptables.TableFilter, kubeForwardChain},
-	{utiliptables.TableFilter, kubeNodePortChain},
-	{utiliptables.TableFilter, kubeProxyFirewallChain},
-	{utiliptables.TableFilter, kubeSourceRangesFirewallChain},
-	{utiliptables.TableFilter, kubeIPVSFilterChain},
-	{utiliptables.TableFilter, kubeIPVSOutFilterChain},
-}
-
-// ipsetInfo is all ipset we needed in ipvs proxier
-var ipsetInfo = []struct {
-	name    string
-	setType utilipset.Type
-	comment string
-}{
-	{kubeLoopBackIPSet, utilipset.HashIPPortIP, kubeLoopBackIPSetComment},
-	{kubeClusterIPSet, utilipset.HashIPPort, kubeClusterIPSetComment},
-	{kubeExternalIPSet, utilipset.HashIPPort, kubeExternalIPSetComment},
-	{kubeExternalIPLocalSet, utilipset.HashIPPort, kubeExternalIPLocalSetComment},
-	{kubeLoadBalancerSet, utilipset.HashIPPort, kubeLoadBalancerSetComment},
-	{kubeLoadBalancerFWSet, utilipset.HashIPPort, kubeLoadBalancerFWSetComment},
-	{kubeLoadBalancerLocalSet, utilipset.HashIPPort, kubeLoadBalancerLocalSetComment},
-	{kubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, kubeLoadBalancerSourceIPSetComment},
-	{kubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment},
-	{kubeNodePortSetTCP, utilipset.BitmapPort, kubeNodePortSetTCPComment},
-	{kubeNodePortLocalSetTCP, utilipset.BitmapPort, kubeNodePortLocalSetTCPComment},
-	{kubeNodePortSetUDP, utilipset.BitmapPort, kubeNodePortSetUDPComment},
-	{kubeNodePortLocalSetUDP, utilipset.BitmapPort, kubeNodePortLocalSetUDPComment},
-	{kubeNodePortSetSCTP, utilipset.HashIPPort, kubeNodePortSetSCTPComment},
-	{kubeNodePortLocalSetSCTP, utilipset.HashIPPort, kubeNodePortLocalSetSCTPComment},
-	{kubeHealthCheckNodePortSet, utilipset.BitmapPort, kubeHealthCheckNodePortSetComment},
-	{kubeIPVSSet, utilipset.HashIP, kubeIPVSSetComment},
-}
-
-// ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to
-// `iptables -t nat -A <from> -m set --match-set <name> <matchType> -j <to>`
-// example: iptables -t nat -A KUBE-SERVICES -m set --match-set KUBE-NODE-PORT-TCP dst -j KUBE-NODE-PORT
-// ipsets with other match rules will be created Individually.
-// Note: kubeNodePortLocalSetTCP must be prior to kubeNodePortSetTCP, the same for UDP.
-var ipsetWithIptablesChain = []struct {
-	name          string
-	table         utiliptables.Table
-	from          string
-	to            string
-	matchType     string
-	protocolMatch string
-}{
-	{kubeLoopBackIPSet, utiliptables.TableNAT, string(kubePostroutingChain), "MASQUERADE", "dst,dst,src", ""},
-	{kubeLoadBalancerSet, utiliptables.TableNAT, string(kubeServicesChain), string(kubeLoadBalancerChain), "dst,dst", ""},
-	{kubeLoadBalancerLocalSet, utiliptables.TableNAT, string(kubeLoadBalancerChain), "RETURN", "dst,dst", ""},
-	{kubeNodePortLocalSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolTCP},
-	{kubeNodePortSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolTCP},
-	{kubeNodePortLocalSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolUDP},
-	{kubeNodePortSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolUDP},
-	{kubeNodePortLocalSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst,dst", utilipset.ProtocolSCTP},
-	{kubeNodePortSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst,dst", utilipset.ProtocolSCTP},
-
-	{kubeLoadBalancerFWSet, utiliptables.TableFilter, string(kubeProxyFirewallChain), string(kubeSourceRangesFirewallChain), "dst,dst", ""},
-	{kubeLoadBalancerSourceCIDRSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""},
-	{kubeLoadBalancerSourceIPSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""},
-}
-
 // In IPVS proxy mode, the following flags need to be set
 const (
 	sysctlVSConnTrack             = "net/ipv4/vs/conntrack"
@@ -215,6 +109,58 @@ const (
 	sysctlArpAnnounce             = "net/ipv4/conf/all/arp_announce"
 )
 
+// NewDualStackProxier returns a new Proxier for dual-stack operation
+func NewDualStackProxier(
+	ipt [2]utiliptables.Interface,
+	ipvs utilipvs.Interface,
+	ipset utilipset.Interface,
+	sysctl utilsysctl.Interface,
+	exec utilexec.Interface,
+	syncPeriod time.Duration,
+	minSyncPeriod time.Duration,
+	excludeCIDRs []string,
+	strictARP bool,
+	tcpTimeout time.Duration,
+	tcpFinTimeout time.Duration,
+	udpTimeout time.Duration,
+	masqueradeAll bool,
+	masqueradeBit int,
+	localDetectors [2]proxyutiliptables.LocalTrafficDetector,
+	hostname string,
+	nodeIPs map[v1.IPFamily]net.IP,
+	recorder events.EventRecorder,
+	healthzServer *healthcheck.ProxierHealthServer,
+	scheduler string,
+	nodePortAddresses []string,
+	initOnly bool,
+) (proxy.Provider, error) {
+	// Create an ipv4 instance of the single-stack proxier
+	ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], ipvs, ipset, sysctl,
+		exec, syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP,
+		tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit,
+		localDetectors[0], hostname, nodeIPs[v1.IPv4Protocol], recorder,
+		healthzServer, scheduler, nodePortAddresses, initOnly)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err)
+	}
+
+	ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], ipvs, ipset, sysctl,
+		exec, syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP,
+		tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit,
+		localDetectors[1], hostname, nodeIPs[v1.IPv6Protocol], recorder,
+		healthzServer, scheduler, nodePortAddresses, initOnly)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err)
+	}
+	if initOnly {
+		return nil, nil
+	}
+
+	// Return a meta-proxier that dispatch calls between the two
+	// single-stack proxier instances
+	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
+}
+
 // Proxier is an ipvs based proxy for connections between a localhost:lport
 // and services that provide the actual backends.
 type Proxier struct {
@@ -466,59 +412,6 @@ func NewProxier(ipFamily v1.IPFamily,
 	return proxier, nil
 }
 
-// NewDualStackProxier returns a new Proxier for dual-stack operation
-func NewDualStackProxier(
-	ipt [2]utiliptables.Interface,
-	ipvs utilipvs.Interface,
-	ipset utilipset.Interface,
-	sysctl utilsysctl.Interface,
-	exec utilexec.Interface,
-	syncPeriod time.Duration,
-	minSyncPeriod time.Duration,
-	excludeCIDRs []string,
-	strictARP bool,
-	tcpTimeout time.Duration,
-	tcpFinTimeout time.Duration,
-	udpTimeout time.Duration,
-	masqueradeAll bool,
-	masqueradeBit int,
-	localDetectors [2]proxyutiliptables.LocalTrafficDetector,
-	hostname string,
-	nodeIPs map[v1.IPFamily]net.IP,
-	recorder events.EventRecorder,
-	healthzServer *healthcheck.ProxierHealthServer,
-	scheduler string,
-	nodePortAddresses []string,
-	initOnly bool,
-) (proxy.Provider, error) {
-
-	// Create an ipv4 instance of the single-stack proxier
-	ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], ipvs, ipset, sysctl,
-		exec, syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP,
-		tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit,
-		localDetectors[0], hostname, nodeIPs[v1.IPv4Protocol], recorder,
-		healthzServer, scheduler, nodePortAddresses, initOnly)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err)
-	}
-
-	ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], ipvs, ipset, sysctl,
-		exec, syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP,
-		tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit,
-		localDetectors[1], hostname, nodeIPs[v1.IPv6Protocol], recorder,
-		healthzServer, scheduler, nodePortAddresses, initOnly)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err)
-	}
-	if initOnly {
-		return nil, nil
-	}
-
-	// Return a meta-proxier that dispatch calls between the two
-	// single-stack proxier instances
-	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
-}
-
 func filterCIDRs(wantIPv6 bool, cidrs []string) []string {
 	var filteredCIDRs []string
 	for _, cidr := range cidrs {
@@ -529,6 +422,112 @@ func filterCIDRs(wantIPv6 bool, cidrs []string) []string {
 	return filteredCIDRs
 }
 
+// iptablesJumpChain is tables of iptables chains that ipvs proxier used to install iptables or cleanup iptables.
+// `to` is the iptables chain we want to operate.
+// `from` is the source iptables chain
+var iptablesJumpChain = []struct {
+	table   utiliptables.Table
+	from    utiliptables.Chain
+	to      utiliptables.Chain
+	comment string
+}{
+	{utiliptables.TableNAT, utiliptables.ChainOutput, kubeServicesChain, "kubernetes service portals"},
+	{utiliptables.TableNAT, utiliptables.ChainPrerouting, kubeServicesChain, "kubernetes service portals"},
+	{utiliptables.TableNAT, utiliptables.ChainPostrouting, kubePostroutingChain, "kubernetes postrouting rules"},
+	{utiliptables.TableFilter, utiliptables.ChainForward, kubeForwardChain, "kubernetes forwarding rules"},
+	{utiliptables.TableFilter, utiliptables.ChainInput, kubeNodePortChain, "kubernetes health check rules"},
+	{utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"},
+	{utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"},
+	{utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"},
+	{utiliptables.TableFilter, utiliptables.ChainOutput, kubeIPVSOutFilterChain, "kubernetes ipvs access filter"},
+}
+
+var iptablesChains = []struct {
+	table utiliptables.Table
+	chain utiliptables.Chain
+}{
+	{utiliptables.TableNAT, kubeServicesChain},
+	{utiliptables.TableNAT, kubePostroutingChain},
+	{utiliptables.TableNAT, kubeNodePortChain},
+	{utiliptables.TableNAT, kubeLoadBalancerChain},
+	{utiliptables.TableNAT, kubeMarkMasqChain},
+	{utiliptables.TableFilter, kubeForwardChain},
+	{utiliptables.TableFilter, kubeNodePortChain},
+	{utiliptables.TableFilter, kubeProxyFirewallChain},
+	{utiliptables.TableFilter, kubeSourceRangesFirewallChain},
+	{utiliptables.TableFilter, kubeIPVSFilterChain},
+	{utiliptables.TableFilter, kubeIPVSOutFilterChain},
+}
+
+var iptablesCleanupChains = []struct {
+	table utiliptables.Table
+	chain utiliptables.Chain
+}{
+	{utiliptables.TableNAT, kubeServicesChain},
+	{utiliptables.TableNAT, kubePostroutingChain},
+	{utiliptables.TableNAT, kubeNodePortChain},
+	{utiliptables.TableNAT, kubeLoadBalancerChain},
+	{utiliptables.TableFilter, kubeForwardChain},
+	{utiliptables.TableFilter, kubeNodePortChain},
+	{utiliptables.TableFilter, kubeProxyFirewallChain},
+	{utiliptables.TableFilter, kubeSourceRangesFirewallChain},
+	{utiliptables.TableFilter, kubeIPVSFilterChain},
+	{utiliptables.TableFilter, kubeIPVSOutFilterChain},
+}
+
+// ipsetInfo is all ipset we needed in ipvs proxier
+var ipsetInfo = []struct {
+	name    string
+	setType utilipset.Type
+	comment string
+}{
+	{kubeLoopBackIPSet, utilipset.HashIPPortIP, kubeLoopBackIPSetComment},
+	{kubeClusterIPSet, utilipset.HashIPPort, kubeClusterIPSetComment},
+	{kubeExternalIPSet, utilipset.HashIPPort, kubeExternalIPSetComment},
+	{kubeExternalIPLocalSet, utilipset.HashIPPort, kubeExternalIPLocalSetComment},
+	{kubeLoadBalancerSet, utilipset.HashIPPort, kubeLoadBalancerSetComment},
+	{kubeLoadBalancerFWSet, utilipset.HashIPPort, kubeLoadBalancerFWSetComment},
+	{kubeLoadBalancerLocalSet, utilipset.HashIPPort, kubeLoadBalancerLocalSetComment},
+	{kubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, kubeLoadBalancerSourceIPSetComment},
+	{kubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment},
+	{kubeNodePortSetTCP, utilipset.BitmapPort, kubeNodePortSetTCPComment},
+	{kubeNodePortLocalSetTCP, utilipset.BitmapPort, kubeNodePortLocalSetTCPComment},
+	{kubeNodePortSetUDP, utilipset.BitmapPort, kubeNodePortSetUDPComment},
+	{kubeNodePortLocalSetUDP, utilipset.BitmapPort, kubeNodePortLocalSetUDPComment},
+	{kubeNodePortSetSCTP, utilipset.HashIPPort, kubeNodePortSetSCTPComment},
+	{kubeNodePortLocalSetSCTP, utilipset.HashIPPort, kubeNodePortLocalSetSCTPComment},
+	{kubeHealthCheckNodePortSet, utilipset.BitmapPort, kubeHealthCheckNodePortSetComment},
+	{kubeIPVSSet, utilipset.HashIP, kubeIPVSSetComment},
+}
+
+// ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to
+// `iptables -t nat -A <from> -m set --match-set <name> <matchType> -j <to>`
+// example: iptables -t nat -A KUBE-SERVICES -m set --match-set KUBE-NODE-PORT-TCP dst -j KUBE-NODE-PORT
+// ipsets with other match rules will be created Individually.
+// Note: kubeNodePortLocalSetTCP must be prior to kubeNodePortSetTCP, the same for UDP.
+var ipsetWithIptablesChain = []struct {
+	name          string
+	table         utiliptables.Table
+	from          string
+	to            string
+	matchType     string
+	protocolMatch string
+}{
+	{kubeLoopBackIPSet, utiliptables.TableNAT, string(kubePostroutingChain), "MASQUERADE", "dst,dst,src", ""},
+	{kubeLoadBalancerSet, utiliptables.TableNAT, string(kubeServicesChain), string(kubeLoadBalancerChain), "dst,dst", ""},
+	{kubeLoadBalancerLocalSet, utiliptables.TableNAT, string(kubeLoadBalancerChain), "RETURN", "dst,dst", ""},
+	{kubeNodePortLocalSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolTCP},
+	{kubeNodePortSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolTCP},
+	{kubeNodePortLocalSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolUDP},
+	{kubeNodePortSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolUDP},
+	{kubeNodePortLocalSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst,dst", utilipset.ProtocolSCTP},
+	{kubeNodePortSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst,dst", utilipset.ProtocolSCTP},
+
+	{kubeLoadBalancerFWSet, utiliptables.TableFilter, string(kubeProxyFirewallChain), string(kubeSourceRangesFirewallChain), "dst,dst", ""},
+	{kubeLoadBalancerSourceCIDRSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""},
+	{kubeLoadBalancerSourceIPSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""},
+}
+
 // internal struct for string service information
 type servicePortInfo struct {
 	*proxy.BaseServicePortInfo

pkg/proxy/nftables/proxier.go

@@ -104,51 +104,39 @@ const (
 	masqueradingChain = "masquerading"
 )
 
-// internal struct for string service information
+// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies.
-type servicePortInfo struct {
+func NewDualStackProxier(
-	*proxy.BaseServicePortInfo
+	sysctl utilsysctl.Interface,
-	// The following fields are computed and stored for performance reasons.
+	syncPeriod time.Duration,
-	nameString             string
+	minSyncPeriod time.Duration,
-	clusterPolicyChainName string
+	masqueradeAll bool,
-	localPolicyChainName   string
+	masqueradeBit int,
-	externalChainName      string
+	localDetectors [2]proxyutiliptables.LocalTrafficDetector,
-	firewallChainName      string
+	hostname string,
-}
+	nodeIPs map[v1.IPFamily]net.IP,
-
+	recorder events.EventRecorder,
-// returns a new proxy.ServicePort which abstracts a serviceInfo
+	healthzServer *healthcheck.ProxierHealthServer,
-func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort {
+	nodePortAddresses []string,
-	svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo}
+	initOnly bool,
-
+) (proxy.Provider, error) {
-	// Store the following for performance reasons.
+	// Create an ipv4 instance of the single-stack proxier
-	svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	ipv4Proxier, err := NewProxier(v1.IPv4Protocol, sysctl,
-	svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name}
+		syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[0], hostname,
-	svcPort.nameString = svcPortName.String()
+		nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
-
+	if err != nil {
-	chainNameBase := servicePortChainNameBase(&svcPortName, strings.ToLower(string(svcPort.Protocol())))
+		return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err)
-	svcPort.clusterPolicyChainName = servicePortPolicyClusterChainNamePrefix + chainNameBase
-	svcPort.localPolicyChainName = servicePortPolicyLocalChainNamePrefix + chainNameBase
-	svcPort.externalChainName = serviceExternalChainNamePrefix + chainNameBase
-	svcPort.firewallChainName = servicePortFirewallChainNamePrefix + chainNameBase
-
-	return svcPort
-}
-
-// internal struct for endpoints information
-type endpointInfo struct {
-	*proxy.BaseEndpointInfo
-
-	chainName       string
-	affinitySetName string
-}
-
-// returns a new proxy.Endpoint which abstracts a endpointInfo
-func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint {
-	chainNameBase := servicePortEndpointChainNameBase(svcPortName, strings.ToLower(string(svcPortName.Protocol)), baseInfo.String())
-	return &endpointInfo{
-		BaseEndpointInfo: baseInfo,
-		chainName:        servicePortEndpointChainNamePrefix + chainNameBase,
-		affinitySetName:  servicePortEndpointAffinityNamePrefix + chainNameBase,
 	}
+
+	ipv6Proxier, err := NewProxier(v1.IPv6Protocol, sysctl,
+		syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[1], hostname,
+		nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err)
+	}
+	if initOnly {
+		return nil, nil
+	}
+	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
 }
 
 // Proxier is an nftables based proxy
@@ -277,39 +265,51 @@ func NewProxier(ipFamily v1.IPFamily,
 	return proxier, nil
 }
 
-// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies.
+// internal struct for string service information
-func NewDualStackProxier(
+type servicePortInfo struct {
-	sysctl utilsysctl.Interface,
+	*proxy.BaseServicePortInfo
-	syncPeriod time.Duration,
+	// The following fields are computed and stored for performance reasons.
-	minSyncPeriod time.Duration,
+	nameString             string
-	masqueradeAll bool,
+	clusterPolicyChainName string
-	masqueradeBit int,
+	localPolicyChainName   string
-	localDetectors [2]proxyutiliptables.LocalTrafficDetector,
+	externalChainName      string
-	hostname string,
+	firewallChainName      string
-	nodeIPs map[v1.IPFamily]net.IP,
+}
-	recorder events.EventRecorder,
-	healthzServer *healthcheck.ProxierHealthServer,
-	nodePortAddresses []string,
-	initOnly bool,
-) (proxy.Provider, error) {
-	// Create an ipv4 instance of the single-stack proxier
-	ipv4Proxier, err := NewProxier(v1.IPv4Protocol, sysctl,
-		syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[0], hostname,
-		nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err)
-	}
 
-	ipv6Proxier, err := NewProxier(v1.IPv6Protocol, sysctl,
+// returns a new proxy.ServicePort which abstracts a serviceInfo
-		syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[1], hostname,
+func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort {
-		nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly)
+	svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo}
-	if err != nil {
+
-		return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err)
+	// Store the following for performance reasons.
+	svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name}
+	svcPort.nameString = svcPortName.String()
+
+	chainNameBase := servicePortChainNameBase(&svcPortName, strings.ToLower(string(svcPort.Protocol())))
+	svcPort.clusterPolicyChainName = servicePortPolicyClusterChainNamePrefix + chainNameBase
+	svcPort.localPolicyChainName = servicePortPolicyLocalChainNamePrefix + chainNameBase
+	svcPort.externalChainName = serviceExternalChainNamePrefix + chainNameBase
+	svcPort.firewallChainName = servicePortFirewallChainNamePrefix + chainNameBase
+
+	return svcPort
+}
+
+// internal struct for endpoints information
+type endpointInfo struct {
+	*proxy.BaseEndpointInfo
+
+	chainName       string
+	affinitySetName string
+}
+
+// returns a new proxy.Endpoint which abstracts a endpointInfo
+func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint {
+	chainNameBase := servicePortEndpointChainNameBase(svcPortName, strings.ToLower(string(svcPortName.Protocol)), baseInfo.String())
+	return &endpointInfo{
+		BaseEndpointInfo: baseInfo,
+		chainName:        servicePortEndpointChainNamePrefix + chainNameBase,
+		affinitySetName:  servicePortEndpointAffinityNamePrefix + chainNameBase,
 	}
-	if initOnly {
-		return nil, nil
-	}
-	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
 }
 
 // nftablesBaseChains lists our "base chains"; those that are directly connected to the