diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 75280851ac5..2771801b69c 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -95,47 +95,42 @@ const ( const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet" const sysctlNFConntrackTCPBeLiberal = "net/netfilter/nf_conntrack_tcp_be_liberal" -// internal struct for string service information -type servicePortInfo struct { - *proxy.BaseServicePortInfo - // The following fields are computed and stored for performance reasons. - nameString string - clusterPolicyChainName utiliptables.Chain - localPolicyChainName utiliptables.Chain - firewallChainName utiliptables.Chain - externalChainName utiliptables.Chain -} - -// returns a new proxy.ServicePort which abstracts a serviceInfo -func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort { - svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo} - - // Store the following for performance reasons. - svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} - protocol := strings.ToLower(string(svcPort.Protocol())) - svcPort.nameString = svcPortName.String() - svcPort.clusterPolicyChainName = servicePortPolicyClusterChain(svcPort.nameString, protocol) - svcPort.localPolicyChainName = servicePortPolicyLocalChainName(svcPort.nameString, protocol) - svcPort.firewallChainName = serviceFirewallChainName(svcPort.nameString, protocol) - svcPort.externalChainName = serviceExternalChainName(svcPort.nameString, protocol) - - return svcPort -} - -// internal struct for endpoints information -type endpointInfo struct { - *proxy.BaseEndpointInfo - - ChainName utiliptables.Chain -} - -// returns a new proxy.Endpoint which abstracts a endpointInfo -func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint { - return &endpointInfo{ - BaseEndpointInfo: baseInfo, - ChainName: servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(svcPortName.Protocol)), baseInfo.String()), +// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies. +func NewDualStackProxier( + ipt [2]utiliptables.Interface, + sysctl utilsysctl.Interface, + exec utilexec.Interface, + syncPeriod time.Duration, + minSyncPeriod time.Duration, + masqueradeAll bool, + localhostNodePorts bool, + masqueradeBit int, + localDetectors [2]proxyutiliptables.LocalTrafficDetector, + hostname string, + nodeIPs map[v1.IPFamily]net.IP, + recorder events.EventRecorder, + healthzServer *healthcheck.ProxierHealthServer, + nodePortAddresses []string, + initOnly bool, +) (proxy.Provider, error) { + // Create an ipv4 instance of the single-stack proxier + ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], sysctl, + exec, syncPeriod, minSyncPeriod, masqueradeAll, localhostNodePorts, masqueradeBit, localDetectors[0], hostname, + nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly) + if err != nil { + return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) } + + ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], sysctl, + exec, syncPeriod, minSyncPeriod, masqueradeAll, false, masqueradeBit, localDetectors[1], hostname, + nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly) + if err != nil { + return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) + } + if initOnly { + return nil, nil + } + return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil } // Proxier is an iptables based proxy for connections between a localhost:lport @@ -322,42 +317,47 @@ func NewProxier(ipFamily v1.IPFamily, return proxier, nil } -// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies. -func NewDualStackProxier( - ipt [2]utiliptables.Interface, - sysctl utilsysctl.Interface, - exec utilexec.Interface, - syncPeriod time.Duration, - minSyncPeriod time.Duration, - masqueradeAll bool, - localhostNodePorts bool, - masqueradeBit int, - localDetectors [2]proxyutiliptables.LocalTrafficDetector, - hostname string, - nodeIPs map[v1.IPFamily]net.IP, - recorder events.EventRecorder, - healthzServer *healthcheck.ProxierHealthServer, - nodePortAddresses []string, - initOnly bool, -) (proxy.Provider, error) { - // Create an ipv4 instance of the single-stack proxier - ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], sysctl, - exec, syncPeriod, minSyncPeriod, masqueradeAll, localhostNodePorts, masqueradeBit, localDetectors[0], hostname, - nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly) - if err != nil { - return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) - } +// internal struct for string service information +type servicePortInfo struct { + *proxy.BaseServicePortInfo + // The following fields are computed and stored for performance reasons. + nameString string + clusterPolicyChainName utiliptables.Chain + localPolicyChainName utiliptables.Chain + firewallChainName utiliptables.Chain + externalChainName utiliptables.Chain +} - ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], sysctl, - exec, syncPeriod, minSyncPeriod, masqueradeAll, false, masqueradeBit, localDetectors[1], hostname, - nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly) - if err != nil { - return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) +// returns a new proxy.ServicePort which abstracts a serviceInfo +func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort { + svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo} + + // Store the following for performance reasons. + svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} + protocol := strings.ToLower(string(svcPort.Protocol())) + svcPort.nameString = svcPortName.String() + svcPort.clusterPolicyChainName = servicePortPolicyClusterChain(svcPort.nameString, protocol) + svcPort.localPolicyChainName = servicePortPolicyLocalChainName(svcPort.nameString, protocol) + svcPort.firewallChainName = serviceFirewallChainName(svcPort.nameString, protocol) + svcPort.externalChainName = serviceExternalChainName(svcPort.nameString, protocol) + + return svcPort +} + +// internal struct for endpoints information +type endpointInfo struct { + *proxy.BaseEndpointInfo + + ChainName utiliptables.Chain +} + +// returns a new proxy.Endpoint which abstracts a endpointInfo +func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint { + return &endpointInfo{ + BaseEndpointInfo: baseInfo, + ChainName: servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(svcPortName.Protocol)), baseInfo.String()), } - if initOnly { - return nil, nil - } - return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil } type iptablesJumpChain struct { diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index c67ecf14a0c..daf28fee6be 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -98,112 +98,6 @@ const ( defaultDummyDevice = "kube-ipvs0" ) -// iptablesJumpChain is tables of iptables chains that ipvs proxier used to install iptables or cleanup iptables. -// `to` is the iptables chain we want to operate. -// `from` is the source iptables chain -var iptablesJumpChain = []struct { - table utiliptables.Table - from utiliptables.Chain - to utiliptables.Chain - comment string -}{ - {utiliptables.TableNAT, utiliptables.ChainOutput, kubeServicesChain, "kubernetes service portals"}, - {utiliptables.TableNAT, utiliptables.ChainPrerouting, kubeServicesChain, "kubernetes service portals"}, - {utiliptables.TableNAT, utiliptables.ChainPostrouting, kubePostroutingChain, "kubernetes postrouting rules"}, - {utiliptables.TableFilter, utiliptables.ChainForward, kubeForwardChain, "kubernetes forwarding rules"}, - {utiliptables.TableFilter, utiliptables.ChainInput, kubeNodePortChain, "kubernetes health check rules"}, - {utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"}, - {utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"}, - {utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"}, - {utiliptables.TableFilter, utiliptables.ChainOutput, kubeIPVSOutFilterChain, "kubernetes ipvs access filter"}, -} - -var iptablesChains = []struct { - table utiliptables.Table - chain utiliptables.Chain -}{ - {utiliptables.TableNAT, kubeServicesChain}, - {utiliptables.TableNAT, kubePostroutingChain}, - {utiliptables.TableNAT, kubeNodePortChain}, - {utiliptables.TableNAT, kubeLoadBalancerChain}, - {utiliptables.TableNAT, kubeMarkMasqChain}, - {utiliptables.TableFilter, kubeForwardChain}, - {utiliptables.TableFilter, kubeNodePortChain}, - {utiliptables.TableFilter, kubeProxyFirewallChain}, - {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, - {utiliptables.TableFilter, kubeIPVSFilterChain}, - {utiliptables.TableFilter, kubeIPVSOutFilterChain}, -} - -var iptablesCleanupChains = []struct { - table utiliptables.Table - chain utiliptables.Chain -}{ - {utiliptables.TableNAT, kubeServicesChain}, - {utiliptables.TableNAT, kubePostroutingChain}, - {utiliptables.TableNAT, kubeNodePortChain}, - {utiliptables.TableNAT, kubeLoadBalancerChain}, - {utiliptables.TableFilter, kubeForwardChain}, - {utiliptables.TableFilter, kubeNodePortChain}, - {utiliptables.TableFilter, kubeProxyFirewallChain}, - {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, - {utiliptables.TableFilter, kubeIPVSFilterChain}, - {utiliptables.TableFilter, kubeIPVSOutFilterChain}, -} - -// ipsetInfo is all ipset we needed in ipvs proxier -var ipsetInfo = []struct { - name string - setType utilipset.Type - comment string -}{ - {kubeLoopBackIPSet, utilipset.HashIPPortIP, kubeLoopBackIPSetComment}, - {kubeClusterIPSet, utilipset.HashIPPort, kubeClusterIPSetComment}, - {kubeExternalIPSet, utilipset.HashIPPort, kubeExternalIPSetComment}, - {kubeExternalIPLocalSet, utilipset.HashIPPort, kubeExternalIPLocalSetComment}, - {kubeLoadBalancerSet, utilipset.HashIPPort, kubeLoadBalancerSetComment}, - {kubeLoadBalancerFWSet, utilipset.HashIPPort, kubeLoadBalancerFWSetComment}, - {kubeLoadBalancerLocalSet, utilipset.HashIPPort, kubeLoadBalancerLocalSetComment}, - {kubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, kubeLoadBalancerSourceIPSetComment}, - {kubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment}, - {kubeNodePortSetTCP, utilipset.BitmapPort, kubeNodePortSetTCPComment}, - {kubeNodePortLocalSetTCP, utilipset.BitmapPort, kubeNodePortLocalSetTCPComment}, - {kubeNodePortSetUDP, utilipset.BitmapPort, kubeNodePortSetUDPComment}, - {kubeNodePortLocalSetUDP, utilipset.BitmapPort, kubeNodePortLocalSetUDPComment}, - {kubeNodePortSetSCTP, utilipset.HashIPPort, kubeNodePortSetSCTPComment}, - {kubeNodePortLocalSetSCTP, utilipset.HashIPPort, kubeNodePortLocalSetSCTPComment}, - {kubeHealthCheckNodePortSet, utilipset.BitmapPort, kubeHealthCheckNodePortSetComment}, - {kubeIPVSSet, utilipset.HashIP, kubeIPVSSetComment}, -} - -// ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to -// `iptables -t nat -A -m set --match-set -j ` -// example: iptables -t nat -A KUBE-SERVICES -m set --match-set KUBE-NODE-PORT-TCP dst -j KUBE-NODE-PORT -// ipsets with other match rules will be created Individually. -// Note: kubeNodePortLocalSetTCP must be prior to kubeNodePortSetTCP, the same for UDP. -var ipsetWithIptablesChain = []struct { - name string - table utiliptables.Table - from string - to string - matchType string - protocolMatch string -}{ - {kubeLoopBackIPSet, utiliptables.TableNAT, string(kubePostroutingChain), "MASQUERADE", "dst,dst,src", ""}, - {kubeLoadBalancerSet, utiliptables.TableNAT, string(kubeServicesChain), string(kubeLoadBalancerChain), "dst,dst", ""}, - {kubeLoadBalancerLocalSet, utiliptables.TableNAT, string(kubeLoadBalancerChain), "RETURN", "dst,dst", ""}, - {kubeNodePortLocalSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolTCP}, - {kubeNodePortSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolTCP}, - {kubeNodePortLocalSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolUDP}, - {kubeNodePortSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolUDP}, - {kubeNodePortLocalSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst,dst", utilipset.ProtocolSCTP}, - {kubeNodePortSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst,dst", utilipset.ProtocolSCTP}, - - {kubeLoadBalancerFWSet, utiliptables.TableFilter, string(kubeProxyFirewallChain), string(kubeSourceRangesFirewallChain), "dst,dst", ""}, - {kubeLoadBalancerSourceCIDRSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""}, - {kubeLoadBalancerSourceIPSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""}, -} - // In IPVS proxy mode, the following flags need to be set const ( sysctlVSConnTrack = "net/ipv4/vs/conntrack" @@ -215,6 +109,58 @@ const ( sysctlArpAnnounce = "net/ipv4/conf/all/arp_announce" ) +// NewDualStackProxier returns a new Proxier for dual-stack operation +func NewDualStackProxier( + ipt [2]utiliptables.Interface, + ipvs utilipvs.Interface, + ipset utilipset.Interface, + sysctl utilsysctl.Interface, + exec utilexec.Interface, + syncPeriod time.Duration, + minSyncPeriod time.Duration, + excludeCIDRs []string, + strictARP bool, + tcpTimeout time.Duration, + tcpFinTimeout time.Duration, + udpTimeout time.Duration, + masqueradeAll bool, + masqueradeBit int, + localDetectors [2]proxyutiliptables.LocalTrafficDetector, + hostname string, + nodeIPs map[v1.IPFamily]net.IP, + recorder events.EventRecorder, + healthzServer *healthcheck.ProxierHealthServer, + scheduler string, + nodePortAddresses []string, + initOnly bool, +) (proxy.Provider, error) { + // Create an ipv4 instance of the single-stack proxier + ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], ipvs, ipset, sysctl, + exec, syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP, + tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit, + localDetectors[0], hostname, nodeIPs[v1.IPv4Protocol], recorder, + healthzServer, scheduler, nodePortAddresses, initOnly) + if err != nil { + return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) + } + + ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], ipvs, ipset, sysctl, + exec, syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP, + tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit, + localDetectors[1], hostname, nodeIPs[v1.IPv6Protocol], recorder, + healthzServer, scheduler, nodePortAddresses, initOnly) + if err != nil { + return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) + } + if initOnly { + return nil, nil + } + + // Return a meta-proxier that dispatch calls between the two + // single-stack proxier instances + return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil +} + // Proxier is an ipvs based proxy for connections between a localhost:lport // and services that provide the actual backends. type Proxier struct { @@ -466,59 +412,6 @@ func NewProxier(ipFamily v1.IPFamily, return proxier, nil } -// NewDualStackProxier returns a new Proxier for dual-stack operation -func NewDualStackProxier( - ipt [2]utiliptables.Interface, - ipvs utilipvs.Interface, - ipset utilipset.Interface, - sysctl utilsysctl.Interface, - exec utilexec.Interface, - syncPeriod time.Duration, - minSyncPeriod time.Duration, - excludeCIDRs []string, - strictARP bool, - tcpTimeout time.Duration, - tcpFinTimeout time.Duration, - udpTimeout time.Duration, - masqueradeAll bool, - masqueradeBit int, - localDetectors [2]proxyutiliptables.LocalTrafficDetector, - hostname string, - nodeIPs map[v1.IPFamily]net.IP, - recorder events.EventRecorder, - healthzServer *healthcheck.ProxierHealthServer, - scheduler string, - nodePortAddresses []string, - initOnly bool, -) (proxy.Provider, error) { - - // Create an ipv4 instance of the single-stack proxier - ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], ipvs, ipset, sysctl, - exec, syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP, - tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit, - localDetectors[0], hostname, nodeIPs[v1.IPv4Protocol], recorder, - healthzServer, scheduler, nodePortAddresses, initOnly) - if err != nil { - return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) - } - - ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], ipvs, ipset, sysctl, - exec, syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP, - tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit, - localDetectors[1], hostname, nodeIPs[v1.IPv6Protocol], recorder, - healthzServer, scheduler, nodePortAddresses, initOnly) - if err != nil { - return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) - } - if initOnly { - return nil, nil - } - - // Return a meta-proxier that dispatch calls between the two - // single-stack proxier instances - return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil -} - func filterCIDRs(wantIPv6 bool, cidrs []string) []string { var filteredCIDRs []string for _, cidr := range cidrs { @@ -529,6 +422,112 @@ func filterCIDRs(wantIPv6 bool, cidrs []string) []string { return filteredCIDRs } +// iptablesJumpChain is tables of iptables chains that ipvs proxier used to install iptables or cleanup iptables. +// `to` is the iptables chain we want to operate. +// `from` is the source iptables chain +var iptablesJumpChain = []struct { + table utiliptables.Table + from utiliptables.Chain + to utiliptables.Chain + comment string +}{ + {utiliptables.TableNAT, utiliptables.ChainOutput, kubeServicesChain, "kubernetes service portals"}, + {utiliptables.TableNAT, utiliptables.ChainPrerouting, kubeServicesChain, "kubernetes service portals"}, + {utiliptables.TableNAT, utiliptables.ChainPostrouting, kubePostroutingChain, "kubernetes postrouting rules"}, + {utiliptables.TableFilter, utiliptables.ChainForward, kubeForwardChain, "kubernetes forwarding rules"}, + {utiliptables.TableFilter, utiliptables.ChainInput, kubeNodePortChain, "kubernetes health check rules"}, + {utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"}, + {utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"}, + {utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"}, + {utiliptables.TableFilter, utiliptables.ChainOutput, kubeIPVSOutFilterChain, "kubernetes ipvs access filter"}, +} + +var iptablesChains = []struct { + table utiliptables.Table + chain utiliptables.Chain +}{ + {utiliptables.TableNAT, kubeServicesChain}, + {utiliptables.TableNAT, kubePostroutingChain}, + {utiliptables.TableNAT, kubeNodePortChain}, + {utiliptables.TableNAT, kubeLoadBalancerChain}, + {utiliptables.TableNAT, kubeMarkMasqChain}, + {utiliptables.TableFilter, kubeForwardChain}, + {utiliptables.TableFilter, kubeNodePortChain}, + {utiliptables.TableFilter, kubeProxyFirewallChain}, + {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, + {utiliptables.TableFilter, kubeIPVSFilterChain}, + {utiliptables.TableFilter, kubeIPVSOutFilterChain}, +} + +var iptablesCleanupChains = []struct { + table utiliptables.Table + chain utiliptables.Chain +}{ + {utiliptables.TableNAT, kubeServicesChain}, + {utiliptables.TableNAT, kubePostroutingChain}, + {utiliptables.TableNAT, kubeNodePortChain}, + {utiliptables.TableNAT, kubeLoadBalancerChain}, + {utiliptables.TableFilter, kubeForwardChain}, + {utiliptables.TableFilter, kubeNodePortChain}, + {utiliptables.TableFilter, kubeProxyFirewallChain}, + {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, + {utiliptables.TableFilter, kubeIPVSFilterChain}, + {utiliptables.TableFilter, kubeIPVSOutFilterChain}, +} + +// ipsetInfo is all ipset we needed in ipvs proxier +var ipsetInfo = []struct { + name string + setType utilipset.Type + comment string +}{ + {kubeLoopBackIPSet, utilipset.HashIPPortIP, kubeLoopBackIPSetComment}, + {kubeClusterIPSet, utilipset.HashIPPort, kubeClusterIPSetComment}, + {kubeExternalIPSet, utilipset.HashIPPort, kubeExternalIPSetComment}, + {kubeExternalIPLocalSet, utilipset.HashIPPort, kubeExternalIPLocalSetComment}, + {kubeLoadBalancerSet, utilipset.HashIPPort, kubeLoadBalancerSetComment}, + {kubeLoadBalancerFWSet, utilipset.HashIPPort, kubeLoadBalancerFWSetComment}, + {kubeLoadBalancerLocalSet, utilipset.HashIPPort, kubeLoadBalancerLocalSetComment}, + {kubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, kubeLoadBalancerSourceIPSetComment}, + {kubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment}, + {kubeNodePortSetTCP, utilipset.BitmapPort, kubeNodePortSetTCPComment}, + {kubeNodePortLocalSetTCP, utilipset.BitmapPort, kubeNodePortLocalSetTCPComment}, + {kubeNodePortSetUDP, utilipset.BitmapPort, kubeNodePortSetUDPComment}, + {kubeNodePortLocalSetUDP, utilipset.BitmapPort, kubeNodePortLocalSetUDPComment}, + {kubeNodePortSetSCTP, utilipset.HashIPPort, kubeNodePortSetSCTPComment}, + {kubeNodePortLocalSetSCTP, utilipset.HashIPPort, kubeNodePortLocalSetSCTPComment}, + {kubeHealthCheckNodePortSet, utilipset.BitmapPort, kubeHealthCheckNodePortSetComment}, + {kubeIPVSSet, utilipset.HashIP, kubeIPVSSetComment}, +} + +// ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to +// `iptables -t nat -A -m set --match-set -j ` +// example: iptables -t nat -A KUBE-SERVICES -m set --match-set KUBE-NODE-PORT-TCP dst -j KUBE-NODE-PORT +// ipsets with other match rules will be created Individually. +// Note: kubeNodePortLocalSetTCP must be prior to kubeNodePortSetTCP, the same for UDP. +var ipsetWithIptablesChain = []struct { + name string + table utiliptables.Table + from string + to string + matchType string + protocolMatch string +}{ + {kubeLoopBackIPSet, utiliptables.TableNAT, string(kubePostroutingChain), "MASQUERADE", "dst,dst,src", ""}, + {kubeLoadBalancerSet, utiliptables.TableNAT, string(kubeServicesChain), string(kubeLoadBalancerChain), "dst,dst", ""}, + {kubeLoadBalancerLocalSet, utiliptables.TableNAT, string(kubeLoadBalancerChain), "RETURN", "dst,dst", ""}, + {kubeNodePortLocalSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolTCP}, + {kubeNodePortSetTCP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolTCP}, + {kubeNodePortLocalSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst", utilipset.ProtocolUDP}, + {kubeNodePortSetUDP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst", utilipset.ProtocolUDP}, + {kubeNodePortLocalSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), "RETURN", "dst,dst", utilipset.ProtocolSCTP}, + {kubeNodePortSetSCTP, utiliptables.TableNAT, string(kubeNodePortChain), string(kubeMarkMasqChain), "dst,dst", utilipset.ProtocolSCTP}, + + {kubeLoadBalancerFWSet, utiliptables.TableFilter, string(kubeProxyFirewallChain), string(kubeSourceRangesFirewallChain), "dst,dst", ""}, + {kubeLoadBalancerSourceCIDRSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""}, + {kubeLoadBalancerSourceIPSet, utiliptables.TableFilter, string(kubeSourceRangesFirewallChain), "RETURN", "dst,dst,src", ""}, +} + // internal struct for string service information type servicePortInfo struct { *proxy.BaseServicePortInfo diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index a823c8e7f08..c8732c545f8 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -104,51 +104,39 @@ const ( masqueradingChain = "masquerading" ) -// internal struct for string service information -type servicePortInfo struct { - *proxy.BaseServicePortInfo - // The following fields are computed and stored for performance reasons. - nameString string - clusterPolicyChainName string - localPolicyChainName string - externalChainName string - firewallChainName string -} - -// returns a new proxy.ServicePort which abstracts a serviceInfo -func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort { - svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo} - - // Store the following for performance reasons. - svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} - svcPort.nameString = svcPortName.String() - - chainNameBase := servicePortChainNameBase(&svcPortName, strings.ToLower(string(svcPort.Protocol()))) - svcPort.clusterPolicyChainName = servicePortPolicyClusterChainNamePrefix + chainNameBase - svcPort.localPolicyChainName = servicePortPolicyLocalChainNamePrefix + chainNameBase - svcPort.externalChainName = serviceExternalChainNamePrefix + chainNameBase - svcPort.firewallChainName = servicePortFirewallChainNamePrefix + chainNameBase - - return svcPort -} - -// internal struct for endpoints information -type endpointInfo struct { - *proxy.BaseEndpointInfo - - chainName string - affinitySetName string -} - -// returns a new proxy.Endpoint which abstracts a endpointInfo -func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint { - chainNameBase := servicePortEndpointChainNameBase(svcPortName, strings.ToLower(string(svcPortName.Protocol)), baseInfo.String()) - return &endpointInfo{ - BaseEndpointInfo: baseInfo, - chainName: servicePortEndpointChainNamePrefix + chainNameBase, - affinitySetName: servicePortEndpointAffinityNamePrefix + chainNameBase, +// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies. +func NewDualStackProxier( + sysctl utilsysctl.Interface, + syncPeriod time.Duration, + minSyncPeriod time.Duration, + masqueradeAll bool, + masqueradeBit int, + localDetectors [2]proxyutiliptables.LocalTrafficDetector, + hostname string, + nodeIPs map[v1.IPFamily]net.IP, + recorder events.EventRecorder, + healthzServer *healthcheck.ProxierHealthServer, + nodePortAddresses []string, + initOnly bool, +) (proxy.Provider, error) { + // Create an ipv4 instance of the single-stack proxier + ipv4Proxier, err := NewProxier(v1.IPv4Protocol, sysctl, + syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[0], hostname, + nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly) + if err != nil { + return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) } + + ipv6Proxier, err := NewProxier(v1.IPv6Protocol, sysctl, + syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[1], hostname, + nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly) + if err != nil { + return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) + } + if initOnly { + return nil, nil + } + return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil } // Proxier is an nftables based proxy @@ -277,39 +265,51 @@ func NewProxier(ipFamily v1.IPFamily, return proxier, nil } -// NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies. -func NewDualStackProxier( - sysctl utilsysctl.Interface, - syncPeriod time.Duration, - minSyncPeriod time.Duration, - masqueradeAll bool, - masqueradeBit int, - localDetectors [2]proxyutiliptables.LocalTrafficDetector, - hostname string, - nodeIPs map[v1.IPFamily]net.IP, - recorder events.EventRecorder, - healthzServer *healthcheck.ProxierHealthServer, - nodePortAddresses []string, - initOnly bool, -) (proxy.Provider, error) { - // Create an ipv4 instance of the single-stack proxier - ipv4Proxier, err := NewProxier(v1.IPv4Protocol, sysctl, - syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[0], hostname, - nodeIPs[v1.IPv4Protocol], recorder, healthzServer, nodePortAddresses, initOnly) - if err != nil { - return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) - } +// internal struct for string service information +type servicePortInfo struct { + *proxy.BaseServicePortInfo + // The following fields are computed and stored for performance reasons. + nameString string + clusterPolicyChainName string + localPolicyChainName string + externalChainName string + firewallChainName string +} - ipv6Proxier, err := NewProxier(v1.IPv6Protocol, sysctl, - syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[1], hostname, - nodeIPs[v1.IPv6Protocol], recorder, healthzServer, nodePortAddresses, initOnly) - if err != nil { - return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) +// returns a new proxy.ServicePort which abstracts a serviceInfo +func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *proxy.BaseServicePortInfo) proxy.ServicePort { + svcPort := &servicePortInfo{BaseServicePortInfo: bsvcPortInfo} + + // Store the following for performance reasons. + svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} + svcPort.nameString = svcPortName.String() + + chainNameBase := servicePortChainNameBase(&svcPortName, strings.ToLower(string(svcPort.Protocol()))) + svcPort.clusterPolicyChainName = servicePortPolicyClusterChainNamePrefix + chainNameBase + svcPort.localPolicyChainName = servicePortPolicyLocalChainNamePrefix + chainNameBase + svcPort.externalChainName = serviceExternalChainNamePrefix + chainNameBase + svcPort.firewallChainName = servicePortFirewallChainNamePrefix + chainNameBase + + return svcPort +} + +// internal struct for endpoints information +type endpointInfo struct { + *proxy.BaseEndpointInfo + + chainName string + affinitySetName string +} + +// returns a new proxy.Endpoint which abstracts a endpointInfo +func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo, svcPortName *proxy.ServicePortName) proxy.Endpoint { + chainNameBase := servicePortEndpointChainNameBase(svcPortName, strings.ToLower(string(svcPortName.Protocol)), baseInfo.String()) + return &endpointInfo{ + BaseEndpointInfo: baseInfo, + chainName: servicePortEndpointChainNamePrefix + chainNameBase, + affinitySetName: servicePortEndpointAffinityNamePrefix + chainNameBase, } - if initOnly { - return nil, nil - } - return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil } // nftablesBaseChains lists our "base chains"; those that are directly connected to the