diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go index 5f2891c5893..6cb4736f7a8 100644 --- a/pkg/proxy/ipvs/ipset.go +++ b/pkg/proxy/ipvs/ipset.go @@ -78,6 +78,9 @@ const ( kubeHealthCheckNodePortSetComment = "Kubernetes health check node port" kubeHealthCheckNodePortSet = "KUBE-HEALTH-CHECK-NODE-PORT" + + kubeIPVSSetComment = "Addresses on the ipvs interface" + kubeIPVSSet = "KUBE-IPVS-IPS" ) // IPSetVersioner can query the current ipset version. diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 1dcb1830dd1..0ce180f4e24 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -84,6 +84,10 @@ const ( // kubeLoadBalancerChain is the kubernetes chain for loadbalancer type service kubeLoadBalancerChain utiliptables.Chain = "KUBE-LOAD-BALANCER" + // kubeIPVSFilterChain filters external access to main netns + // https://github.com/kubernetes/kubernetes/issues/72236 + kubeIPVSFilterChain utiliptables.Chain = "KUBE-IPVS-FILTER" + // defaultScheduler is the default ipvs scheduler algorithm - round robin. defaultScheduler = "rr" @@ -112,6 +116,7 @@ var iptablesJumpChain = []struct { {utiliptables.TableFilter, utiliptables.ChainInput, kubeNodePortChain, "kubernetes health check rules"}, {utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"}, {utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"}, + {utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"}, } var iptablesChains = []struct { @@ -127,6 +132,7 @@ var iptablesChains = []struct { {utiliptables.TableFilter, kubeNodePortChain}, {utiliptables.TableFilter, kubeProxyFirewallChain}, {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, + {utiliptables.TableFilter, kubeIPVSFilterChain}, } var iptablesCleanupChains = []struct { @@ -141,6 +147,7 @@ var iptablesCleanupChains = []struct { {utiliptables.TableFilter, kubeNodePortChain}, {utiliptables.TableFilter, kubeProxyFirewallChain}, {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, + {utiliptables.TableFilter, kubeIPVSFilterChain}, } // ipsetInfo is all ipset we needed in ipvs proxier @@ -165,6 +172,7 @@ var ipsetInfo = []struct { {kubeNodePortSetSCTP, utilipset.HashIPPort, kubeNodePortSetSCTPComment}, {kubeNodePortLocalSetSCTP, utilipset.HashIPPort, kubeNodePortLocalSetSCTPComment}, {kubeHealthCheckNodePortSet, utilipset.BitmapPort, kubeHealthCheckNodePortSetComment}, + {kubeIPVSSet, utilipset.HashIP, kubeIPVSSetComment}, } // ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to @@ -1549,6 +1557,9 @@ func (proxier *Proxier) syncProxyRules() { } } + // Set the KUBE-IPVS-IPS set to the "activeBindAddrs" + proxier.ipsetList[kubeIPVSSet].activeEntries = sets.StringKeySet(activeBindAddrs) + // sync ipset entries for _, set := range proxier.ipsetList { set.syncIPSetEntries() @@ -1792,6 +1803,22 @@ func (proxier *Proxier) writeIptablesRules() { "-j", "ACCEPT", ) + // Add rules to the filter/KUBE-IPVS-FILTER chain to prevent access to ports on the host through VIP addresses. + // https://github.com/kubernetes/kubernetes/issues/72236 + proxier.filterRules.Write( + "-A", string(kubeIPVSFilterChain), + "-m", "set", "--match-set", proxier.ipsetList[kubeLoadBalancerSet].Name, "dst,dst", "-j", "ACCEPT") + proxier.filterRules.Write( + "-A", string(kubeIPVSFilterChain), + "-m", "set", "--match-set", proxier.ipsetList[kubeClusterIPSet].Name, "dst,dst", "-j", "ACCEPT") + proxier.filterRules.Write( + "-A", string(kubeIPVSFilterChain), + "-m", "set", "--match-set", proxier.ipsetList[kubeExternalIPSet].Name, "dst,dst", "-j", "ACCEPT") + proxier.filterRules.Write( + "-A", string(kubeIPVSFilterChain), + "-m", "conntrack", "--ctstate", "NEW", + "-m", "set", "--match-set", proxier.ipsetList[kubeIPVSSet].Name, "dst", "-j", "REJECT") + // Install the kubernetes-specific postrouting rules. We use a whole chain for // this so that it is easier to flush and change, for example if the mark // value should ever change. diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 13584429212..30f87d39432 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -4772,6 +4772,7 @@ func TestCreateAndLinkKubeChain(t *testing.T) { :KUBE-NODE-PORT - [0:0] :KUBE-PROXY-FIREWALL - [0:0] :KUBE-SOURCE-RANGES-FIREWALL - [0:0] +:KUBE-IPVS-FILTER - [0:0] ` assert.Equal(t, expectedNATChains, string(fp.natChains.Bytes())) assert.Equal(t, expectedFilterChains, string(fp.filterChains.Bytes())) diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go index 05fa90be5db..c82fe0c310f 100644 --- a/pkg/util/ipset/ipset.go +++ b/pkg/util/ipset/ipset.go @@ -169,6 +169,11 @@ func (e *Entry) Validate(set *IPSet) bool { return false } switch e.SetType { + case HashIP: + //check if IP of Entry is valid. + if valid := e.checkIP(set); !valid { + return false + } case HashIPPort: //check if IP and Protocol of Entry is valid. if valid := e.checkIPandProtocol(set); !valid { @@ -219,6 +224,9 @@ func (e *Entry) Validate(set *IPSet) bool { // String returns the string format for ipset entry. func (e *Entry) String() string { switch e.SetType { + case HashIP: + // Entry{192.168.1.1} -> 192.168.1.1 + return fmt.Sprintf("%s", e.IP) case HashIPPort: // Entry{192.168.1.1, udp, 53} -> 192.168.1.1,udp:53 // Entry{192.168.1.2, tcp, 8080} -> 192.168.1.2,tcp:8080 @@ -247,7 +255,11 @@ func (e *Entry) checkIPandProtocol(set *IPSet) bool { } else if !validateProtocol(e.Protocol) { return false } + return e.checkIP(set) +} +// checkIP checks if IP of Entry is valid. +func (e *Entry) checkIP(set *IPSet) bool { if netutils.ParseIPSloppy(e.IP) == nil { klog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set) return false @@ -283,7 +295,7 @@ func (runner *runner) CreateSet(set *IPSet, ignoreExistErr bool) error { // otherwise raised when the same set (setname and create parameters are identical) already exists. func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error { args := []string{"create", set.Name, string(set.SetType)} - if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet { + if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet || set.SetType == HashIP { args = append(args, "family", set.HashFamily, "hashsize", strconv.Itoa(set.HashSize), diff --git a/pkg/util/ipset/types.go b/pkg/util/ipset/types.go index 0c2d16daacc..11f98d712c4 100644 --- a/pkg/util/ipset/types.go +++ b/pkg/util/ipset/types.go @@ -35,6 +35,8 @@ const ( // BitmapPort represents the `bitmap:port` type ipset. The bitmap:port set type uses a memory range, where each bit // represents one TCP/UDP port. A bitmap:port type of set can store up to 65535 ports. BitmapPort Type = "bitmap:port" + // HashIP represents the `hash:ip` type ipset. + HashIP Type = "hash:ip" ) // DefaultPortRange defines the default bitmap:port valid port range. @@ -59,4 +61,5 @@ var ValidIPSetTypes = []Type{ HashIPPortIP, BitmapPort, HashIPPortNet, + HashIP, }