From 62683c8d9595f6791549cd887334b5e75f414175 Mon Sep 17 00:00:00 2001 From: Daman Arora Date: Mon, 7 Aug 2023 13:40:15 +0530 Subject: [PATCH] pkg/proxy/ipvs: hardening lb source ranges filtering Signed-off-by: Daman Arora --- pkg/proxy/ipvs/proxier.go | 30 ++++++++++++++++++++++++++++++ pkg/proxy/ipvs/proxier_test.go | 1 + 2 files changed, 31 insertions(+) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 19c354e9ac1..f6641ec4f6b 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -84,6 +84,10 @@ const ( // https://github.com/kubernetes/kubernetes/issues/72236 kubeIPVSFilterChain utiliptables.Chain = "KUBE-IPVS-FILTER" + // kubeIPVSOutFilterChain filters access to load balancer services from node. + // https://github.com/kubernetes/kubernetes/issues/119656 + kubeIPVSOutFilterChain utiliptables.Chain = "KUBE-IPVS-OUT-FILTER" + // defaultScheduler is the default ipvs scheduler algorithm - round robin. defaultScheduler = "rr" @@ -113,6 +117,7 @@ var iptablesJumpChain = []struct { {utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"}, {utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"}, {utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"}, + {utiliptables.TableFilter, utiliptables.ChainOutput, kubeIPVSOutFilterChain, "kubernetes ipvs access filter"}, } var iptablesChains = []struct { @@ -129,6 +134,7 @@ var iptablesChains = []struct { {utiliptables.TableFilter, kubeProxyFirewallChain}, {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, {utiliptables.TableFilter, kubeIPVSFilterChain}, + {utiliptables.TableFilter, kubeIPVSOutFilterChain}, } var iptablesCleanupChains = []struct { @@ -144,6 +150,7 @@ var iptablesCleanupChains = []struct { {utiliptables.TableFilter, kubeProxyFirewallChain}, {utiliptables.TableFilter, kubeSourceRangesFirewallChain}, {utiliptables.TableFilter, kubeIPVSFilterChain}, + {utiliptables.TableFilter, kubeIPVSOutFilterChain}, } // ipsetInfo is all ipset we needed in ipvs proxier @@ -296,6 +303,11 @@ type Proxier struct { // A Set is used here since we end up calculating endpoint topology multiple times for the same Service // if it has multiple ports but each Service should only be counted once. serviceNoLocalEndpointsExternal sets.Set[string] + // lbNoNodeAccessIPPortProtocolEntries represents the set of loadBalancers IP + Port + Protocol that should not be accessible from K8s nodes + // We cannot directly restrict LB access from node using LoadBalancerSourceRanges, we need to install + // additional iptables rules. + // (ref: https://github.com/kubernetes/kubernetes/issues/119656) + lbNoNodeAccessIPPortProtocolEntries []*utilipset.Entry } // Proxier implements proxy.Provider @@ -949,6 +961,9 @@ func (proxier *Proxier) syncProxyRules() { proxier.serviceNoLocalEndpointsInternal = sets.New[string]() proxier.serviceNoLocalEndpointsExternal = sets.New[string]() + + proxier.lbNoNodeAccessIPPortProtocolEntries = make([]*utilipset.Entry, 0) + // Begin install iptables // Reset all buffers used later. @@ -1246,6 +1261,10 @@ func (proxier *Proxier) syncProxyRules() { continue } proxier.ipsetList[kubeLoadBalancerSourceIPSet].activeEntries.Insert(entry.String()) + } else { + // since nodeIP is not covered in any of SourceRange we need to explicitly block the lbIP access from k8s nodes. + proxier.lbNoNodeAccessIPPortProtocolEntries = append(proxier.lbNoNodeAccessIPPortProtocolEntries, entry) + } } // ipvs call @@ -1637,6 +1656,17 @@ func (proxier *Proxier) writeIptablesRules() { "-j", "DROP", ) + // disable LB access from node + // for IPVS src and dst both would be lbIP + for _, entry := range proxier.lbNoNodeAccessIPPortProtocolEntries { + proxier.filterRules.Write( + "-A", string(kubeIPVSOutFilterChain), + "-s", entry.IP, + "-m", "ipvs", "--vaddr", entry.IP, "--vproto", entry.Protocol, "--vport", strconv.Itoa(entry.Port), + "-j", "DROP", + ) + } + // Accept all traffic with destination of ipvs virtual service, in case other iptables rules // block the traffic, that may result in ipvs rules invalid. // Those rules must be in the end of KUBE-SERVICE chain diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index d536647945d..086c5652aa3 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -4662,6 +4662,7 @@ func TestCreateAndLinkKubeChain(t *testing.T) { :KUBE-PROXY-FIREWALL - [0:0] :KUBE-SOURCE-RANGES-FIREWALL - [0:0] :KUBE-IPVS-FILTER - [0:0] +:KUBE-IPVS-OUT-FILTER - [0:0] ` assert.Equal(t, expectedNATChains, fp.natChains.String()) assert.Equal(t, expectedFilterChains, fp.filterChains.String())