From 8a5801996b801126a203e23b30b35c2b616909f1 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 9 Jul 2022 06:46:48 -0400 Subject: [PATCH 1/6] proxy/iptables: belatedly simplify local traffic policy metrics We figure out early on whether we're going to end up outputting no endpoints, so update the metrics then. (Also remove a redundant feature gate check; svcInfo already checks the ServiceInternalTrafficPolicy feature gate itself and so svcInfo.InternalPolicyLocal() will always return false if the gate is not enabled.) --- pkg/proxy/iptables/proxier.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 6b759ece813..aea8d91c94d 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -38,11 +38,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/events" utilsysctl "k8s.io/component-helpers/node/util/sysctl" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/healthcheck" "k8s.io/kubernetes/pkg/proxy/metaproxier" @@ -1082,6 +1080,7 @@ func (proxier *Proxier) syncProxyRules() { // external traffic may still be accepted. internalTrafficFilterTarget = "DROP" internalTrafficFilterComment = fmt.Sprintf(`"%s has no local endpoints"`, svcPortNameString) + serviceNoLocalEndpointsTotalInternal++ } if !hasExternalEndpoints { // The externalTrafficPolicy is "Local" but there are no @@ -1090,6 +1089,7 @@ func (proxier *Proxier) syncProxyRules() { // the cluster may still be accepted. externalTrafficFilterTarget = "DROP" externalTrafficFilterComment = fmt.Sprintf(`"%s has no local endpoints"`, svcPortNameString) + serviceNoLocalEndpointsTotalExternal++ } } @@ -1367,17 +1367,8 @@ func (proxier *Proxier) syncProxyRules() { } if svcInfo.UsesLocalEndpoints() { - if len(localEndpoints) != 0 { - // Write rules jumping from localPolicyChain to localEndpointChains - proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, localPolicyChain, localEndpoints, args) - } else if hasEndpoints { - if svcInfo.InternalPolicyLocal() && utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) { - serviceNoLocalEndpointsTotalInternal++ - } - if svcInfo.ExternalPolicyLocal() { - serviceNoLocalEndpointsTotalExternal++ - } - } + // Write rules jumping from localPolicyChain to localEndpointChains + proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, localPolicyChain, localEndpoints, args) } } From da14a12fe5e196e501c842e7b9e3d22040e822a9 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 6 Apr 2022 08:34:08 -0400 Subject: [PATCH 2/6] proxy/iptables: move endpoint chain rule creation to the end Part of reorganizing the syncProxyRules loop to do: 1. figure out what chains are needed, mark them in activeNATChains 2. write servicePort jump rules to KUBE-SERVICES/KUBE-NODEPORTS 3. write servicePort-specific chains (SVC, SVL, EXT, FW, SEP) This fixes the handling of the endpoint chains. Previously they were handled entirely at the top of the loop. Now we record which ones are in use at the top but don't create them and fill them in until the bottom. --- pkg/proxy/iptables/proxier.go | 62 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index aea8d91c94d..b619ec7ea18 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -984,42 +984,18 @@ func (proxier *Proxier) syncProxyRules() { protocol := strings.ToLower(string(svcInfo.Protocol())) svcPortNameString := svcInfo.nameString - allEndpoints := proxier.endpointsMap[svcName] - // Figure out the endpoints for Cluster and Local traffic policy. // allLocallyReachableEndpoints is the set of all endpoints that can be routed to // from this node, given the service's traffic policies. hasEndpoints is true // if the service has any usable endpoints on any node, not just this one. + allEndpoints := proxier.endpointsMap[svcName] clusterEndpoints, localEndpoints, allLocallyReachableEndpoints, hasEndpoints := proxy.CategorizeEndpoints(allEndpoints, svcInfo, proxier.nodeLabels) - // Generate the per-endpoint chains. + // Note the endpoint chains that will be used for _, ep := range allLocallyReachableEndpoints { - epInfo, ok := ep.(*endpointsInfo) - if !ok { - klog.ErrorS(err, "Failed to cast endpointsInfo", "endpointsInfo", ep) - continue + if epInfo, ok := ep.(*endpointsInfo); ok { + activeNATChains[epInfo.ChainName] = true } - - endpointChain := epInfo.ChainName - - // Create the endpoint chain - proxier.natChains.Write(utiliptables.MakeChainLine(endpointChain)) - activeNATChains[endpointChain] = true - - args = append(args[:0], "-A", string(endpointChain)) - args = proxier.appendServiceCommentLocked(args, svcPortNameString) - // Handle traffic that loops back to the originator with SNAT. - proxier.natRules.Write( - args, - "-s", epInfo.IP(), - "-j", string(kubeMarkMasqChain)) - // Update client-affinity lists. - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - args = append(args, "-m", "recent", "--name", string(endpointChain), "--set") - } - // DNAT to final destination. - args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", epInfo.Endpoint) - proxier.natRules.Write(args) } // These chains represent the sets of endpoints to use when internal or @@ -1370,6 +1346,36 @@ func (proxier *Proxier) syncProxyRules() { // Write rules jumping from localPolicyChain to localEndpointChains proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, localPolicyChain, localEndpoints, args) } + + // Generate the per-endpoint chains. + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointsInfo) + if !ok { + klog.ErrorS(err, "Failed to cast endpointsInfo", "endpointsInfo", ep) + continue + } + + endpointChain := epInfo.ChainName + + // Create the endpoint chain + proxier.natChains.Write(utiliptables.MakeChainLine(endpointChain)) + activeNATChains[endpointChain] = true + + args = append(args[:0], "-A", string(endpointChain)) + args = proxier.appendServiceCommentLocked(args, svcPortNameString) + // Handle traffic that loops back to the originator with SNAT. + proxier.natRules.Write( + args, + "-s", epInfo.IP(), + "-j", string(kubeMarkMasqChain)) + // Update client-affinity lists. + if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { + args = append(args, "-m", "recent", "--name", string(endpointChain), "--set") + } + // DNAT to final destination. + args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", epInfo.Endpoint) + proxier.natRules.Write(args) + } } // Delete chains no longer in use. Since "iptables-save" can take several seconds From 8906ab390eaed6c0ec46855ef379954e55b4084d Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 6 Apr 2022 10:52:22 -0400 Subject: [PATCH 3/6] proxy/iptables: reorganize cluster/local chain creation Part of reorganizing the syncProxyRules loop to do: 1. figure out what chains are needed, mark them in activeNATChains 2. write servicePort jump rules to KUBE-SERVICES/KUBE-NODEPORTS 3. write servicePort-specific chains (SVC, SVL, EXT, FW, SEP) This fixes the handling of the SVC and SVL chains. We were already filling them in at the end of the loop; this fixes it to create them at the bottom of the loop as well. --- pkg/proxy/iptables/proxier.go | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index b619ec7ea18..b7dcb50f6bc 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -998,10 +998,19 @@ func (proxier *Proxier) syncProxyRules() { } } - // These chains represent the sets of endpoints to use when internal or - // external traffic policy is "Cluster" vs "Local". + // clusterPolicyChain contains the endpoints used with "Cluster" traffic policy clusterPolicyChain := svcInfo.clusterPolicyChainName + usesClusterPolicyChain := len(clusterEndpoints) > 0 && svcInfo.UsesClusterEndpoints() + if usesClusterPolicyChain { + activeNATChains[clusterPolicyChain] = true + } + + // localPolicyChain contains the endpoints used with "Local" traffic policy localPolicyChain := svcInfo.localPolicyChainName + usesLocalPolicyChain := len(localEndpoints) > 0 && svcInfo.UsesLocalEndpoints() + if usesLocalPolicyChain { + activeNATChains[localPolicyChain] = true + } // internalPolicyChain is the chain containing the endpoints for // "internal" (ClusterIP) traffic. internalTrafficChain is the chain that @@ -1069,19 +1078,6 @@ func (proxier *Proxier) syncProxyRules() { } } - // Declare the clusterPolicyChain if needed. - if len(clusterEndpoints) > 0 && svcInfo.UsesClusterEndpoints() { - // Create the Cluster traffic policy chain - proxier.natChains.Write(utiliptables.MakeChainLine(clusterPolicyChain)) - activeNATChains[clusterPolicyChain] = true - } - - // Declare the localPolicyChain if needed. - if len(localEndpoints) > 0 && svcInfo.UsesLocalEndpoints() { - proxier.natChains.Write(utiliptables.MakeChainLine(localPolicyChain)) - activeNATChains[localPolicyChain] = true - } - // If any "external" destinations are enabled, set up external traffic // handling. All captured traffic for all external destinations should // jump to externalTrafficChain, which will handle some special-cases @@ -1337,13 +1333,17 @@ func (proxier *Proxier) syncProxyRules() { ) } - if svcInfo.UsesClusterEndpoints() { - // Write rules jumping from clusterPolicyChain to clusterEndpoints + // If Cluster policy is in use, create the chain and create rules jumping + // from clusterPolicyChain to the clusterEndpoints + if usesClusterPolicyChain { + proxier.natChains.Write(utiliptables.MakeChainLine(clusterPolicyChain)) proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, clusterPolicyChain, clusterEndpoints, args) } - if svcInfo.UsesLocalEndpoints() { - // Write rules jumping from localPolicyChain to localEndpointChains + // If Local policy is in use, create the chain and create rules jumping + // from localPolicyChain to the localEndpoints + if usesLocalPolicyChain { + proxier.natChains.Write(utiliptables.MakeChainLine(localPolicyChain)) proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, localPolicyChain, localEndpoints, args) } From 00f789cd8df8ebace902cf4a348a352d79d8ad91 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 6 Apr 2022 10:05:00 -0400 Subject: [PATCH 4/6] proxy/iptables: move EXT chain rule creation to the end Part of reorganizing the syncProxyRules loop to do: 1. figure out what chains are needed, mark them in activeNATChains 2. write servicePort jump rules to KUBE-SERVICES/KUBE-NODEPORTS 3. write servicePort-specific chains (SVC, SVL, EXT, FW, SEP) This fixes the handling of the EXT chain. --- pkg/proxy/iptables/proxier.go | 125 ++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index b7dcb50f6bc..54d3dea14be 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1046,6 +1046,14 @@ func (proxier *Proxier) syncProxyRules() { } externalTrafficChain := svcInfo.externalChainName // eventually jumps to externalPolicyChain + // usesExternalTrafficChain is based on hasEndpoints, not hasExternalEndpoints, + // because we need the local-traffic-short-circuiting rules even when there + // are no externally-usable endpoints. + usesExternalTrafficChain := hasEndpoints && svcInfo.ExternallyAccessible() + if usesExternalTrafficChain { + activeNATChains[externalTrafficChain] = true + } + var internalTrafficFilterTarget, internalTrafficFilterComment string var externalTrafficFilterTarget, externalTrafficFilterComment string if !hasEndpoints { @@ -1078,66 +1086,6 @@ func (proxier *Proxier) syncProxyRules() { } } - // If any "external" destinations are enabled, set up external traffic - // handling. All captured traffic for all external destinations should - // jump to externalTrafficChain, which will handle some special-cases - // and then jump to externalPolicyChain. (We check hasEndpoints here not - // hasExternalEndpoints because we need the short-circuit rules in the - // EXT chain even when there are no external endpoints.) - if hasEndpoints && svcInfo.ExternallyAccessible() { - proxier.natChains.Write(utiliptables.MakeChainLine(externalTrafficChain)) - activeNATChains[externalTrafficChain] = true - - if !svcInfo.ExternalPolicyLocal() { - // If we are using non-local endpoints we need to masquerade, - // in case we cross nodes. - proxier.natRules.Write( - "-A", string(externalTrafficChain), - "-m", "comment", "--comment", fmt.Sprintf(`"masquerade traffic for %s external destinations"`, svcPortNameString), - "-j", string(kubeMarkMasqChain)) - } else { - // If we are only using same-node endpoints, we can retain the - // source IP in most cases. - - if proxier.localDetector.IsImplemented() { - // Treat all locally-originated pod -> external destination - // traffic as a special-case. It is subject to neither - // form of traffic policy, which simulates going up-and-out - // to an external load-balancer and coming back in. - proxier.natRules.Write( - "-A", string(externalTrafficChain), - "-m", "comment", "--comment", fmt.Sprintf(`"pod traffic for %s external destinations"`, svcPortNameString), - proxier.localDetector.IfLocal(), - "-j", string(clusterPolicyChain)) - } - - // Locally originated traffic (not a pod, but the host node) - // still needs masquerade because the LBIP itself is a local - // address, so that will be the chosen source IP. - proxier.natRules.Write( - "-A", string(externalTrafficChain), - "-m", "comment", "--comment", fmt.Sprintf(`"masquerade LOCAL traffic for %s external destinations"`, svcPortNameString), - "-m", "addrtype", "--src-type", "LOCAL", - "-j", string(kubeMarkMasqChain)) - - // Redirect all src-type=LOCAL -> external destination to the - // policy=cluster chain. This allows traffic originating - // from the host to be redirected to the service correctly. - proxier.natRules.Write( - "-A", string(externalTrafficChain), - "-m", "comment", "--comment", fmt.Sprintf(`"route LOCAL traffic for %s external destinations"`, svcPortNameString), - "-m", "addrtype", "--src-type", "LOCAL", - "-j", string(clusterPolicyChain)) - } - - // Anything else falls thru to the appropriate policy chain. - if hasExternalEndpoints { - proxier.natRules.Write( - "-A", string(externalTrafficChain), - "-j", string(externalPolicyChain)) - } - } - // Capture the clusterIP. if hasInternalEndpoints { args = append(args[:0], @@ -1333,6 +1281,63 @@ func (proxier *Proxier) syncProxyRules() { ) } + // Set up external traffic handling (if any "external" destinations are + // enabled). All captured traffic for all external destinations should + // jump to externalTrafficChain, which will handle some special cases and + // then jump to externalPolicyChain. + if usesExternalTrafficChain { + proxier.natChains.Write(utiliptables.MakeChainLine(externalTrafficChain)) + + if !svcInfo.ExternalPolicyLocal() { + // If we are using non-local endpoints we need to masquerade, + // in case we cross nodes. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"masquerade traffic for %s external destinations"`, svcPortNameString), + "-j", string(kubeMarkMasqChain)) + } else { + // If we are only using same-node endpoints, we can retain the + // source IP in most cases. + + if proxier.localDetector.IsImplemented() { + // Treat all locally-originated pod -> external destination + // traffic as a special-case. It is subject to neither + // form of traffic policy, which simulates going up-and-out + // to an external load-balancer and coming back in. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"pod traffic for %s external destinations"`, svcPortNameString), + proxier.localDetector.IfLocal(), + "-j", string(clusterPolicyChain)) + } + + // Locally originated traffic (not a pod, but the host node) + // still needs masquerade because the LBIP itself is a local + // address, so that will be the chosen source IP. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"masquerade LOCAL traffic for %s external destinations"`, svcPortNameString), + "-m", "addrtype", "--src-type", "LOCAL", + "-j", string(kubeMarkMasqChain)) + + // Redirect all src-type=LOCAL -> external destination to the + // policy=cluster chain. This allows traffic originating + // from the host to be redirected to the service correctly. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"route LOCAL traffic for %s external destinations"`, svcPortNameString), + "-m", "addrtype", "--src-type", "LOCAL", + "-j", string(clusterPolicyChain)) + } + + // Anything else falls thru to the appropriate policy chain. + if hasExternalEndpoints { + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-j", string(externalPolicyChain)) + } + } + // If Cluster policy is in use, create the chain and create rules jumping // from clusterPolicyChain to the clusterEndpoints if usesClusterPolicyChain { From 2030591ce7b2b4e71b01274fb8a83af4cad366a6 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 6 Apr 2022 10:10:12 -0400 Subject: [PATCH 5/6] proxy/iptables: move internal traffic setup code Part of reorganizing the syncProxyRules loop to do: 1. figure out what chains are needed, mark them in activeNATChains 2. write servicePort jump rules to KUBE-SERVICES/KUBE-NODEPORTS 3. write servicePort-specific chains (SVC, SVL, EXT, FW, SEP) This fixes the jump rules for internal traffic. Previously we were handling "jumping from kubeServices to internalTrafficChain" and "adding masquerade rules to internalTrafficChain" in the same place. --- pkg/proxy/iptables/proxier.go | 50 ++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 54d3dea14be..416ab5cab2f 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1088,31 +1088,12 @@ func (proxier *Proxier) syncProxyRules() { // Capture the clusterIP. if hasInternalEndpoints { - args = append(args[:0], + proxier.natRules.Write( + "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcPortNameString), "-m", protocol, "-p", protocol, "-d", svcInfo.ClusterIP().String(), "--dport", strconv.Itoa(svcInfo.Port()), - ) - if proxier.masqueradeAll { - proxier.natRules.Write( - "-A", string(internalTrafficChain), - args, - "-j", string(kubeMarkMasqChain)) - } else if proxier.localDetector.IsImplemented() { - // This masquerades off-cluster traffic to a service VIP. The idea - // is that you can establish a static route for your Service range, - // routing to any node, and that node will bridge into the Service - // for you. Since that might bounce off-node, we masquerade here. - proxier.natRules.Write( - "-A", string(internalTrafficChain), - args, - proxier.localDetector.IfNotLocal(), - "-j", string(kubeMarkMasqChain)) - } - proxier.natRules.Write( - "-A", string(kubeServicesChain), - args, "-j", string(internalTrafficChain)) } else { // No endpoints. @@ -1281,6 +1262,33 @@ func (proxier *Proxier) syncProxyRules() { ) } + // Set up internal traffic handling. + if hasInternalEndpoints { + args = append(args[:0], + "-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcPortNameString), + "-m", protocol, "-p", protocol, + "-d", svcInfo.ClusterIP().String(), + "--dport", strconv.Itoa(svcInfo.Port()), + ) + if proxier.masqueradeAll { + proxier.natRules.Write( + "-A", string(internalTrafficChain), + args, + "-j", string(kubeMarkMasqChain)) + } else if proxier.localDetector.IsImplemented() { + // This masquerades off-cluster traffic to a service VIP. The + // idea is that you can establish a static route for your + // Service range, routing to any node, and that node will + // bridge into the Service for you. Since that might bounce + // off-node, we masquerade here. + proxier.natRules.Write( + "-A", string(internalTrafficChain), + args, + proxier.localDetector.IfNotLocal(), + "-j", string(kubeMarkMasqChain)) + } + } + // Set up external traffic handling (if any "external" destinations are // enabled). All captured traffic for all external destinations should // jump to externalTrafficChain, which will handle some special cases and From 367f18c49b9e91b91f839511a6872849d413ba54 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 6 Apr 2022 10:17:36 -0400 Subject: [PATCH 6/6] proxy/iptables: move firewall chain setup Part of reorganizing the syncProxyRules loop to do: 1. figure out what chains are needed, mark them in activeNATChains 2. write servicePort jump rules to KUBE-SERVICES/KUBE-NODEPORTS 3. write servicePort-specific chains (SVC, SVL, EXT, FW, SEP) This moves the FW chain creation to the end (rather than having it in the middle of adding the jump rules for the LB IPs). --- pkg/proxy/iptables/proxier.go | 118 +++++++++++++++++----------------- 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 416ab5cab2f..f007525024d 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1054,6 +1054,17 @@ func (proxier *Proxier) syncProxyRules() { activeNATChains[externalTrafficChain] = true } + // Traffic to LoadBalancer IPs can go directly to externalTrafficChain + // unless LoadBalancerSourceRanges is in use in which case we will + // create a firewall chain. + loadBalancerTrafficChain := externalTrafficChain + fwChain := svcInfo.firewallChainName + usesFWChain := hasEndpoints && len(svcInfo.LoadBalancerIPStrings()) > 0 && len(svcInfo.LoadBalancerSourceRanges()) > 0 + if usesFWChain { + activeNATChains[fwChain] = true + loadBalancerTrafficChain = fwChain + } + var internalTrafficFilterTarget, internalTrafficFilterComment string var externalTrafficFilterTarget, externalTrafficFilterComment string if !hasEndpoints { @@ -1136,72 +1147,15 @@ func (proxier *Proxier) syncProxyRules() { } // Capture load-balancer ingress. - if len(svcInfo.LoadBalancerIPStrings()) > 0 && hasEndpoints { - // Normally we send LB matches to the "external destination" chain. - nextChain := externalTrafficChain - - // If the service specifies any LB source ranges, we need to insert - // a firewall chain first. - if len(svcInfo.LoadBalancerSourceRanges()) > 0 { - fwChain := svcInfo.firewallChainName - - // Declare the service firewall chain. - proxier.natChains.Write(utiliptables.MakeChainLine(fwChain)) - activeNATChains[fwChain] = true - - // The firewall chain will jump to the "external destination" - // chain. - nextChain = svcInfo.firewallChainName - - // The service firewall rules are created based on the - // loadBalancerSourceRanges field. This only works for - // VIP-like loadbalancers that preserve source IPs. For - // loadbalancers which direct traffic to service NodePort, the - // firewall rules will not apply. - args = append(args[:0], - "-A", string(nextChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcPortNameString), - ) - - // firewall filter based on each source range - allowFromNode := false - for _, src := range svcInfo.LoadBalancerSourceRanges() { - proxier.natRules.Write(args, "-s", src, "-j", string(externalTrafficChain)) - _, cidr, err := netutils.ParseCIDRSloppy(src) - if err != nil { - klog.ErrorS(err, "Error parsing CIDR in LoadBalancerSourceRanges, dropping it", "cidr", cidr) - } else if cidr.Contains(proxier.nodeIP) { - allowFromNode = true - } - } - // For VIP-like LBs, the VIP is often added as a local - // address (via an IP route rule). In that case, a request - // from a node to the VIP will not hit the loadbalancer but - // will loop back with the source IP set to the VIP. We - // need the following rules to allow requests from this node. - if allowFromNode { - for _, lbip := range svcInfo.LoadBalancerIPStrings() { - proxier.natRules.Write( - args, - "-s", lbip, - "-j", string(externalTrafficChain)) - } - } - - // If the packet was able to reach the end of firewall chain, - // then it did not get DNATed. It means the packet cannot go - // thru the firewall, then mark it for DROP. - proxier.natRules.Write(args, "-j", string(kubeMarkDropChain)) - } - - for _, lbip := range svcInfo.LoadBalancerIPStrings() { + for _, lbip := range svcInfo.LoadBalancerIPStrings() { + if hasEndpoints { proxier.natRules.Write( "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcPortNameString), "-m", protocol, "-p", protocol, "-d", lbip, "--dport", strconv.Itoa(svcInfo.Port()), - "-j", string(nextChain)) + "-j", string(loadBalancerTrafficChain)) } } @@ -1346,6 +1300,50 @@ func (proxier *Proxier) syncProxyRules() { } } + // Set up firewall chain, if needed + if usesFWChain { + proxier.natChains.Write(utiliptables.MakeChainLine(fwChain)) + + // The service firewall rules are created based on the + // loadBalancerSourceRanges field. This only works for VIP-like + // loadbalancers that preserve source IPs. For loadbalancers which + // direct traffic to service NodePort, the firewall rules will not + // apply. + args = append(args[:0], + "-A", string(fwChain), + "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcPortNameString), + ) + + // firewall filter based on each source range + allowFromNode := false + for _, src := range svcInfo.LoadBalancerSourceRanges() { + proxier.natRules.Write(args, "-s", src, "-j", string(externalTrafficChain)) + _, cidr, err := netutils.ParseCIDRSloppy(src) + if err != nil { + klog.ErrorS(err, "Error parsing CIDR in LoadBalancerSourceRanges, dropping it", "cidr", cidr) + } else if cidr.Contains(proxier.nodeIP) { + allowFromNode = true + } + } + // For VIP-like LBs, the VIP is often added as a local + // address (via an IP route rule). In that case, a request + // from a node to the VIP will not hit the loadbalancer but + // will loop back with the source IP set to the VIP. We + // need the following rules to allow requests from this node. + if allowFromNode { + for _, lbip := range svcInfo.LoadBalancerIPStrings() { + proxier.natRules.Write( + args, + "-s", lbip, + "-j", string(externalTrafficChain)) + } + } + // If the packet was able to reach the end of firewall chain, + // then it did not get DNATed. It means the packet cannot go + // thru the firewall, then mark it for DROP. + proxier.natRules.Write(args, "-j", string(kubeMarkDropChain)) + } + // If Cluster policy is in use, create the chain and create rules jumping // from clusterPolicyChain to the clusterEndpoints if usesClusterPolicyChain {