From 5c16940508bed1f1c1501c9ac7844e30bde94525 Mon Sep 17 00:00:00 2001 From: Cezar Sa Espinola Date: Wed, 26 Jun 2019 16:24:13 -0300 Subject: [PATCH 1/2] proxy/ipvs: Only compute node ip addresses once per sync Previously the same ip addresses would be computed for each nodePort service and this could be CPU intensive for a large number of nodePort services with a large number of ipaddresses on the node. --- pkg/proxy/ipvs/proxier.go | 61 ++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 64aa1ddee84..a64408bb2cd 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -30,7 +30,7 @@ import ( "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -808,6 +808,45 @@ func (proxier *Proxier) syncProxyRules() { // activeBindAddrs represents ip address successfully bind to DefaultDummyDevice in this round of sync activeBindAddrs := map[string]bool{} + hasNodePort := false + for _, svc := range proxier.serviceMap { + svcInfo, ok := svc.(*serviceInfo) + if ok && svcInfo.NodePort() != 0 { + hasNodePort = true + break + } + } + + // Both nodeAddresses and nodeIPs can be reused for all nodePort services + // and only need to be computed if we have at least one nodePort service. + var ( + // List of node addresses to listen on if a nodePort is set. + nodeAddresses []string + // List of node IP addresses to be used as IPVS services if nodePort is set. + nodeIPs []net.IP + ) + + if hasNodePort { + nodeAddrSet, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) + if err != nil { + klog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) + } + if err == nil && nodeAddrSet.Len() > 0 { + nodeAddresses = nodeAddrSet.List() + for _, address := range nodeAddresses { + if !utilproxy.IsZeroCIDR(address) { + nodeIPs = append(nodeIPs, net.ParseIP(address)) + continue + } + // zero cidr + nodeIPs, err = proxier.ipGetter.NodeIPs() + if err != nil { + klog.Errorf("Failed to list all node IPs from host, err: %v", err) + } + } + } + } + // Build IPVS rules for each service. for svcName, svc := range proxier.serviceMap { svcInfo, ok := svc.(*serviceInfo) @@ -1064,14 +1103,14 @@ func (proxier *Proxier) syncProxyRules() { } if svcInfo.NodePort() != 0 { - addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) - if err != nil { - klog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) + if len(nodeAddresses) == 0 || len(nodeIPs) == 0 { + // Skip nodePort configuration since an error occurred when + // computing nodeAddresses or nodeIPs. continue } var lps []utilproxy.LocalPort - for address := range addresses { + for _, address := range nodeAddresses { lp := utilproxy.LocalPort{ Description: "nodePort for " + svcNameString, IP: address, @@ -1174,18 +1213,6 @@ func (proxier *Proxier) syncProxyRules() { } // Build ipvs kernel routes for each node ip address - var nodeIPs []net.IP - for address := range addresses { - if !utilproxy.IsZeroCIDR(address) { - nodeIPs = append(nodeIPs, net.ParseIP(address)) - continue - } - // zero cidr - nodeIPs, err = proxier.ipGetter.NodeIPs() - if err != nil { - klog.Errorf("Failed to list all node IPs from host, err: %v", err) - } - } for _, nodeIP := range nodeIPs { // ipvs call serv := &utilipvs.VirtualServer{ From c25763e159a4560b6a7b794c109255eebbf6385d Mon Sep 17 00:00:00 2001 From: Cezar Sa Espinola Date: Tue, 23 Jul 2019 10:55:38 -0300 Subject: [PATCH 2/2] proxy/ipvs: Compute all node ips only once when a zero cidr is used Computing all node ips twice would always happen when no node port addresses were explicitly set. The GetNodeAddresses call would return two zero cidrs (ipv4 and ipv6) and we would then retrieve all node IPs twice because the loop wouldn't break after the first time. Also, it is possible for the user to set explicit node port addresses including both a zero and a non-zero cidr, but this wouldn't make sense for nodeIPs since the zero cidr would already cause nodeIPs to include all IPs on the node. --- pkg/proxy/ipvs/proxier.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index a64408bb2cd..6e665586b56 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -834,15 +834,14 @@ func (proxier *Proxier) syncProxyRules() { if err == nil && nodeAddrSet.Len() > 0 { nodeAddresses = nodeAddrSet.List() for _, address := range nodeAddresses { - if !utilproxy.IsZeroCIDR(address) { - nodeIPs = append(nodeIPs, net.ParseIP(address)) - continue - } - // zero cidr - nodeIPs, err = proxier.ipGetter.NodeIPs() - if err != nil { - klog.Errorf("Failed to list all node IPs from host, err: %v", err) + if utilproxy.IsZeroCIDR(address) { + nodeIPs, err = proxier.ipGetter.NodeIPs() + if err != nil { + klog.Errorf("Failed to list all node IPs from host, err: %v", err) + } + break } + nodeIPs = append(nodeIPs, net.ParseIP(address)) } } }