From 1f24a198e7632857495c00f5d4d5619ee1ee4198 Mon Sep 17 00:00:00 2001 From: chenyw1990 Date: Wed, 23 Jun 2021 19:45:24 +0800 Subject: [PATCH] reduce cpu usage of kube-proxy with iptables mode --- pkg/proxy/iptables/proxier_test.go | 10 +++---- pkg/proxy/ipvs/proxier_test.go | 8 ++--- pkg/proxy/util/network.go | 14 +++------ pkg/proxy/util/testing/fake.go | 30 +++++-------------- pkg/proxy/util/utils.go | 39 +++++++++++------------- pkg/proxy/util/utils_test.go | 48 +++++++++++++++--------------- 6 files changed, 61 insertions(+), 88 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index ec4328604b0..8e78d4d233e 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1128,9 +1128,9 @@ func TestNodePort(t *testing.T) { ) itf := net.Interface{Index: 0, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0} - addrs := []net.Addr{utilproxytest.AddrStruct{Val: "127.0.0.1/16"}} + addrs := []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(16, 32)}} itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} - addrs1 := []net.Addr{utilproxytest.AddrStruct{Val: "::1/128"}} + addrs1 := []net.Addr{&net.IPNet{IP: net.ParseIP("::1/128"), Mask: net.CIDRMask(128, 128)}} fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs) fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) fp.nodePortAddresses = []string{} @@ -1180,9 +1180,9 @@ func TestHealthCheckNodePort(t *testing.T) { ) itf := net.Interface{Index: 0, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0} - addrs := []net.Addr{utilproxytest.AddrStruct{Val: "127.0.0.1/16"}} + addrs := []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(16, 32)}} itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} - addrs1 := []net.Addr{utilproxytest.AddrStruct{Val: "::1/128"}} + addrs1 := []net.Addr{&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}} fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs) fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) fp.nodePortAddresses = []string{"127.0.0.1/16"} @@ -1629,7 +1629,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable ) itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} - addrs := []net.Addr{utilproxytest.AddrStruct{Val: "10.20.30.51/24"}} + addrs := []net.Addr{&net.IPNet{IP: net.ParseIP("10.20.30.51"), Mask: net.CIDRMask(24, 32)}} fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs) fp.nodePortAddresses = []string{"10.20.30.0/24"} diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 62d4b91fc25..48e82de6e13 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -1879,9 +1879,9 @@ func TestOnlyLocalNodePorts(t *testing.T) { ) itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} - addrs := []net.Addr{proxyutiltest.AddrStruct{Val: "100.101.102.103/24"}} + addrs := []net.Addr{&net.IPNet{IP: net.ParseIP("100.101.102.103"), Mask: net.CIDRMask(24, 32)}} itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} - addrs1 := []net.Addr{proxyutiltest.AddrStruct{Val: "2001:db8::0/64"}} + addrs1 := []net.Addr{&net.IPNet{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(64, 128)}} fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs) fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) fp.nodePortAddresses = []string{"100.101.102.0/24", "2001:db8::0/64"} @@ -1959,9 +1959,9 @@ func TestHealthCheckNodePort(t *testing.T) { ) itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} - addrs := []net.Addr{proxyutiltest.AddrStruct{Val: "100.101.102.103/24"}} + addrs := []net.Addr{&net.IPNet{IP: net.ParseIP("100.101.102.103"), Mask: net.CIDRMask(24, 32)}} itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} - addrs1 := []net.Addr{proxyutiltest.AddrStruct{Val: "2001:db8::0/64"}} + addrs1 := []net.Addr{&net.IPNet{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(64, 128)}} fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs) fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) fp.nodePortAddresses = []string{"100.101.102.0/24", "2001:db8::0/64"} diff --git a/pkg/proxy/util/network.go b/pkg/proxy/util/network.go index 504ad939ace..9d46da0397a 100644 --- a/pkg/proxy/util/network.go +++ b/pkg/proxy/util/network.go @@ -24,22 +24,16 @@ import ( // code will forward to net library functions, and unit tests will override the methods // for testing purposes. type NetworkInterfacer interface { - Addrs(intf *net.Interface) ([]net.Addr, error) - Interfaces() ([]net.Interface, error) + InterfaceAddrs() ([]net.Addr, error) } // RealNetwork implements the NetworkInterfacer interface for production code, just // wrapping the underlying net library function calls. type RealNetwork struct{} -// Addrs wraps net.Interface.Addrs(), it's a part of NetworkInterfacer interface. -func (RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { - return intf.Addrs() -} - -// Interfaces wraps net.Interfaces(), it's a part of NetworkInterfacer interface. -func (RealNetwork) Interfaces() ([]net.Interface, error) { - return net.Interfaces() +// InterfaceAddrs wraps net.InterfaceAddrs(), it's a part of NetworkInterfacer interface. +func (RealNetwork) InterfaceAddrs() ([]net.Addr, error) { + return net.InterfaceAddrs() } var _ NetworkInterfacer = &RealNetwork{} diff --git a/pkg/proxy/util/testing/fake.go b/pkg/proxy/util/testing/fake.go index 44fd56d6301..ee6f39a876b 100644 --- a/pkg/proxy/util/testing/fake.go +++ b/pkg/proxy/util/testing/fake.go @@ -39,27 +39,11 @@ func (f *FakeNetwork) AddInterfaceAddr(intf *net.Interface, addrs []net.Addr) { f.Address[intf.Name] = addrs } -// Addrs is part of NetworkInterfacer interface. -func (f *FakeNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { - return f.Address[intf.Name], nil +// InterfaceAddrs is part of NetworkInterfacer interface. +func (f *FakeNetwork) InterfaceAddrs() ([]net.Addr, error) { + addrs := make([]net.Addr, 0) + for _, value := range f.Address { + addrs = append(addrs, value...) + } + return addrs, nil } - -// Interfaces is part of NetworkInterfacer interface. -func (f *FakeNetwork) Interfaces() ([]net.Interface, error) { - return f.NetworkInterfaces, nil -} - -// AddrStruct implements the net.Addr for test purpose. -type AddrStruct struct{ Val string } - -// Network is part of net.Addr interface. -func (a AddrStruct) Network() string { - return a.Val -} - -// String is part of net.Addr interface. -func (a AddrStruct) String() string { - return a.Val -} - -var _ net.Addr = &AddrStruct{} diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index 57744f10abb..d10d3637ce5 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -209,9 +209,9 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) } } - itfs, err := nw.Interfaces() + addrs, err := nw.InterfaceAddrs() if err != nil { - return nil, fmt.Errorf("error listing all interfaces from host, error: %v", err) + return nil, fmt.Errorf("error listing all interfaceAddrs from host, error: %v", err) } // Second round of iteration to parse IPs based on cidr. @@ -221,29 +221,24 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) } _, ipNet, _ := net.ParseCIDR(cidr) - for _, itf := range itfs { - addrs, err := nw.Addrs(&itf) - if err != nil { - return nil, fmt.Errorf("error getting address from interface %s, error: %v", itf.Name, err) + for _, addr := range addrs { + var ip net.IP + // nw.InterfaceAddrs may return net.IPAddr or net.IPNet on windows, and it will return net.IPNet on linux. + switch v := addr.(type) { + case *net.IPAddr: + ip = v.IP + case *net.IPNet: + ip = v.IP + default: + continue } - for _, addr := range addrs { - if addr == nil { - continue + if ipNet.Contains(ip) { + if utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) { + uniqueAddressList.Insert(ip.String()) } - - ip, _, err := net.ParseCIDR(addr.String()) - if err != nil { - return nil, fmt.Errorf("error parsing CIDR for interface %s, error: %v", itf.Name, err) - } - - if ipNet.Contains(ip) { - if utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) { - uniqueAddressList.Insert(ip.String()) - } - if !utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) { - uniqueAddressList.Insert(ip.String()) - } + if !utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) { + uniqueAddressList.Insert(ip.String()) } } } diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go index d33e95f0593..a0ee3761ee5 100644 --- a/pkg/proxy/util/utils_test.go +++ b/pkg/proxy/util/utils_test.go @@ -417,11 +417,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("10.20.30.51"), Mask: net.CIDRMask(24, 32)}}, }, { itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "100.200.201.1/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("100.200.201.1"), Mask: net.CIDRMask(24, 32)}}, }, }, expected: sets.NewString("10.20.30.51"), @@ -432,11 +432,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("10.20.30.51"), Mask: net.CIDRMask(24, 32)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)}}, }, }, expected: sets.NewString("0.0.0.0/0"), @@ -447,11 +447,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(32, 128)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}}, }, }, expected: sets.NewString("2001:db8::1", "::1"), @@ -462,11 +462,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(32, 128)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}}, }, }, expected: sets.NewString("::/0"), @@ -477,11 +477,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("10.20.30.51"), Mask: net.CIDRMask(24, 32)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)}}, }, }, expected: sets.NewString("127.0.0.1"), @@ -492,7 +492,7 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "127.0.1.1/8"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.1.1"), Mask: net.CIDRMask(8, 32)}}, }, }, expected: sets.NewString("127.0.1.1"), @@ -503,11 +503,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("10.20.30.51"), Mask: net.CIDRMask(24, 32)}}, }, { itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "100.200.201.1/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("100.200.201.1"), Mask: net.CIDRMask(24, 32)}}, }, }, expected: sets.NewString("10.20.30.51", "100.200.201.1"), @@ -518,11 +518,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "192.168.1.2/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("192.168.1.2"), Mask: net.CIDRMask(24, 32)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)}}, }, }, expected: nil, @@ -534,11 +534,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "192.168.1.2/24"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("192.168.1.2"), Mask: net.CIDRMask(24, 32)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)}}, }, }, expected: sets.NewString("0.0.0.0/0", "::/0"), @@ -549,11 +549,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(32, 128)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}}, }, }, expected: sets.NewString("0.0.0.0/0", "::/0"), @@ -564,7 +564,7 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("1.2.3.4"), Mask: net.CIDRMask(30, 32)}}, }, }, expected: sets.NewString("0.0.0.0/0"), @@ -575,11 +575,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("1.2.3.4"), Mask: net.CIDRMask(30, 32)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}}, }, }, expected: sets.NewString("0.0.0.0/0", "::1"), @@ -590,11 +590,11 @@ func TestGetNodeAddresses(t *testing.T) { itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("1.2.3.4"), Mask: net.CIDRMask(30, 32)}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}}, }, }, expected: sets.NewString("::/0", "1.2.3.4"),