From a2a99c786ac01fa7ec32e829c777c6d335c24528 Mon Sep 17 00:00:00 2001 From: Paul Michali Date: Thu, 18 May 2017 14:11:42 -0400 Subject: [PATCH] IPv6 support for getting node IP As part of ChooseHostInterface(), it will call a function to try to get the global IP for the host, by looking at all the system interfaces and select the first IP that is not a loopback, link-local, or point-to-point IP. This commit does the following: - Allows IPv6 non-local IPs to be selected. - IPv4 takes priority (checks all interfaces for IPv4 addresses and then checks all interfaces for IPv6), for backward compatibility. - Adds UTs for code coverage (was no coverage of underlying function), increasing from 62% to 84%. - Improved logging and reporting for error conditions. - Minor renaming of functions and variables for readability. --- .../apimachinery/pkg/util/net/interface.go | 114 +++++--- .../pkg/util/net/interface_test.go | 249 +++++++++++++++--- 2 files changed, 290 insertions(+), 73 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go index a1e53d2e436..95d398c86e5 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go @@ -29,6 +29,13 @@ import ( "github.com/golang/glog" ) +type AddressFamily uint + +const ( + familyIPv4 AddressFamily = 4 + familyIPv6 AddressFamily = 6 +) + type Route struct { Interface string Destination net.IP @@ -96,6 +103,10 @@ func isInterfaceUp(intf *net.Interface) bool { return false } +func isLoopbackOrPointToPoint(intf *net.Interface) bool { + return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) != 0 +} + //getFinalIP method receives all the IP addrs of a Interface //and returns a nil if the address is Loopback, Ipv6, link-local or nil. //It returns a valid IPv4 if an Ipv4 address is found in the array. @@ -149,50 +160,65 @@ func getIPFromInterface(intfName string, nw networkInterfacer) (net.IP, error) { return nil, nil } -func flagsSet(flags net.Flags, test net.Flags) bool { - return flags&test != 0 +// memberOF tells if the IP is of the desired family. Used for checking interface addresses. +func memberOf(ip net.IP, family AddressFamily) bool { + if ip.To4() != nil { + return family == familyIPv4 + } else { + return family == familyIPv6 + } } -func flagsClear(flags net.Flags, test net.Flags) bool { - return flags&test == 0 -} - -func chooseHostInterfaceNativeGo() (net.IP, error) { - intfs, err := net.Interfaces() +// chooseIPFromHostInterfaces looks at all system interfaces, trying to find one that is up that +// has a global unicast address (non-loopback, non-link local, non-point2point), and returns the IP. +// Searches for IPv4 addresses, and then IPv6 addresses. +func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { + intfs, err := nw.Interfaces() if err != nil { return nil, err } - i := 0 - var ip net.IP - for i = range intfs { - if flagsSet(intfs[i].Flags, net.FlagUp) && flagsClear(intfs[i].Flags, net.FlagLoopback|net.FlagPointToPoint) { - addrs, err := intfs[i].Addrs() + if len(intfs) == 0 { + return nil, fmt.Errorf("no interfaces found on host.") + } + for _, family := range []AddressFamily{familyIPv4, familyIPv6} { + glog.V(4).Infof("Looking for system interface with a global IPv%d address", uint(family)) + for _, intf := range intfs { + if !isInterfaceUp(&intf) { + glog.V(4).Infof("Skipping: down interface %q", intf.Name) + continue + } + if isLoopbackOrPointToPoint(&intf) { + glog.V(4).Infof("Skipping: LB or P2P interface %q", intf.Name) + continue + } + addrs, err := nw.Addrs(&intf) if err != nil { return nil, err } - if len(addrs) > 0 { - for _, addr := range addrs { - if addrIP, _, err := net.ParseCIDR(addr.String()); err == nil { - if addrIP.To4() != nil { - ip = addrIP.To4() - if !ip.IsLinkLocalMulticast() && !ip.IsLinkLocalUnicast() { - break - } - } - } + if len(addrs) == 0 { + glog.V(4).Infof("Skipping: no addresses on interface %q", intf.Name) + continue + } + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, fmt.Errorf("Unable to parse CIDR for interface %q: %s", intf.Name, err) } - if ip != nil { - // This interface should suffice. - break + if !memberOf(ip, family) { + glog.V(4).Infof("Skipping: no address family match for %q on interface %q.", ip, intf.Name) + continue } + // TODO: Decide if should open up to allow IPv6 LLAs in future. + if !ip.IsGlobalUnicast() { + glog.V(4).Infof("Skipping: non-global address %q on interface %q.", ip, intf.Name) + continue + } + glog.V(4).Infof("Found global unicast address %q on interface %q.", ip, intf.Name) + return ip, nil } } } - if ip == nil { - return nil, fmt.Errorf("no acceptable interface from host") - } - glog.V(4).Infof("Choosing interface %s (IP %v) as default", intfs[i].Name, ip) - return ip, nil + return nil, fmt.Errorf("no acceptable interface with global unicast address found on host") } //ChooseHostInterface is a method used fetch an IP for a daemon. @@ -200,39 +226,41 @@ func chooseHostInterfaceNativeGo() (net.IP, error) { //For a node with no internet connection ,it returns error //For a multi n/w interface node it returns the IP of the interface with gateway on it. func ChooseHostInterface() (net.IP, error) { + var nw networkInterfacer = networkInterface{} inFile, err := os.Open("/proc/net/route") if err != nil { if os.IsNotExist(err) { - return chooseHostInterfaceNativeGo() + return chooseIPFromHostInterfaces(nw) } return nil, err } defer inFile.Close() - var nw networkInterfacer = networkInterface{} return chooseHostInterfaceFromRoute(inFile, nw) } +// networkInterfacer defines an interface for several net library functions. Production +// code will forward to net library functions, and unit tests will override the methods +// for testing purposes. type networkInterfacer interface { InterfaceByName(intfName string) (*net.Interface, error) Addrs(intf *net.Interface) ([]net.Addr, error) + Interfaces() ([]net.Interface, error) } +// networkInterface implements the networkInterfacer interface for production code, just +// wrapping the underlying net library function calls. type networkInterface struct{} func (_ networkInterface) InterfaceByName(intfName string) (*net.Interface, error) { - intf, err := net.InterfaceByName(intfName) - if err != nil { - return nil, err - } - return intf, nil + return net.InterfaceByName(intfName) } func (_ networkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { - addrs, err := intf.Addrs() - if err != nil { - return nil, err - } - return addrs, nil + return intf.Addrs() +} + +func (_ networkInterface) Interfaces() ([]net.Interface, error) { + return net.Interfaces() } func chooseHostInterfaceFromRoute(inFile io.Reader, nw networkInterfacer) (net.IP, error) { diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go index fc66904c57e..ab810cca5cd 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go @@ -73,6 +73,30 @@ eth0 00000000 0120372D 0001 0 0 0 00000000 eth0 00000000 00000000 0001 0 0 2048 00000000 0 0 0 ` +const ( + flagUp = net.FlagUp | net.FlagBroadcast | net.FlagMulticast + flagDown = net.FlagBroadcast | net.FlagMulticast + flagLoopback = net.FlagUp | net.FlagLoopback + flagP2P = net.FlagUp | net.FlagPointToPoint +) + +func makeIntf(index int, name string, flags net.Flags) net.Interface { + mac := net.HardwareAddr{0, 0x32, 0x7d, 0x69, 0xf7, byte(0x30 + index)} + return net.Interface{ + Index: index, + MTU: 1500, + Name: name, + HardwareAddr: mac, + Flags: flags} +} + +var ( + downIntf = makeIntf(1, "eth3", flagDown) + loopbackIntf = makeIntf(1, "lo", flagLoopback) + p2pIntf = makeIntf(1, "lo", flagP2P) + upIntf = makeIntf(1, "eth3", flagUp) +) + func TestGetRoutes(t *testing.T) { testCases := []struct { tcase string @@ -189,15 +213,19 @@ func (_ validNetworkInterface) InterfaceByName(intfName string) (*net.Interface, func (_ validNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { var ifat []net.Addr ifat = []net.Addr{ - addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}, addrStruct{val: "10.254.71.145/17"}} + addrStruct{val: "2001::200/64"}, addrStruct{val: "10.254.71.145/17"}} return ifat, nil } +func (_ validNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil +} +// First is link-local, second is not. type validNetworkInterfaceWithLinkLocal struct { } func (_ validNetworkInterfaceWithLinkLocal) InterfaceByName(intfName string) (*net.Interface, error) { - c := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: net.FlagUp} + c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp} return &c, nil } func (_ validNetworkInterfaceWithLinkLocal) Addrs(intf *net.Interface) ([]net.Addr, error) { @@ -205,20 +233,57 @@ func (_ validNetworkInterfaceWithLinkLocal) Addrs(intf *net.Interface) ([]net.Ad ifat = []net.Addr{addrStruct{val: "169.254.162.166/16"}, addrStruct{val: "45.55.47.146/19"}} return ifat, nil } - -type validNetworkInterfacewithIpv6Only struct { +func (_ validNetworkInterfaceWithLinkLocal) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil } -func (_ validNetworkInterfacewithIpv6Only) InterfaceByName(intfName string) (*net.Interface, error) { - c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp} - return &c, nil +// Interface with only IPv6 address +type ipv6NetworkInterface struct { } -func (_ validNetworkInterfacewithIpv6Only) Addrs(intf *net.Interface) ([]net.Addr, error) { + +func (_ ipv6NetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + return &upIntf, nil +} +func (_ ipv6NetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { var ifat []net.Addr - ifat = []net.Addr{addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}} + ifat = []net.Addr{addrStruct{val: "2001::200/64"}} return ifat, nil } +func (_ ipv6NetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil +} +// Only with link local addresses +type networkInterfaceWithOnlyLinkLocals struct { +} + +func (_ networkInterfaceWithOnlyLinkLocals) InterfaceByName(intfName string) (*net.Interface, error) { + return &upIntf, nil +} +func (_ networkInterfaceWithOnlyLinkLocals) Addrs(intf *net.Interface) ([]net.Addr, error) { + var ifat []net.Addr + ifat = []net.Addr{addrStruct{val: "169.254.162.166/16"}, addrStruct{val: "fe80::200/10"}} + return ifat, nil +} +func (_ networkInterfaceWithOnlyLinkLocals) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil +} + +// Unable to get interface(s) +type failGettingNetworkInterface struct { +} + +func (_ failGettingNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + return nil, fmt.Errorf("unable get Interface") +} +func (_ failGettingNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { + return nil, nil +} +func (_ failGettingNetworkInterface) Interfaces() ([]net.Interface, error) { + return nil, fmt.Errorf("mock failed getting all interfaces") +} + +// No interfaces type noNetworkInterface struct { } @@ -228,30 +293,105 @@ func (_ noNetworkInterface) InterfaceByName(intfName string) (*net.Interface, er func (_ noNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { return nil, nil } - -type networkInterfacewithNoAddrs struct { +func (_ noNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{}, nil } -func (_ networkInterfacewithNoAddrs) InterfaceByName(intfName string) (*net.Interface, error) { - c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp} - return &c, nil -} -func (_ networkInterfacewithNoAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) { - return nil, fmt.Errorf("unable get Addrs") +// Interface is down +type downNetworkInterface struct { } -type networkInterfacewithIpv6addrs struct { +func (_ downNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + return &downIntf, nil } - -func (_ networkInterfacewithIpv6addrs) InterfaceByName(intfName string) (*net.Interface, error) { - c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp} - return &c, nil -} -func (_ networkInterfacewithIpv6addrs) Addrs(intf *net.Interface) ([]net.Addr, error) { +func (_ downNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { var ifat []net.Addr - ifat = []net.Addr{addrStruct{val: "fe80::2f7:6ffff:fe6e:2956/64"}} + ifat = []net.Addr{ + addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}, addrStruct{val: "10.254.71.145/17"}} return ifat, nil } +func (_ downNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{downIntf}, nil +} + +// Loopback interface +type loopbackNetworkInterface struct { +} + +func (_ loopbackNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + return &loopbackIntf, nil +} +func (_ loopbackNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { + var ifat []net.Addr + ifat = []net.Addr{ + addrStruct{val: "::1/128"}, addrStruct{val: "127.0.0.1/8"}} + return ifat, nil +} +func (_ loopbackNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{loopbackIntf}, nil +} + +// Point to point interface +type p2pNetworkInterface struct { +} + +func (_ p2pNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + return &p2pIntf, nil +} +func (_ p2pNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { + var ifat []net.Addr + ifat = []net.Addr{ + addrStruct{val: "::1/128"}, addrStruct{val: "127.0.0.1/8"}} + return ifat, nil +} +func (_ p2pNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{p2pIntf}, nil +} + +// Unable to get IP addresses for interface +type networkInterfaceFailGetAddrs struct { +} + +func (_ networkInterfaceFailGetAddrs) InterfaceByName(intfName string) (*net.Interface, error) { + return &upIntf, nil +} +func (_ networkInterfaceFailGetAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) { + return nil, fmt.Errorf("unable to get Addrs") +} +func (_ networkInterfaceFailGetAddrs) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil +} + +// No addresses for interface +type networkInterfaceWithNoAddrs struct { +} + +func (_ networkInterfaceWithNoAddrs) InterfaceByName(intfName string) (*net.Interface, error) { + return &upIntf, nil +} +func (_ networkInterfaceWithNoAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) { + ifat := []net.Addr{} + return ifat, nil +} +func (_ networkInterfaceWithNoAddrs) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil +} + +// Invalid addresses for interface +type networkInterfaceWithInvalidAddr struct { +} + +func (_ networkInterfaceWithInvalidAddr) InterfaceByName(intfName string) (*net.Interface, error) { + return &upIntf, nil +} +func (_ networkInterfaceWithInvalidAddr) Addrs(intf *net.Interface) ([]net.Addr, error) { + var ifat []net.Addr + ifat = []net.Addr{addrStruct{val: "10.20.30.40.50/24"}} + return ifat, nil +} +func (_ networkInterfaceWithInvalidAddr) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf}, nil +} func TestGetIPFromInterface(t *testing.T) { testCases := []struct { @@ -261,7 +401,7 @@ func TestGetIPFromInterface(t *testing.T) { expected net.IP }{ {"valid", "eth3", validNetworkInterface{}, net.ParseIP("10.254.71.145")}, - {"ipv6", "eth3", validNetworkInterfacewithIpv6Only{}, nil}, + {"ipv6", "eth3", ipv6NetworkInterface{}, nil}, {"nothing", "eth3", noNetworkInterface{}, nil}, } for _, tc := range testCases { @@ -282,14 +422,14 @@ func TestChooseHostInterfaceFromRoute(t *testing.T) { {"valid_routefirst", strings.NewReader(gatewayfirst), validNetworkInterface{}, net.ParseIP("10.254.71.145")}, {"valid_routelast", strings.NewReader(gatewaylast), validNetworkInterface{}, net.ParseIP("10.254.71.145")}, {"valid_routemiddle", strings.NewReader(gatewaymiddle), validNetworkInterface{}, net.ParseIP("10.254.71.145")}, - {"valid_routemiddle_ipv6", strings.NewReader(gatewaymiddle), validNetworkInterfacewithIpv6Only{}, nil}, + {"valid_routemiddle_ipv6", strings.NewReader(gatewaymiddle), ipv6NetworkInterface{}, nil}, {"no internet connection", strings.NewReader(noInternetConnection), validNetworkInterface{}, nil}, - {"no non-link-local ip", strings.NewReader(gatewayfirstLinkLocal), validNetworkInterfaceWithLinkLocal{}, net.ParseIP("45.55.47.146")}, + {"1st ip link-local", strings.NewReader(gatewayfirstLinkLocal), validNetworkInterfaceWithLinkLocal{}, net.ParseIP("45.55.47.146")}, {"no route", strings.NewReader(nothing), validNetworkInterface{}, nil}, {"no route file", nil, validNetworkInterface{}, nil}, {"no interfaces", nil, noNetworkInterface{}, nil}, - {"no interface Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithNoAddrs{}, nil}, - {"Invalid Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithIpv6addrs{}, nil}, + {"no interface Addrs", strings.NewReader(gatewaymiddle), networkInterfaceWithNoAddrs{}, nil}, + {"Invalid Addrs", strings.NewReader(gatewaymiddle), ipv6NetworkInterface{}, nil}, } for _, tc := range testCases { ip, err := chooseHostInterfaceFromRoute(tc.inFile, tc.nw) @@ -298,3 +438,52 @@ func TestChooseHostInterfaceFromRoute(t *testing.T) { } } } +func TestMemberOf(t *testing.T) { + testCases := []struct { + tcase string + ip net.IP + family AddressFamily + expected bool + }{ + {"ipv4 is 4", net.ParseIP("10.20.30.40"), familyIPv4, true}, + {"ipv4 is 6", net.ParseIP("10.10.10.10"), familyIPv6, false}, + {"ipv6 is 4", net.ParseIP("2001::100"), familyIPv4, false}, + {"ipv6 is 6", net.ParseIP("2001::100"), familyIPv6, true}, + } + for _, tc := range testCases { + if memberOf(tc.ip, tc.family) != tc.expected { + t.Errorf("case[%s]: expected %+v", tc.tcase, tc.expected) + } + } +} + +func TestGetIPFromHostInterfaces(t *testing.T) { + testCases := []struct { + tcase string + nw networkInterfacer + expected net.IP + errStrFrag string + }{ + {"fail get I/Fs", failGettingNetworkInterface{}, nil, "failed getting all interfaces"}, + {"no interfaces", noNetworkInterface{}, nil, "no interfaces"}, + {"I/F not up", downNetworkInterface{}, nil, "no acceptable"}, + {"loopback only", loopbackNetworkInterface{}, nil, "no acceptable"}, + {"P2P I/F only", p2pNetworkInterface{}, nil, "no acceptable"}, + {"fail get addrs", networkInterfaceFailGetAddrs{}, nil, "unable to get Addrs"}, + {"no addresses", networkInterfaceWithNoAddrs{}, nil, "no acceptable"}, + {"invalid addr", networkInterfaceWithInvalidAddr{}, nil, "invalid CIDR"}, + {"no matches", networkInterfaceWithOnlyLinkLocals{}, nil, "no acceptable"}, + {"ipv4", validNetworkInterface{}, net.ParseIP("10.254.71.145"), ""}, + {"ipv6", ipv6NetworkInterface{}, net.ParseIP("2001::200"), ""}, + } + + for _, tc := range testCases { + ip, err := chooseIPFromHostInterfaces(tc.nw) + if !ip.Equal(tc.expected) { + t.Errorf("case[%s]: expected %+v, got %+v with err : %v", tc.tcase, tc.expected, ip, err) + } + if err != nil && !strings.Contains(err.Error(), tc.errStrFrag) { + t.Errorf("case[%s]: unable to find %q in error string %q", tc.tcase, tc.errStrFrag, err.Error()) + } + } +}