From 078d2e65b5b654c9b2c8a6141a853290856a57fa Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 22 Oct 2020 12:46:30 +0200 Subject: [PATCH 1/2] autodetect global addresses on loopback interfaces There are some network scenarios that, in order to have high availability based on routing protocols, you may have multiple interfaces in the host and you use a routing protocol to configure the routing in the host. It is common to not use global addresses on those interfaces because you don't want them to be reachable, so you assign the global address to the loopback interface. Loopback interfaces are always up, regardless of the states of physical interfaces. They are not subject to interface problems, i.e., if the interface is down or flapping you can not reach the IP despite you have connectivity through another interface. We should consider global ip addresses on loopback interfaces when: - the host has default routes - there are no global IPs on those interfaces. There can be more cases in which you have global addresses on the interfaces too, but that will open an explosion of scenarios hard to support and to "autodetect" It will be a cluster admin responsability to configure the network in such scenarios. --- .../apimachinery/pkg/util/net/interface.go | 16 ++++- .../pkg/util/net/interface_test.go | 59 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go index 204e223caf0..6d22d38c2ff 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go @@ -393,8 +393,9 @@ func getAllDefaultRoutes() ([]Route, error) { } // chooseHostInterfaceFromRoute cycles through each default route provided, looking for a -// global IP address from the interface for the route. addressFamilies determines whether it -// prefers IPv4 or IPv6 +// global IP address from the interface for the route. If there are routes but no global +// address is obtained from the interfaces, it checks if the loopback interface has a global address. +// addressFamilies determines whether it prefers IPv4 or IPv6 func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer, addressFamilies AddressFamilyPreference) (net.IP, error) { for _, family := range addressFamilies { klog.V(4).Infof("Looking for default routes with IPv%d addresses", uint(family)) @@ -411,6 +412,17 @@ func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer, addressF klog.V(4).Infof("Found active IP %v ", finalIP) return finalIP, nil } + // In case of network setups where default routes are present, but network + // interfaces use only link-local addresses (e.g. as described in RFC5549). + // the global IP is assigned to the loopback interface + loopbackIP, err := getIPFromInterface(LoopbackInterfaceName, family, nw) + if err != nil { + return nil, err + } + if loopbackIP != nil { + klog.V(4).Infof("Found active IP %v on Loopback interface", loopbackIP) + return loopbackIP, nil + } } } klog.V(4).Infof("No active IP found by looking at default routes") diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go index 7cdd2b74972..fb4702603c8 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go @@ -453,6 +453,55 @@ func (_ p2pNetworkInterface) Interfaces() ([]net.Interface, error) { return []net.Interface{p2pIntf}, nil } +// Interface with link locals and loopback interface with global addresses +type linkLocalLoopbackNetworkInterface struct { +} + +func (_ linkLocalLoopbackNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + if intfName == LoopbackInterfaceName { + return &loopbackIntf, nil + } + return &upIntf, nil +} +func (_ linkLocalLoopbackNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { + var ifat []net.Addr + ifat = []net.Addr{addrStruct{val: "169.254.162.166/16"}, addrStruct{val: "fe80::200/10"}} + if intf.Name == LoopbackInterfaceName { + ifat = []net.Addr{addrStruct{val: "::1/128"}, addrStruct{val: "127.0.0.1/8"}, + // global addresses on loopback interface + addrStruct{val: "10.1.1.1/32"}, addrStruct{val: "fd00:1:1::1/128"}} + } + return ifat, nil +} +func (_ linkLocalLoopbackNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf, loopbackIntf}, nil +} + +// Interface and loopback interface with global addresses +type globalsNetworkInterface struct { +} + +func (_ globalsNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) { + if intfName == LoopbackInterfaceName { + return &loopbackIntf, nil + } + return &upIntf, nil +} +func (_ globalsNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) { + var ifat []net.Addr + ifat = []net.Addr{addrStruct{val: "169.254.162.166/16"}, addrStruct{val: "fe80::200/10"}, + addrStruct{val: "192.168.1.1/31"}, addrStruct{val: "fd00::200/127"}} + if intf.Name == LoopbackInterfaceName { + ifat = []net.Addr{addrStruct{val: "::1/128"}, addrStruct{val: "127.0.0.1/8"}, + // global addresses on loopback interface + addrStruct{val: "10.1.1.1/32"}, addrStruct{val: "fd00:1:1::1/128"}} + } + return ifat, nil +} +func (_ globalsNetworkInterface) Interfaces() ([]net.Interface, error) { + return []net.Interface{upIntf, loopbackIntf}, nil +} + // Unable to get IP addresses for interface type networkInterfaceFailGetAddrs struct { } @@ -544,6 +593,16 @@ func TestChooseHostInterfaceFromRoute(t *testing.T) { {"single-stack ipv6, prefer v6", routeV6, ipv6NetworkInterface{}, preferIPv6, net.ParseIP("2001::200")}, {"dual stack", bothRoutes, v4v6NetworkInterface{}, preferIPv4, net.ParseIP("10.254.71.145")}, {"dual stack, prefer v6", bothRoutes, v4v6NetworkInterface{}, preferIPv6, net.ParseIP("2001::10")}, + {"LLA and loopback with global, IPv4", routeV4, linkLocalLoopbackNetworkInterface{}, preferIPv4, net.ParseIP("10.1.1.1")}, + {"LLA and loopback with global, IPv6", routeV6, linkLocalLoopbackNetworkInterface{}, preferIPv6, net.ParseIP("fd00:1:1::1")}, + {"LLA and loopback with global, dual stack prefer IPv4", bothRoutes, linkLocalLoopbackNetworkInterface{}, preferIPv4, net.ParseIP("10.1.1.1")}, + {"LLA and loopback with global, dual stack prefer IPv6", bothRoutes, linkLocalLoopbackNetworkInterface{}, preferIPv6, net.ParseIP("fd00:1:1::1")}, + {"LLA and loopback with global, no routes", noRoutes, linkLocalLoopbackNetworkInterface{}, preferIPv6, nil}, + {"interface and loopback with global, IPv4", routeV4, globalsNetworkInterface{}, preferIPv4, net.ParseIP("192.168.1.1")}, + {"interface and loopback with global, IPv6", routeV6, globalsNetworkInterface{}, preferIPv6, net.ParseIP("fd00::200")}, + {"interface and loopback with global, dual stack prefer IPv4", bothRoutes, globalsNetworkInterface{}, preferIPv4, net.ParseIP("192.168.1.1")}, + {"interface and loopback with global, dual stack prefer IPv6", bothRoutes, globalsNetworkInterface{}, preferIPv6, net.ParseIP("fd00::200")}, + {"interface and loopback with global, no routes", noRoutes, globalsNetworkInterface{}, preferIPv6, nil}, {"all LLA", routeV4, networkInterfaceWithOnlyLinkLocals{}, preferIPv4, nil}, {"no routes", noRoutes, validNetworkInterface{}, preferIPv4, nil}, {"fail get IP", routeV4, networkInterfaceFailGetAddrs{}, preferIPv4, nil}, From 432e74d24583bb70b90d47477505f6e3b2a24b0c Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sat, 14 Nov 2020 22:11:41 +0100 Subject: [PATCH 2/2] get ips from loopback interfaces --- .../apimachinery/pkg/util/net/interface.go | 34 +++++++++++++++++-- .../pkg/util/net/interface_test.go | 27 +++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go index 6d22d38c2ff..9adf4cfe477 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go @@ -266,6 +266,36 @@ func getIPFromInterface(intfName string, forFamily AddressFamily, nw networkInte return nil, nil } +// getIPFromLoopbackInterface gets the IPs on a loopback interface and returns a global unicast address, if any. +// The loopback interface must be up, the IP must in the family requested, and the IP must be a global unicast address. +func getIPFromLoopbackInterface(forFamily AddressFamily, nw networkInterfacer) (net.IP, error) { + intfs, err := nw.Interfaces() + if err != nil { + return nil, err + } + for _, intf := range intfs { + if !isInterfaceUp(&intf) { + continue + } + if intf.Flags&(net.FlagLoopback) != 0 { + addrs, err := nw.Addrs(&intf) + if err != nil { + return nil, err + } + klog.V(4).Infof("Interface %q has %d addresses :%v.", intf.Name, len(addrs), addrs) + matchingIP, err := getMatchingGlobalIP(addrs, forFamily) + if err != nil { + return nil, err + } + if matchingIP != nil { + klog.V(4).Infof("Found valid IPv%d address %v for interface %q.", int(forFamily), matchingIP, intf.Name) + return matchingIP, nil + } + } + } + return nil, nil +} + // memberOf tells if the IP is of the desired family. Used for checking interface addresses. func memberOf(ip net.IP, family AddressFamily) bool { if ip.To4() != nil { @@ -414,8 +444,8 @@ func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer, addressF } // In case of network setups where default routes are present, but network // interfaces use only link-local addresses (e.g. as described in RFC5549). - // the global IP is assigned to the loopback interface - loopbackIP, err := getIPFromInterface(LoopbackInterfaceName, family, nw) + // the global IP is assigned to the loopback interface, and we should use it + loopbackIP, err := getIPFromLoopbackInterface(family, nw) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go index fb4702603c8..fac078d20f2 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go @@ -579,6 +579,33 @@ func TestGetIPFromInterface(t *testing.T) { } } +func TestGetIPFromLoopbackInterface(t *testing.T) { + testCases := []struct { + tcase string + family AddressFamily + nw networkInterfacer + expected net.IP + errStrFrag string + }{ + {"ipv4", familyIPv4, linkLocalLoopbackNetworkInterface{}, net.ParseIP("10.1.1.1"), ""}, + {"ipv6", familyIPv6, linkLocalLoopbackNetworkInterface{}, net.ParseIP("fd00:1:1::1"), ""}, + {"no global ipv4", familyIPv4, loopbackNetworkInterface{}, nil, ""}, + {"no global ipv6", familyIPv6, loopbackNetworkInterface{}, nil, ""}, + } + for _, tc := range testCases { + ip, err := getIPFromLoopbackInterface(tc.family, tc.nw) + if err != nil { + if !strings.Contains(err.Error(), tc.errStrFrag) { + t.Errorf("case[%s]: Error string %q does not contain %q", tc.tcase, err, tc.errStrFrag) + } + } else if tc.errStrFrag != "" { + t.Errorf("case[%s]: Error %q expected, but seen %v", tc.tcase, tc.errStrFrag, err) + } else if !ip.Equal(tc.expected) { + t.Errorf("case[%v]: expected %v, got %+v .err : %v", tc.tcase, tc.expected, ip, err) + } + } +} + func TestChooseHostInterfaceFromRoute(t *testing.T) { testCases := []struct { tcase string