From afa0b808f873b515c9d58a9ead788972ea7d2533 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 30 Oct 2019 10:46:46 -0400 Subject: [PATCH] Fix apiserver to advertise IPv6 endpoints if bound to IPv6 Also rename utilnet.ChooseBindAddress() to ResolveBindAddress(), to better describe its functionality. --- cmd/kubeadm/app/util/config/common.go | 4 +- pkg/kubeapiserver/options/serving.go | 2 +- .../apimachinery/pkg/util/net/interface.go | 52 +++++++++++++------ .../pkg/util/net/interface_test.go | 47 ++++++++++------- .../apiserver/pkg/server/options/serving.go | 4 +- 5 files changed, 68 insertions(+), 41 deletions(-) diff --git a/cmd/kubeadm/app/util/config/common.go b/cmd/kubeadm/app/util/config/common.go index 6d8663e495b..6b474f56fea 100644 --- a/cmd/kubeadm/app/util/config/common.go +++ b/cmd/kubeadm/app/util/config/common.go @@ -139,10 +139,10 @@ func VerifyAPIServerBindAddress(address string) error { return nil } -// ChooseAPIServerBindAddress is a wrapper for netutil.ChooseBindAddress that also handles +// ChooseAPIServerBindAddress is a wrapper for netutil.ResolveBindAddress that also handles // the case where no default routes were found and an IP for the API server could not be obtained. func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) { - ip, err := netutil.ChooseBindAddress(bindAddress) + ip, err := netutil.ResolveBindAddress(bindAddress) if err != nil { if netutil.IsNoRoutesError(err) { klog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress) diff --git a/pkg/kubeapiserver/options/serving.go b/pkg/kubeapiserver/options/serving.go index 5d561807fd0..c659fc8cd6d 100644 --- a/pkg/kubeapiserver/options/serving.go +++ b/pkg/kubeapiserver/options/serving.go @@ -60,7 +60,7 @@ func DefaultAdvertiseAddress(s *genericoptions.ServerRunOptions, insecure *gener } if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { - hostIP, err := utilnet.ChooseBindAddress(insecure.BindAddress) + hostIP, err := utilnet.ResolveBindAddress(insecure.BindAddress) if err != nil { return fmt.Errorf("unable to find suitable network address.error='%v'. "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this", err) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go index fa1ebb947f1..836494d579a 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go @@ -36,6 +36,13 @@ const ( familyIPv6 AddressFamily = 6 ) +type AddressFamilyPreference []AddressFamily + +var ( + preferIPv4 = AddressFamilyPreference{familyIPv4, familyIPv6} + preferIPv6 = AddressFamilyPreference{familyIPv6, familyIPv4} +) + const ( // LoopbackInterfaceName is the default name of the loopback interface LoopbackInterfaceName = "lo" @@ -58,7 +65,7 @@ type RouteFile struct { parse func(input io.Reader) ([]Route, error) } -// noRoutesError can be returned by ChooseBindAddress() in case of no routes +// noRoutesError can be returned in case of no routes type noRoutesError struct { message string } @@ -259,7 +266,7 @@ func getIPFromInterface(intfName string, forFamily AddressFamily, nw networkInte return nil, nil } -// memberOF tells if the IP is of the desired family. Used for checking interface addresses. +// memberOf tells if the IP is of the desired family. Used for checking interface addresses. func memberOf(ip net.IP, family AddressFamily) bool { if ip.To4() != nil { return family == familyIPv4 @@ -270,8 +277,8 @@ func memberOf(ip net.IP, family AddressFamily) bool { // chooseIPFromHostInterfaces looks at all system interfaces, trying to find one that is up that // has a global unicast address (non-loopback, non-link local, non-point2point), and returns the IP. -// Searches for IPv4 addresses, and then IPv6 addresses. -func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { +// addressFamilies determines whether it prefers IPv4 or IPv6 +func chooseIPFromHostInterfaces(nw networkInterfacer, addressFamilies AddressFamilyPreference) (net.IP, error) { intfs, err := nw.Interfaces() if err != nil { return nil, err @@ -279,7 +286,7 @@ func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { if len(intfs) == 0 { return nil, fmt.Errorf("no interfaces found on host.") } - for _, family := range []AddressFamily{familyIPv4, familyIPv6} { + for _, family := range addressFamilies { klog.V(4).Infof("Looking for system interface with a global IPv%d address", uint(family)) for _, intf := range intfs { if !isInterfaceUp(&intf) { @@ -326,15 +333,19 @@ func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { // IP of the interface with a gateway on it (with priority given to IPv4). For a node // with no internet connection, it returns error. func ChooseHostInterface() (net.IP, error) { + return chooseHostInterface(preferIPv4) +} + +func chooseHostInterface(addressFamilies AddressFamilyPreference) (net.IP, error) { var nw networkInterfacer = networkInterface{} if _, err := os.Stat(ipv4RouteFile); os.IsNotExist(err) { - return chooseIPFromHostInterfaces(nw) + return chooseIPFromHostInterfaces(nw, addressFamilies) } routes, err := getAllDefaultRoutes() if err != nil { return nil, err } - return chooseHostInterfaceFromRoute(routes, nw) + return chooseHostInterfaceFromRoute(routes, nw, addressFamilies) } // networkInterfacer defines an interface for several net library functions. Production @@ -382,10 +393,10 @@ func getAllDefaultRoutes() ([]Route, error) { } // chooseHostInterfaceFromRoute cycles through each default route provided, looking for a -// global IP address from the interface for the route. Will first look all each IPv4 route for -// an IPv4 IP, and then will look at each IPv6 route for an IPv6 IP. -func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer) (net.IP, error) { - for _, family := range []AddressFamily{familyIPv4, familyIPv6} { +// global IP address from the interface for the route. addressFamilies determines whether it +// prefers IPv4 or IPv6 +func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer, addressFamilies AddressFamilyPreference) (net.IP, error) { + for _, family := range addressFamilies { klog.V(4).Infof("Looking for default routes with IPv%d addresses", uint(family)) for _, route := range routes { if route.Family != family { @@ -406,12 +417,19 @@ func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer) (net.IP, return nil, fmt.Errorf("unable to select an IP from default routes.") } -// If bind-address is usable, return it directly -// If bind-address is not usable (unset, 0.0.0.0, or loopback), we will use the host's default -// interface. -func ChooseBindAddress(bindAddress net.IP) (net.IP, error) { +// ResolveBindAddress returns the IP address of a daemon, based on the given bindAddress: +// If bindAddress is unset, it returns the host's default IP, as with ChooseHostInterface(). +// If bindAddress is unspecified or loopback, it returns the default IP of the same +// address family as bindAddress. +// Otherwise, it just returns bindAddress. +func ResolveBindAddress(bindAddress net.IP) (net.IP, error) { + addressFamilies := preferIPv4 + if bindAddress != nil && memberOf(bindAddress, familyIPv6) { + addressFamilies = preferIPv6 + } + if bindAddress == nil || bindAddress.IsUnspecified() || bindAddress.IsLoopback() { - hostIP, err := ChooseHostInterface() + hostIP, err := chooseHostInterface(addressFamilies) if err != nil { return nil, err } @@ -426,7 +444,7 @@ func ChooseBindAddress(bindAddress net.IP) (net.IP, error) { // e.g when using BGP to announce a host IP over link-local ip addresses and this ip address is attached to the lo interface. func ChooseBindAddressForInterface(intfName string) (net.IP, error) { var nw networkInterfacer = networkInterface{} - for _, family := range []AddressFamily{familyIPv4, familyIPv6} { + for _, family := range preferIPv4 { ip, err := getIPFromInterface(intfName, family, nw) if err != nil { return nil, err diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go index d652f479d2c..7cdd2b74972 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go @@ -535,17 +535,21 @@ func TestChooseHostInterfaceFromRoute(t *testing.T) { tcase string routes []Route nw networkInterfacer + order AddressFamilyPreference expected net.IP }{ - {"ipv4", routeV4, validNetworkInterface{}, net.ParseIP("10.254.71.145")}, - {"ipv6", routeV6, ipv6NetworkInterface{}, net.ParseIP("2001::200")}, - {"prefer ipv4", bothRoutes, v4v6NetworkInterface{}, net.ParseIP("10.254.71.145")}, - {"all LLA", routeV4, networkInterfaceWithOnlyLinkLocals{}, nil}, - {"no routes", noRoutes, validNetworkInterface{}, nil}, - {"fail get IP", routeV4, networkInterfaceFailGetAddrs{}, nil}, + {"single-stack ipv4", routeV4, validNetworkInterface{}, preferIPv4, net.ParseIP("10.254.71.145")}, + {"single-stack ipv4, prefer v6", routeV4, validNetworkInterface{}, preferIPv6, net.ParseIP("10.254.71.145")}, + {"single-stack ipv6", routeV6, ipv6NetworkInterface{}, preferIPv4, net.ParseIP("2001::200")}, + {"single-stack ipv6, prefer v6", routeV6, ipv6NetworkInterface{}, preferIPv6, net.ParseIP("2001::200")}, + {"dual stack", bothRoutes, v4v6NetworkInterface{}, preferIPv4, net.ParseIP("10.254.71.145")}, + {"dual stack, prefer v6", bothRoutes, v4v6NetworkInterface{}, preferIPv6, net.ParseIP("2001::10")}, + {"all LLA", routeV4, networkInterfaceWithOnlyLinkLocals{}, preferIPv4, nil}, + {"no routes", noRoutes, validNetworkInterface{}, preferIPv4, nil}, + {"fail get IP", routeV4, networkInterfaceFailGetAddrs{}, preferIPv4, nil}, } for _, tc := range testCases { - ip, err := chooseHostInterfaceFromRoute(tc.routes, tc.nw) + ip, err := chooseHostInterfaceFromRoute(tc.routes, tc.nw, tc.order) if !ip.Equal(tc.expected) { t.Errorf("case[%v]: expected %v, got %+v .err : %v", tc.tcase, tc.expected, ip, err) } @@ -575,24 +579,29 @@ func TestGetIPFromHostInterfaces(t *testing.T) { testCases := []struct { tcase string nw networkInterfacer + order AddressFamilyPreference expected net.IP errStrFrag string }{ - {"fail get I/Fs", failGettingNetworkInterface{}, nil, "failed getting all interfaces"}, - {"no interfaces", noNetworkInterface{}, nil, "no interfaces"}, - {"I/F not up", downNetworkInterface{}, nil, "no acceptable"}, - {"loopback only", loopbackNetworkInterface{}, nil, "no acceptable"}, - {"P2P I/F only", p2pNetworkInterface{}, nil, "no acceptable"}, - {"fail get addrs", networkInterfaceFailGetAddrs{}, nil, "unable to get Addrs"}, - {"no addresses", networkInterfaceWithNoAddrs{}, nil, "no acceptable"}, - {"invalid addr", networkInterfaceWithInvalidAddr{}, nil, "invalid CIDR"}, - {"no matches", networkInterfaceWithOnlyLinkLocals{}, nil, "no acceptable"}, - {"ipv4", validNetworkInterface{}, net.ParseIP("10.254.71.145"), ""}, - {"ipv6", ipv6NetworkInterface{}, net.ParseIP("2001::200"), ""}, + {"fail get I/Fs", failGettingNetworkInterface{}, preferIPv4, nil, "failed getting all interfaces"}, + {"no interfaces", noNetworkInterface{}, preferIPv4, nil, "no interfaces"}, + {"I/F not up", downNetworkInterface{}, preferIPv4, nil, "no acceptable"}, + {"loopback only", loopbackNetworkInterface{}, preferIPv4, nil, "no acceptable"}, + {"P2P I/F only", p2pNetworkInterface{}, preferIPv4, nil, "no acceptable"}, + {"fail get addrs", networkInterfaceFailGetAddrs{}, preferIPv4, nil, "unable to get Addrs"}, + {"no addresses", networkInterfaceWithNoAddrs{}, preferIPv4, nil, "no acceptable"}, + {"invalid addr", networkInterfaceWithInvalidAddr{}, preferIPv4, nil, "invalid CIDR"}, + {"no matches", networkInterfaceWithOnlyLinkLocals{}, preferIPv4, nil, "no acceptable"}, + {"single-stack ipv4", validNetworkInterface{}, preferIPv4, net.ParseIP("10.254.71.145"), ""}, + {"single-stack ipv4, prefer ipv6", validNetworkInterface{}, preferIPv6, net.ParseIP("10.254.71.145"), ""}, + {"single-stack ipv6", ipv6NetworkInterface{}, preferIPv4, net.ParseIP("2001::200"), ""}, + {"single-stack ipv6, prefer ipv6", ipv6NetworkInterface{}, preferIPv6, net.ParseIP("2001::200"), ""}, + {"dual stack", v4v6NetworkInterface{}, preferIPv4, net.ParseIP("10.254.71.145"), ""}, + {"dual stack, prefer ipv6", v4v6NetworkInterface{}, preferIPv6, net.ParseIP("2001::10"), ""}, } for _, tc := range testCases { - ip, err := chooseIPFromHostInterfaces(tc.nw) + ip, err := chooseIPFromHostInterfaces(tc.nw, tc.order) if !ip.Equal(tc.expected) { t.Errorf("case[%s]: expected %+v, got %+v with err : %v", tc.tcase, tc.expected, ip, err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index ea9faab6232..06edf361e91 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -109,10 +109,10 @@ func NewSecureServingOptions() *SecureServingOptions { } func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) { - if !s.ExternalAddress.IsUnspecified() { + if s.ExternalAddress != nil && !s.ExternalAddress.IsUnspecified() { return s.ExternalAddress, nil } - return utilnet.ChooseBindAddress(s.BindAddress) + return utilnet.ResolveBindAddress(s.BindAddress) } func (s *SecureServingOptions) Validate() []error {