Merge pull request #4865 from sub-mod/proxier_go_change

use ChooseHostInterface in kube-proxy

pkg/proxy/proxier.go

@@ -319,12 +319,12 @@ func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.In
 		return nil
 	}
 
-	hostIP, err := chooseHostInterface()
+	hostIP, err := util.ChooseHostInterface()
 	if err != nil {
 		glog.Errorf("Failed to select a host interface: %v", err)
 		return nil
 	}
-
+	glog.Infof("Setting Proxy IP to %v", hostIP)
 	glog.Infof("Initializing iptables")
 	// Clean up old messes.  Ignore erors.
 	iptablesDeleteOld(iptables)
@@ -761,45 +761,3 @@ func (proxier *Proxier) iptablesHostPortalArgs(destIP net.IP, destPort int, prot
 	args = append(args, "-j", "DNAT", "--to-destination", net.JoinHostPort(proxyIP.String(), strconv.Itoa(proxyPort)))
 	return args
 }
-
-func chooseHostInterface() (net.IP, error) {
-	intfs, err := net.Interfaces()
-	if err != nil {
-		return nil, err
-	}
-	i := 0
-	for i = range intfs {
-		if flagsSet(intfs[i].Flags, net.FlagUp) && flagsClear(intfs[i].Flags, net.FlagLoopback|net.FlagPointToPoint) {
-			addrs, err := intfs[i].Addrs()
-			if err != nil {
-				return nil, err
-			}
-			if len(addrs) > 0 {
-				// This interface should suffice.
-				break
-			}
-		}
-	}
-	if i == len(intfs) {
-		return nil, err
-	}
-	glog.V(2).Infof("Choosing interface %s for from-host portals", intfs[i].Name)
-	addrs, err := intfs[i].Addrs()
-	if err != nil {
-		return nil, err
-	}
-	glog.V(2).Infof("Interface %s = %s", intfs[i].Name, addrs[0].String())
-	ip, _, err := net.ParseCIDR(addrs[0].String())
-	if err != nil {
-		return nil, err
-	}
-	return ip, nil
-}
-
-func flagsSet(flags net.Flags, test net.Flags) bool {
-	return flags&test != 0
-}
-
-func flagsClear(flags net.Flags, test net.Flags) bool {
-	return flags&test == 0
-}

pkg/util/util.go

@@ -17,9 +17,14 @@ limitations under the License.
 package util
 
 import (
+	"bufio"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net"
+	"os"
 	"path"
 	"reflect"
 	"regexp"
@@ -252,3 +257,188 @@ func SplitQualifiedName(str string) (string, string) {
 func JoinQualifiedName(namespace, name string) string {
 	return path.Join(namespace, name)
 }
+
+type Route struct {
+	Interface   string
+	Destination net.IP
+	Gateway     net.IP
+	// TODO: add more fields here if needed
+}
+
+func getRoutes(input io.Reader) ([]Route, error) {
+	routes := []Route{}
+	if input == nil {
+		return nil, fmt.Errorf("input is nil")
+	}
+	scanner := bufio.NewReader(input)
+	for {
+		line, err := scanner.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		//ignore the headers in the route info
+		if strings.HasPrefix(line, "Iface") {
+			continue
+		}
+		fields := strings.Fields(line)
+		routes = append(routes, Route{})
+		route := &routes[len(routes)-1]
+		route.Interface = fields[0]
+		ip, err := parseIP(fields[1])
+		if err != nil {
+			return nil, err
+		}
+		route.Destination = ip
+		ip, err = parseIP(fields[2])
+		if err != nil {
+			return nil, err
+		}
+		route.Gateway = ip
+	}
+	return routes, nil
+}
+
+func parseIP(str string) (net.IP, error) {
+	if str == "" {
+		return nil, fmt.Errorf("input is nil")
+	}
+	bytes, err := hex.DecodeString(str)
+	if err != nil {
+		return nil, err
+	}
+	//TODO add ipv6 support
+	if len(bytes) != net.IPv4len {
+		return nil, fmt.Errorf("only IPv4 is supported")
+	}
+	bytes[0], bytes[1], bytes[2], bytes[3] = bytes[3], bytes[2], bytes[1], bytes[0]
+	return net.IP(bytes), nil
+}
+
+func isInterfaceUp(intf *net.Interface) bool {
+	if intf == nil {
+		return false
+	}
+	if intf.Flags&net.FlagUp != 0 {
+		glog.V(4).Infof("Interface %v is up", intf.Name)
+		return true
+	}
+	return false
+}
+
+//getFinalIP method receives all the IP addrs of a Interface
+//and returns a nil if the address is Loopback , Ipv6  or nil.
+//It returns a valid IPv4 if an Ipv4 address is found in the array.
+func getFinalIP(addrs []net.Addr) (net.IP, error) {
+	if len(addrs) > 0 {
+		for i := range addrs {
+			glog.V(4).Infof("Checking addr  %s.", addrs[i].String())
+			ip, _, err := net.ParseCIDR(addrs[i].String())
+			if err != nil {
+				return nil, err
+			}
+			//Only IPv4
+			//TODO : add IPv6 support
+			if ip.To4() != nil {
+				if !ip.IsLoopback() {
+					glog.V(4).Infof("IP found %v", ip)
+					return ip, nil
+				} else {
+					glog.V(4).Infof("Loopback found %v", ip)
+				}
+			} else {
+				glog.V(4).Infof("%v is not a valid IPv4 address", ip)
+			}
+
+		}
+	}
+	return nil, nil
+}
+
+func getIPFromInterface(intfName string, nw networkInterfacer) (net.IP, error) {
+	intf, err := nw.InterfaceByName(intfName)
+	if err != nil {
+		return nil, err
+	}
+	if isInterfaceUp(intf) {
+		addrs, err := nw.Addrs(intf)
+		if err != nil {
+			return nil, err
+		}
+		glog.V(4).Infof("Interface %q has %d addresses :%v.", intfName, len(addrs), addrs)
+		finalIP, err := getFinalIP(addrs)
+		if err != nil {
+			return nil, err
+		}
+		if finalIP != nil {
+			glog.V(4).Infof("valid IPv4 address for interface %q found as %v.", intfName, finalIP)
+			return finalIP, nil
+		}
+	}
+
+	return nil, nil
+}
+
+//ChooseHostInterface is a method used fetch an IP for a daemon.
+//It uses data from /proc/net/route file.
+//For a node with no internet connection ,it returns error
+//For a multi n/w interface node it returns the IP of the interface with gateway on it.
+func ChooseHostInterface() (net.IP, error) {
+	inFile, err := os.Open("/proc/net/route")
+	if err != nil {
+		return nil, err
+	}
+	defer inFile.Close()
+	var nw networkInterfacer = networkInterface{}
+	return chooseHostInterfaceFromRoute(inFile, nw)
+}
+
+type networkInterfacer interface {
+	InterfaceByName(intfName string) (*net.Interface, error)
+	Addrs(intf *net.Interface) ([]net.Addr, error)
+}
+
+type networkInterface struct{}
+
+func (_ networkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	intf, err := net.InterfaceByName(intfName)
+	if err != nil {
+		return nil, err
+	}
+	return intf, nil
+}
+
+func (_ networkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	addrs, err := intf.Addrs()
+	if err != nil {
+		return nil, err
+	}
+	return addrs, nil
+}
+
+func chooseHostInterfaceFromRoute(inFile io.Reader, nw networkInterfacer) (net.IP, error) {
+	routes, err := getRoutes(inFile)
+	if err != nil {
+		return nil, err
+	}
+	zero := net.IP{0, 0, 0, 0}
+	var finalIP net.IP
+	for i := range routes {
+		//find interface with gateway
+		if routes[i].Destination.Equal(zero) {
+			glog.V(4).Infof("Default route transits interface %q", routes[i].Interface)
+			finalIP, err := getIPFromInterface(routes[i].Interface, nw)
+			if err != nil {
+				return nil, err
+			}
+			if finalIP != nil {
+				glog.V(4).Infof("Choosing IP %v ", finalIP)
+				return finalIP, nil
+			}
+		}
+	}
+	glog.V(4).Infof("No valid IP found")
+	if finalIP == nil {
+		return nil, fmt.Errorf("Unable to select an IP.")
+	}
+	return nil, nil
+}

pkg/util/util_test.go

@@ -19,7 +19,10 @@ package util
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"net"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/ghodss/yaml"
@@ -332,3 +335,258 @@ func TestJoinQualifiedName(t *testing.T) {
 		}
 	}
 }
+
+const gatewayfirst = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+eth3	00000000	0100FE0A	0003	0	0	1024	00000000	0	0	0                                                                   
+eth3	0000FE0A	00000000	0001	0	0	0	0080FFFF	0	0	0                                                                      
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0                                                                            
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`
+const gatewaylast = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT  
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0                                                                            
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0                                                                                                                     
+eth3	0000FE0A	00000000	0001	0	0	0	0080FFFF	0	0	0       
+eth3	00000000	0100FE0A	0003	0	0	1024	00000000	0	0	0                                                                 
+`
+const gatewaymiddle = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                                                                                     
+eth3	0000FE0A	00000000	0001	0	0	0	0080FFFF	0	0	0                                                                      
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0       
+eth3	00000000	0100FE0A	0003	0	0	1024	00000000	0	0	0                                                                         
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`
+const noInternetConnection = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0                                                                            
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0            
+`
+const nothing = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                            
+`
+const gatewayfirstIpv6_1 = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+eth3	00000000	0100FE0A	0003	0	0	1024	00000000	0	0	0                                                                   
+eth3	0000FE0AA1	00000000	0001	0	0	0	0080FFFF	0	0	0                                                                      
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0                                                                            
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`
+const gatewayfirstIpv6_2 = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+eth3	00000000	0100FE0AA1	0003	0	0	1024	00000000	0	0	0                                                                   
+eth3	0000FE0A	00000000	0001	0	0	0	0080FFFF	0	0	0                                                                      
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0                                                                            
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`
+const route_Invalidhex = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+eth3	00000000	0100FE0AA	0003	0	0	1024	00000000	0	0	0                                                                   
+eth3	0000FE0A	00000000	0001	0	0	0	0080FFFF	0	0	0                                                                      
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0                                                                            
+virbr0	007AA8C0	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`
+
+func TestGetRoutes(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		route    string
+		expected int
+	}{
+		{"gatewayfirst", gatewayfirst, 4},
+		{"gatewaymiddle", gatewaymiddle, 4},
+		{"gatewaylast", gatewaylast, 4},
+		{"nothing", nothing, 0},
+		{"gatewayfirstIpv6_1", gatewayfirstIpv6_1, 0},
+		{"gatewayfirstIpv6_2", gatewayfirstIpv6_2, 0},
+		{"route_Invalidhex", route_Invalidhex, 0},
+	}
+	for _, tc := range testCases {
+		r := strings.NewReader(tc.route)
+		routes, err := getRoutes(r)
+		if len(routes) != tc.expected {
+			t.Errorf("case[%v]: expected %v, got %v .err : %v", tc.tcase, tc.expected, len(routes), err)
+		}
+	}
+}
+
+func TestParseIP(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		ip       string
+		success  bool
+		expected net.IP
+	}{
+		{"empty", "", false, nil},
+		{"too short", "AA", false, nil},
+		{"too long", "0011223344", false, nil},
+		{"invalid", "invalid!", false, nil},
+		{"zero", "00000000", true, net.IP{0, 0, 0, 0}},
+		{"ffff", "FFFFFFFF", true, net.IP{0xff, 0xff, 0xff, 0xff}},
+		{"valid", "12345678", true, net.IP{120, 86, 52, 18}},
+	}
+	for _, tc := range testCases {
+		ip, err := parseIP(tc.ip)
+		if !ip.Equal(tc.expected) {
+			t.Errorf("case[%v]: expected %q, got %q . err : %v", tc.tcase, tc.expected, ip, err)
+		}
+	}
+}
+
+func TestIsInterfaceUp(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		intf     net.Interface
+		expected bool
+	}{
+		{"up", net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}, true},
+		{"down", net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: 0}, false},
+		{"nothing", net.Interface{}, false},
+	}
+	for _, tc := range testCases {
+		it := isInterfaceUp(&tc.intf)
+		if it != tc.expected {
+			t.Errorf("case[%v]: expected %v, got %v .", tc.tcase, tc.expected, it)
+		}
+	}
+}
+
+type addrStruct struct{ val string }
+
+func (a addrStruct) Network() string {
+	return a.val
+}
+func (a addrStruct) String() string {
+	return a.val
+}
+
+func TestFinalIP(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		addr     []net.Addr
+		expected net.IP
+	}{
+		{"ipv6", []net.Addr{addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}}, nil},
+		{"invalidCIDR", []net.Addr{addrStruct{val: "fe80::2f7:67fff:fe6e:2956/64"}}, nil},
+		{"loopback", []net.Addr{addrStruct{val: "127.0.0.1/24"}}, nil},
+		{"ip4", []net.Addr{addrStruct{val: "10.254.12.132/17"}}, net.ParseIP("10.254.12.132")},
+
+		{"nothing", []net.Addr{}, nil},
+	}
+	for _, tc := range testCases {
+		ip, err := getFinalIP(tc.addr)
+		if !ip.Equal(tc.expected) {
+			t.Errorf("case[%v]: expected %v, got %v .err : %v", tc.tcase, tc.expected, ip, err)
+		}
+	}
+}
+
+func TestAddrs(t *testing.T) {
+	var nw networkInterfacer = validNetworkInterface{}
+	intf := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: 0}
+	addrs, err := nw.Addrs(&intf)
+	if err != nil {
+		t.Errorf("expected no error got : %v", err)
+	}
+	if len(addrs) != 2 {
+		t.Errorf("expected addrs: 2 got null")
+	}
+}
+
+type validNetworkInterface struct {
+}
+
+func (_ validNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
+	return &c, nil
+}
+func (_ validNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{
+		addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}, addrStruct{val: "10.254.71.145/17"}}
+	return ifat, nil
+}
+
+type validNetworkInterfacewithIpv6Only struct {
+}
+
+func (_ validNetworkInterfacewithIpv6Only) InterfaceByName(intfName string) (*net.Interface, error) {
+	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
+	return &c, nil
+}
+func (_ validNetworkInterfacewithIpv6Only) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}}
+	return ifat, nil
+}
+
+type noNetworkInterface struct {
+}
+
+func (_ noNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	return nil, fmt.Errorf("unable get Interface")
+}
+func (_ noNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return nil, nil
+}
+
+type networkInterfacewithNoAddrs struct {
+}
+
+func (_ networkInterfacewithNoAddrs) InterfaceByName(intfName string) (*net.Interface, error) {
+	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
+	return &c, nil
+}
+func (_ networkInterfacewithNoAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return nil, fmt.Errorf("unable get Addrs")
+}
+
+type networkInterfacewithIpv6addrs struct {
+}
+
+func (_ networkInterfacewithIpv6addrs) InterfaceByName(intfName string) (*net.Interface, error) {
+	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
+	return &c, nil
+}
+func (_ networkInterfacewithIpv6addrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{addrStruct{val: "fe80::2f7:6ffff:fe6e:2956/64"}}
+	return ifat, nil
+}
+
+func TestGetIPFromInterface(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		nwname   string
+		nw       networkInterfacer
+		expected net.IP
+	}{
+		{"valid", "eth3", validNetworkInterface{}, net.ParseIP("10.254.71.145")},
+		{"ipv6", "eth3", validNetworkInterfacewithIpv6Only{}, nil},
+		{"nothing", "eth3", noNetworkInterface{}, nil},
+	}
+	for _, tc := range testCases {
+		ip, err := getIPFromInterface(tc.nwname, tc.nw)
+		if !ip.Equal(tc.expected) {
+			t.Errorf("case[%v]: expected %v, got %+v .err : %v", tc.tcase, tc.expected, ip, err)
+		}
+	}
+}
+
+func TestChooseHostInterfaceFromRoute(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		inFile   io.Reader
+		nw       networkInterfacer
+		expected net.IP
+	}{
+		{"valid_routefirst", strings.NewReader(gatewayfirst), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
+		{"valid_routelast", strings.NewReader(gatewaylast), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
+		{"valid_routemiddle", strings.NewReader(gatewaymiddle), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
+		{"valid_routemiddle_ipv6", strings.NewReader(gatewaymiddle), validNetworkInterfacewithIpv6Only{}, nil},
+		{"no internet connection", strings.NewReader(noInternetConnection), validNetworkInterface{}, nil},
+		{"no route", strings.NewReader(nothing), validNetworkInterface{}, nil},
+		{"no route file", nil, validNetworkInterface{}, nil},
+		{"no interfaces", nil, noNetworkInterface{}, nil},
+		{"no interface Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithNoAddrs{}, nil},
+		{"Invalid Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithIpv6addrs{}, nil},
+	}
+	for _, tc := range testCases {
+		ip, err := chooseHostInterfaceFromRoute(tc.inFile, tc.nw)
+		if !ip.Equal(tc.expected) {
+			t.Errorf("case[%v]: expected %v, got %+v .err : %v", tc.tcase, tc.expected, ip, err)
+		}
+	}
+}