Merge pull request #4865 from sub-mod/proxier_go_change

use ChooseHostInterface in kube-proxy
This commit is contained in:
Tim Hockin 2015-03-11 08:09:32 -07:00
commit 52298ab851
3 changed files with 450 additions and 44 deletions

View File

@ -319,12 +319,12 @@ func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.In
return nil
}
hostIP, err := chooseHostInterface()
hostIP, err := util.ChooseHostInterface()
if err != nil {
glog.Errorf("Failed to select a host interface: %v", err)
return nil
}
glog.Infof("Setting Proxy IP to %v", hostIP)
glog.Infof("Initializing iptables")
// Clean up old messes. Ignore erors.
iptablesDeleteOld(iptables)
@ -761,45 +761,3 @@ func (proxier *Proxier) iptablesHostPortalArgs(destIP net.IP, destPort int, prot
args = append(args, "-j", "DNAT", "--to-destination", net.JoinHostPort(proxyIP.String(), strconv.Itoa(proxyPort)))
return args
}
func chooseHostInterface() (net.IP, error) {
intfs, err := net.Interfaces()
if err != nil {
return nil, err
}
i := 0
for i = range intfs {
if flagsSet(intfs[i].Flags, net.FlagUp) && flagsClear(intfs[i].Flags, net.FlagLoopback|net.FlagPointToPoint) {
addrs, err := intfs[i].Addrs()
if err != nil {
return nil, err
}
if len(addrs) > 0 {
// This interface should suffice.
break
}
}
}
if i == len(intfs) {
return nil, err
}
glog.V(2).Infof("Choosing interface %s for from-host portals", intfs[i].Name)
addrs, err := intfs[i].Addrs()
if err != nil {
return nil, err
}
glog.V(2).Infof("Interface %s = %s", intfs[i].Name, addrs[0].String())
ip, _, err := net.ParseCIDR(addrs[0].String())
if err != nil {
return nil, err
}
return ip, nil
}
func flagsSet(flags net.Flags, test net.Flags) bool {
return flags&test != 0
}
func flagsClear(flags net.Flags, test net.Flags) bool {
return flags&test == 0
}

View File

@ -17,9 +17,14 @@ limitations under the License.
package util
import (
"bufio"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path"
"reflect"
"regexp"
@ -252,3 +257,188 @@ func SplitQualifiedName(str string) (string, string) {
func JoinQualifiedName(namespace, name string) string {
return path.Join(namespace, name)
}
type Route struct {
Interface string
Destination net.IP
Gateway net.IP
// TODO: add more fields here if needed
}
func getRoutes(input io.Reader) ([]Route, error) {
routes := []Route{}
if input == nil {
return nil, fmt.Errorf("input is nil")
}
scanner := bufio.NewReader(input)
for {
line, err := scanner.ReadString('\n')
if err == io.EOF {
break
}
//ignore the headers in the route info
if strings.HasPrefix(line, "Iface") {
continue
}
fields := strings.Fields(line)
routes = append(routes, Route{})
route := &routes[len(routes)-1]
route.Interface = fields[0]
ip, err := parseIP(fields[1])
if err != nil {
return nil, err
}
route.Destination = ip
ip, err = parseIP(fields[2])
if err != nil {
return nil, err
}
route.Gateway = ip
}
return routes, nil
}
func parseIP(str string) (net.IP, error) {
if str == "" {
return nil, fmt.Errorf("input is nil")
}
bytes, err := hex.DecodeString(str)
if err != nil {
return nil, err
}
//TODO add ipv6 support
if len(bytes) != net.IPv4len {
return nil, fmt.Errorf("only IPv4 is supported")
}
bytes[0], bytes[1], bytes[2], bytes[3] = bytes[3], bytes[2], bytes[1], bytes[0]
return net.IP(bytes), nil
}
func isInterfaceUp(intf *net.Interface) bool {
if intf == nil {
return false
}
if intf.Flags&net.FlagUp != 0 {
glog.V(4).Infof("Interface %v is up", intf.Name)
return true
}
return false
}
//getFinalIP method receives all the IP addrs of a Interface
//and returns a nil if the address is Loopback , Ipv6 or nil.
//It returns a valid IPv4 if an Ipv4 address is found in the array.
func getFinalIP(addrs []net.Addr) (net.IP, error) {
if len(addrs) > 0 {
for i := range addrs {
glog.V(4).Infof("Checking addr %s.", addrs[i].String())
ip, _, err := net.ParseCIDR(addrs[i].String())
if err != nil {
return nil, err
}
//Only IPv4
//TODO : add IPv6 support
if ip.To4() != nil {
if !ip.IsLoopback() {
glog.V(4).Infof("IP found %v", ip)
return ip, nil
} else {
glog.V(4).Infof("Loopback found %v", ip)
}
} else {
glog.V(4).Infof("%v is not a valid IPv4 address", ip)
}
}
}
return nil, nil
}
func getIPFromInterface(intfName string, nw networkInterfacer) (net.IP, error) {
intf, err := nw.InterfaceByName(intfName)
if err != nil {
return nil, err
}
if isInterfaceUp(intf) {
addrs, err := nw.Addrs(intf)
if err != nil {
return nil, err
}
glog.V(4).Infof("Interface %q has %d addresses :%v.", intfName, len(addrs), addrs)
finalIP, err := getFinalIP(addrs)
if err != nil {
return nil, err
}
if finalIP != nil {
glog.V(4).Infof("valid IPv4 address for interface %q found as %v.", intfName, finalIP)
return finalIP, nil
}
}
return nil, nil
}
//ChooseHostInterface is a method used fetch an IP for a daemon.
//It uses data from /proc/net/route file.
//For a node with no internet connection ,it returns error
//For a multi n/w interface node it returns the IP of the interface with gateway on it.
func ChooseHostInterface() (net.IP, error) {
inFile, err := os.Open("/proc/net/route")
if err != nil {
return nil, err
}
defer inFile.Close()
var nw networkInterfacer = networkInterface{}
return chooseHostInterfaceFromRoute(inFile, nw)
}
type networkInterfacer interface {
InterfaceByName(intfName string) (*net.Interface, error)
Addrs(intf *net.Interface) ([]net.Addr, error)
}
type networkInterface struct{}
func (_ networkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
intf, err := net.InterfaceByName(intfName)
if err != nil {
return nil, err
}
return intf, nil
}
func (_ networkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
addrs, err := intf.Addrs()
if err != nil {
return nil, err
}
return addrs, nil
}
func chooseHostInterfaceFromRoute(inFile io.Reader, nw networkInterfacer) (net.IP, error) {
routes, err := getRoutes(inFile)
if err != nil {
return nil, err
}
zero := net.IP{0, 0, 0, 0}
var finalIP net.IP
for i := range routes {
//find interface with gateway
if routes[i].Destination.Equal(zero) {
glog.V(4).Infof("Default route transits interface %q", routes[i].Interface)
finalIP, err := getIPFromInterface(routes[i].Interface, nw)
if err != nil {
return nil, err
}
if finalIP != nil {
glog.V(4).Infof("Choosing IP %v ", finalIP)
return finalIP, nil
}
}
}
glog.V(4).Infof("No valid IP found")
if finalIP == nil {
return nil, fmt.Errorf("Unable to select an IP.")
}
return nil, nil
}

View File

@ -19,7 +19,10 @@ package util
import (
"encoding/json"
"fmt"
"io"
"net"
"reflect"
"strings"
"testing"
"github.com/ghodss/yaml"
@ -332,3 +335,258 @@ func TestJoinQualifiedName(t *testing.T) {
}
}
}
const gatewayfirst = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
eth3 00000000 0100FE0A 0003 0 0 1024 00000000 0 0 0
eth3 0000FE0A 00000000 0001 0 0 0 0080FFFF 0 0 0
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
`
const gatewaylast = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
eth3 0000FE0A 00000000 0001 0 0 0 0080FFFF 0 0 0
eth3 00000000 0100FE0A 0003 0 0 1024 00000000 0 0 0
`
const gatewaymiddle = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
eth3 0000FE0A 00000000 0001 0 0 0 0080FFFF 0 0 0
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
eth3 00000000 0100FE0A 0003 0 0 1024 00000000 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
`
const noInternetConnection = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
`
const nothing = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
`
const gatewayfirstIpv6_1 = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
eth3 00000000 0100FE0A 0003 0 0 1024 00000000 0 0 0
eth3 0000FE0AA1 00000000 0001 0 0 0 0080FFFF 0 0 0
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
`
const gatewayfirstIpv6_2 = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
eth3 00000000 0100FE0AA1 0003 0 0 1024 00000000 0 0 0
eth3 0000FE0A 00000000 0001 0 0 0 0080FFFF 0 0 0
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
`
const route_Invalidhex = `Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
eth3 00000000 0100FE0AA 0003 0 0 1024 00000000 0 0 0
eth3 0000FE0A 00000000 0001 0 0 0 0080FFFF 0 0 0
docker0 000011AC 00000000 0001 0 0 0 0000FFFF 0 0 0
virbr0 007AA8C0 00000000 0001 0 0 0 00FFFFFF 0 0 0
`
func TestGetRoutes(t *testing.T) {
testCases := []struct {
tcase string
route string
expected int
}{
{"gatewayfirst", gatewayfirst, 4},
{"gatewaymiddle", gatewaymiddle, 4},
{"gatewaylast", gatewaylast, 4},
{"nothing", nothing, 0},
{"gatewayfirstIpv6_1", gatewayfirstIpv6_1, 0},
{"gatewayfirstIpv6_2", gatewayfirstIpv6_2, 0},
{"route_Invalidhex", route_Invalidhex, 0},
}
for _, tc := range testCases {
r := strings.NewReader(tc.route)
routes, err := getRoutes(r)
if len(routes) != tc.expected {
t.Errorf("case[%v]: expected %v, got %v .err : %v", tc.tcase, tc.expected, len(routes), err)
}
}
}
func TestParseIP(t *testing.T) {
testCases := []struct {
tcase string
ip string
success bool
expected net.IP
}{
{"empty", "", false, nil},
{"too short", "AA", false, nil},
{"too long", "0011223344", false, nil},
{"invalid", "invalid!", false, nil},
{"zero", "00000000", true, net.IP{0, 0, 0, 0}},
{"ffff", "FFFFFFFF", true, net.IP{0xff, 0xff, 0xff, 0xff}},
{"valid", "12345678", true, net.IP{120, 86, 52, 18}},
}
for _, tc := range testCases {
ip, err := parseIP(tc.ip)
if !ip.Equal(tc.expected) {
t.Errorf("case[%v]: expected %q, got %q . err : %v", tc.tcase, tc.expected, ip, err)
}
}
}
func TestIsInterfaceUp(t *testing.T) {
testCases := []struct {
tcase string
intf net.Interface
expected bool
}{
{"up", net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}, true},
{"down", net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: 0}, false},
{"nothing", net.Interface{}, false},
}
for _, tc := range testCases {
it := isInterfaceUp(&tc.intf)
if it != tc.expected {
t.Errorf("case[%v]: expected %v, got %v .", tc.tcase, tc.expected, it)
}
}
}
type addrStruct struct{ val string }
func (a addrStruct) Network() string {
return a.val
}
func (a addrStruct) String() string {
return a.val
}
func TestFinalIP(t *testing.T) {
testCases := []struct {
tcase string
addr []net.Addr
expected net.IP
}{
{"ipv6", []net.Addr{addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}}, nil},
{"invalidCIDR", []net.Addr{addrStruct{val: "fe80::2f7:67fff:fe6e:2956/64"}}, nil},
{"loopback", []net.Addr{addrStruct{val: "127.0.0.1/24"}}, nil},
{"ip4", []net.Addr{addrStruct{val: "10.254.12.132/17"}}, net.ParseIP("10.254.12.132")},
{"nothing", []net.Addr{}, nil},
}
for _, tc := range testCases {
ip, err := getFinalIP(tc.addr)
if !ip.Equal(tc.expected) {
t.Errorf("case[%v]: expected %v, got %v .err : %v", tc.tcase, tc.expected, ip, err)
}
}
}
func TestAddrs(t *testing.T) {
var nw networkInterfacer = validNetworkInterface{}
intf := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: 0}
addrs, err := nw.Addrs(&intf)
if err != nil {
t.Errorf("expected no error got : %v", err)
}
if len(addrs) != 2 {
t.Errorf("expected addrs: 2 got null")
}
}
type validNetworkInterface struct {
}
func (_ validNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
return &c, nil
}
func (_ validNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
var ifat []net.Addr
ifat = []net.Addr{
addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}, addrStruct{val: "10.254.71.145/17"}}
return ifat, nil
}
type validNetworkInterfacewithIpv6Only struct {
}
func (_ validNetworkInterfacewithIpv6Only) InterfaceByName(intfName string) (*net.Interface, error) {
c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
return &c, nil
}
func (_ validNetworkInterfacewithIpv6Only) Addrs(intf *net.Interface) ([]net.Addr, error) {
var ifat []net.Addr
ifat = []net.Addr{addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}}
return ifat, nil
}
type noNetworkInterface struct {
}
func (_ noNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
return nil, fmt.Errorf("unable get Interface")
}
func (_ noNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
return nil, nil
}
type networkInterfacewithNoAddrs struct {
}
func (_ networkInterfacewithNoAddrs) InterfaceByName(intfName string) (*net.Interface, error) {
c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
return &c, nil
}
func (_ networkInterfacewithNoAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
return nil, fmt.Errorf("unable get Addrs")
}
type networkInterfacewithIpv6addrs struct {
}
func (_ networkInterfacewithIpv6addrs) InterfaceByName(intfName string) (*net.Interface, error) {
c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
return &c, nil
}
func (_ networkInterfacewithIpv6addrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
var ifat []net.Addr
ifat = []net.Addr{addrStruct{val: "fe80::2f7:6ffff:fe6e:2956/64"}}
return ifat, nil
}
func TestGetIPFromInterface(t *testing.T) {
testCases := []struct {
tcase string
nwname string
nw networkInterfacer
expected net.IP
}{
{"valid", "eth3", validNetworkInterface{}, net.ParseIP("10.254.71.145")},
{"ipv6", "eth3", validNetworkInterfacewithIpv6Only{}, nil},
{"nothing", "eth3", noNetworkInterface{}, nil},
}
for _, tc := range testCases {
ip, err := getIPFromInterface(tc.nwname, tc.nw)
if !ip.Equal(tc.expected) {
t.Errorf("case[%v]: expected %v, got %+v .err : %v", tc.tcase, tc.expected, ip, err)
}
}
}
func TestChooseHostInterfaceFromRoute(t *testing.T) {
testCases := []struct {
tcase string
inFile io.Reader
nw networkInterfacer
expected net.IP
}{
{"valid_routefirst", strings.NewReader(gatewayfirst), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
{"valid_routelast", strings.NewReader(gatewaylast), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
{"valid_routemiddle", strings.NewReader(gatewaymiddle), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
{"valid_routemiddle_ipv6", strings.NewReader(gatewaymiddle), validNetworkInterfacewithIpv6Only{}, nil},
{"no internet connection", strings.NewReader(noInternetConnection), validNetworkInterface{}, nil},
{"no route", strings.NewReader(nothing), validNetworkInterface{}, nil},
{"no route file", nil, validNetworkInterface{}, nil},
{"no interfaces", nil, noNetworkInterface{}, nil},
{"no interface Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithNoAddrs{}, nil},
{"Invalid Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithIpv6addrs{}, nil},
}
for _, tc := range testCases {
ip, err := chooseHostInterfaceFromRoute(tc.inFile, tc.nw)
if !ip.Equal(tc.expected) {
t.Errorf("case[%v]: expected %v, got %+v .err : %v", tc.tcase, tc.expected, ip, err)
}
}
}