diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 17b7e64faf1..5971ef20282 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -235,7 +235,6 @@ go_test( "//pkg/proxy/apis/config:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/component-base/config:go_default_library", - "//staging/src/k8s.io/component-base/configz:go_default_library", "//vendor/github.com/google/go-cmp/cmp:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", @@ -246,6 +245,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ "//pkg/proxy/ipvs:go_default_library", @@ -253,6 +253,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ "//pkg/proxy/ipvs:go_default_library", @@ -260,6 +261,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ "//pkg/proxy/ipvs:go_default_library", @@ -267,6 +269,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:ios": [ "//pkg/proxy/ipvs:go_default_library", @@ -274,6 +277,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ "//pkg/proxy/ipvs:go_default_library", @@ -281,6 +285,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/proxy/ipvs:go_default_library", @@ -288,6 +293,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ "//pkg/proxy/ipvs:go_default_library", @@ -295,6 +301,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ "//pkg/proxy/ipvs:go_default_library", @@ -302,6 +309,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ "//pkg/proxy/ipvs:go_default_library", @@ -309,6 +317,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ "//pkg/proxy/ipvs:go_default_library", @@ -316,6 +325,7 @@ go_test( "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], "//conditions:default": [], }), diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 2692a64c211..eb69ea27fb9 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -91,10 +91,24 @@ func newProxyServer( return nil, fmt.Errorf("unable to register configz: %s", err) } + hostname, err := utilnode.GetHostname(config.HostnameOverride) + if err != nil { + return nil, err + } + + client, eventClient, err := createClients(config.ClientConnection, master) + if err != nil { + return nil, err + } + + nodeIP := detectNodeIP(client, hostname, config.BindAddress) + protocol := utiliptables.ProtocolIPv4 - if net.ParseIP(config.BindAddress).To4() == nil { - klog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress) + if utilsnet.IsIPv6(nodeIP) { + klog.V(0).Infof("kube-proxy node IP is an IPv6 address (%s), assume IPv6 operation", nodeIP.String()) protocol = utiliptables.ProtocolIPv6 + } else { + klog.V(0).Infof("kube-proxy node IP is an IPv4 address (%s), assume IPv4 operation", nodeIP.String()) } var iptInterface utiliptables.Interface @@ -131,16 +145,7 @@ func newProxyServer( metrics.SetShowHidden() } - client, eventClient, err := createClients(config.ClientConnection, master) - if err != nil { - return nil, err - } - // Create event recorder - hostname, err := utilnode.GetHostname(config.HostnameOverride) - if err != nil { - return nil, err - } eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, v1.EventSource{Component: "kube-proxy", Host: hostname}) @@ -175,15 +180,6 @@ func newProxyServer( klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs) } - nodeIP := net.ParseIP(config.BindAddress) - if nodeIP.IsUnspecified() { - nodeIP = utilnode.GetNodeIP(client, hostname) - if nodeIP == nil { - klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag") - nodeIP = net.ParseIP("127.0.0.1") - } - } - klog.V(2).Info("DetectLocalMode: '", string(detectLocalMode), "'") if proxyMode == proxyModeIPTables { @@ -422,6 +418,23 @@ func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, erro return nil, fmt.Errorf("event object not of type node") } +// detectNodeIP returns the nodeIP used by the proxier +// The order of precedence is: +// 1. config.bindAddress if bindAddress is not 0.0.0.0 or :: +// 2. the primary IP from the Node object, if set +// 3. if no IP is found it defaults to 127.0.0.1 and IPv4 +func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP { + nodeIP := net.ParseIP(bindAddress) + if nodeIP.IsUnspecified() { + nodeIP = utilnode.GetNodeIP(client, hostname) + } + if nodeIP == nil { + klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag") + nodeIP = net.ParseIP("127.0.0.1") + } + return nodeIP +} + func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) { mode := config.DetectLocalMode switch mode { diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 30a09c8576a..b7c492a1446 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -20,10 +20,15 @@ package app import ( "fmt" + "net" "reflect" "testing" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clientsetfake "k8s.io/client-go/kubernetes/fake" + proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" "k8s.io/kubernetes/pkg/proxy/ipvs" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" @@ -194,6 +199,111 @@ func Test_getDetectLocalMode(t *testing.T) { } } +func Test_detectNodeIP(t *testing.T) { + cases := []struct { + name string + nodeInfo *v1.Node + hostname string + bindAddress string + expectedIP net.IP + }{ + { + name: "Bind address IPv4 unicast address and no Node object", + nodeInfo: makeNodeWithAddresses("", "", ""), + hostname: "fakeHost", + bindAddress: "10.0.0.1", + expectedIP: net.ParseIP("10.0.0.1"), + }, + { + name: "Bind address IPv6 unicast address and no Node object", + nodeInfo: makeNodeWithAddresses("", "", ""), + hostname: "fakeHost", + bindAddress: "fd00:4321::2", + expectedIP: net.ParseIP("fd00:4321::2"), + }, + { + name: "No Valid IP found", + nodeInfo: makeNodeWithAddresses("", "", ""), + hostname: "fakeHost", + bindAddress: "", + expectedIP: net.ParseIP("127.0.0.1"), + }, + // Disabled because the GetNodeIP method has a backoff retry mechanism + // and the test takes more than 30 seconds + // ok k8s.io/kubernetes/cmd/kube-proxy/app 34.136s + // { + // name: "No Valid IP found and unspecified bind address", + // nodeInfo: makeNodeWithAddresses("", "", ""), + // hostname: "fakeHost", + // bindAddress: "0.0.0.0", + // expectedIP: net.ParseIP("127.0.0.1"), + // }, + { + name: "Bind address 0.0.0.0 and node with IPv4 InternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"), + hostname: "fakeHost", + bindAddress: "0.0.0.0", + expectedIP: net.ParseIP("192.168.1.1"), + }, + { + name: "Bind address :: and node with IPv4 InternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"), + hostname: "fakeHost", + bindAddress: "::", + expectedIP: net.ParseIP("192.168.1.1"), + }, + { + name: "Bind address 0.0.0.0 and node with IPv6 InternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"), + hostname: "fakeHost", + bindAddress: "0.0.0.0", + expectedIP: net.ParseIP("fd00:1234::1"), + }, + { + name: "Bind address :: and node with IPv6 InternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"), + hostname: "fakeHost", + bindAddress: "::", + expectedIP: net.ParseIP("fd00:1234::1"), + }, + { + name: "Bind address 0.0.0.0 and node with only IPv4 ExternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "", "90.90.90.90"), + hostname: "fakeHost", + bindAddress: "0.0.0.0", + expectedIP: net.ParseIP("90.90.90.90"), + }, + { + name: "Bind address :: and node with only IPv4 ExternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "", "90.90.90.90"), + hostname: "fakeHost", + bindAddress: "::", + expectedIP: net.ParseIP("90.90.90.90"), + }, + { + name: "Bind address 0.0.0.0 and node with only IPv6 ExternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "", "2001:db8::2"), + hostname: "fakeHost", + bindAddress: "0.0.0.0", + expectedIP: net.ParseIP("2001:db8::2"), + }, + { + name: "Bind address :: and node with only IPv6 ExternalIP set", + nodeInfo: makeNodeWithAddresses("fakeHost", "", "2001:db8::2"), + hostname: "fakeHost", + bindAddress: "::", + expectedIP: net.ParseIP("2001:db8::2"), + }, + } + for _, c := range cases { + client := clientsetfake.NewSimpleClientset(c.nodeInfo) + ip := detectNodeIP(client, c.hostname, c.bindAddress) + if !ip.Equal(c.expectedIP) { + t.Errorf("Case[%s] Expected IP %q got %q", c.name, c.expectedIP, ip) + } + } +} + func Test_getLocalDetector(t *testing.T) { cases := []struct { mode proxyconfigapi.LocalMode @@ -474,6 +584,35 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { } } +func makeNodeWithAddresses(name, internal, external string) *v1.Node { + if name == "" { + return &v1.Node{} + } + + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{}, + }, + } + + if internal != "" { + node.Status.Addresses = append(node.Status.Addresses, + v1.NodeAddress{Type: v1.NodeInternalIP, Address: internal}, + ) + } + + if external != "" { + node.Status.Addresses = append(node.Status.Addresses, + v1.NodeAddress{Type: v1.NodeExternalIP, Address: external}, + ) + } + + return node +} + func makeNodeWithPodCIDRs(cidrs ...string) *v1.Node { if len(cidrs) == 0 { return &v1.Node{} diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index ff542d7e7de..17ec2ce7640 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -36,36 +36,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/configz" kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" ) -// This test verifies that NewProxyServer does not crash when CleanupAndExit is true. -func TestProxyServerWithCleanupAndExit(t *testing.T) { - // Each bind address below is a separate test case - bindAddresses := []string{ - "0.0.0.0", - "::", - } - for _, addr := range bindAddresses { - options := NewOptions() - - options.config = &kubeproxyconfig.KubeProxyConfiguration{ - BindAddress: addr, - } - options.CleanupAndExit = true - - proxyserver, err := NewProxyServer(options) - - assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr) - assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr) - assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr) - - // Clean up config for next test case - configz.Delete(kubeproxyconfig.GroupName) - } -} - func TestGetConntrackMax(t *testing.T) { ncores := runtime.NumCPU() testCases := []struct {