From bc850adce41e42c431c5bc349d05ed2c96faef2a Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:15:43 +0800 Subject: [PATCH 1/9] add nodeport-addresses flag for kube-proxy --- cmd/kube-proxy/app/server.go | 2 ++ cmd/kube-proxy/app/server_others.go | 3 +++ cmd/kube-proxy/app/server_test.go | 4 ++++ pkg/kubemark/hollow_proxy.go | 1 + pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go | 4 +++- pkg/proxy/apis/kubeproxyconfig/types.go | 8 ++++++++ pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go | 8 ++++++++ 7 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 46d715ed498..ff30980da27 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -169,6 +169,8 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "NAT timeout for TCP connections in the CLOSE_WAIT state") fs.BoolVar(&o.config.EnableProfiling, "profiling", o.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.") fs.StringVar(&o.config.IPVS.Scheduler, "ipvs-scheduler", o.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs") + fs.StringSliceVar(&o.config.NodePortAddresses, "nodeport-addresses", o.config.NodePortAddresses, + "A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.") fs.Var(flag.NewMapStringBool(&o.config.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) } diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 30a8ac9bbf9..46b1acbe286 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -161,6 +161,7 @@ func newProxyServer( nodeIP, recorder, healthzUpdater, + config.NodePortAddresses, ) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) @@ -196,6 +197,7 @@ func newProxyServer( recorder, healthzServer, config.IPVS.Scheduler, + config.NodePortAddresses, ) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) @@ -226,6 +228,7 @@ func newProxyServer( config.IPTables.SyncPeriod.Duration, config.IPTables.MinSyncPeriod.Duration, config.UDPIdleTimeout.Duration, + config.NodePortAddresses, ) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 431ef73f53d..8d547f825b9 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -433,6 +433,9 @@ oomScoreAdj: 17 portRange: "2-7" resourceContainer: /foo udpIdleTimeout: 123ms +nodePortAddresses: + - "10.20.30.40/16" + - "fd00:1::0/64" ` testCases := []struct { @@ -545,6 +548,7 @@ udpIdleTimeout: 123ms PortRange: "2-7", ResourceContainer: "/foo", UDPIdleTimeout: metav1.Duration{Duration: 123 * time.Millisecond}, + NodePortAddresses: []string{"10.20.30.40/16", "fd00:1::0/64"}, } options := NewOptions() diff --git a/pkg/kubemark/hollow_proxy.go b/pkg/kubemark/hollow_proxy.go index 27ca1128926..8ed70fec124 100644 --- a/pkg/kubemark/hollow_proxy.go +++ b/pkg/kubemark/hollow_proxy.go @@ -92,6 +92,7 @@ func NewHollowProxyOrDie( getNodeIP(client, nodeName), recorder, nil, + []string{}, ) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) diff --git a/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go b/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go index 2f3347fcf2a..b96464c44c7 100644 --- a/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go +++ b/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go @@ -44,7 +44,9 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.IPTables.MasqueradeBit = utilpointer.Int32Ptr(c.Int31()) obj.MetricsBindAddress = fmt.Sprintf("%d.%d.%d.%d:%d", c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(65536)) obj.OOMScoreAdj = utilpointer.Int32Ptr(c.Int31()) - obj.ResourceContainer = c.RandString() + obj.ResourceContainer = "foo" + obj.ClientConnection.ContentType = "bar" + obj.NodePortAddresses = []string{"1.2.3.0/24"} }, } } diff --git a/pkg/proxy/apis/kubeproxyconfig/types.go b/pkg/proxy/apis/kubeproxyconfig/types.go index fd4d78f8b7b..b9e662826ae 100644 --- a/pkg/proxy/apis/kubeproxyconfig/types.go +++ b/pkg/proxy/apis/kubeproxyconfig/types.go @@ -144,6 +144,14 @@ type KubeProxyConfiguration struct { // configSyncPeriod is how often configuration from the apiserver is refreshed. Must be greater // than 0. ConfigSyncPeriod metav1.Duration + // nodePortAddresses is the --nodeport-addresses value for kube-proxy process. Values must be valid + // IP blocks. These values are as a parameter to select the interfaces where nodeport works. + // In case someone would like to expose a service on localhost for local visit and some other interfaces for + // particular purpose, a list of IP blocks would do that. + // If set it to "127.0.0.0/8", kube-proxy will only select the loopback interface for NodePort. + // If set it to a non-zero IP block, kube-proxy will filter that down to just the IPs that applied to the node. + // An empty string slice is meant to select all network interfaces. + NodePortAddresses []string } // Currently, three modes of proxy are available in Linux platform: 'userspace' (older, going to be EOL), 'iptables' diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go index fabec668f00..b0e44c1fe12 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go @@ -140,6 +140,14 @@ type KubeProxyConfiguration struct { // configSyncPeriod is how often configuration from the apiserver is refreshed. Must be greater // than 0. ConfigSyncPeriod metav1.Duration `json:"configSyncPeriod"` + // nodePortAddresses is the --nodeport-addresses value for kube-proxy process. Values must be valid + // IP blocks. These values are as a parameter to select the interfaces where nodeport works. + // In case someone would like to expose a service on localhost for local visit and some other interfaces for + // particular purpose, a list of IP blocks would do that. + // If set it to "127.0.0.0/8", kube-proxy will only select the loopback interface for NodePort. + // If set it to a non-zero IP block, kube-proxy will filter that down to just the IPs that applied to the node. + // An empty string slice is meant to select all network interfaces. + NodePortAddresses []string `json:"nodePortAddresses"` } // Currently, three modes of proxy are available in Linux platform: 'userspace' (older, going to be EOL), 'iptables' From dbcb2c9b27c16da8bd35e0f0898af552bb5ba9fd Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:17:20 +0800 Subject: [PATCH 2/9] validate nodeport-addresses --- .../kubeproxyconfig/validation/validation.go | 15 ++++ .../validation/validation_test.go | 70 +++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/pkg/proxy/apis/kubeproxyconfig/validation/validation.go b/pkg/proxy/apis/kubeproxyconfig/validation/validation.go index e2453af72c4..d28453e2bb1 100644 --- a/pkg/proxy/apis/kubeproxyconfig/validation/validation.go +++ b/pkg/proxy/apis/kubeproxyconfig/validation/validation.go @@ -73,6 +73,8 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList { allErrs = append(allErrs, field.Invalid(newPath.Child("PortRange"), config.PortRange, "must be a valid port range (e.g. 300-2000)")) } + allErrs = append(allErrs, validateKubeProxyNodePortAddress(config.NodePortAddresses, newPath.Child("NodePortAddresses"))...) + return allErrs } @@ -238,3 +240,16 @@ func validateIPVSSchedulerMethod(scheduler kubeproxyconfig.IPVSSchedulerMethod, } return allErrs } + +func validateKubeProxyNodePortAddress(nodePortAddresses []string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + for i := range nodePortAddresses { + if _, _, err := net.ParseCIDR(nodePortAddresses[i]); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, nodePortAddresses, "must be a valid IP block")) + break + } + } + + return allErrs +} diff --git a/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go b/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go index 8815e0af077..aeb53a3e5a2 100644 --- a/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go +++ b/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go @@ -679,3 +679,73 @@ func TestValidateIPVSSchedulerMethod(t *testing.T) { } } } + +func TestValidateKubeProxyNodePortAddress(t *testing.T) { + newPath := field.NewPath("KubeProxyConfiguration") + + successCases := []struct { + addresses []string + }{ + {[]string{}}, + {[]string{"127.0.0.0/8"}}, + {[]string{"0.0.0.0/0"}}, + {[]string{"::/0"}}, + {[]string{"127.0.0.1/32", "1.2.3.0/24"}}, + {[]string{"127.0.0.0/8"}}, + {[]string{"127.0.0.1/32"}}, + {[]string{"::1/128"}}, + {[]string{"1.2.3.4/32"}}, + {[]string{"10.20.30.0/24"}}, + {[]string{"10.20.0.0/16", "100.200.0.0/16"}}, + {[]string{"10.0.0.0/8"}}, + {[]string{"2001:db8::/32"}}, + } + + for _, successCase := range successCases { + if errs := validateKubeProxyNodePortAddress(successCase.addresses, newPath.Child("NodePortAddresses")); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + } + + errorCases := []struct { + addresses []string + msg string + }{ + { + addresses: []string{"foo"}, + msg: "must be a valid IP block", + }, + { + addresses: []string{"1.2.3"}, + msg: "must be a valid IP block", + }, + { + addresses: []string{""}, + msg: "must be a valid IP block", + }, + { + addresses: []string{"10.20.30.40"}, + msg: "must be a valid IP block", + }, + { + addresses: []string{"::1"}, + msg: "must be a valid IP block", + }, + { + addresses: []string{"2001:db8:1"}, + msg: "must be a valid IP block", + }, + { + addresses: []string{"2001:db8:xyz/64"}, + msg: "must be a valid IP block", + }, + } + + for _, errorCase := range errorCases { + if errs := validateKubeProxyNodePortAddress(errorCase.addresses, newPath.Child("NodePortAddresses")); len(errs) == 0 { + t.Errorf("expected failure for %s", errorCase.msg) + } else if !strings.Contains(errs[0].Error(), errorCase.msg) { + t.Errorf("unexpected error: %v, expected: %s", errs[0], errorCase.msg) + } + } +} From bf565305eee684f32fe920ecccc9786da40901f4 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:23:33 +0800 Subject: [PATCH 3/9] create netwowrk interface util --- pkg/proxy/util/interface.go | 83 ++++++++++++++ pkg/proxy/util/utils.go | 84 ++++++++++++++ pkg/proxy/util/utils_test.go | 217 +++++++++++++++++++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 pkg/proxy/util/interface.go diff --git a/pkg/proxy/util/interface.go b/pkg/proxy/util/interface.go new file mode 100644 index 00000000000..2c6f02c9134 --- /dev/null +++ b/pkg/proxy/util/interface.go @@ -0,0 +1,83 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "net" +) + +// NetworkInterface defines an interface for several net library functions. Production +// code will forward to net library functions, and unit tests will override the methods +// for testing purposes. +type NetworkInterface interface { + Addrs(intf *net.Interface) ([]net.Addr, error) + Interfaces() ([]net.Interface, error) +} + +// RealNetwork implements the NetworkInterface interface for production code, just +// wrapping the underlying net library function calls. +type RealNetwork struct{} + +// Addrs wraps net.Interface.Addrs() +func (_ RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { + return intf.Addrs() +} + +// Interfaces wraps net.Interfaces() +func (_ RealNetwork) Interfaces() ([]net.Interface, error) { + return net.Interfaces() +} + +// RealNetwork implements the NetworkInterface interface for production code, just +// wrapping the underlying net library function calls. +type FakeNetwork struct { + NetworkInterfaces []net.Interface + // The key of map Addrs is interface name + Address map[string][]net.Addr +} + +func NewFakeNetwork() *FakeNetwork { + return &FakeNetwork{ + NetworkInterfaces: make([]net.Interface, 0), + Address: make(map[string][]net.Addr), + } +} + +// AddInterfaceAddr create an interface and its associated addresses for FakeNetwork implementation. +func (f *FakeNetwork) AddInterfaceAddr(intf *net.Interface, addrs []net.Addr) { + f.NetworkInterfaces = append(f.NetworkInterfaces, *intf) + f.Address[intf.Name] = addrs +} + +// Addrs is part of FakeNetwork interface. +func (f *FakeNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { + return f.Address[intf.Name], nil +} + +// Interfaces is part of FakeNetwork interface. +func (f *FakeNetwork) Interfaces() ([]net.Interface, error) { + return f.NetworkInterfaces, nil +} + +type AddrStruct struct{ Val string } + +func (a AddrStruct) Network() string { + return a.Val +} +func (a AddrStruct) String() string { + return a.Val +} diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index cac0140c386..63ae1f7c7b6 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -17,15 +17,32 @@ limitations under the License. package util import ( + "fmt" "net" "k8s.io/apimachinery/pkg/types" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/helper" + "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" + "k8s.io/kubernetes/pkg/util/conntrack" "github.com/golang/glog" ) +const ( + IPv4ZeroCIDR = "0.0.0.0/0" + IPv6ZeroCIDR = "::/0" +) + +func IsZeroCIDR(cidr string) bool { + if cidr == IPv4ZeroCIDR || cidr == IPv6ZeroCIDR { + return true + } + return false +} + func IsLocalIP(ip string) (bool, error) { addrs, err := net.InterfaceAddrs() if err != nil { @@ -56,3 +73,70 @@ func ShouldSkipService(svcName types.NamespacedName, service *api.Service) bool } return false } + +// GetNodeAddresses list all match node IP addresses based on given cidr list. Inject NetworkInterface for test purpose. +// We expect the cidr list passed in is already validated. +// Given `default-route`, it will the IP of host interface having default route. +// Given an empty input `[]`, it will return `0.0.0.0/0` and `::/0` directly. +func GetNodeAddresses(cidrs []string, nw NetworkInterface) (sets.String, error) { + uniqueAddressList := sets.NewString() + if len(cidrs) == 0 { + uniqueAddressList.Insert(IPv4ZeroCIDR) + uniqueAddressList.Insert(IPv6ZeroCIDR) + return uniqueAddressList, nil + } + // First round of iteration to pick out `0.0.0.0/0` or `::/0` for the sake of excluding non-zero IPs. + for _, cidr := range cidrs { + if IsZeroCIDR(cidr) { + uniqueAddressList.Insert(cidr) + } + } + // Second round of iteration to parse IPs based on cidr. + for _, cidr := range cidrs { + if IsZeroCIDR(cidr) { + continue + } + if cidr == string(kubeproxyconfig.DefaultRoute) { + hostIP, err := utilnet.ChooseHostInterface() + if err != nil { + return nil, fmt.Errorf("error selecting a host interface having default route, error: %v", err) + } + if conntrack.IsIPv6(hostIP) && !uniqueAddressList.Has(IPv6ZeroCIDR) { + uniqueAddressList.Insert(hostIP.String()) + } + if !conntrack.IsIPv6(hostIP) && !uniqueAddressList.Has(IPv4ZeroCIDR) { + uniqueAddressList.Insert(hostIP.String()) + } + continue + } + _, ipNet, _ := net.ParseCIDR(cidr) + itfs, err := nw.Interfaces() + if err != nil { + return nil, fmt.Errorf("error listing all interfaces from host, error: %v", err) + } + for _, itf := range itfs { + addrs, err := nw.Addrs(&itf) + if err != nil { + return nil, fmt.Errorf("error getting address from interface %s, error: %v", itf.Name, err) + } + for _, addr := range addrs { + if addr == nil { + continue + } + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, fmt.Errorf("error parsing CIDR for interface %s, error: %v", itf.Name, err) + } + if ipNet.Contains(ip) { + if conntrack.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) { + uniqueAddressList.Insert(ip.String()) + } + if !conntrack.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) { + uniqueAddressList.Insert(ip.String()) + } + } + } + } + } + return uniqueAddressList, nil +} diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go index 5810788714c..74221c2d9e8 100644 --- a/pkg/proxy/util/utils_test.go +++ b/pkg/proxy/util/utils_test.go @@ -17,10 +17,12 @@ limitations under the License. package util import ( + "net" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kubernetes/pkg/apis/core" ) @@ -109,3 +111,218 @@ func TestShouldSkipService(t *testing.T) { } } } + +type InterfaceAddrsPair struct { + itf net.Interface + addrs []net.Addr +} + +func TestGetNodeAddressses(t *testing.T) { + testCases := []struct { + cidrs []string + nw *FakeNetwork + itfAddrsPairs []InterfaceAddrsPair + expected sets.String + }{ + { // case 0 + cidrs: []string{"10.20.30.0/24"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + }, + { + itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "100.200.201.1/24"}}, + }, + }, + expected: sets.NewString("10.20.30.51"), + }, + { // case 1 + cidrs: []string{"0.0.0.0/0"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + }, + }, + expected: sets.NewString("0.0.0.0/0"), + }, + { // case 2 + cidrs: []string{"2001:db8::/32", "::1/128"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + }, + }, + expected: sets.NewString("2001:db8::1", "::1"), + }, + { // case 3 + cidrs: []string{"::/0"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + }, + }, + expected: sets.NewString("::/0"), + }, + { // case 4 + cidrs: []string{"127.0.0.1/32"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + }, + }, + expected: sets.NewString("127.0.0.1"), + }, + { // case 5 + cidrs: []string{"127.0.0.0/8"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "127.0.1.1/8"}}, + }, + }, + expected: sets.NewString("127.0.1.1"), + }, + { // case 6 + cidrs: []string{"10.20.30.0/24", "100.200.201.0/24"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + }, + { + itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "100.200.201.1/24"}}, + }, + }, + expected: sets.NewString("10.20.30.51", "100.200.201.1"), + }, + { // case 7 + cidrs: []string{"10.20.30.0/24", "100.200.201.0/24"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "192.168.1.2/24"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + }, + }, + expected: sets.NewString(), + }, + { // case 8 + cidrs: []string{}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "192.168.1.2/24"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + }, + }, + expected: sets.NewString("0.0.0.0/0", "::/0"), + }, + { // case 9 + cidrs: []string{}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + }, + }, + expected: sets.NewString("0.0.0.0/0", "::/0"), + }, + { // case 9 + cidrs: []string{"1.2.3.0/24", "0.0.0.0/0"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}}, + }, + }, + expected: sets.NewString("0.0.0.0/0"), + }, + { // case 10 + cidrs: []string{"0.0.0.0/0", "1.2.3.0/24", "::1/128"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + }, + }, + expected: sets.NewString("0.0.0.0/0", "::1"), + }, + { // case 11 + cidrs: []string{"::/0", "1.2.3.0/24", "::1/128"}, + nw: NewFakeNetwork(), + itfAddrsPairs: []InterfaceAddrsPair{ + { + itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}}, + }, + { + itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, + addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + }, + }, + expected: sets.NewString("::/0", "1.2.3.4"), + }, + } + + for i := range testCases { + for _, pair := range testCases[i].itfAddrsPairs { + testCases[i].nw.AddInterfaceAddr(&pair.itf, pair.addrs) + } + addrList, err := GetNodeAddresses(testCases[i].cidrs, testCases[i].nw) + if err != nil { + t.Errorf("case [%d], unexpected error: %v", i, err) + } + if !addrList.Equal(testCases[i].expected) { + t.Errorf("case [%d], unexpected mismatch, expected: %v, got: %v", i, testCases[i].expected, addrList) + } + } +} From ddfa04e8f4d795453a7f8cbf04f533181da29032 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:30:19 +0800 Subject: [PATCH 4/9] iptables part implementation --- pkg/proxy/iptables/proxier.go | 45 +++++++++++++++-- pkg/proxy/iptables/proxier_test.go | 22 ++++++++- pkg/proxy/util/network.go | 45 +++++++++++++++++ pkg/proxy/util/testing/fake.go | 64 +++++++++++++++++++++++++ pkg/proxy/util/utils.go | 27 ++++------- pkg/proxy/util/utils_test.go | 77 +++++++++++++++--------------- 6 files changed, 217 insertions(+), 63 deletions(-) create mode 100644 pkg/proxy/util/network.go create mode 100644 pkg/proxy/util/testing/fake.go diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 02b9eee3be0..3ec67c6e50f 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -341,6 +341,12 @@ type Proxier struct { filterRules *bytes.Buffer natChains *bytes.Buffer natRules *bytes.Buffer + + // Values are as a parameter to select the interfaces where nodeport works. + nodePortAddresses []string + // networkInterfacer defines an interface for several net library functions. + // Inject for test purpose. + networkInterfacer utilproxy.NetworkInterfacer } // listenPortOpener opens ports by calling bind() and listen(). @@ -371,6 +377,7 @@ func NewProxier(ipt utiliptables.Interface, nodeIP net.IP, recorder record.EventRecorder, healthzServer healthcheck.HealthzUpdater, + nodePortAddresses []string, ) (*Proxier, error) { // Set the route_localnet sysctl we need for if err := sysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil { @@ -422,6 +429,8 @@ func NewProxier(ipt utiliptables.Interface, filterRules: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), natRules: bytes.NewBuffer(nil), + nodePortAddresses: nodePortAddresses, + networkInterfacer: utilproxy.RealNetwork{}, } burstSyncs := 2 glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) @@ -1289,11 +1298,37 @@ func (proxier *Proxier) syncProxyRules() { // Finally, tail-call to the nodeports chain. This needs to be after all // other service portal rules. - writeLine(proxier.natRules, - "-A", string(kubeServicesChain), - "-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`, - "-m", "addrtype", "--dst-type", "LOCAL", - "-j", string(kubeNodePortsChain)) + addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) + if err != nil { + glog.Errorf("Failed to get node ip address matching nodeport cidr") + } else { + isIPv6 := proxier.iptables.IsIpv6() + for address := range addresses { + // TODO(thockin, m1093782566): If/when we have dual-stack support we will want to distinguish v4 from v6 zero-CIDRs. + if utilproxy.IsZeroCIDR(address) { + args = append(args[:0], + "-A", string(kubeServicesChain), + "-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`, + "-m", "addrtype", "--dst-type", "LOCAL", + "-j", string(kubeNodePortsChain)) + writeLine(proxier.natRules, args...) + // Nothing else matters after the zero CIDR. + break + } + // Ignore IP addresses with incorrect version + if isIPv6 && !conntrack.IsIPv6String(address) || !isIPv6 && conntrack.IsIPv6String(address) { + glog.Errorf("IP address %s has incorrect IP version", address) + continue + } + // create nodeport rules for each IP one by one + args = append(args[:0], + "-A", string(kubeServicesChain), + "-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`, + "-d", address, + "-j", string(kubeNodePortsChain)) + writeLine(proxier.natRules, args...) + } + } // If the masqueradeMark has been added then we want to forward that same // traffic, this allows NodePort traffic to be forwarded even if the default diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 046411d745f..e335df23ab6 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -34,6 +34,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/proxy" utilproxy "k8s.io/kubernetes/pkg/proxy/util" + utilproxytest "k8s.io/kubernetes/pkg/proxy/util/testing" "k8s.io/kubernetes/pkg/util/async" "k8s.io/kubernetes/pkg/util/conntrack" utiliptables "k8s.io/kubernetes/pkg/util/iptables" @@ -405,6 +406,8 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { filterRules: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), natRules: bytes.NewBuffer(nil), + nodePortAddresses: make([]string, 0), + networkInterfacer: utilproxytest.NewFakeNetwork(), } p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1) return p @@ -769,6 +772,14 @@ func TestNodePort(t *testing.T) { }), ) + itf := net.Interface{Index: 0, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0} + addrs := []net.Addr{utilproxytest.AddrStruct{Val: "127.0.0.1/16"}} + itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} + addrs1 := []net.Addr{utilproxytest.AddrStruct{Val: "::1/128"}} + fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs) + fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) + fp.nodePortAddresses = []string{} + fp.syncProxyRules() proto := strings.ToLower(string(api.ProtocolTCP)) @@ -1001,6 +1012,11 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable }), ) + itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} + addrs := []net.Addr{utilproxytest.AddrStruct{Val: "10.20.30.51/24"}} + fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs) + fp.nodePortAddresses = []string{"10.20.30.0/24"} + fp.syncProxyRules() proto := strings.ToLower(string(api.ProtocolTCP)) @@ -1013,11 +1029,15 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable if !hasJump(kubeNodePortRules, lbChain, "", svcNodePort) { errorf(fmt.Sprintf("Failed to find jump to lb chain %v", lbChain), kubeNodePortRules, t) } - if !hasJump(kubeNodePortRules, string(KubeMarkMasqChain), "", svcNodePort) { errorf(fmt.Sprintf("Failed to find jump to %s chain for destination IP %d", KubeMarkMasqChain, svcNodePort), kubeNodePortRules, t) } + kubeServiceRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeServiceRules, string(kubeNodePortsChain), "10.20.30.51", 0) { + errorf(fmt.Sprintf("Failed to find jump to KUBE-NODEPORTS chain %v", string(kubeNodePortsChain)), kubeServiceRules, t) + } + svcChain := string(servicePortChainName(svcPortName.String(), proto)) lbRules := ipt.GetRules(lbChain) if hasJump(lbRules, nonLocalEpChain, "", 0) { diff --git a/pkg/proxy/util/network.go b/pkg/proxy/util/network.go new file mode 100644 index 00000000000..8a4461247e3 --- /dev/null +++ b/pkg/proxy/util/network.go @@ -0,0 +1,45 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "net" +) + +// NetworkInterfacer defines an interface for several net library functions. Production +// code will forward to net library functions, and unit tests will override the methods +// for testing purposes. +type NetworkInterfacer interface { + Addrs(intf *net.Interface) ([]net.Addr, error) + Interfaces() ([]net.Interface, error) +} + +// RealNetwork implements the NetworkInterfacer interface for production code, just +// wrapping the underlying net library function calls. +type RealNetwork struct{} + +// Addrs wraps net.Interface.Addrs(), it's a part of NetworkInterfacer interface. +func (_ RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { + return intf.Addrs() +} + +// Interfaces wraps net.Interfaces(), it's a part of NetworkInterfacer interface. +func (_ RealNetwork) Interfaces() ([]net.Interface, error) { + return net.Interfaces() +} + +var _ NetworkInterfacer = &RealNetwork{} diff --git a/pkg/proxy/util/testing/fake.go b/pkg/proxy/util/testing/fake.go new file mode 100644 index 00000000000..a041785288e --- /dev/null +++ b/pkg/proxy/util/testing/fake.go @@ -0,0 +1,64 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import "net" + +// FakeNetwork implements the NetworkInterfacer interface for test purpose. +type FakeNetwork struct { + NetworkInterfaces []net.Interface + // The key of map Addrs is the network interface name + Address map[string][]net.Addr +} + +func NewFakeNetwork() *FakeNetwork { + return &FakeNetwork{ + NetworkInterfaces: make([]net.Interface, 0), + Address: make(map[string][]net.Addr), + } +} + +// AddInterfaceAddr create an interface and its associated addresses for FakeNetwork implementation. +func (f *FakeNetwork) AddInterfaceAddr(intf *net.Interface, addrs []net.Addr) { + f.NetworkInterfaces = append(f.NetworkInterfaces, *intf) + f.Address[intf.Name] = addrs +} + +// Addrs is part of NetworkInterfacer interface. +func (f *FakeNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { + return f.Address[intf.Name], nil +} + +// Interfaces is part of NetworkInterfacer interface. +func (f *FakeNetwork) Interfaces() ([]net.Interface, error) { + return f.NetworkInterfaces, nil +} + +// AddrStruct implements the net.Addr for test purpose. +type AddrStruct struct{ Val string } + +// Network is part of net.Addr interface. +func (a AddrStruct) Network() string { + return a.Val +} + +// String is part of net.Addr interface. +func (a AddrStruct) String() string { + return a.Val +} + +var _ net.Addr = &AddrStruct{} diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index 63ae1f7c7b6..037cbdd1e9c 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -21,11 +21,9 @@ import ( "net" "k8s.io/apimachinery/pkg/types" - utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/helper" - "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" "k8s.io/kubernetes/pkg/util/conntrack" "github.com/golang/glog" @@ -74,11 +72,15 @@ func ShouldSkipService(svcName types.NamespacedName, service *api.Service) bool return false } -// GetNodeAddresses list all match node IP addresses based on given cidr list. Inject NetworkInterface for test purpose. -// We expect the cidr list passed in is already validated. -// Given `default-route`, it will the IP of host interface having default route. +// GetNodeAddresses return all matched node IP addresses based on given cidr slice. +// Some callers, e.g. IPVS proxier, need concrete IPs, not ranges, which is why this exists. +// NetworkInterfacer is injected for test purpose. +// We expect the cidrs passed in is already validated. // Given an empty input `[]`, it will return `0.0.0.0/0` and `::/0` directly. -func GetNodeAddresses(cidrs []string, nw NetworkInterface) (sets.String, error) { +// If multiple cidrs is given, it will return the minimal IP sets, e.g. given input `[1.2.0.0/16, 0.0.0.0/0]`, it will +// only return `0.0.0.0/0`. +// NOTE: GetNodeAddresses only accepts CIDRs, if you want concrete IPs, e.g. 1.2.3.4, then the input should be 1.2.3.4/32. +func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) { uniqueAddressList := sets.NewString() if len(cidrs) == 0 { uniqueAddressList.Insert(IPv4ZeroCIDR) @@ -96,19 +98,6 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterface) (sets.String, error) if IsZeroCIDR(cidr) { continue } - if cidr == string(kubeproxyconfig.DefaultRoute) { - hostIP, err := utilnet.ChooseHostInterface() - if err != nil { - return nil, fmt.Errorf("error selecting a host interface having default route, error: %v", err) - } - if conntrack.IsIPv6(hostIP) && !uniqueAddressList.Has(IPv6ZeroCIDR) { - uniqueAddressList.Insert(hostIP.String()) - } - if !conntrack.IsIPv6(hostIP) && !uniqueAddressList.Has(IPv4ZeroCIDR) { - uniqueAddressList.Insert(hostIP.String()) - } - continue - } _, ipNet, _ := net.ParseCIDR(cidr) itfs, err := nw.Interfaces() if err != nil { diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go index 74221c2d9e8..aaac610ed48 100644 --- a/pkg/proxy/util/utils_test.go +++ b/pkg/proxy/util/utils_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kubernetes/pkg/apis/core" + fake "k8s.io/kubernetes/pkg/proxy/util/testing" ) func TestShouldSkipService(t *testing.T) { @@ -120,193 +121,193 @@ type InterfaceAddrsPair struct { func TestGetNodeAddressses(t *testing.T) { testCases := []struct { cidrs []string - nw *FakeNetwork + nw *fake.FakeNetwork itfAddrsPairs []InterfaceAddrsPair expected sets.String }{ { // case 0 cidrs: []string{"10.20.30.0/24"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, }, { itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "100.200.201.1/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "100.200.201.1/24"}}, }, }, expected: sets.NewString("10.20.30.51"), }, { // case 1 cidrs: []string{"0.0.0.0/0"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, }, }, expected: sets.NewString("0.0.0.0/0"), }, { // case 2 cidrs: []string{"2001:db8::/32", "::1/128"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, }, }, expected: sets.NewString("2001:db8::1", "::1"), }, { // case 3 cidrs: []string{"::/0"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, }, }, expected: sets.NewString("::/0"), }, { // case 4 cidrs: []string{"127.0.0.1/32"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, }, }, expected: sets.NewString("127.0.0.1"), }, { // case 5 cidrs: []string{"127.0.0.0/8"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "127.0.1.1/8"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "127.0.1.1/8"}}, }, }, expected: sets.NewString("127.0.1.1"), }, { // case 6 cidrs: []string{"10.20.30.0/24", "100.200.201.0/24"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}}, }, { itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "100.200.201.1/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "100.200.201.1/24"}}, }, }, expected: sets.NewString("10.20.30.51", "100.200.201.1"), }, { // case 7 cidrs: []string{"10.20.30.0/24", "100.200.201.0/24"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "192.168.1.2/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "192.168.1.2/24"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, }, }, expected: sets.NewString(), }, { // case 8 cidrs: []string{}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "192.168.1.2/24"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "192.168.1.2/24"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}}, }, }, expected: sets.NewString("0.0.0.0/0", "::/0"), }, { // case 9 cidrs: []string{}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, }, }, expected: sets.NewString("0.0.0.0/0", "::/0"), }, { // case 9 cidrs: []string{"1.2.3.0/24", "0.0.0.0/0"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}}, }, }, expected: sets.NewString("0.0.0.0/0"), }, { // case 10 cidrs: []string{"0.0.0.0/0", "1.2.3.0/24", "::1/128"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, }, }, expected: sets.NewString("0.0.0.0/0", "::1"), }, { // case 11 cidrs: []string{"::/0", "1.2.3.0/24", "::1/128"}, - nw: NewFakeNetwork(), + nw: fake.NewFakeNetwork(), itfAddrsPairs: []InterfaceAddrsPair{ { itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}}, }, { itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}, - addrs: []net.Addr{AddrStruct{Val: "::1/128"}}, + addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}}, }, }, expected: sets.NewString("::/0", "1.2.3.4"), From c537ff54e7abe5594a957b92d781ac1710352346 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:30:33 +0800 Subject: [PATCH 5/9] ipvs part implementation --- pkg/proxy/ipvs/proxier.go | 67 +++++++++++++++++---------- pkg/proxy/ipvs/proxier_test.go | 21 +++++++-- pkg/proxy/util/interface.go | 83 ---------------------------------- 3 files changed, 62 insertions(+), 109 deletions(-) delete mode 100644 pkg/proxy/util/interface.go diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 40e50680f5f..ad2f74449cb 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -170,6 +170,11 @@ type Proxier struct { // lbWhiteListIPSet is the hash:ip,port,net type ipset where stores all service load balancer ingress IP:Port,sourceCIDR pair, any packets // from the source CIDR visit ingress IP:Port can pass through. lbWhiteListCIDRSet *IPSet + // Values are as a parameter to select the interfaces where nodeport works. + nodePortAddresses []string + // networkInterfacer defines an interface for several net library functions. + // Inject for test purpose. + networkInterfacer utilproxy.NetworkInterfacer } // IPGetter helps get node network interface IP @@ -253,6 +258,7 @@ func NewProxier(ipt utiliptables.Interface, recorder record.EventRecorder, healthzServer healthcheck.HealthzUpdater, scheduler string, + nodePortAddresses []string, ) (*Proxier, error) { // Set the route_localnet sysctl we need for if err := sysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil { @@ -336,6 +342,8 @@ func NewProxier(ipt utiliptables.Interface, lbWhiteListCIDRSet: NewIPSet(ipset, KubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, isIPv6), nodePortSetTCP: NewIPSet(ipset, KubeNodePortSetTCP, utilipset.BitmapPort, false), nodePortSetUDP: NewIPSet(ipset, KubeNodePortSetUDP, utilipset.BitmapPort, false), + nodePortAddresses: nodePortAddresses, + networkInterfacer: utilproxy.RealNetwork{}, } burstSyncs := 2 glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) @@ -1134,6 +1142,7 @@ func (proxier *Proxier) syncProxyRules() { } var nodePortSet *IPSet switch protocol { + case "tcp": nodePortSet = proxier.nodePortSetTCP case "udp": @@ -1152,31 +1161,43 @@ func (proxier *Proxier) syncProxyRules() { } // Build ipvs kernel routes for each node ip address - nodeIPs, err := proxier.ipGetter.NodeIPs() + nodeIPs := make([]net.IP, 0) + addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) if err != nil { - glog.Errorf("Failed to get node IP, err: %v", err) - } else { - for _, nodeIP := range nodeIPs { - // ipvs call - serv := &utilipvs.VirtualServer{ - Address: nodeIP, - Port: uint16(svcInfo.nodePort), - Protocol: string(svcInfo.protocol), - Scheduler: proxier.ipvsScheduler, - } - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { - serv.Flags |= utilipvs.FlagPersistent - serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds) - } - // There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`. - if err := proxier.syncService(svcNameString, serv, false); err == nil { - activeIPVSServices[serv.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil { - glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) - } - } else { - glog.Errorf("Failed to sync service: %v, err: %v", serv, err) + glog.Errorf("Failed to get node ip address matching nodeport cidr") + continue + } + for address := range addresses { + if !utilproxy.IsZeroCIDR(address) { + nodeIPs = append(nodeIPs, net.ParseIP(address)) + continue + } + // zero cidr + nodeIPs, err = proxier.ipGetter.NodeIPs() + if err != nil { + glog.Errorf("Failed to list all node IPs from host, err: %v", err) + } + } + for _, nodeIP := range nodeIPs { + // ipvs call + serv := &utilipvs.VirtualServer{ + Address: nodeIP, + Port: uint16(svcInfo.nodePort), + Protocol: string(svcInfo.protocol), + Scheduler: proxier.ipvsScheduler, + } + if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + serv.Flags |= utilipvs.FlagPersistent + serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds) + } + // There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`. + if err := proxier.syncService(svcNameString, serv, false); err == nil { + activeIPVSServices[serv.String()] = true + if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil { + glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } + } else { + glog.Errorf("Failed to sync service: %v, err: %v", serv, err) } } } diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 2f1e8418a08..a26e7a38b49 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing" proxyutil "k8s.io/kubernetes/pkg/proxy/util" + proxyutiltest "k8s.io/kubernetes/pkg/proxy/util/testing" utilipset "k8s.io/kubernetes/pkg/util/ipset" ipsettest "k8s.io/kubernetes/pkg/util/ipset/testing" utiliptables "k8s.io/kubernetes/pkg/util/iptables" @@ -146,6 +147,8 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u lbWhiteListCIDRSet: NewIPSet(ipset, KubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, false), nodePortSetTCP: NewIPSet(ipset, KubeNodePortSetTCP, utilipset.BitmapPort, false), nodePortSetUDP: NewIPSet(ipset, KubeNodePortSetUDP, utilipset.BitmapPort, false), + nodePortAddresses: make([]string, 0), + networkInterfacer: proxyutiltest.NewFakeNetwork(), } } @@ -386,6 +389,8 @@ func TestNodePort(t *testing.T) { }), ) + fp.nodePortAddresses = []string{"0.0.0.0/0"} + fp.syncProxyRules() // Check ipvs service and destinations @@ -445,6 +450,8 @@ func TestNodePortNoEndpoint(t *testing.T) { ) makeEndpointsMap(fp) + fp.nodePortAddresses = []string{"0.0.0.0/0"} + fp.syncProxyRules() // Check ipvs service and destinations @@ -847,15 +854,23 @@ func TestOnlyLocalNodePorts(t *testing.T) { }), ) + itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} + addrs := []net.Addr{proxyutiltest.AddrStruct{Val: "100.101.102.103/24"}} + itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} + addrs1 := []net.Addr{proxyutiltest.AddrStruct{Val: "2001:db8::0/64"}} + fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs) + fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) + fp.nodePortAddresses = []string{"100.101.102.0/24", "2001:db8::0/64"} + fp.syncProxyRules() - // Expect 2 services and 1 destination + // Expect 3 services and 1 destination services, err := ipvs.GetVirtualServers() if err != nil { t.Errorf("Failed to get ipvs services, err: %v", err) } - if len(services) != 2 { - t.Errorf("Expect 2 ipvs services, got %d", len(services)) + if len(services) != 3 { + t.Errorf("Expect 3 ipvs services, got %d", len(services)) } found := false for _, svc := range services { diff --git a/pkg/proxy/util/interface.go b/pkg/proxy/util/interface.go deleted file mode 100644 index 2c6f02c9134..00000000000 --- a/pkg/proxy/util/interface.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "net" -) - -// NetworkInterface defines an interface for several net library functions. Production -// code will forward to net library functions, and unit tests will override the methods -// for testing purposes. -type NetworkInterface interface { - Addrs(intf *net.Interface) ([]net.Addr, error) - Interfaces() ([]net.Interface, error) -} - -// RealNetwork implements the NetworkInterface interface for production code, just -// wrapping the underlying net library function calls. -type RealNetwork struct{} - -// Addrs wraps net.Interface.Addrs() -func (_ RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { - return intf.Addrs() -} - -// Interfaces wraps net.Interfaces() -func (_ RealNetwork) Interfaces() ([]net.Interface, error) { - return net.Interfaces() -} - -// RealNetwork implements the NetworkInterface interface for production code, just -// wrapping the underlying net library function calls. -type FakeNetwork struct { - NetworkInterfaces []net.Interface - // The key of map Addrs is interface name - Address map[string][]net.Addr -} - -func NewFakeNetwork() *FakeNetwork { - return &FakeNetwork{ - NetworkInterfaces: make([]net.Interface, 0), - Address: make(map[string][]net.Addr), - } -} - -// AddInterfaceAddr create an interface and its associated addresses for FakeNetwork implementation. -func (f *FakeNetwork) AddInterfaceAddr(intf *net.Interface, addrs []net.Addr) { - f.NetworkInterfaces = append(f.NetworkInterfaces, *intf) - f.Address[intf.Name] = addrs -} - -// Addrs is part of FakeNetwork interface. -func (f *FakeNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) { - return f.Address[intf.Name], nil -} - -// Interfaces is part of FakeNetwork interface. -func (f *FakeNetwork) Interfaces() ([]net.Interface, error) { - return f.NetworkInterfaces, nil -} - -type AddrStruct struct{ Val string } - -func (a AddrStruct) Network() string { - return a.Val -} -func (a AddrStruct) String() string { - return a.Val -} From b498a33927cec31c6623f267279f4737030bf86b Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:37:37 +0800 Subject: [PATCH 6/9] userspace part changes --- pkg/proxy/userspace/proxier.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go index 139a3665367..7fdf1024db0 100644 --- a/pkg/proxy/userspace/proxier.go +++ b/pkg/proxy/userspace/proxier.go @@ -155,18 +155,15 @@ func IsProxyLocked(err error) bool { // if iptables fails to update or acquire the initial lock. Once a proxier is // created, it will keep iptables up to date in the background and will not // terminate if a particular iptables call fails. -func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.Interface, exec utilexec.Interface, pr utilnet.PortRange, syncPeriod, minSyncPeriod, udpIdleTimeout time.Duration) (*Proxier, error) { - return NewCustomProxier(loadBalancer, listenIP, iptables, exec, pr, syncPeriod, minSyncPeriod, udpIdleTimeout, newProxySocket) +func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.Interface, exec utilexec.Interface, pr utilnet.PortRange, syncPeriod, minSyncPeriod, udpIdleTimeout time.Duration, nodePortAddresses []string) (*Proxier, error) { + return NewCustomProxier(loadBalancer, listenIP, iptables, exec, pr, syncPeriod, minSyncPeriod, udpIdleTimeout, nodePortAddresses, newProxySocket) } // NewCustomProxier functions similarly to NewProxier, returning a new Proxier // for the given LoadBalancer and address. The new proxier is constructed using // the ProxySocket constructor provided, however, instead of constructing the // default ProxySockets. -func NewCustomProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.Interface, exec utilexec.Interface, pr utilnet.PortRange, syncPeriod, minSyncPeriod, udpIdleTimeout time.Duration, makeProxySocket ProxySocketFunc) (*Proxier, error) { - if listenIP == nil { - return nil, fmt.Errorf("invalid listen ip for kube-proxy") - } +func NewCustomProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables.Interface, exec utilexec.Interface, pr utilnet.PortRange, syncPeriod, minSyncPeriod, udpIdleTimeout time.Duration, nodePortAddresses []string, makeProxySocket ProxySocketFunc) (*Proxier, error) { if listenIP.Equal(localhostIPv4) || listenIP.Equal(localhostIPv6) { return nil, ErrProxyOnLocalhost } From df7df811a4c577d0d67dc80feccc40626ca9a828 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Wed, 17 Jan 2018 10:34:07 +0800 Subject: [PATCH 7/9] auto generated codes --- .../apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go | 2 ++ .../apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go | 5 +++++ pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go index 474c79e1c06..90a25711876 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go @@ -104,6 +104,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_kubeproxyconfig_KubeProxyCon return err } out.ConfigSyncPeriod = in.ConfigSyncPeriod + out.NodePortAddresses = *(*[]string)(unsafe.Pointer(&in.NodePortAddresses)) return nil } @@ -138,6 +139,7 @@ func autoConvert_kubeproxyconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon return err } out.ConfigSyncPeriod = in.ConfigSyncPeriod + out.NodePortAddresses = *(*[]string)(unsafe.Pointer(&in.NodePortAddresses)) return nil } diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go index 6373c015f75..5f510d0bcd2 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go @@ -67,6 +67,11 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { out.UDPIdleTimeout = in.UDPIdleTimeout in.Conntrack.DeepCopyInto(&out.Conntrack) out.ConfigSyncPeriod = in.ConfigSyncPeriod + if in.NodePortAddresses != nil { + in, out := &in.NodePortAddresses, &out.NodePortAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go b/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go index a0bd25a10dc..ae9563dc847 100644 --- a/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go +++ b/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go @@ -67,6 +67,11 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { out.UDPIdleTimeout = in.UDPIdleTimeout in.Conntrack.DeepCopyInto(&out.Conntrack) out.ConfigSyncPeriod = in.ConfigSyncPeriod + if in.NodePortAddresses != nil { + in, out := &in.NodePortAddresses, &out.NodePortAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } return } From 9bb4807e251efb1f2aa97dec065b215fb51f5f18 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Mon, 26 Feb 2018 17:03:26 +0800 Subject: [PATCH 8/9] update bazel --- pkg/proxy/iptables/BUILD | 1 + pkg/proxy/ipvs/BUILD | 1 + pkg/proxy/util/BUILD | 10 +++++++++- pkg/proxy/util/testing/BUILD | 22 ++++++++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 pkg/proxy/util/testing/BUILD diff --git a/pkg/proxy/iptables/BUILD b/pkg/proxy/iptables/BUILD index 24adfcb0590..8fba4ce5105 100644 --- a/pkg/proxy/iptables/BUILD +++ b/pkg/proxy/iptables/BUILD @@ -42,6 +42,7 @@ go_test( "//pkg/apis/core:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/util:go_default_library", + "//pkg/proxy/util/testing:go_default_library", "//pkg/util/async:go_default_library", "//pkg/util/conntrack:go_default_library", "//pkg/util/iptables:go_default_library", diff --git a/pkg/proxy/ipvs/BUILD b/pkg/proxy/ipvs/BUILD index ae5d09d0839..d55681bda13 100644 --- a/pkg/proxy/ipvs/BUILD +++ b/pkg/proxy/ipvs/BUILD @@ -18,6 +18,7 @@ go_test( "//pkg/proxy:go_default_library", "//pkg/proxy/ipvs/testing:go_default_library", "//pkg/proxy/util:go_default_library", + "//pkg/proxy/util/testing:go_default_library", "//pkg/util/ipset:go_default_library", "//pkg/util/ipset/testing:go_default_library", "//pkg/util/iptables:go_default_library", diff --git a/pkg/proxy/util/BUILD b/pkg/proxy/util/BUILD index 681c510bdfe..91f6feaad8b 100644 --- a/pkg/proxy/util/BUILD +++ b/pkg/proxy/util/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "endpoints.go", + "network.go", "port.go", "utils.go", ], @@ -12,8 +13,10 @@ go_library( deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/core/helper:go_default_library", + "//pkg/util/conntrack:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], ) @@ -27,8 +30,10 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", + "//pkg/proxy/util/testing:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], ) @@ -41,7 +46,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//pkg/proxy/util/testing:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/pkg/proxy/util/testing/BUILD b/pkg/proxy/util/testing/BUILD new file mode 100644 index 00000000000..b176e7826ab --- /dev/null +++ b/pkg/proxy/util/testing/BUILD @@ -0,0 +1,22 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["fake.go"], + importpath = "k8s.io/kubernetes/pkg/proxy/util/testing", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) From 2edc97bf17cc7e8c6fba1f5b91e270296ea56fab Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Mon, 26 Feb 2018 17:56:16 +0800 Subject: [PATCH 9/9] fix static checks --- pkg/proxy/util/network.go | 2 +- pkg/proxy/util/testing/fake.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/proxy/util/network.go b/pkg/proxy/util/network.go index 8a4461247e3..fa5c9fc7535 100644 --- a/pkg/proxy/util/network.go +++ b/pkg/proxy/util/network.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/proxy/util/testing/fake.go b/pkg/proxy/util/testing/fake.go index a041785288e..44fd56d6301 100644 --- a/pkg/proxy/util/testing/fake.go +++ b/pkg/proxy/util/testing/fake.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ type FakeNetwork struct { Address map[string][]net.Addr } +// NewFakeNetwork initializes a FakeNetwork. func NewFakeNetwork() *FakeNetwork { return &FakeNetwork{ NetworkInterfaces: make([]net.Interface, 0),