From dc2e57ba7495d70a4731238e0e8a3c0c1c1cbbb1 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Sun, 26 Nov 2017 21:47:50 +0800 Subject: [PATCH 1/4] refactor canUseIPVSMode and test it --- cmd/kube-proxy/app/server_others.go | 14 ++--- cmd/kube-proxy/app/server_test.go | 13 ++++- pkg/proxy/ipvs/proxier.go | 44 +++++++++++---- pkg/proxy/ipvs/proxier_test.go | 84 +++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 16 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 380e83e2a0a..30a8ac9bbf9 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -83,6 +83,7 @@ func newProxyServer( var iptInterface utiliptables.Interface var ipvsInterface utilipvs.Interface + var kernelHandler ipvs.KernelHandler var ipsetInterface utilipset.Interface var dbus utildbus.Interface @@ -92,6 +93,7 @@ func newProxyServer( dbus = utildbus.New() iptInterface = utiliptables.New(execer, dbus, protocol) ipvsInterface = utilipvs.New(execer) + kernelHandler = ipvs.NewLinuxKernelHandler() ipsetInterface = utilipset.New(execer) // We omit creation of pretty much everything if we run in cleanup mode @@ -133,7 +135,7 @@ func newProxyServer( var serviceEventHandler proxyconfig.ServiceHandler var endpointsEventHandler proxyconfig.EndpointsHandler - proxyMode := getProxyMode(string(config.Mode), iptInterface, ipsetInterface, iptables.LinuxKernelCompatTester{}) + proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{}) if proxyMode == proxyModeIPTables { glog.V(0).Info("Using iptables Proxier.") nodeIP := net.ParseIP(config.BindAddress) @@ -269,7 +271,7 @@ func newProxyServer( }, nil } -func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { +func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { if proxyMode == proxyModeUserspace { return proxyModeUserspace } @@ -280,7 +282,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) { if proxyMode == proxyModeIPVS { - return tryIPVSProxy(iptver, ipsetver, kcompat) + return tryIPVSProxy(iptver, khandle, ipsetver, kcompat) } else { glog.Warningf("Can't use ipvs proxier, trying iptables proxier") return tryIPTablesProxy(iptver, kcompat) @@ -290,10 +292,10 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver return tryIPTablesProxy(iptver, kcompat) } -func tryIPVSProxy(iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { +func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { // guaranteed false on error, error only necessary for debugging - // IPVS Proxier relies on ipset - useIPVSProxy, err := ipvs.CanUseIPVSProxier(ipsetver) + // IPVS Proxier relies on ip_vs_* kernel modules and ipset + useIPVSProxy, err := ipvs.CanUseIPVSProxier(khandle, ipsetver) if err != nil { // Try to fallback to iptables before falling back to userspace utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, error: %v", err)) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index e957e262e75..f50898ae623 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -72,6 +72,15 @@ func (fake *fakeKernelCompatTester) IsCompatible() error { return nil } +// fakeKernelHandler implements KernelHandler. +type fakeKernelHandler struct { + modules []string +} + +func (fake *fakeKernelHandler) GetModules() ([]string, error) { + return fake.modules, nil +} + func Test_getProxyMode(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("skipping on non-Linux") @@ -82,6 +91,7 @@ func Test_getProxyMode(t *testing.T) { annotationVal string iptablesVersion string ipsetVersion string + kmods []string kernelCompat bool iptablesError error ipsetError error @@ -140,7 +150,8 @@ func Test_getProxyMode(t *testing.T) { versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError} kcompater := &fakeKernelCompatTester{c.kernelCompat} ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError} - r := getProxyMode(c.flag, versioner, ipsetver, kcompater) + khandler := &fakeKernelHandler{c.kmods} + r := getProxyMode(c.flag, versioner, khandler, ipsetver, kcompater) if r != c.expected { t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r) } diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 5f5a09d2447..841ce1fd041 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -688,14 +688,28 @@ func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap) { } } -// CanUseIPVSProxier returns true if we can use the ipvs Proxier. -// This is determined by checking if all the required kernel modules can be loaded. It may -// return an error if it fails to get the kernel modules information without error, in which -// case it will also return false. -func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) { - // Try to load IPVS required kernel modules using modprobe +// KernelHandler can handle the current installed kernel modules. +type KernelHandler interface { + GetModules() ([]string, error) +} + +// LinuxKernelHandler implements KernelHandler interface. +type LinuxKernelHandler struct { + executor utilexec.Interface +} + +// NewLinuxKernelHandler initializes LinuxKernelHandler with exec. +func NewLinuxKernelHandler() *LinuxKernelHandler { + return &LinuxKernelHandler{ + executor: utilexec.New(), + } +} + +// GetModules returns all installed kernel modules. +func (handle *LinuxKernelHandler) GetModules() ([]string, error) { + // Try to load IPVS required kernel modules using modprobe first for _, kmod := range ipvsModules { - err := utilexec.New().Command("modprobe", "--", kmod).Run() + err := handle.executor.Command("modprobe", "--", kmod).Run() if err != nil { glog.Warningf("Failed to load kernel module %v with modprobe. "+ "You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", kmod) @@ -703,12 +717,24 @@ func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) { } // Find out loaded kernel modules - out, err := utilexec.New().Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() + out, err := handle.executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() if err != nil { - return false, err + return nil, err } mods := strings.Split(string(out), "\n") + return mods, nil +} + +// CanUseIPVSProxier returns true if we can use the ipvs Proxier. +// This is determined by checking if all the required kernel modules can be loaded. It may +// return an error if it fails to get the kernel modules information without error, in which +// case it will also return false. +func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) { + mods, err := handle.GetModules() + if err != nil { + return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err) + } wantModules := sets.NewString() loadModules := sets.NewString() wantModules.Insert(ipvsModules...) diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index b490f1a37e2..b7864a8b88e 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -18,6 +18,7 @@ package ipvs import ( "bytes" + "fmt" "net" "reflect" "testing" @@ -87,6 +88,25 @@ func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedNa return nil } +// fakeKernelHandler implements KernelHandler. +type fakeKernelHandler struct { + modules []string +} + +func (fake *fakeKernelHandler) GetModules() ([]string, error) { + return fake.modules, nil +} + +// fakeKernelHandler implements KernelHandler. +type fakeIPSetVersioner struct { + version string + err error +} + +func (fake *fakeIPSetVersioner) GetVersion() (string, error) { + return fake.version, fake.err +} + func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP) *Proxier { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ @@ -180,6 +200,70 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *ap return ept } +func TestCanUseIPVSProxier(t *testing.T) { + testCases := []struct { + mods []string + kernelErr error + ipsetVersion string + ipsetErr error + ok bool + }{ + // case 0, kernel error + { + mods: []string{"foo", "bar", "baz"}, + kernelErr: fmt.Errorf("oops"), + ipsetVersion: "0.0", + ok: false, + }, + // case 1, ipset error + { + mods: []string{"foo", "bar", "baz"}, + ipsetVersion: MinIPSetCheckVersion, + ipsetErr: fmt.Errorf("oops"), + ok: false, + }, + // case 2, missing required kernel modules and ipset version too low + { + mods: []string{"foo", "bar", "baz"}, + ipsetVersion: "1.1", + ok: false, + }, + // case 3, missing required ip_vs_* kernel modules + { + mods: []string{"ip_vs", "a", "bc", "def"}, + ipsetVersion: MinIPSetCheckVersion, + ok: false, + }, + // case 4, ipset version too low + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "4.3.0", + ok: false, + }, + // case 5 + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: MinIPSetCheckVersion, + ok: true, + }, + // case 6 + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4", "foo", "bar"}, + ipsetVersion: "6.19", + ok: true, + }, + } + + for i := range testCases { + handle := &fakeKernelHandler{modules: testCases[i].mods} + versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr} + ok, _ := CanUseIPVSProxier(handle, versioner) + if ok != testCases[i].ok { + t.Errorf("Case [%d], expect %v, got %v", i, testCases[i].ok, ok) + } + } +} + func TestNodePort(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() From 7b372143c5e70fe046e95a0e23897e010e040963 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Mon, 27 Nov 2017 19:31:39 +0800 Subject: [PATCH 2/4] test ipvs proxy mode when feature gateway unset --- cmd/kube-proxy/app/server_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index f50898ae623..0fc92c259c8 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -145,6 +145,23 @@ func Test_getProxyMode(t *testing.T) { kernelCompat: true, expected: proxyModeIPTables, }, + { // specify ipvs, feature gateway disabled, iptables version ok, kernel is compatible + flag: "ipvs", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // specify ipvs, feature gateway disabled, iptables version too low + flag: "ipvs", + iptablesVersion: "0.0.0", + expected: proxyModeUserspace, + }, + { // specify ipvs, feature gateway disabled, iptables version ok, kernel is not compatible + flag: "ipvs", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: false, + expected: proxyModeUserspace, + }, } for i, c := range cases { versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError} From 1a10652173f2fb48bc4d59a8a746313076258544 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Mon, 27 Nov 2017 19:34:59 +0800 Subject: [PATCH 3/4] test ipvs proxy mode when feature gateway set --- cmd/kube-proxy/app/server_test.go | 134 ++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 0fc92c259c8..ba470950556 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -28,8 +28,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + utilfeature "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" + "k8s.io/kubernetes/pkg/proxy/ipvs" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/iptables" utilpointer "k8s.io/kubernetes/pkg/util/pointer" @@ -175,6 +177,138 @@ func Test_getProxyMode(t *testing.T) { } } +// This is a coarse test, but it offers some modicum of confidence as the code is evolved. +func Test_getProxyModeEnableFeatureGateway(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("skipping on non-Linux") + } + + // enable IPVS feature gateway + utilfeature.DefaultFeatureGate.Set("SupportIPVSProxyMode=true") + + var cases = []struct { + flag string + iptablesVersion string + ipsetVersion string + kernelCompat bool + iptablesError error + ipsetError error + mods []string + expected string + }{ + { // flag says userspace + flag: "userspace", + expected: proxyModeUserspace, + }, + { // flag says iptables, error detecting version + flag: "iptables", + iptablesError: fmt.Errorf("oops!"), + expected: proxyModeUserspace, + }, + { // flag says iptables, version too low + flag: "iptables", + iptablesVersion: "0.0.0", + expected: proxyModeUserspace, + }, + { // flag says iptables, version ok, kernel not compatible + flag: "iptables", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: false, + expected: proxyModeUserspace, + }, + { // flag says iptables, version ok, kernel is compatible + flag: "iptables", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // detect, error + flag: "", + iptablesError: fmt.Errorf("oops!"), + expected: proxyModeUserspace, + }, + { // detect, version too low + flag: "", + iptablesVersion: "0.0.0", + expected: proxyModeUserspace, + }, + { // detect, version ok, kernel not compatible + flag: "", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: false, + expected: proxyModeUserspace, + }, + { // detect, version ok, kernel is compatible + flag: "", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // detect, version ok, kernel is compatible + flag: "", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // flag says ipvs, ipset version ok, kernel modules installed + flag: "ipvs", + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: ipvs.MinIPSetCheckVersion, + expected: proxyModeIPVS, + }, + { // flag says ipvs, ipset version too low, fallback on iptables mode + flag: "ipvs", + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "0.0", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // flag says ipvs, bad ipset version, fallback on iptables mode + flag: "ipvs", + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "a.b.c", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // flag says ipvs, required kernel modules are not installed, fallback on iptables mode + flag: "ipvs", + mods: []string{"foo", "bar", "baz"}, + ipsetVersion: ipvs.MinIPSetCheckVersion, + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode + flag: "ipvs", + mods: []string{"foo", "bar", "baz"}, + ipsetVersion: ipvs.MinIPSetCheckVersion, + iptablesVersion: "0.0.0", + kernelCompat: true, + expected: proxyModeUserspace, + }, + { // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode + flag: "ipvs", + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "0.0", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: false, + expected: proxyModeUserspace, + }, + } + for i, c := range cases { + versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError} + kcompater := &fakeKernelCompatTester{c.kernelCompat} + ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError} + khandle := &fakeKernelHandler{c.mods} + r := getProxyMode(c.flag, versioner, khandle, ipsetver, kcompater) + if r != c.expected { + t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r) + } + } +} + // This test verifies that NewProxyServer does not crash when CleanupAndExit is true. func TestProxyServerWithCleanupAndExit(t *testing.T) { // Each bind address below is a separate test case From 10aea7e88c6f57235bffc2b5d778d9cfb0cc5bc4 Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Mon, 27 Nov 2017 20:10:43 +0800 Subject: [PATCH 4/4] update bazel BUILD file --- cmd/kube-proxy/app/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 4a4cb061112..02bbd56e48a 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -92,12 +92,14 @@ go_test( deps = [ "//pkg/apis/core:go_default_library", "//pkg/proxy/apis/kubeproxyconfig:go_default_library", + "//pkg/proxy/ipvs:go_default_library", "//pkg/util/configz:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/pointer:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], )