From e2a0eddaf02ba3680c044ba038d1bcb28491bf61 Mon Sep 17 00:00:00 2001 From: "Christopher M. Luciano" Date: Mon, 13 Jul 2020 15:20:59 -0400 Subject: [PATCH] ipvs: ensure selected scheduler kernel modules are loaded Signed-off-by: Christopher M. Luciano --- cmd/kube-proxy/app/server_others.go | 2 +- cmd/kube-proxy/app/server_others_test.go | 15 ++++++++++++- pkg/proxy/ipvs/proxier.go | 3 ++- pkg/proxy/ipvs/proxier_test.go | 27 ++++++++++++++++++++++- pkg/util/ipvs/ipvs.go | 28 ++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index eb69ea27fb9..e4cf210cbdf 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -122,7 +122,7 @@ func newProxyServer( iptInterface = utiliptables.New(execer, protocol) kernelHandler = ipvs.NewLinuxKernelHandler() ipsetInterface = utilipset.New(execer) - canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface) + canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler) if string(config.Mode) == proxyModeIPVS && err != nil { klog.Errorf("Can't use the IPVS proxier: %v", err) } diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index b7c492a1446..b1ea7c36869 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -79,6 +79,7 @@ func Test_getProxyMode(t *testing.T) { kernelCompat bool ipsetError error expected string + scheduler string }{ { // flag says userspace flag: "userspace", @@ -142,6 +143,14 @@ func Test_getProxyMode(t *testing.T) { kernelCompat: true, expected: proxyModeIPTables, }, + { // flag says ipvs, ipset version ok, kernel modules installed for sed scheduler + flag: "ipvs", + kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_sed"}, + kernelVersion: "4.19", + ipsetVersion: ipvs.MinIPSetCheckVersion, + expected: proxyModeIPVS, + scheduler: "sed", + }, } for i, c := range cases { kcompater := &fakeKernelCompatTester{c.kernelCompat} @@ -150,7 +159,11 @@ func Test_getProxyMode(t *testing.T) { modules: c.kmods, kernelVersion: c.kernelVersion, } - canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver) + scheduler := cases[i].scheduler + if scheduler == "" { + scheduler = "rr" + } + canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver, scheduler) r := getProxyMode(c.flag, canUseIPVS, kcompater) if r != c.expected { t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 872cbe990e7..9d9ad3da8fd 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -693,7 +693,7 @@ func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) { // This is determined by checking if all the required kernel modules can be loaded. It may // return an error if it fails to get the kernel modules information without error, in which // case it will also return false. -func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) { +func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) (bool, error) { mods, err := handle.GetModules() if err != nil { return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err) @@ -711,6 +711,7 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err } mods = utilipvs.GetRequiredIPVSModules(kernelVersion) wantModules := sets.NewString() + mods = append(mods, utilipvs.GetRequiredSchedulerModules(scheduler)) wantModules.Insert(mods...) modules := wantModules.Difference(loadModules).UnsortedList() diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index ec027608f57..77b5e75317b 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -259,6 +259,7 @@ func TestCleanupLeftovers(t *testing.T) { func TestCanUseIPVSProxier(t *testing.T) { testCases := []struct { mods []string + scheduler string kernelVersion string kernelErr error ipsetVersion string @@ -268,6 +269,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 0, kernel error { mods: []string{"foo", "bar", "baz"}, + scheduler: "", kernelVersion: "4.19", kernelErr: fmt.Errorf("oops"), ipsetVersion: "0.0", @@ -276,6 +278,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 1, ipset error { mods: []string{"foo", "bar", "baz"}, + scheduler: "", kernelVersion: "4.19", ipsetVersion: MinIPSetCheckVersion, ipsetErr: fmt.Errorf("oops"), @@ -284,6 +287,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 2, missing required kernel modules and ipset version too low { mods: []string{"foo", "bar", "baz"}, + scheduler: "rr", kernelVersion: "4.19", ipsetVersion: "1.1", ok: false, @@ -291,6 +295,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 3, missing required ip_vs_* kernel modules { mods: []string{"ip_vs", "a", "bc", "def"}, + scheduler: "sed", kernelVersion: "4.19", ipsetVersion: MinIPSetCheckVersion, ok: false, @@ -298,6 +303,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 4, ipset version too low { mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, + scheduler: "rr", kernelVersion: "4.19", ipsetVersion: "4.3.0", ok: false, @@ -305,6 +311,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 5, ok for linux kernel 4.19 { mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, + scheduler: "rr", kernelVersion: "4.19", ipsetVersion: MinIPSetCheckVersion, ok: true, @@ -312,6 +319,7 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 6, ok for linux kernel 4.18 { mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + scheduler: "rr", kernelVersion: "4.18", ipsetVersion: MinIPSetCheckVersion, ok: true, @@ -319,16 +327,33 @@ func TestCanUseIPVSProxier(t *testing.T) { // case 7. ok when module list has extra modules { mods: []string{"foo", "ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "bar"}, + scheduler: "rr", kernelVersion: "4.19", ipsetVersion: "6.19", ok: true, }, + // case 8, not ok for sed based IPVS scheduling + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, + scheduler: "sed", + kernelVersion: "4.19", + ipsetVersion: MinIPSetCheckVersion, + ok: false, + }, + // case 9, ok for dh based IPVS scheduling + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"}, + scheduler: "dh", + kernelVersion: "4.19", + ipsetVersion: MinIPSetCheckVersion, + ok: true, + }, } for i := range testCases { handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion} versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr} - ok, err := CanUseIPVSProxier(handle, versioner) + ok, err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler) if ok != testCases[i].ok { t.Errorf("Case [%d], expect %v, got %v: err: %v", i, testCases[i].ok, ok, err) } diff --git a/pkg/util/ipvs/ipvs.go b/pkg/util/ipvs/ipvs.go index a9bb32f1be6..b331a052848 100644 --- a/pkg/util/ipvs/ipvs.go +++ b/pkg/util/ipvs/ipvs.go @@ -83,6 +83,14 @@ const ( KernelModuleIPVSWRR string = "ip_vs_wrr" // KernelModuleIPVSSH is the kernel module "ip_vs_sh" KernelModuleIPVSSH string = "ip_vs_sh" + // KernelModuleIPVSLC is the kernel module "ip_vs_lc" + KernelModuleIPVSLC string = "ip_vs_lc" + // KernelModuleIPVSNQ is the kernel module "ip_vs_nq" + KernelModuleIPVSNQ string = "ip_vs_nq" + // KernelModuleIPVSDH is the kernel module "ip_vs_dh" + KernelModuleIPVSDH string = "ip_vs_dh" + // KernelModuleIPVSSED is the kernel module "ip_vs_sed" + KernelModuleIPVSSED string = "ip_vs_sed" // KernelModuleNfConntrackIPV4 is the module "nf_conntrack_ipv4" KernelModuleNfConntrackIPV4 string = "nf_conntrack_ipv4" // KernelModuleNfConntrack is the kernel module "nf_conntrack" @@ -139,3 +147,23 @@ func GetRequiredIPVSModules(kernelVersion *version.Version) []string { func IsRsGracefulTerminationNeeded(proto string) bool { return !strings.EqualFold(proto, "UDP") && !strings.EqualFold(proto, "SCTP") } + +// GetRequiredSchedulerModules returns the required ipvs scheduler module for configured scheduler +func GetRequiredSchedulerModules(scheduler string) string { + switch s := scheduler; s { + case "rr": + return KernelModuleIPVSRR + case "lc": + return KernelModuleIPVSLC + case "nq": + return KernelModuleIPVSNQ + case "dh": + return KernelModuleIPVSDH + case "sh": + return KernelModuleIPVSSH + case "sed": + return KernelModuleIPVSSED + default: + return KernelModuleIPVSRR + } +}