Merge pull request #93040 from cmluciano/cml/ipvsschedmodules

ipvs: ensure selected scheduler kernel modules are loaded

cmd/kube-proxy/app/server_others.go

@@ -101,7 +101,7 @@ func newProxyServer(
 
 	kernelHandler = ipvs.NewLinuxKernelHandler()
 	ipsetInterface = utilipset.New(execer)
-	canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface)
+	canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler)
 	if string(config.Mode) == proxyModeIPVS && err != nil {
 		klog.Errorf("Can't use the IPVS proxier: %v", err)
 	}

cmd/kube-proxy/app/server_others_test.go

@@ -79,6 +79,7 @@ func Test_getProxyMode(t *testing.T) {
 		kernelCompat  bool
 		ipsetError    error
 		expected      string
+		scheduler     string
 	}{
 		{ // flag says userspace
 			flag:     "userspace",
@@ -110,6 +111,7 @@ func Test_getProxyMode(t *testing.T) {
 			kernelVersion: "4.18",
 			ipsetVersion:  ipvs.MinIPSetCheckVersion,
 			expected:      proxyModeIPVS,
+			scheduler:     "rr",
 		},
 		{ // flag says ipvs, ipset version ok, kernel modules installed for linux kernel 4.19
 			flag:          "ipvs",
@@ -117,6 +119,7 @@ func Test_getProxyMode(t *testing.T) {
 			kernelVersion: "4.19",
 			ipsetVersion:  ipvs.MinIPSetCheckVersion,
 			expected:      proxyModeIPVS,
+			scheduler:     "rr",
 		},
 		{ // flag says ipvs, ipset version too low, fallback on iptables mode
 			flag:          "ipvs",
@@ -142,6 +145,23 @@ func Test_getProxyMode(t *testing.T) {
 			kernelCompat:  true,
 			expected:      proxyModeIPTables,
 		},
+		{ // flag says ipvs, ipset version ok, kernel modules installed for sed scheduler
+			flag:          "ipvs",
+			kmods:         []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_sed"},
+			kernelVersion: "4.19",
+			ipsetVersion:  ipvs.MinIPSetCheckVersion,
+			expected:      proxyModeIPVS,
+			scheduler:     "sed",
+		},
+		{ // flag says ipvs, kernel modules not installed for sed scheduler, fallback to iptables
+			flag:          "ipvs",
+			kmods:         []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
+			kernelVersion: "4.19",
+			ipsetVersion:  ipvs.MinIPSetCheckVersion,
+			expected:      proxyModeIPTables,
+			kernelCompat:  true,
+			scheduler:     "sed",
+		},
 	}
 	for i, c := range cases {
 		kcompater := &fakeKernelCompatTester{c.kernelCompat}
@@ -150,7 +170,7 @@ func Test_getProxyMode(t *testing.T) {
 			modules:       c.kmods,
 			kernelVersion: c.kernelVersion,
 		}
-		canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver)
+		canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver, cases[i].scheduler)
 		r := getProxyMode(c.flag, canUseIPVS, kcompater)
 		if r != c.expected {
 			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)

pkg/proxy/ipvs/proxier.go

@@ -699,7 +699,7 @@ func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) {
 // This is determined by checking if all the required kernel modules can be loaded. It may
 // return an error if it fails to get the kernel modules information without error, in which
 // case it will also return false.
-func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) {
+func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) (bool, error) {
 	mods, err := handle.GetModules()
 	if err != nil {
 		return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
@@ -717,6 +717,12 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err
 	}
 	mods = utilipvs.GetRequiredIPVSModules(kernelVersion)
 	wantModules := sets.NewString()
+	// We check for the existence of the scheduler mod and will trigger a missingMods error if not found
+	if scheduler == "" {
+		scheduler = DefaultScheduler
+	}
+	schedulerMod := "ip_vs_" + scheduler
+	mods = append(mods, schedulerMod)
 	wantModules.Insert(mods...)
 
 	modules := wantModules.Difference(loadModules).UnsortedList()

pkg/proxy/ipvs/proxier_test.go

@@ -256,6 +256,7 @@ func TestCleanupLeftovers(t *testing.T) {
 func TestCanUseIPVSProxier(t *testing.T) {
 	testCases := []struct {
 		mods          []string
+		scheduler     string
 		kernelVersion string
 		kernelErr     error
 		ipsetVersion  string
@@ -265,6 +266,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 0, kernel error
 		{
 			mods:          []string{"foo", "bar", "baz"},
+			scheduler:     "",
 			kernelVersion: "4.19",
 			kernelErr:     fmt.Errorf("oops"),
 			ipsetVersion:  "0.0",
@@ -273,6 +275,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 1, ipset error
 		{
 			mods:          []string{"foo", "bar", "baz"},
+			scheduler:     "",
 			kernelVersion: "4.19",
 			ipsetVersion:  MinIPSetCheckVersion,
 			ipsetErr:      fmt.Errorf("oops"),
@@ -281,6 +284,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 2, missing required kernel modules and ipset version too low
 		{
 			mods:          []string{"foo", "bar", "baz"},
+			scheduler:     "rr",
 			kernelVersion: "4.19",
 			ipsetVersion:  "1.1",
 			ok:            false,
@@ -288,6 +292,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 3, missing required ip_vs_* kernel modules
 		{
 			mods:          []string{"ip_vs", "a", "bc", "def"},
+			scheduler:     "sed",
 			kernelVersion: "4.19",
 			ipsetVersion:  MinIPSetCheckVersion,
 			ok:            false,
@@ -295,6 +300,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 4, ipset version too low
 		{
 			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
+			scheduler:     "rr",
 			kernelVersion: "4.19",
 			ipsetVersion:  "4.3.0",
 			ok:            false,
@@ -302,6 +308,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 5, ok for linux kernel 4.19
 		{
 			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
+			scheduler:     "rr",
 			kernelVersion: "4.19",
 			ipsetVersion:  MinIPSetCheckVersion,
 			ok:            true,
@@ -309,6 +316,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 6, ok for linux kernel 4.18
 		{
 			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			scheduler:     "rr",
 			kernelVersion: "4.18",
 			ipsetVersion:  MinIPSetCheckVersion,
 			ok:            true,
@@ -316,16 +324,41 @@ func TestCanUseIPVSProxier(t *testing.T) {
 		// case 7. ok when module list has extra modules
 		{
 			mods:          []string{"foo", "ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "bar"},
+			scheduler:     "rr",
 			kernelVersion: "4.19",
 			ipsetVersion:  "6.19",
 			ok:            true,
 		},
+		// case 8, not ok for sed based IPVS scheduling
+		{
+			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
+			scheduler:     "sed",
+			kernelVersion: "4.19",
+			ipsetVersion:  MinIPSetCheckVersion,
+			ok:            false,
+		},
+		// case 9, ok for dh based IPVS scheduling
+		{
+			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
+			scheduler:     "dh",
+			kernelVersion: "4.19",
+			ipsetVersion:  MinIPSetCheckVersion,
+			ok:            true,
+		},
+		// case 10, non-existent scheduler, error due to modules not existing
+		{
+			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
+			scheduler:     "foobar",
+			kernelVersion: "4.19",
+			ipsetVersion:  MinIPSetCheckVersion,
+			ok:            false,
+		},
 	}
 
 	for i := range testCases {
 		handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion}
 		versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
-		ok, err := CanUseIPVSProxier(handle, versioner)
+		ok, err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler)
 		if ok != testCases[i].ok {
 			t.Errorf("Case [%d], expect %v, got %v: err: %v", i, testCases[i].ok, ok, err)
 		}