Merge pull request #114669 from Nordix/ipvs-remove-module-check

Ipvs: remove module check

cmd/kube-proxy/app/server_others.go

@@ -240,10 +240,10 @@ func newProxyServer(
 	} else if proxyMode == proxyconfigapi.ProxyModeIPVS {
 		kernelHandler := ipvs.NewLinuxKernelHandler()
 		ipsetInterface = utilipset.New(execer)
-		if err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler); err != nil {
+		ipvsInterface = utilipvs.New()
+		if err := ipvs.CanUseIPVSProxier(ipvsInterface, ipsetInterface, config.IPVS.Scheduler); err != nil {
 			return nil, fmt.Errorf("can't use the IPVS proxier: %v", err)
 		}
-		ipvsInterface = utilipvs.New()
 
 		klog.InfoS("Using ipvs Proxier")
 		if dualStack {

pkg/proxy/ipvs/proxier.go

@@ -24,7 +24,6 @@ import (
 	"net"
 	"os"
 	"reflect"
-	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -606,7 +605,6 @@ func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *pro
 
 // KernelHandler can handle the current installed kernel modules.
 type KernelHandler interface {
-	GetModules() ([]string, error)
 	GetKernelVersion() (string, error)
 }
 
@@ -622,73 +620,6 @@ func NewLinuxKernelHandler() *LinuxKernelHandler {
 	}
 }
 
-// GetModules returns all installed kernel modules.
-func (handle *LinuxKernelHandler) GetModules() ([]string, error) {
-	// Check whether IPVS required kernel modules are built-in
-	kernelVersionStr, err := handle.GetKernelVersion()
-	if err != nil {
-		return nil, err
-	}
-	kernelVersion, err := version.ParseGeneric(kernelVersionStr)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err)
-	}
-	ipvsModules := utilipvs.GetRequiredIPVSModules(kernelVersion)
-
-	var bmods, lmods []string
-
-	// Find out loaded kernel modules. If this is a full static kernel it will try to verify if the module is compiled using /boot/config-KERNELVERSION
-	modulesFile, err := os.Open("/proc/modules")
-	if err == os.ErrNotExist {
-		klog.ErrorS(err, "Failed to read file /proc/modules, assuming this is a kernel without loadable modules support enabled")
-		kernelConfigFile := fmt.Sprintf("/boot/config-%s", kernelVersionStr)
-		kConfig, err := os.ReadFile(kernelConfigFile)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read Kernel Config file %s with error %w", kernelConfigFile, err)
-		}
-		for _, module := range ipvsModules {
-			if match, _ := regexp.Match("CONFIG_"+strings.ToUpper(module)+"=y", kConfig); match {
-				bmods = append(bmods, module)
-			}
-		}
-		return bmods, nil
-	}
-	if err != nil {
-		return nil, fmt.Errorf("failed to read file /proc/modules with error %w", err)
-	}
-	defer modulesFile.Close()
-
-	mods, err := getFirstColumn(modulesFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to find loaded kernel modules: %v", err)
-	}
-
-	builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersionStr)
-	b, err := os.ReadFile(builtinModsFilePath)
-	if err != nil {
-		klog.ErrorS(err, "Failed to read builtin modules file, you can ignore this message when kube-proxy is running inside container without mounting /lib/modules", "filePath", builtinModsFilePath)
-	}
-
-	for _, module := range ipvsModules {
-		if match, _ := regexp.Match(module+".ko", b); match {
-			bmods = append(bmods, module)
-		} else {
-			// Try to load the required IPVS kernel modules if not built in
-			err := handle.executor.Command("modprobe", "--", module).Run()
-			if err != nil {
-				klog.InfoS("Failed to load kernel module with modprobe, "+
-					"you can ignore this message when kube-proxy is running inside container without mounting /lib/modules", "moduleName", module)
-			} else {
-				lmods = append(lmods, module)
-			}
-		}
-	}
-
-	mods = append(mods, bmods...)
-	mods = append(mods, lmods...)
-	return mods, nil
-}
-
 // getFirstColumn reads all the content from r into memory and return a
 // slice which consists of the first word from each line.
 func getFirstColumn(r io.Reader) ([]string, error) {
@@ -720,51 +651,18 @@ func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) {
 }
 
 // CanUseIPVSProxier checks if we can use the ipvs Proxier.
-// This is determined by checking if all the required kernel modules can be loaded. It may
+// The ipset version and the scheduler are checked. If any virtual servers (VS)
-// return an error if it fails to get the kernel modules information without error, in which
+// already exist with the configured scheduler, we just return. Otherwise
-// case it will also return false.
+// we check if a dummy VS can be configured with the configured scheduler.
-func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) error {
+// Kernel modules will be loaded automatically if necessary.
-	mods, err := handle.GetModules()
+func CanUseIPVSProxier(ipvs utilipvs.Interface, ipsetver IPSetVersioner, scheduler string) error {
-	if err != nil {
+	// BUG: https://github.com/moby/ipvs/issues/27
-		return fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
+	// If ipvs is not compiled into the kernel no error is returned and handle==nil.
-	}
+	// This in turn causes ipvs.GetVirtualServers and ipvs.AddVirtualServer
-	loadModules := sets.NewString()
+	// to return ok (err==nil). If/when this bug is fixed parameter "ipvs" will be nil
-	loadModules.Insert(mods...)
+	// if ipvs is not supported by the kernel. Until then a re-read work-around is used.
-
+	if ipvs == nil {
-	kernelVersionStr, err := handle.GetKernelVersion()
+		return fmt.Errorf("Ipvs not supported by the kernel")
-	if err != nil {
-		return fmt.Errorf("error determining kernel version to find required kernel modules for ipvs support: %v", err)
-	}
-	kernelVersion, err := version.ParseGeneric(kernelVersionStr)
-	if err != nil {
-		return fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err)
-	}
-	mods = utilipvs.GetRequiredIPVSModules(kernelVersion)
-	wantModules := sets.NewString()
-	// We check for the existence of the scheduler mod and will trigger a missingMods error if not found
-	if scheduler == "" {
-		scheduler = defaultScheduler
-	}
-	schedulerMod := "ip_vs_" + scheduler
-	mods = append(mods, schedulerMod)
-	wantModules.Insert(mods...)
-
-	modules := wantModules.Difference(loadModules).UnsortedList()
-	var missingMods []string
-	ConntrackiMissingCounter := 0
-	for _, mod := range modules {
-		if strings.Contains(mod, "nf_conntrack") {
-			ConntrackiMissingCounter++
-		} else {
-			missingMods = append(missingMods, mod)
-		}
-	}
-	if ConntrackiMissingCounter == 2 {
-		missingMods = append(missingMods, "nf_conntrack_ipv4(or nf_conntrack for Linux kernel 4.19 and later)")
-	}
-
-	if len(missingMods) != 0 {
-		return fmt.Errorf("IPVS proxier will not be used because the following required kernel modules are not loaded: %v", missingMods)
 	}
 
 	// Check ipset version
@@ -775,6 +673,70 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler
 	if !checkMinVersion(versionString) {
 		return fmt.Errorf("ipset version: %s is less than min required version: %s", versionString, MinIPSetCheckVersion)
 	}
+
+	if scheduler == "" {
+		scheduler = defaultScheduler
+	}
+
+	// If any virtual server (VS) using the scheduler exist we skip the checks.
+	vservers, err := ipvs.GetVirtualServers()
+	if err != nil {
+		klog.ErrorS(err, "Can't read the ipvs")
+		return err
+	}
+	klog.V(5).InfoS("Virtual Servers", "count", len(vservers))
+	if len(vservers) > 0 {
+		// This is most likely a kube-proxy re-start. We know that ipvs works
+		// and if any VS uses the configured scheduler, we are done.
+		for _, vs := range vservers {
+			if vs.Scheduler == scheduler {
+				klog.V(5).InfoS("VS exist, Skipping checks")
+				return nil
+			}
+		}
+		klog.V(5).InfoS("No existing VS uses the configured scheduler", "scheduler", scheduler)
+	}
+
+	// Try to insert a dummy VS with the passed scheduler.
+	// We should use a VIP address that is not used on the node.
+	// An address "198.51.100.0" from the TEST-NET-2 rage in https://datatracker.ietf.org/doc/html/rfc5737
+	// is used. These addresses are reserved for documentation. If the user is using
+	// this address for a VS anyway we *will* mess up, but that would be an invalid configuration.
+	// If the user have configured the address to an interface on the node (but not a VS)
+	// then traffic will temporary be routed to ipvs during the probe and dropped.
+	// The later case is also and invalid configuration, but the traffic impact will be minor.
+	// This should not be a problem if users honors reserved addresses, but cut/paste
+	// from documentation is not unheard of, so the restriction to not use the TEST-NET-2 range
+	// must be documented.
+	vs := utilipvs.VirtualServer{
+		Address:   netutils.ParseIPSloppy("198.51.100.0"),
+		Protocol:  "TCP",
+		Port:      20000,
+		Scheduler: scheduler,
+	}
+	if err := ipvs.AddVirtualServer(&vs); err != nil {
+		klog.ErrorS(err, "Could not create dummy VS", "scheduler", scheduler)
+		return err
+	}
+
+	// To overcome the BUG described above we check that the VS is *really* added.
+	vservers, err = ipvs.GetVirtualServers()
+	if err != nil {
+		klog.ErrorS(err, "ipvs.GetVirtualServers")
+		return err
+	}
+	klog.V(5).InfoS("Virtual Servers after adding dummy", "count", len(vservers))
+	if len(vservers) == 0 {
+		klog.InfoS("Dummy VS not created", "scheduler", scheduler)
+		return fmt.Errorf("Ipvs not supported") // This is a BUG work-around
+	}
+	klog.V(5).InfoS("Dummy VS created", "vs", vs)
+
+	if err := ipvs.DeleteVirtualServer(&vs); err != nil {
+		klog.ErrorS(err, "Could not delete dummy VS")
+		return err
+	}
+
 	return nil
 }
 

pkg/proxy/ipvs/proxier_test.go

@@ -75,21 +75,61 @@ func (f *fakeIPGetter) BindedIPs() (sets.String, error) {
 	return f.bindedIPs, nil
 }
 
-// fakeKernelHandler implements KernelHandler.
+// fakeIpvs implements utilipvs.Interface
-type fakeKernelHandler struct {
+type fakeIpvs struct {
-	modules       []string
+	ipvsErr   string
-	kernelVersion string
+	vsCreated bool
 }
 
-func (fake *fakeKernelHandler) GetModules() ([]string, error) {
+func (f *fakeIpvs) Flush() error {
-	return fake.modules, nil
+	return nil
+}
+func (f *fakeIpvs) AddVirtualServer(*utilipvs.VirtualServer) error {
+	if f.ipvsErr == "AddVirtualServer" {
+		return fmt.Errorf("oops")
+	}
+	f.vsCreated = true
+	return nil
+}
+func (f *fakeIpvs) UpdateVirtualServer(*utilipvs.VirtualServer) error {
+	return nil
+}
+func (f *fakeIpvs) DeleteVirtualServer(*utilipvs.VirtualServer) error {
+	if f.ipvsErr == "DeleteVirtualServer" {
+		return fmt.Errorf("oops")
+	}
+	return nil
+}
+func (f *fakeIpvs) GetVirtualServer(*utilipvs.VirtualServer) (*utilipvs.VirtualServer, error) {
+	return nil, nil
+}
+func (f *fakeIpvs) GetVirtualServers() ([]*utilipvs.VirtualServer, error) {
+	if f.ipvsErr == "GetVirtualServers" {
+		return nil, fmt.Errorf("oops")
+	}
+	if f.vsCreated {
+		vs := []*utilipvs.VirtualServer{{}}
+		return vs, nil
+	}
+	return nil, nil
+}
+func (f *fakeIpvs) AddRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
+	return nil
+}
+func (f *fakeIpvs) GetRealServers(*utilipvs.VirtualServer) ([]*utilipvs.RealServer, error) {
+	return nil, nil
+}
+func (f *fakeIpvs) DeleteRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
+	return nil
+}
+func (f *fakeIpvs) UpdateRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
+	return nil
+}
+func (f *fakeIpvs) ConfigureTimeouts(time.Duration, time.Duration, time.Duration) error {
+	return nil
 }
 
-func (fake *fakeKernelHandler) GetKernelVersion() (string, error) {
+// fakeIPSetVersioner implements IPSetVersioner.
-	return fake.kernelVersion, nil
-}
-
-// fakeKernelHandler implements KernelHandler.
 type fakeIPSetVersioner struct {
 	version string
 	err     error
@@ -273,112 +313,57 @@ func TestCleanupLeftovers(t *testing.T) {
 
 func TestCanUseIPVSProxier(t *testing.T) {
 	testCases := []struct {
-		mods          []string
+		name         string
-		scheduler     string
+		scheduler    string
-		kernelVersion string
+		ipsetVersion string
-		kernelErr     error
+		ipsetErr     error
-		ipsetVersion  string
+		ipvsErr      string
-		ipsetErr      error
+		ok           bool
-		ok            bool
 	}{
-		// case 0, kernel error
 		{
-			mods:          []string{"foo", "bar", "baz"},
+			name:         "happy days",
-			scheduler:     "",
+			ipsetVersion: MinIPSetCheckVersion,
-			kernelVersion: "4.19",
+			ok:           true,
-			kernelErr:     fmt.Errorf("oops"),
-			ipsetVersion:  "0.0",
-			ok:            false,
 		},
-		// case 1, ipset error
 		{
-			mods:          []string{"foo", "bar", "baz"},
+			name:         "ipset error",
-			scheduler:     "",
+			scheduler:    "",
-			kernelVersion: "4.19",
+			ipsetVersion: MinIPSetCheckVersion,
-			ipsetVersion:  MinIPSetCheckVersion,
+			ipsetErr:     fmt.Errorf("oops"),
-			ipsetErr:      fmt.Errorf("oops"),
+			ok:           false,
-			ok:            false,
 		},
-		// case 2, missing required kernel modules and ipset version too low
 		{
-			mods:          []string{"foo", "bar", "baz"},
+			name:         "ipset version too low",
-			scheduler:     "rr",
+			scheduler:    "rr",
-			kernelVersion: "4.19",
+			ipsetVersion: "4.3.0",
-			ipsetVersion:  "1.1",
+			ok:           false,
-			ok:            false,
 		},
-		// case 3, missing required ip_vs_* kernel modules
 		{
-			mods:          []string{"ip_vs", "a", "bc", "def"},
+			name:         "GetVirtualServers fail",
-			scheduler:     "sed",
+			ipsetVersion: MinIPSetCheckVersion,
-			kernelVersion: "4.19",
+			ipvsErr:      "GetVirtualServers",
-			ipsetVersion:  MinIPSetCheckVersion,
+			ok:           false,
-			ok:            false,
 		},
-		// case 4, ipset version too low
 		{
-			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
+			name:         "AddVirtualServer fail",
-			scheduler:     "rr",
+			ipsetVersion: MinIPSetCheckVersion,
-			kernelVersion: "4.19",
+			ipvsErr:      "AddVirtualServer",
-			ipsetVersion:  "4.3.0",
+			ok:           false,
-			ok:            false,
 		},
-		// case 5, ok for linux kernel 4.19
 		{
-			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
+			name:         "DeleteVirtualServer fail",
-			scheduler:     "rr",
+			ipsetVersion: MinIPSetCheckVersion,
-			kernelVersion: "4.19",
+			ipvsErr:      "DeleteVirtualServer",
-			ipsetVersion:  MinIPSetCheckVersion,
+			ok:           false,
-			ok:            true,
-		},
-		// case 6, ok for linux kernel 4.18
-		{
-			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
-			scheduler:     "rr",
-			kernelVersion: "4.18",
-			ipsetVersion:  MinIPSetCheckVersion,
-			ok:            true,
-		},
-		// case 7. ok when module list has extra modules
-		{
-			mods:          []string{"foo", "ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "bar"},
-			scheduler:     "rr",
-			kernelVersion: "4.19",
-			ipsetVersion:  "6.19",
-			ok:            true,
-		},
-		// case 8, not ok for sed based IPVS scheduling
-		{
-			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
-			scheduler:     "sed",
-			kernelVersion: "4.19",
-			ipsetVersion:  MinIPSetCheckVersion,
-			ok:            false,
-		},
-		// case 9, ok for dh based IPVS scheduling
-		{
-			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
-			scheduler:     "dh",
-			kernelVersion: "4.19",
-			ipsetVersion:  MinIPSetCheckVersion,
-			ok:            true,
-		},
-		// case 10, non-existent scheduler, error due to modules not existing
-		{
-			mods:          []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
-			scheduler:     "foobar",
-			kernelVersion: "4.19",
-			ipsetVersion:  MinIPSetCheckVersion,
-			ok:            false,
 		},
 	}
 
-	for i := range testCases {
+	for _, tc := range testCases {
-		handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion}
+		ipvs := &fakeIpvs{tc.ipvsErr, false}
-		versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
+		versioner := &fakeIPSetVersioner{version: tc.ipsetVersion, err: tc.ipsetErr}
-		err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler)
+		err := CanUseIPVSProxier(ipvs, versioner, tc.scheduler)
-		if (err == nil) != testCases[i].ok {
+		if (err == nil) != tc.ok {
-			t.Errorf("Case [%d], expect %v, got err: %v", i, testCases[i].ok, err)
+			t.Errorf("Case [%s], expect %v, got err: %v", tc.name, tc.ok, err)
 		}
 	}
 }

pkg/util/ipvs/ipvs.go

@@ -21,8 +21,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"k8s.io/apimachinery/pkg/util/version"
 )
 
 // Interface is an injectable interface for running ipvs commands.  Implementations must be goroutine-safe.
@@ -71,22 +69,6 @@ const (
 	FlagHashed = 0x2
 )
 
-// IPVS required kernel modules.
-const (
-	// KernelModuleIPVS is the kernel module "ip_vs"
-	KernelModuleIPVS string = "ip_vs"
-	// KernelModuleIPVSRR is the kernel module "ip_vs_rr"
-	KernelModuleIPVSRR string = "ip_vs_rr"
-	// KernelModuleIPVSWRR is the kernel module "ip_vs_wrr"
-	KernelModuleIPVSWRR string = "ip_vs_wrr"
-	// KernelModuleIPVSSH is the kernel module "ip_vs_sh"
-	KernelModuleIPVSSH string = "ip_vs_sh"
-	// KernelModuleNfConntrackIPV4 is the module "nf_conntrack_ipv4"
-	KernelModuleNfConntrackIPV4 string = "nf_conntrack_ipv4"
-	// KernelModuleNfConntrack is the kernel module "nf_conntrack"
-	KernelModuleNfConntrack string = "nf_conntrack"
-)
-
 // Equal check the equality of virtual server.
 // We don't use struct == since it doesn't work because of slice.
 func (svc *VirtualServer) Equal(other *VirtualServer) bool {
@@ -122,17 +104,6 @@ func (rs *RealServer) Equal(other *RealServer) bool {
 		rs.Port == other.Port
 }
 
-// GetRequiredIPVSModules returns the required ipvs modules for the given linux kernel version.
-func GetRequiredIPVSModules(kernelVersion *version.Version) []string {
-	// "nf_conntrack_ipv4" has been removed since v4.19
-	// see https://github.com/torvalds/linux/commit/a0ae2562c6c4b2721d9fddba63b7286c13517d9f
-	if kernelVersion.LessThan(version.MustParseGeneric("4.19")) {
-		return []string{KernelModuleIPVS, KernelModuleIPVSRR, KernelModuleIPVSWRR, KernelModuleIPVSSH, KernelModuleNfConntrackIPV4}
-	}
-	return []string{KernelModuleIPVS, KernelModuleIPVSRR, KernelModuleIPVSWRR, KernelModuleIPVSSH, KernelModuleNfConntrack}
-
-}
-
 // IsRsGracefulTerminationNeeded returns true if protocol requires graceful termination for the stale connections
 func IsRsGracefulTerminationNeeded(proto string) bool {
 	return !strings.EqualFold(proto, "UDP") && !strings.EqualFold(proto, "SCTP")

pkg/util/ipvs/ipvs_test.go

@@ -17,11 +17,8 @@ limitations under the License.
 package ipvs
 
 import (
-	"reflect"
-	"sort"
 	"testing"
 
-	"k8s.io/apimachinery/pkg/util/version"
 	netutils "k8s.io/utils/net"
 )
 
@@ -384,32 +381,3 @@ func TestFrontendDestinationString(t *testing.T) {
 		}
 	}
 }
-
-func TestGetRequiredIPVSModules(t *testing.T) {
-	Tests := []struct {
-		name          string
-		kernelVersion *version.Version
-		want          []string
-	}{
-		{
-			name:          "kernel version < 4.19",
-			kernelVersion: version.MustParseGeneric("4.18"),
-			want:          []string{KernelModuleIPVS, KernelModuleIPVSRR, KernelModuleIPVSWRR, KernelModuleIPVSSH, KernelModuleNfConntrackIPV4},
-		},
-		{
-			name:          "kernel version 4.19",
-			kernelVersion: version.MustParseGeneric("4.19"),
-			want:          []string{KernelModuleIPVS, KernelModuleIPVSRR, KernelModuleIPVSWRR, KernelModuleIPVSSH, KernelModuleNfConntrack},
-		},
-	}
-	for _, test := range Tests {
-		t.Run(test.name, func(t *testing.T) {
-			got := GetRequiredIPVSModules(test.kernelVersion)
-			sort.Strings(got)
-			sort.Strings(test.want)
-			if !reflect.DeepEqual(got, test.want) {
-				t.Errorf("GetRequiredIPVSMods() = %v for kenel version: %s, want %v", got, test.kernelVersion, test.want)
-			}
-		})
-	}
-}