diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 380e83e2a0a..30a8ac9bbf9 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -83,6 +83,7 @@ func newProxyServer( var iptInterface utiliptables.Interface var ipvsInterface utilipvs.Interface + var kernelHandler ipvs.KernelHandler var ipsetInterface utilipset.Interface var dbus utildbus.Interface @@ -92,6 +93,7 @@ func newProxyServer( dbus = utildbus.New() iptInterface = utiliptables.New(execer, dbus, protocol) ipvsInterface = utilipvs.New(execer) + kernelHandler = ipvs.NewLinuxKernelHandler() ipsetInterface = utilipset.New(execer) // We omit creation of pretty much everything if we run in cleanup mode @@ -133,7 +135,7 @@ func newProxyServer( var serviceEventHandler proxyconfig.ServiceHandler var endpointsEventHandler proxyconfig.EndpointsHandler - proxyMode := getProxyMode(string(config.Mode), iptInterface, ipsetInterface, iptables.LinuxKernelCompatTester{}) + proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{}) if proxyMode == proxyModeIPTables { glog.V(0).Info("Using iptables Proxier.") nodeIP := net.ParseIP(config.BindAddress) @@ -269,7 +271,7 @@ func newProxyServer( }, nil } -func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { +func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { if proxyMode == proxyModeUserspace { return proxyModeUserspace } @@ -280,7 +282,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) { if proxyMode == proxyModeIPVS { - return tryIPVSProxy(iptver, ipsetver, kcompat) + return tryIPVSProxy(iptver, khandle, ipsetver, kcompat) } else { glog.Warningf("Can't use ipvs proxier, trying iptables proxier") return tryIPTablesProxy(iptver, kcompat) @@ -290,10 +292,10 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver return tryIPTablesProxy(iptver, kcompat) } -func tryIPVSProxy(iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { +func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { // guaranteed false on error, error only necessary for debugging - // IPVS Proxier relies on ipset - useIPVSProxy, err := ipvs.CanUseIPVSProxier(ipsetver) + // IPVS Proxier relies on ip_vs_* kernel modules and ipset + useIPVSProxy, err := ipvs.CanUseIPVSProxier(khandle, ipsetver) if err != nil { // Try to fallback to iptables before falling back to userspace utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, error: %v", err)) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index e957e262e75..f50898ae623 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -72,6 +72,15 @@ func (fake *fakeKernelCompatTester) IsCompatible() error { return nil } +// fakeKernelHandler implements KernelHandler. +type fakeKernelHandler struct { + modules []string +} + +func (fake *fakeKernelHandler) GetModules() ([]string, error) { + return fake.modules, nil +} + func Test_getProxyMode(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("skipping on non-Linux") @@ -82,6 +91,7 @@ func Test_getProxyMode(t *testing.T) { annotationVal string iptablesVersion string ipsetVersion string + kmods []string kernelCompat bool iptablesError error ipsetError error @@ -140,7 +150,8 @@ func Test_getProxyMode(t *testing.T) { versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError} kcompater := &fakeKernelCompatTester{c.kernelCompat} ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError} - r := getProxyMode(c.flag, versioner, ipsetver, kcompater) + khandler := &fakeKernelHandler{c.kmods} + r := getProxyMode(c.flag, versioner, khandler, ipsetver, kcompater) if r != c.expected { t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r) } diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 5f5a09d2447..841ce1fd041 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -688,14 +688,28 @@ func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap) { } } -// CanUseIPVSProxier returns true if we can use the ipvs Proxier. -// This is determined by checking if all the required kernel modules can be loaded. It may -// return an error if it fails to get the kernel modules information without error, in which -// case it will also return false. -func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) { - // Try to load IPVS required kernel modules using modprobe +// KernelHandler can handle the current installed kernel modules. +type KernelHandler interface { + GetModules() ([]string, error) +} + +// LinuxKernelHandler implements KernelHandler interface. +type LinuxKernelHandler struct { + executor utilexec.Interface +} + +// NewLinuxKernelHandler initializes LinuxKernelHandler with exec. +func NewLinuxKernelHandler() *LinuxKernelHandler { + return &LinuxKernelHandler{ + executor: utilexec.New(), + } +} + +// GetModules returns all installed kernel modules. +func (handle *LinuxKernelHandler) GetModules() ([]string, error) { + // Try to load IPVS required kernel modules using modprobe first for _, kmod := range ipvsModules { - err := utilexec.New().Command("modprobe", "--", kmod).Run() + err := handle.executor.Command("modprobe", "--", kmod).Run() if err != nil { glog.Warningf("Failed to load kernel module %v with modprobe. "+ "You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", kmod) @@ -703,12 +717,24 @@ func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) { } // Find out loaded kernel modules - out, err := utilexec.New().Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() + out, err := handle.executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() if err != nil { - return false, err + return nil, err } mods := strings.Split(string(out), "\n") + return mods, nil +} + +// CanUseIPVSProxier returns true if we can use the ipvs Proxier. +// This is determined by checking if all the required kernel modules can be loaded. It may +// return an error if it fails to get the kernel modules information without error, in which +// case it will also return false. +func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) { + mods, err := handle.GetModules() + if err != nil { + return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err) + } wantModules := sets.NewString() loadModules := sets.NewString() wantModules.Insert(ipvsModules...) diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index b490f1a37e2..b7864a8b88e 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -18,6 +18,7 @@ package ipvs import ( "bytes" + "fmt" "net" "reflect" "testing" @@ -87,6 +88,25 @@ func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedNa return nil } +// fakeKernelHandler implements KernelHandler. +type fakeKernelHandler struct { + modules []string +} + +func (fake *fakeKernelHandler) GetModules() ([]string, error) { + return fake.modules, nil +} + +// fakeKernelHandler implements KernelHandler. +type fakeIPSetVersioner struct { + version string + err error +} + +func (fake *fakeIPSetVersioner) GetVersion() (string, error) { + return fake.version, fake.err +} + func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP) *Proxier { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ @@ -180,6 +200,70 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *ap return ept } +func TestCanUseIPVSProxier(t *testing.T) { + testCases := []struct { + mods []string + kernelErr error + ipsetVersion string + ipsetErr error + ok bool + }{ + // case 0, kernel error + { + mods: []string{"foo", "bar", "baz"}, + kernelErr: fmt.Errorf("oops"), + ipsetVersion: "0.0", + ok: false, + }, + // case 1, ipset error + { + mods: []string{"foo", "bar", "baz"}, + ipsetVersion: MinIPSetCheckVersion, + ipsetErr: fmt.Errorf("oops"), + ok: false, + }, + // case 2, missing required kernel modules and ipset version too low + { + mods: []string{"foo", "bar", "baz"}, + ipsetVersion: "1.1", + ok: false, + }, + // case 3, missing required ip_vs_* kernel modules + { + mods: []string{"ip_vs", "a", "bc", "def"}, + ipsetVersion: MinIPSetCheckVersion, + ok: false, + }, + // case 4, ipset version too low + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "4.3.0", + ok: false, + }, + // case 5 + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: MinIPSetCheckVersion, + ok: true, + }, + // case 6 + { + mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4", "foo", "bar"}, + ipsetVersion: "6.19", + ok: true, + }, + } + + for i := range testCases { + handle := &fakeKernelHandler{modules: testCases[i].mods} + versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr} + ok, _ := CanUseIPVSProxier(handle, versioner) + if ok != testCases[i].ok { + t.Errorf("Case [%d], expect %v, got %v", i, testCases[i].ok, ok) + } + } +} + func TestNodePort(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake()