Merge pull request #75036 from bart0sh/PR0065-kubeadm-replace-RequiredIPVSKernelModulesAvailable-check

kubeadm: reimplement IPVS check

cmd/kubeadm/.import-restrictions

@@ -43,7 +43,8 @@
 				"k8s.io/utils/integer",
 				"k8s.io/utils/path",
 				"k8s.io/utils/pointer",
-				"k8s.io/utils/net"
+				"k8s.io/utils/net",
+				"k8s.io/utils/trace"
 			]
 		},
 		{
@@ -57,6 +58,7 @@
 				"k8s.io/kubernetes/pkg/apis/rbac",
 				"k8s.io/kubernetes/pkg/apis/scheduling",
 				"k8s.io/kubernetes/pkg/api/v1/pod",
+				"k8s.io/kubernetes/pkg/api/v1/service",
 				"k8s.io/kubernetes/pkg/capabilities",
 				"k8s.io/kubernetes/pkg/controller",
 				"k8s.io/kubernetes/pkg/features",
@@ -67,20 +69,26 @@
 				"k8s.io/kubernetes/pkg/kubelet/types",
 				"k8s.io/kubernetes/pkg/master/ports",
 				"k8s.io/kubernetes/pkg/proxy/apis/config",
+				"k8s.io/kubernetes/pkg/proxy",
 				"k8s.io/kubernetes/pkg/registry/core/service/allocator",
 				"k8s.io/kubernetes/pkg/registry/core/service/ipallocator",
 				"k8s.io/kubernetes/pkg/security/apparmor",
 				"k8s.io/kubernetes/pkg/serviceaccount",
+				"k8s.io/kubernetes/pkg/util/async",
+				"k8s.io/kubernetes/pkg/util/conntrack",
+				"k8s.io/kubernetes/pkg/util/dbus",
 				"k8s.io/kubernetes/pkg/util/hash",
 				"k8s.io/kubernetes/pkg/util/initsystem",
+				"k8s.io/kubernetes/pkg/util/ipset",
+				"k8s.io/kubernetes/pkg/util/iptables",
 				"k8s.io/kubernetes/pkg/util/ipvs",
 				"k8s.io/kubernetes/pkg/util/metrics",
 				"k8s.io/kubernetes/pkg/util/node",
 				"k8s.io/kubernetes/pkg/util/normalizer",
 				"k8s.io/kubernetes/pkg/util/parsers",
 				"k8s.io/kubernetes/pkg/util/procfs",
+				"k8s.io/kubernetes/pkg/util/sysctl",
 				"k8s.io/kubernetes/pkg/util/taints",
-				"k8s.io/kubernetes/pkg/util/ipvs",
 				"k8s.io/kubernetes/pkg/version"
 			],
 			"ForbiddenPrefixes": [
@@ -120,6 +128,7 @@
 				"github.com/docker/go-connections/tlsconfig",
 				"github.com/docker/go-units",
 				"github.com/docker/libnetwork/ipvs",
+				"github.com/godbus/dbus",
 				"github.com/gogo/protobuf/proto",
 				"github.com/gogo/protobuf/sortkeys",
 				"github.com/golang/groupcache/lru",
@@ -156,7 +165,8 @@
 				"github.com/russross/blackfriday",
 				"github.com/shurcooL/sanitized_anchor_name",
 				"github.com/spf13/cobra",
-				"github.com/spf13/pflag"
+				"github.com/spf13/pflag",
+				"github.com/vishvananda/netlink"
 			]
 		},
 		{

cmd/kubeadm/app/preflight/BUILD

@@ -39,6 +39,8 @@ go_library(
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
             "//cmd/kubeadm/app/util:go_default_library",
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/ipset:go_default_library",
         ],
         "//conditions:default": [],
     }),

cmd/kubeadm/app/preflight/checks.go

@@ -872,6 +872,16 @@ func (ncc NumCPUCheck) Check() (warnings, errorList []error) {
 	return warnings, errorList
 }
 
+// IPVSProxierCheck tests if IPVS proxier can be used.
+type IPVSProxierCheck struct {
+	exec utilsexec.Interface
+}
+
+// Name returns label for IPVSProxierCheck
+func (r IPVSProxierCheck) Name() string {
+	return "IPVSProxierCheck"
+}
+
 // RunInitNodeChecks executes all individual, applicable to control-plane node checks.
 // The boolean flag 'isSecondaryControlPlane' controls whether we are running checks in a --join-control-plane scenario.
 // If the flag is set to true we should skip checks already executed by RunJoinNodeChecks and RunOptionalJoinNodeChecks.
@@ -903,11 +913,9 @@ func RunInitNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.InitConfigura
 	if !isSecondaryControlPlane {
 		checks = addCommonChecks(execer, cfg.KubernetesVersion, &cfg.NodeRegistration, checks)
 
-		// Check IVPS required kernel module once we use IVPS kube-proxy mode
+		// Check if IVPS kube-proxy mode is supported
 		if cfg.ComponentConfigs.KubeProxy != nil && cfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode {
-			checks = append(checks,
+			checks = append(checks, IPVSProxierCheck{exec: execer})
-				ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer},
-			)
 		}
 
 		// Check if Bridge-netfilter and IPv6 relevant flags are set
@@ -994,11 +1002,9 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura
 func RunOptionalJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.ClusterConfiguration, ignorePreflightErrors sets.String) error {
 	checks := []Checker{}
 
-	// Check ipvs required kernel module if we use ipvs kube-proxy mode
+	// Check if IVPS kube-proxy mode is supported
 	if cfg.ComponentConfigs.KubeProxy != nil && cfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode {
-		checks = append(checks,
+		checks = append(checks, IPVSProxierCheck{exec: execer})
-			ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer},
-		)
 	}
 
 	return RunChecks(checks, os.Stderr, ignorePreflightErrors)

cmd/kubeadm/app/preflight/checks_darwin.go

@@ -25,3 +25,9 @@ package preflight
 func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) {
 	return nil, nil
 }
+
+// Check determines if IPVS proxier can be used or not
+// No-op for for Darwin (MacOS).
+func (ipvspc IPVSProxierCheck) Check() (warnings, errors []error) {
+	return nil, nil
+}

cmd/kubeadm/app/preflight/checks_linux.go

@@ -21,7 +21,10 @@ package preflight
 import (
 	"github.com/pkg/errors"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/pkg/proxy/ipvs"
 	"k8s.io/utils/exec"
+
+	utilipset "k8s.io/kubernetes/pkg/util/ipset"
 )
 
 // Check validates if Docker is setup to use systemd as the cgroup driver.
@@ -42,3 +45,13 @@ func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) {
 	}
 	return warnings, nil
 }
+
+// Check determines if IPVS proxier can be used or not
+func (ipvspc IPVSProxierCheck) Check() (warnings, errors []error) {
+	ipsetInterface := utilipset.New(ipvspc.exec)
+	kernelHandler := ipvs.NewLinuxKernelHandler()
+	if _, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface); err != nil {
+		return nil, append(errors, err)
+	}
+	return nil, nil
+}

cmd/kubeadm/app/preflight/checks_windows.go

@@ -49,3 +49,9 @@ func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) {
 func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) {
 	return nil, nil
 }
+
+// Check determines if IPVS proxier can be used or not
+// No-op for Windows.
+func (ipvspc IPVSProxierCheck) Check() (warnings, errors []error) {
+	return nil, nil
+}

pkg/util/ipvs/BUILD

@@ -11,14 +11,11 @@ go_test(
     srcs = [
         "ipvs_linux_test.go",
         "ipvs_test.go",
-        "kernelcheck_linux_test.go",
     ],
     embed = [":go_default_library"],
     deps = select({
         "@io_bazel_rules_go//go/platform:linux": [
             "//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
-            "//vendor/k8s.io/utils/exec:go_default_library",
-            "//vendor/k8s.io/utils/exec/testing:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -30,7 +27,6 @@ go_library(
         "ipvs.go",
         "ipvs_linux.go",
         "ipvs_unsupported.go",
-        "kernelcheck_linux.go",
         "kernelcheck_unsupported.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/util/ipvs",
@@ -39,9 +35,7 @@ go_library(
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
-            "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
             "//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
-            "//vendor/github.com/lithammer/dedent:go_default_library",
             "//vendor/k8s.io/klog:go_default_library",
         ],
         "//conditions:default": [],

pkg/util/ipvs/kernelcheck_linux.go

@@ -1,102 +0,0 @@
-// +build linux
-
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package ipvs
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/util/sets"
-	utilsexec "k8s.io/utils/exec"
-
-	"github.com/lithammer/dedent"
-	"k8s.io/klog"
-)
-
-// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules.
-type RequiredIPVSKernelModulesAvailableCheck struct {
-	Executor utilsexec.Interface
-}
-
-// Name returns label for RequiredIPVSKernelModulesAvailableCheck
-func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
-	return "RequiredIPVSKernelModulesAvailable"
-}
-
-// Check try to validates IPVS required kernel modules exists or not.
-// The name of function can not be changed.
-func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
-	klog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not")
-
-	kernelVersion, ipvsModules, err := GetKernelVersionAndIPVSMods(r.Executor)
-	if err != nil {
-		errors = append(errors, err)
-	}
-
-	// Find out loaded kernel modules
-	out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
-	if err != nil {
-		errors = append(errors, fmt.Errorf("error getting installed ipvs required kernel modules: %v(%s)", err, out))
-		return nil, errors
-	}
-	mods := strings.Split(string(out), "\n")
-
-	wantModules := sets.NewString()
-	loadModules := sets.NewString()
-	wantModules.Insert(ipvsModules...)
-	loadModules.Insert(mods...)
-	modules := wantModules.Difference(loadModules).UnsortedList()
-
-	// Check builtin modules exist or not
-	if len(modules) != 0 {
-		builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
-		out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput()
-		if err != nil {
-			errors = append(errors, fmt.Errorf("error getting required builtin kernel modules: %v(%s)", err, out))
-			return nil, errors
-		}
-
-		builtInModules := sets.NewString()
-		for _, builtInMode := range ipvsModules {
-			match, _ := regexp.Match(builtInMode+".ko", out)
-			if !match {
-				builtInModules.Insert(string(builtInMode))
-			}
-		}
-		if len(builtInModules) != 0 {
-			warnings = append(warnings, fmt.Errorf(dedent.Dedent(`
-
-				The IPVS proxier may not be used because the following required kernel modules are not loaded: %v
-				or no builtin kernel IPVS support was found: %v.
-				However, these modules may be loaded automatically by kube-proxy if they are available on your system.
-				To verify IPVS support:
-
-				   Run "lsmod | grep 'ip_vs|nf_conntrack'" and verify each of the above modules are listed.
-
-				If they are not listed, you can use the following methods to load them:
-
-				1. For each missing module run 'modprobe $modulename' (e.g., 'modprobe ip_vs', 'modprobe ip_vs_rr', ...)
-				2. If 'modprobe $modulename' returns an error, you will need to install the missing module support for your kernel.
-				`), modules, builtInModules))
-		}
-	}
-
-	return warnings, errors
-}

pkg/util/ipvs/kernelcheck_linux_test.go

@@ -1,130 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package ipvs
-
-import (
-	"testing"
-
-	utilsexec "k8s.io/utils/exec"
-	fakeexec "k8s.io/utils/exec/testing"
-)
-
-func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) {
-	cases := []struct {
-		caseName string
-
-		loadedKernel  string
-		kernelVersion string
-		builtinKernel string
-
-		expectErrors   bool
-		expectWarnings bool
-	}{
-		{
-			caseName:       "no installed kernel modules and no builtin kernel modules",
-			loadedKernel:   "",
-			kernelVersion:  "3.13.0-24-generic",
-			builtinKernel:  "",
-			expectErrors:   false,
-			expectWarnings: true,
-		},
-		{
-			caseName:      "no installed kernel modules and missing builtin kernel modules",
-			loadedKernel:  "",
-			kernelVersion: "3.13.0-24-generic",
-			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
-				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
-			expectErrors:   false,
-			expectWarnings: true,
-		},
-		{
-			caseName:      "no installed kernel modules and own all builtin kernel modules",
-			loadedKernel:  "",
-			kernelVersion: "3.13.0-24-generic",
-			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
-				"kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" +
-				"kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" +
-				"kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" +
-				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
-			expectErrors:   false,
-			expectWarnings: false,
-		},
-		{
-			caseName:       "missing installed kernel modules and no builtin kernel modules",
-			loadedKernel:   "ip_vs",
-			kernelVersion:  "3.13.0-24-generic",
-			builtinKernel:  "",
-			expectErrors:   false,
-			expectWarnings: true,
-		},
-		{
-			caseName: "own all installed kernel modules and no builtin kernel modules",
-			loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" +
-				"ip_vs_rr\n" + "ip_vs_sh",
-			kernelVersion:  "3.13.0-24-generic",
-			builtinKernel:  "",
-			expectErrors:   false,
-			expectWarnings: false,
-		},
-		{
-			caseName:      "own all installed kernel modules and all builtin kernel modules",
-			loadedKernel:  "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + "ip_vs_rr\n" + "ip_vs_sh",
-			kernelVersion: "3.13.0-24-generic",
-			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
-				"kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" +
-				"kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" +
-				"kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" +
-				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
-			expectErrors:   false,
-			expectWarnings: false,
-		},
-	}
-
-	for i, tc := range cases {
-		fcmd := fakeexec.FakeCmd{
-			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
-				func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil },
-				func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
-				func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil },
-			},
-		}
-
-		fexec := fakeexec.FakeExec{
-			CommandScript: []fakeexec.FakeCommandAction{
-				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-			},
-		}
-
-		check := RequiredIPVSKernelModulesAvailableCheck{
-			Executor: &fexec,
-		}
-		warnings, errors := check.Check()
-
-		switch {
-		case warnings != nil && !tc.expectWarnings:
-			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected warnings for installed kernel modules %v and builtin kernel modules %v. Warnings: %v", tc.loadedKernel, tc.builtinKernel, warnings)
-		case warnings == nil && tc.expectWarnings:
-			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected warnings for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel)
-		case errors != nil && !tc.expectErrors:
-			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected errors for installed kernel modules %v and builtin kernel modules %v. errors: %v", tc.loadedKernel, tc.builtinKernel, errors)
-		case errors == nil && tc.expectErrors:
-			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected errors for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel)
-		}
-	}
-}