Merge pull request #59566 from stewart-yu/ipvsGAkubeadm

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Automatically load ipvs required kernel modules in kubeadm

**What this PR does / why we need it**:
This PR is part of [https://github.com/kubernetes/kubernetes/issues/59402](https://github.com/kubernetes/kubernetes/issues/59402), aiming to load kernel modules in kubeadm

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #[https://github.com/kubernetes/kubernetes/issues/59402](https://github.com/kubernetes/kubernetes/issues/59402)

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

cmd/kubeadm/.import-restrictions

@@ -140,6 +140,7 @@
 				"k8s.io/kubernetes/pkg/util/slice",
 				"k8s.io/kubernetes/pkg/util/taints",
 				"k8s.io/kubernetes/pkg/util/version",
+				"k8s.io/kubernetes/pkg/util/ipvs",
 				"k8s.io/kubernetes/pkg/version",
 				"k8s.io/kubernetes/pkg/volume",
 				"k8s.io/kubernetes/pkg/volume/util"

cmd/kubeadm/app/preflight/BUILD

@@ -55,6 +55,7 @@ go_library(
         "//pkg/apis/core/validation:go_default_library",
         "//pkg/registry/core/service/ipallocator:go_default_library",
         "//pkg/util/initsystem:go_default_library",
+        "//pkg/util/ipvs:go_default_library",
         "//pkg/util/procfs:go_default_library",
         "//pkg/util/version:go_default_library",
         "//pkg/version:go_default_library",

cmd/kubeadm/app/preflight/checks.go

@@ -49,6 +49,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/util/initsystem"
+	ipvsutil "k8s.io/kubernetes/pkg/util/ipvs"
 	"k8s.io/kubernetes/pkg/util/procfs"
 	versionutil "k8s.io/kubernetes/pkg/util/version"
 	kubeadmversion "k8s.io/kubernetes/pkg/version"
@@ -866,6 +867,13 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfi
 	}
 	checks = addCommonChecks(execer, cfg, checks)
 
+	// Check ipvs required kernel module once we use ipvs kube-proxy mode
+	if cfg.KubeProxy.Config.Mode == ipvsutil.IPVSProxyMode {
+		checks = append(checks,
+			ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer},
+		)
+	}
+
 	if len(cfg.Etcd.Endpoints) == 0 {
 		// Only do etcd related checks when no external endpoints were specified
 		checks = append(checks,
@@ -911,6 +919,7 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.NodeConfigura
 		FileAvailableCheck{Path: cfg.CACertPath},
 		FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)},
 		FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)},
+		ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer},
 	}
 	checks = addCommonChecks(execer, cfg, checks)
 

pkg/util/ipvs/BUILD

@@ -13,6 +13,7 @@ go_test(
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
             "ipvs_linux_test.go",
+            "kernelcheck_linux_test.go",
         ],
         "//conditions:default": [],
     }),
@@ -20,6 +21,8 @@ go_test(
     deps = select({
         "@io_bazel_rules_go//go/platform:linux": [
             "//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+            "//vendor/k8s.io/utils/exec/testing:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -32,36 +35,47 @@ go_library(
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
             "ipvs_linux.go",
+            "kernelcheck_linux.go",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:windows": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "//conditions:default": [],
     }),
@@ -82,6 +96,7 @@ go_library(
         "@io_bazel_rules_go//go/platform:linux": [
             "//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
             "//vendor/github.com/golang/glog:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
             "//vendor/k8s.io/utils/exec:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [

pkg/util/ipvs/ipvs.go

@@ -61,8 +61,19 @@ const (
 	FlagPersistent = 0x1
 	// FlagHashed specify IPVS service hash flag
 	FlagHashed = 0x2
+	// IPVSProxyMode is match set up cluster with ipvs proxy model
+	IPVSProxyMode = "ipvs"
 )
 
+// Sets of IPVS required kernel modules.
+var ipvsModules = []string{
+	"ip_vs",
+	"ip_vs_rr",
+	"ip_vs_wrr",
+	"ip_vs_sh",
+	"nf_conntrack_ipv4",
+}
+
 // Equal check the equality of virtual server.
 // We don't use struct == since it doesn't work because of slice.
 func (svc *VirtualServer) Equal(other *VirtualServer) bool {

pkg/util/ipvs/kernelcheck_linux.go

@@ -0,0 +1,94 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilsexec "k8s.io/utils/exec"
+
+	"github.com/golang/glog"
+)
+
+// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules.
+type RequiredIPVSKernelModulesAvailableCheck struct {
+	Executor utilsexec.Interface
+}
+
+// Name returns label for RequiredIPVSKernelModulesAvailableCheck
+func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
+	return "RequiredIPVSKernelModulesAvailable"
+}
+
+// Check try to validates IPVS required kernel modules exists or not.
+// The name of function can not be changed.
+func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
+	glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not")
+
+	// Find out loaded kernel modules
+	out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
+	if err != nil {
+		errors = append(errors, fmt.Errorf("error getting installed ipvs required kernel modules: %v(%s)", err, out))
+		return nil, errors
+	}
+	mods := strings.Split(string(out), "\n")
+
+	wantModules := sets.NewString()
+	loadModules := sets.NewString()
+	wantModules.Insert(ipvsModules...)
+	loadModules.Insert(mods...)
+	modules := wantModules.Difference(loadModules).UnsortedList()
+
+	// Check builtin modules exist or not
+	if len(modules) != 0 {
+		kernelVersionFile := "/proc/sys/kernel/osrelease"
+		b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput()
+		if err != nil {
+			errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out))
+			return nil, errors
+		}
+
+		kernelVersion := strings.TrimSpace(string(b))
+		builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
+		out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput()
+		if err != nil {
+			errors = append(errors, fmt.Errorf("error getting required builtin kernel modules: %v(%s)", err, out))
+			return nil, errors
+		}
+
+		builtInModules := sets.NewString()
+		for _, builtInMode := range ipvsModules {
+			match, _ := regexp.Match(builtInMode+".ko", out)
+			if !match {
+				builtInModules.Insert(string(builtInMode))
+			}
+		}
+		if len(builtInModules) != 0 {
+			warnings = append(warnings, fmt.Errorf(
+				"the IPVS proxier will not be used, because the following required kernel modules are not loaded: %v or no builtin kernel ipvs support: %v\n"+
+					"you can solve this problem with following methods:\n 1. Run 'modprobe -- ' to load missing kernel modules;\n"+
+					"2. Provide the missing builtin kernel ipvs support\n", modules, builtInModules))
+		}
+	}
+
+	return warnings, errors
+}

pkg/util/ipvs/kernelcheck_linux_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	"testing"
+
+	utilsexec "k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
+)
+
+func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) {
+	cases := []struct {
+		caseName string
+
+		loadedKernel  string
+		kernelVersion string
+		builtinKernel string
+
+		expectErrors   bool
+		expectWarnings bool
+	}{
+		{
+			caseName:       "no installed kernel modules and no builtin kernel modules",
+			loadedKernel:   "",
+			kernelVersion:  "3.13.0-24-generic",
+			builtinKernel:  "",
+			expectErrors:   false,
+			expectWarnings: true,
+		},
+		{
+			caseName:      "no installed kernel modules and missing builtin kernel modules",
+			loadedKernel:  "",
+			kernelVersion: "3.13.0-24-generic",
+			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
+				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
+			expectErrors:   false,
+			expectWarnings: true,
+		},
+		{
+			caseName:      "no installed kernel modules and own all builtin kernel modules",
+			loadedKernel:  "",
+			kernelVersion: "3.13.0-24-generic",
+			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" +
+				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
+			expectErrors:   false,
+			expectWarnings: false,
+		},
+		{
+			caseName:       "missing installed kernel modules and no builtin kernel modules",
+			loadedKernel:   "ip_vs",
+			kernelVersion:  "3.13.0-24-generic",
+			builtinKernel:  "",
+			expectErrors:   false,
+			expectWarnings: true,
+		},
+		{
+			caseName: "own all installed kernel modules and no builtin kernel modules",
+			loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" +
+				"ip_vs_rr\n" + "ip_vs_sh",
+			kernelVersion:  "3.13.0-24-generic",
+			builtinKernel:  "",
+			expectErrors:   false,
+			expectWarnings: false,
+		},
+		{
+			caseName:      "own all installed kernel modules and all builtin kernel modules",
+			loadedKernel:  "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + "ip_vs_rr\n" + "ip_vs_sh",
+			kernelVersion: "3.13.0-24-generic",
+			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" +
+				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
+			expectErrors:   false,
+			expectWarnings: false,
+		},
+	}
+
+	for i, tc := range cases {
+		fcmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
+				func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil },
+				func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil },
+			},
+		}
+
+		fexec := fakeexec.FakeExec{
+			CommandScript: []fakeexec.FakeCommandAction{
+				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			},
+		}
+
+		check := RequiredIPVSKernelModulesAvailableCheck{
+			Executor: &fexec,
+		}
+		warnings, errors := check.Check()
+
+		switch {
+		case warnings != nil && !tc.expectWarnings:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected warnings for installed kernel modules %v and builtin kernel modules %v. Warnings: %v", tc.loadedKernel, tc.builtinKernel, warnings)
+		case warnings == nil && tc.expectWarnings:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected warnings for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel)
+		case errors != nil && !tc.expectErrors:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected errors for installed kernel modules %v and builtin kernel modules %v. errors: %v", tc.loadedKernel, tc.builtinKernel, errors)
+		case errors == nil && tc.expectErrors:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected errors for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel)
+		}
+	}
+}

pkg/util/ipvs/kernelcheck_unsupported.go

@@ -0,0 +1,39 @@
+// +build !linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	utilsexec "k8s.io/utils/exec"
+)
+
+// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules.
+type RequiredIPVSKernelModulesAvailableCheck struct {
+	Executor utilsexec.Interface
+}
+
+// Name returns label for RequiredIPVSKernelModulesAvailableCheck
+func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
+	return "RequiredIPVSKernelModulesAvailable"
+}
+
+// Check try to validates IPVS required kernel modules exists or not.
+func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
+
+	return nil, nil
+}