From 2914171ed42d14f1fecc8865772e93f91c83ea09 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 6 Mar 2019 14:07:01 +0200 Subject: [PATCH] kubeadm: reimplement IPVS check Used existing IPVS Proxier API CanUseIPVSProxier instead of custom implementation. Fixes kubernetes/kubeadm#975 --- cmd/kubeadm/.import-restrictions | 16 ++- cmd/kubeadm/app/preflight/BUILD | 2 + cmd/kubeadm/app/preflight/checks.go | 22 ++-- cmd/kubeadm/app/preflight/checks_darwin.go | 6 + cmd/kubeadm/app/preflight/checks_linux.go | 13 ++ cmd/kubeadm/app/preflight/checks_windows.go | 6 + pkg/util/ipvs/BUILD | 6 - pkg/util/ipvs/kernelcheck_linux.go | 102 --------------- pkg/util/ipvs/kernelcheck_linux_test.go | 130 -------------------- 9 files changed, 54 insertions(+), 249 deletions(-) delete mode 100644 pkg/util/ipvs/kernelcheck_linux.go delete mode 100644 pkg/util/ipvs/kernelcheck_linux_test.go diff --git a/cmd/kubeadm/.import-restrictions b/cmd/kubeadm/.import-restrictions index 17d0203174a..510a28f8525 100644 --- a/cmd/kubeadm/.import-restrictions +++ b/cmd/kubeadm/.import-restrictions @@ -43,7 +43,8 @@ "k8s.io/utils/integer", "k8s.io/utils/path", "k8s.io/utils/pointer", - "k8s.io/utils/net" + "k8s.io/utils/net", + "k8s.io/utils/trace" ] }, { @@ -57,6 +58,7 @@ "k8s.io/kubernetes/pkg/apis/rbac", "k8s.io/kubernetes/pkg/apis/scheduling", "k8s.io/kubernetes/pkg/api/v1/pod", + "k8s.io/kubernetes/pkg/api/v1/service", "k8s.io/kubernetes/pkg/capabilities", "k8s.io/kubernetes/pkg/controller", "k8s.io/kubernetes/pkg/features", @@ -67,20 +69,26 @@ "k8s.io/kubernetes/pkg/kubelet/types", "k8s.io/kubernetes/pkg/master/ports", "k8s.io/kubernetes/pkg/proxy/apis/config", + "k8s.io/kubernetes/pkg/proxy", "k8s.io/kubernetes/pkg/registry/core/service/allocator", "k8s.io/kubernetes/pkg/registry/core/service/ipallocator", "k8s.io/kubernetes/pkg/security/apparmor", "k8s.io/kubernetes/pkg/serviceaccount", + "k8s.io/kubernetes/pkg/util/async", + "k8s.io/kubernetes/pkg/util/conntrack", + "k8s.io/kubernetes/pkg/util/dbus", "k8s.io/kubernetes/pkg/util/hash", "k8s.io/kubernetes/pkg/util/initsystem", + "k8s.io/kubernetes/pkg/util/ipset", + "k8s.io/kubernetes/pkg/util/iptables", "k8s.io/kubernetes/pkg/util/ipvs", "k8s.io/kubernetes/pkg/util/metrics", "k8s.io/kubernetes/pkg/util/node", "k8s.io/kubernetes/pkg/util/normalizer", "k8s.io/kubernetes/pkg/util/parsers", "k8s.io/kubernetes/pkg/util/procfs", + "k8s.io/kubernetes/pkg/util/sysctl", "k8s.io/kubernetes/pkg/util/taints", - "k8s.io/kubernetes/pkg/util/ipvs", "k8s.io/kubernetes/pkg/version" ], "ForbiddenPrefixes": [ @@ -120,6 +128,7 @@ "github.com/docker/go-connections/tlsconfig", "github.com/docker/go-units", "github.com/docker/libnetwork/ipvs", + "github.com/godbus/dbus", "github.com/gogo/protobuf/proto", "github.com/gogo/protobuf/sortkeys", "github.com/golang/groupcache/lru", @@ -156,7 +165,8 @@ "github.com/russross/blackfriday", "github.com/shurcooL/sanitized_anchor_name", "github.com/spf13/cobra", - "github.com/spf13/pflag" + "github.com/spf13/pflag", + "github.com/vishvananda/netlink" ] }, { diff --git a/cmd/kubeadm/app/preflight/BUILD b/cmd/kubeadm/app/preflight/BUILD index bd91dd56b19..f1841b3b79e 100644 --- a/cmd/kubeadm/app/preflight/BUILD +++ b/cmd/kubeadm/app/preflight/BUILD @@ -39,6 +39,8 @@ go_library( ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//cmd/kubeadm/app/util:go_default_library", + "//pkg/proxy/ipvs:go_default_library", + "//pkg/util/ipset:go_default_library", ], "//conditions:default": [], }), diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index f0b652e41c0..584b77bf364 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -872,6 +872,16 @@ func (ncc NumCPUCheck) Check() (warnings, errorList []error) { return warnings, errorList } +// IPVSProxierCheck tests if IPVS proxier can be used. +type IPVSProxierCheck struct { + exec utilsexec.Interface +} + +// Name returns label for IPVSProxierCheck +func (r IPVSProxierCheck) Name() string { + return "IPVSProxierCheck" +} + // RunInitNodeChecks executes all individual, applicable to control-plane node checks. // The boolean flag 'isSecondaryControlPlane' controls whether we are running checks in a --join-control-plane scenario. // If the flag is set to true we should skip checks already executed by RunJoinNodeChecks and RunOptionalJoinNodeChecks. @@ -903,11 +913,9 @@ func RunInitNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.InitConfigura if !isSecondaryControlPlane { checks = addCommonChecks(execer, cfg, checks) - // Check IVPS required kernel module once we use IVPS kube-proxy mode + // Check if IVPS kube-proxy mode is supported if cfg.ComponentConfigs.KubeProxy != nil && cfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode { - checks = append(checks, - ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, - ) + checks = append(checks, IPVSProxierCheck{exec: execer}) } // Check if Bridge-netfilter and IPv6 relevant flags are set @@ -994,11 +1002,9 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura func RunOptionalJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.ClusterConfiguration, ignorePreflightErrors sets.String) error { checks := []Checker{} - // Check ipvs required kernel module if we use ipvs kube-proxy mode + // Check if IVPS kube-proxy mode is supported if cfg.ComponentConfigs.KubeProxy != nil && cfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode { - checks = append(checks, - ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, - ) + checks = append(checks, IPVSProxierCheck{exec: execer}) } return RunChecks(checks, os.Stderr, ignorePreflightErrors) diff --git a/cmd/kubeadm/app/preflight/checks_darwin.go b/cmd/kubeadm/app/preflight/checks_darwin.go index d4cb02628a4..59e9d030be9 100644 --- a/cmd/kubeadm/app/preflight/checks_darwin.go +++ b/cmd/kubeadm/app/preflight/checks_darwin.go @@ -25,3 +25,9 @@ package preflight func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) { return nil, nil } + +// Check determines if IPVS proxier can be used or not +// No-op for for Darwin (MacOS). +func (ipvspc IPVSProxierCheck) Check() (warnings, errors []error) { + return nil, nil +} diff --git a/cmd/kubeadm/app/preflight/checks_linux.go b/cmd/kubeadm/app/preflight/checks_linux.go index 12828815ce0..9ae7147e9c5 100644 --- a/cmd/kubeadm/app/preflight/checks_linux.go +++ b/cmd/kubeadm/app/preflight/checks_linux.go @@ -21,7 +21,10 @@ package preflight import ( "github.com/pkg/errors" "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/proxy/ipvs" "k8s.io/utils/exec" + + utilipset "k8s.io/kubernetes/pkg/util/ipset" ) // Check validates if Docker is setup to use systemd as the cgroup driver. @@ -42,3 +45,13 @@ func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) { } return warnings, nil } + +// Check determines if IPVS proxier can be used or not +func (ipvspc IPVSProxierCheck) Check() (warnings, errors []error) { + ipsetInterface := utilipset.New(ipvspc.exec) + kernelHandler := ipvs.NewLinuxKernelHandler() + if _, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface); err != nil { + return nil, append(errors, err) + } + return nil, nil +} diff --git a/cmd/kubeadm/app/preflight/checks_windows.go b/cmd/kubeadm/app/preflight/checks_windows.go index 85bae087cca..f7c06d49d1d 100644 --- a/cmd/kubeadm/app/preflight/checks_windows.go +++ b/cmd/kubeadm/app/preflight/checks_windows.go @@ -49,3 +49,9 @@ func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) { return nil, nil } + +// Check determines if IPVS proxier can be used or not +// No-op for Windows. +func (ipvspc IPVSProxierCheck) Check() (warnings, errors []error) { + return nil, nil +} diff --git a/pkg/util/ipvs/BUILD b/pkg/util/ipvs/BUILD index 88111db03df..12f84bd5363 100644 --- a/pkg/util/ipvs/BUILD +++ b/pkg/util/ipvs/BUILD @@ -11,14 +11,11 @@ go_test( srcs = [ "ipvs_linux_test.go", "ipvs_test.go", - "kernelcheck_linux_test.go", ], embed = [":go_default_library"], deps = select({ "@io_bazel_rules_go//go/platform:linux": [ "//vendor/github.com/docker/libnetwork/ipvs:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", ], "//conditions:default": [], }), @@ -30,7 +27,6 @@ go_library( "ipvs.go", "ipvs_linux.go", "ipvs_unsupported.go", - "kernelcheck_linux.go", "kernelcheck_unsupported.go", ], importpath = "k8s.io/kubernetes/pkg/util/ipvs", @@ -39,9 +35,7 @@ go_library( "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/docker/libnetwork/ipvs:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], diff --git a/pkg/util/ipvs/kernelcheck_linux.go b/pkg/util/ipvs/kernelcheck_linux.go deleted file mode 100644 index 6389d583667..00000000000 --- a/pkg/util/ipvs/kernelcheck_linux.go +++ /dev/null @@ -1,102 +0,0 @@ -// +build linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvs - -import ( - "fmt" - "regexp" - "strings" - - "k8s.io/apimachinery/pkg/util/sets" - utilsexec "k8s.io/utils/exec" - - "github.com/lithammer/dedent" - "k8s.io/klog" -) - -// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules. -type RequiredIPVSKernelModulesAvailableCheck struct { - Executor utilsexec.Interface -} - -// Name returns label for RequiredIPVSKernelModulesAvailableCheck -func (r RequiredIPVSKernelModulesAvailableCheck) Name() string { - return "RequiredIPVSKernelModulesAvailable" -} - -// Check try to validates IPVS required kernel modules exists or not. -// The name of function can not be changed. -func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) { - klog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not") - - kernelVersion, ipvsModules, err := GetKernelVersionAndIPVSMods(r.Executor) - if err != nil { - errors = append(errors, err) - } - - // Find out loaded kernel modules - out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() - if err != nil { - errors = append(errors, fmt.Errorf("error getting installed ipvs required kernel modules: %v(%s)", err, out)) - return nil, errors - } - mods := strings.Split(string(out), "\n") - - wantModules := sets.NewString() - loadModules := sets.NewString() - wantModules.Insert(ipvsModules...) - loadModules.Insert(mods...) - modules := wantModules.Difference(loadModules).UnsortedList() - - // Check builtin modules exist or not - if len(modules) != 0 { - builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion) - out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput() - if err != nil { - errors = append(errors, fmt.Errorf("error getting required builtin kernel modules: %v(%s)", err, out)) - return nil, errors - } - - builtInModules := sets.NewString() - for _, builtInMode := range ipvsModules { - match, _ := regexp.Match(builtInMode+".ko", out) - if !match { - builtInModules.Insert(string(builtInMode)) - } - } - if len(builtInModules) != 0 { - warnings = append(warnings, fmt.Errorf(dedent.Dedent(` - - The IPVS proxier may not be used because the following required kernel modules are not loaded: %v - or no builtin kernel IPVS support was found: %v. - However, these modules may be loaded automatically by kube-proxy if they are available on your system. - To verify IPVS support: - - Run "lsmod | grep 'ip_vs|nf_conntrack'" and verify each of the above modules are listed. - - If they are not listed, you can use the following methods to load them: - - 1. For each missing module run 'modprobe $modulename' (e.g., 'modprobe ip_vs', 'modprobe ip_vs_rr', ...) - 2. If 'modprobe $modulename' returns an error, you will need to install the missing module support for your kernel. - `), modules, builtInModules)) - } - } - - return warnings, errors -} diff --git a/pkg/util/ipvs/kernelcheck_linux_test.go b/pkg/util/ipvs/kernelcheck_linux_test.go deleted file mode 100644 index 8d2eb76465b..00000000000 --- a/pkg/util/ipvs/kernelcheck_linux_test.go +++ /dev/null @@ -1,130 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvs - -import ( - "testing" - - utilsexec "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) { - cases := []struct { - caseName string - - loadedKernel string - kernelVersion string - builtinKernel string - - expectErrors bool - expectWarnings bool - }{ - { - caseName: "no installed kernel modules and no builtin kernel modules", - loadedKernel: "", - kernelVersion: "3.13.0-24-generic", - builtinKernel: "", - expectErrors: false, - expectWarnings: true, - }, - { - caseName: "no installed kernel modules and missing builtin kernel modules", - loadedKernel: "", - kernelVersion: "3.13.0-24-generic", - builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" + - "kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko", - expectErrors: false, - expectWarnings: true, - }, - { - caseName: "no installed kernel modules and own all builtin kernel modules", - loadedKernel: "", - kernelVersion: "3.13.0-24-generic", - builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" + - "kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" + - "kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" + - "kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" + - "kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko", - expectErrors: false, - expectWarnings: false, - }, - { - caseName: "missing installed kernel modules and no builtin kernel modules", - loadedKernel: "ip_vs", - kernelVersion: "3.13.0-24-generic", - builtinKernel: "", - expectErrors: false, - expectWarnings: true, - }, - { - caseName: "own all installed kernel modules and no builtin kernel modules", - loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + - "ip_vs_rr\n" + "ip_vs_sh", - kernelVersion: "3.13.0-24-generic", - builtinKernel: "", - expectErrors: false, - expectWarnings: false, - }, - { - caseName: "own all installed kernel modules and all builtin kernel modules", - loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + "ip_vs_rr\n" + "ip_vs_sh", - kernelVersion: "3.13.0-24-generic", - builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" + - "kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" + - "kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" + - "kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" + - "kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko", - expectErrors: false, - expectWarnings: false, - }, - } - - for i, tc := range cases { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil }, - func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, - func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil }, - }, - } - - fexec := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } - - check := RequiredIPVSKernelModulesAvailableCheck{ - Executor: &fexec, - } - warnings, errors := check.Check() - - switch { - case warnings != nil && !tc.expectWarnings: - t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected warnings for installed kernel modules %v and builtin kernel modules %v. Warnings: %v", tc.loadedKernel, tc.builtinKernel, warnings) - case warnings == nil && tc.expectWarnings: - t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected warnings for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel) - case errors != nil && !tc.expectErrors: - t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected errors for installed kernel modules %v and builtin kernel modules %v. errors: %v", tc.loadedKernel, tc.builtinKernel, errors) - case errors == nil && tc.expectErrors: - t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected errors for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel) - } - } -}