From a375aa28ee30a089a8c73d1b8069948b4d62774c Mon Sep 17 00:00:00 2001 From: Daman Arora Date: Wed, 25 Oct 2023 23:21:56 +0530 Subject: [PATCH] pkg/proxy: move get kernel version out of ipvs proxier Signed-off-by: Daman Arora --- cmd/kube-proxy/app/server_others.go | 3 -- pkg/kubelet/sysctl/safe_sysctls.go | 38 ++++--------- pkg/proxy/ipvs/proxier.go | 60 ++++----------------- pkg/util/kernel/OWNERS | 8 +++ pkg/util/kernel/constants.go | 45 ++++++++++++++++ pkg/util/kernel/version.go | 48 +++++++++++++++++ pkg/util/kernel/version_test.go | 82 +++++++++++++++++++++++++++++ 7 files changed, 204 insertions(+), 80 deletions(-) create mode 100644 pkg/util/kernel/OWNERS create mode 100644 pkg/util/kernel/constants.go create mode 100644 pkg/util/kernel/version.go create mode 100644 pkg/util/kernel/version_test.go diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index ff3fff7e66b..63cb84576c8 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -210,7 +210,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio return nil, fmt.Errorf("unable to create proxier: %v", err) } } else if config.Mode == proxyconfigapi.ProxyModeIPVS { - kernelHandler := ipvs.NewLinuxKernelHandler() ipsetInterface := utilipset.New(execer) ipvsInterface := utilipvs.New() if err := ipvs.CanUseIPVSProxier(ipvsInterface, ipsetInterface, config.IPVS.Scheduler); err != nil { @@ -248,7 +247,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.HealthzServer, config.IPVS.Scheduler, config.NodePortAddresses, - kernelHandler, initOnly, ) } else { @@ -281,7 +279,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.HealthzServer, config.IPVS.Scheduler, config.NodePortAddresses, - kernelHandler, initOnly, ) } diff --git a/pkg/kubelet/sysctl/safe_sysctls.go b/pkg/kubelet/sysctl/safe_sysctls.go index 098da3434c7..c502be46e2b 100644 --- a/pkg/kubelet/sysctl/safe_sysctls.go +++ b/pkg/kubelet/sysctl/safe_sysctls.go @@ -17,12 +17,11 @@ limitations under the License. package sysctl import ( - "fmt" goruntime "runtime" "k8s.io/apimachinery/pkg/util/version" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/proxy/ipvs" + utilkernel "k8s.io/kubernetes/pkg/util/kernel" ) type sysctl struct { @@ -44,27 +43,22 @@ var safeSysctls = []sysctl{ }, { name: "net.ipv4.ip_unprivileged_port_start", }, { - name: "net.ipv4.ip_local_reserved_ports", - // refer to https://github.com/torvalds/linux/commit/122ff243f5f104194750ecbc76d5946dd1eec934. - kernel: "3.16", + name: "net.ipv4.ip_local_reserved_ports", + kernel: utilkernel.IPLocalReservedPortsNamespacedKernelVersion, }, { - name: "net.ipv4.tcp_keepalive_time", - // refer to https://github.com/torvalds/linux/commit/13b287e8d1cad951634389f85b8c9b816bd3bb1e. - kernel: "4.5", + name: "net.ipv4.tcp_keepalive_time", + kernel: utilkernel.TCPKeepAliveTimeNamespacedKernelVersion, }, { - // refer to https://github.com/torvalds/linux/commit/1e579caa18b96f9eb18f4f5416658cd15f37c062. name: "net.ipv4.tcp_fin_timeout", - kernel: "4.6", + kernel: utilkernel.TCPFinTimeoutNamespacedKernelVersion, }, { - // refer to https://github.com/torvalds/linux/commit/b840d15d39128d08ed4486085e5507d2617b9ae1. name: "net.ipv4.tcp_keepalive_intvl", - kernel: "4.5", + kernel: utilkernel.TCPKeepAliveIntervalNamespacedKernelVersion, }, { - // refer to https://github.com/torvalds/linux/commit/9bd6861bd4326e3afd3f14a9ec8a723771fb20bb. name: "net.ipv4.tcp_keepalive_probes", - kernel: "4.5", + kernel: utilkernel.TCPKeepAliveProbesNamespacedKernelVersion, }, } @@ -77,7 +71,8 @@ func SafeSysctlAllowlist() []string { if goruntime.GOOS != "linux" { return nil } - return getSafeSysctlAllowlist(getKernelVersion) + + return getSafeSysctlAllowlist(utilkernel.GetVersion) } func getSafeSysctlAllowlist(getVersion func() (*version.Version, error)) []string { @@ -101,16 +96,3 @@ func getSafeSysctlAllowlist(getVersion func() (*version.Version, error)) []strin } return safeSysctlAllowlist } - -func getKernelVersion() (*version.Version, error) { - kernelVersionStr, err := ipvs.NewLinuxKernelHandler().GetKernelVersion() - if err != nil { - return nil, fmt.Errorf("failed to get kernel version: %w", err) - } - - kernelVersion, err := version.ParseGeneric(kernelVersionStr) - if err != nil { - return nil, fmt.Errorf("failed to parse kernel version: %w", err) - } - return kernelVersion, nil -} diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index bf68149199b..6c6ff58b500 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "net" - "os" "reflect" "strconv" "strings" @@ -53,6 +52,7 @@ import ( proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" "k8s.io/kubernetes/pkg/util/async" utiliptables "k8s.io/kubernetes/pkg/util/iptables" + utilkernel "k8s.io/kubernetes/pkg/util/kernel" ) const ( @@ -93,11 +93,6 @@ const ( // defaultDummyDevice is the default dummy interface which ipvs service address will bind to it. defaultDummyDevice = "kube-ipvs0" - - connReuseMinSupportedKernelVersion = "4.1" - - // https://github.com/torvalds/linux/commit/35dfb013149f74c2be1ff9c78f14e6a3cd1539d1 - connReuseFixedKernelVersion = "5.9" ) // iptablesJumpChain is tables of iptables chains that ipvs proxier used to install iptables or cleanup iptables. @@ -339,7 +334,6 @@ func NewProxier(ipFamily v1.IPFamily, healthzServer *healthcheck.ProxierHealthServer, scheduler string, nodePortAddressStrings []string, - kernelHandler KernelHandler, initOnly bool, ) (*Proxier, error) { // Set the conntrack sysctl we need for @@ -347,17 +341,14 @@ func NewProxier(ipFamily v1.IPFamily, return nil, err } - kernelVersionStr, err := kernelHandler.GetKernelVersion() + kernelVersion, err := utilkernel.GetVersion() if err != nil { - return nil, fmt.Errorf("error determining kernel version to find required kernel modules for ipvs support: %v", err) + return nil, fmt.Errorf("failed to get kernel version: %w", err) } - kernelVersion, err := version.ParseGeneric(kernelVersionStr) - if err != nil { - return nil, fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err) - } - if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { - klog.ErrorS(nil, "Can't set sysctl, kernel version doesn't satisfy minimum version requirements", "sysctl", sysctlConnReuse, "minimumKernelVersion", connReuseMinSupportedKernelVersion) - } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { + + if kernelVersion.LessThan(version.MustParseGeneric(utilkernel.IPVSConnReuseModeMinSupportedKernelVersion)) { + klog.ErrorS(nil, "Can't set sysctl, kernel version doesn't satisfy minimum version requirements", "sysctl", sysctlConnReuse, "minimumKernelVersion", utilkernel.IPVSConnReuseModeMinSupportedKernelVersion) + } else if kernelVersion.AtLeast(version.MustParseGeneric(utilkernel.IPVSConnReuseModeFixedKernelVersion)) { // https://github.com/kubernetes/kubernetes/issues/93297 klog.V(2).InfoS("Left as-is", "sysctl", sysctlConnReuse) } else { @@ -495,7 +486,6 @@ func NewDualStackProxier( healthzServer *healthcheck.ProxierHealthServer, scheduler string, nodePortAddresses []string, - kernelHandler KernelHandler, initOnly bool, ) (proxy.Provider, error) { @@ -505,8 +495,8 @@ func NewDualStackProxier( ipv4Proxier, err := NewProxier(v1.IPv4Protocol, ipt[0], ipvs, safeIpset, sysctl, exec, syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP, tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit, - localDetectors[0], hostname, nodeIPs[v1.IPv4Protocol], - recorder, healthzServer, scheduler, nodePortAddresses, kernelHandler, initOnly) + localDetectors[0], hostname, nodeIPs[v1.IPv4Protocol], recorder, + healthzServer, scheduler, nodePortAddresses, initOnly) if err != nil { return nil, fmt.Errorf("unable to create ipv4 proxier: %v", err) } @@ -514,8 +504,8 @@ func NewDualStackProxier( ipv6Proxier, err := NewProxier(v1.IPv6Protocol, ipt[1], ipvs, safeIpset, sysctl, exec, syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP, tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit, - localDetectors[1], hostname, nodeIPs[v1.IPv6Protocol], - recorder, healthzServer, scheduler, nodePortAddresses, kernelHandler, initOnly) + localDetectors[1], hostname, nodeIPs[v1.IPv6Protocol], recorder, + healthzServer, scheduler, nodePortAddresses, initOnly) if err != nil { return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) } @@ -557,23 +547,6 @@ func newServiceInfo(port *v1.ServicePort, service *v1.Service, bsvcPortInfo *pro return svcPort } -// KernelHandler can handle the current installed kernel modules. -type KernelHandler interface { - GetKernelVersion() (string, error) -} - -// LinuxKernelHandler implements KernelHandler interface. -type LinuxKernelHandler struct { - executor utilexec.Interface -} - -// NewLinuxKernelHandler initializes LinuxKernelHandler with exec. -func NewLinuxKernelHandler() *LinuxKernelHandler { - return &LinuxKernelHandler{ - executor: utilexec.New(), - } -} - // getFirstColumn reads all the content from r into memory and return a // slice which consists of the first word from each line. func getFirstColumn(r io.Reader) ([]string, error) { @@ -593,17 +566,6 @@ func getFirstColumn(r io.Reader) ([]string, error) { return words, nil } -// GetKernelVersion returns currently running kernel version. -func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) { - kernelVersionFile := "/proc/sys/kernel/osrelease" - fileContent, err := os.ReadFile(kernelVersionFile) - if err != nil { - return "", fmt.Errorf("error reading osrelease file %q: %v", kernelVersionFile, err) - } - - return strings.TrimSpace(string(fileContent)), nil -} - // CanUseIPVSProxier checks if we can use the ipvs Proxier. // The ipset version and the scheduler are checked. If any virtual servers (VS) // already exist with the configured scheduler, we just return. Otherwise diff --git a/pkg/util/kernel/OWNERS b/pkg/util/kernel/OWNERS new file mode 100644 index 00000000000..9437a585813 --- /dev/null +++ b/pkg/util/kernel/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - sig-network-reviewers + - sig-node-reviewers +approvers: + - sig-network-approvers + - sig-node-approvers diff --git a/pkg/util/kernel/constants.go b/pkg/util/kernel/constants.go new file mode 100644 index 00000000000..da512ce3c71 --- /dev/null +++ b/pkg/util/kernel/constants.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kernel + +// IPLocalReservedPortsNamespacedKernelVersion is the kernel version in which net.ipv4.ip_local_reserved_ports was namespaced(netns). +// (ref: https://github.com/torvalds/linux/commit/122ff243f5f104194750ecbc76d5946dd1eec934) +const IPLocalReservedPortsNamespacedKernelVersion = "3.16" + +// IPVSConnReuseModeMinSupportedKernelVersion is the minium kernel version supporting net.ipv4.vs.conn_reuse_mode. +// (ref: https://github.com/torvalds/linux/commit/d752c364571743d696c2a54a449ce77550c35ac5) +const IPVSConnReuseModeMinSupportedKernelVersion = "4.1" + +// TCPKeepAliveTimeNamespacedKernelVersion is the kernel version in which net.ipv4.tcp_keepalive_time was namespaced(netns). +// (ref: https://github.com/torvalds/linux/commit/13b287e8d1cad951634389f85b8c9b816bd3bb1e) +const TCPKeepAliveTimeNamespacedKernelVersion = "4.5" + +// TCPKeepAliveIntervalNamespacedKernelVersion is the kernel version in which net.ipv4.tcp_keepalive_intvl was namespaced(netns). +// (ref: https://github.com/torvalds/linux/commit/b840d15d39128d08ed4486085e5507d2617b9ae1) +const TCPKeepAliveIntervalNamespacedKernelVersion = "4.5" + +// TCPKeepAliveProbesNamespacedKernelVersion is the kernel version in which net.ipv4.tcp_keepalive_probes was namespaced(netns). +// (ref: https://github.com/torvalds/linux/commit/9bd6861bd4326e3afd3f14a9ec8a723771fb20bb) +const TCPKeepAliveProbesNamespacedKernelVersion = "4.5" + +// TCPFinTimeoutNamespacedKernelVersion is the kernel version in which net.ipv4.tcp_fin_timeout was namespaced(netns). +// (ref: https://github.com/torvalds/linux/commit/1e579caa18b96f9eb18f4f5416658cd15f37c062) +const TCPFinTimeoutNamespacedKernelVersion = "4.6" + +// IPVSConnReuseModeFixedKernelVersion is the kernel version in which net.ipv4.vs.conn_reuse_mode was fixed. +// (ref: https://github.com/torvalds/linux/commit/35dfb013149f74c2be1ff9c78f14e6a3cd1539d1) +const IPVSConnReuseModeFixedKernelVersion = "5.9" diff --git a/pkg/util/kernel/version.go b/pkg/util/kernel/version.go new file mode 100644 index 00000000000..79f0bf9a5e9 --- /dev/null +++ b/pkg/util/kernel/version.go @@ -0,0 +1,48 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kernel + +import ( + "fmt" + "os" + "strings" + + "k8s.io/apimachinery/pkg/util/version" +) + +type readFileFunc func(string) ([]byte, error) + +// GetVersion returns currently running kernel version. +func GetVersion() (*version.Version, error) { + return getVersion(os.ReadFile) +} + +// getVersion reads os release file from the give readFile function. +func getVersion(readFile readFileFunc) (*version.Version, error) { + kernelVersionFile := "/proc/sys/kernel/osrelease" + fileContent, err := readFile(kernelVersionFile) + if err != nil { + return nil, fmt.Errorf("failed to read os-release file: %s", err.Error()) + } + + kernelVersion, err := version.ParseGeneric(strings.TrimSpace(string(fileContent))) + if err != nil { + return nil, fmt.Errorf("failed to parse kernel version: %s", err.Error()) + } + + return kernelVersion, nil +} diff --git a/pkg/util/kernel/version_test.go b/pkg/util/kernel/version_test.go new file mode 100644 index 00000000000..cd619db81bb --- /dev/null +++ b/pkg/util/kernel/version_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kernel + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/apimachinery/pkg/util/version" +) + +func TestGetVersion(t *testing.T) { + testCases := []struct { + name string + readFileFunc readFileFunc + expected *version.Version + err error + }{ + { + name: "valid os-release file", + readFileFunc: func(_ string) ([]byte, error) { + return []byte("5.15.0-84-generic"), nil + }, + expected: version.MajorMinor(5, 15), + }, + { + name: "valid os-release file", + readFileFunc: func(_ string) ([]byte, error) { + return []byte("5.4.0-128-generic"), nil + }, + expected: version.MajorMinor(5, 4), + }, + { + name: "failed to read os-release file", + readFileFunc: func(_ string) ([]byte, error) { + return nil, errors.New("open /proc/sys/kernel/osrelease: failed to read file") + }, + err: errors.New("failed to read os-release file: open /proc/sys/kernel/osrelease: failed to read file"), + expected: nil, + }, + { + name: "version not parsable", + readFileFunc: func(_ string) ([]byte, error) { + return []byte("5-15-0"), nil + }, + err: errors.New("failed to parse kernel version: illegal version string \"5-15-0\""), + expected: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kernelVersion, err := getVersion(tc.readFileFunc) + + if tc.err != nil { + assert.Equal(t, tc.err.Error(), err.Error()) + assert.Nil(t, kernelVersion) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expected.Major(), kernelVersion.Major()) + assert.Equal(t, tc.expected.Minor(), kernelVersion.Minor()) + assert.Equal(t, tc.expected.Patch(), kernelVersion.Patch()) + } + }) + } +}