mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-30 15:05:27 +00:00
Merge pull request #111806 from danwinship/kube-proxy-no-mode-fallback
remove kube-proxy mode fallback
This commit is contained in:
commit
da112dda68
@ -19,7 +19,6 @@ limitations under the License.
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
goflag "flag"
|
goflag "flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
@ -75,9 +74,6 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/proxy/apis/config/validation"
|
"k8s.io/kubernetes/pkg/proxy/apis/config/validation"
|
||||||
"k8s.io/kubernetes/pkg/proxy/config"
|
"k8s.io/kubernetes/pkg/proxy/config"
|
||||||
"k8s.io/kubernetes/pkg/proxy/healthcheck"
|
"k8s.io/kubernetes/pkg/proxy/healthcheck"
|
||||||
"k8s.io/kubernetes/pkg/proxy/iptables"
|
|
||||||
"k8s.io/kubernetes/pkg/proxy/ipvs"
|
|
||||||
"k8s.io/kubernetes/pkg/proxy/userspace"
|
|
||||||
proxyutil "k8s.io/kubernetes/pkg/proxy/util"
|
proxyutil "k8s.io/kubernetes/pkg/proxy/util"
|
||||||
"k8s.io/kubernetes/pkg/util/filesystem"
|
"k8s.io/kubernetes/pkg/util/filesystem"
|
||||||
utilflag "k8s.io/kubernetes/pkg/util/flag"
|
utilflag "k8s.io/kubernetes/pkg/util/flag"
|
||||||
@ -90,17 +86,9 @@ import (
|
|||||||
utilpointer "k8s.io/utils/pointer"
|
utilpointer "k8s.io/utils/pointer"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
proxyModeUserspace = "userspace"
|
|
||||||
proxyModeIPTables = "iptables"
|
|
||||||
proxyModeIPVS = "ipvs"
|
|
||||||
proxyModeKernelspace = "kernelspace" //nolint:deadcode,varcheck
|
|
||||||
)
|
|
||||||
|
|
||||||
// proxyRun defines the interface to run a specified ProxyServer
|
// proxyRun defines the interface to run a specified ProxyServer
|
||||||
type proxyRun interface {
|
type proxyRun interface {
|
||||||
Run() error
|
Run() error
|
||||||
CleanupAndExit() error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Options contains everything necessary to create and run a proxy server.
|
// Options contains everything necessary to create and run a proxy server.
|
||||||
@ -314,15 +302,15 @@ func (o *Options) Run() error {
|
|||||||
return o.writeConfigFile()
|
return o.writeConfigFile()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if o.CleanupAndExit {
|
||||||
|
return cleanupAndExit()
|
||||||
|
}
|
||||||
|
|
||||||
proxyServer, err := NewProxyServer(o)
|
proxyServer, err := NewProxyServer(o)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if o.CleanupAndExit {
|
|
||||||
return proxyServer.CleanupAndExit()
|
|
||||||
}
|
|
||||||
|
|
||||||
o.proxyServer = proxyServer
|
o.proxyServer = proxyServer
|
||||||
return o.runLoop()
|
return o.runLoop()
|
||||||
}
|
}
|
||||||
@ -545,7 +533,7 @@ type ProxyServer struct {
|
|||||||
Recorder events.EventRecorder
|
Recorder events.EventRecorder
|
||||||
ConntrackConfiguration kubeproxyconfig.KubeProxyConntrackConfiguration
|
ConntrackConfiguration kubeproxyconfig.KubeProxyConntrackConfiguration
|
||||||
Conntracker Conntracker // if nil, ignored
|
Conntracker Conntracker // if nil, ignored
|
||||||
ProxyMode string
|
ProxyMode kubeproxyconfig.ProxyMode
|
||||||
NodeRef *v1.ObjectReference
|
NodeRef *v1.ObjectReference
|
||||||
MetricsBindAddress string
|
MetricsBindAddress string
|
||||||
BindAddressHardFail bool
|
BindAddressHardFail bool
|
||||||
@ -616,7 +604,7 @@ func serveHealthz(hz healthcheck.ProxierHealthUpdater, errCh chan error) {
|
|||||||
go wait.Until(fn, 5*time.Second, wait.NeverStop)
|
go wait.Until(fn, 5*time.Second, wait.NeverStop)
|
||||||
}
|
}
|
||||||
|
|
||||||
func serveMetrics(bindAddress, proxyMode string, enableProfiling bool, errCh chan error) {
|
func serveMetrics(bindAddress string, proxyMode kubeproxyconfig.ProxyMode, enableProfiling bool, errCh chan error) {
|
||||||
if len(bindAddress) == 0 {
|
if len(bindAddress) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -815,27 +803,6 @@ func getConntrackMax(config kubeproxyconfig.KubeProxyConntrackConfiguration) (in
|
|||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupAndExit remove iptables rules and ipset/ipvs rules in ipvs proxy mode
|
|
||||||
// and exit if success return nil
|
|
||||||
func (s *ProxyServer) CleanupAndExit() error {
|
|
||||||
// cleanup IPv6 and IPv4 iptables rules
|
|
||||||
ipts := []utiliptables.Interface{
|
|
||||||
utiliptables.New(s.execer, utiliptables.ProtocolIPv4),
|
|
||||||
utiliptables.New(s.execer, utiliptables.ProtocolIPv6),
|
|
||||||
}
|
|
||||||
var encounteredError bool
|
|
||||||
for _, ipt := range ipts {
|
|
||||||
encounteredError = userspace.CleanupLeftovers(ipt) || encounteredError
|
|
||||||
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
|
|
||||||
encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, ipt, s.IpsetInterface) || encounteredError
|
|
||||||
}
|
|
||||||
if encounteredError {
|
|
||||||
return errors.New("encountered an error while tearing down rules")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// detectNodeIP returns the nodeIP used by the proxier
|
// detectNodeIP returns the nodeIP used by the proxier
|
||||||
// The order of precedence is:
|
// The order of precedence is:
|
||||||
// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
|
// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
|
||||||
|
@ -43,7 +43,6 @@ import (
|
|||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
"k8s.io/apimachinery/pkg/types"
|
"k8s.io/apimachinery/pkg/types"
|
||||||
utilnet "k8s.io/apimachinery/pkg/util/net"
|
utilnet "k8s.io/apimachinery/pkg/util/net"
|
||||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
toolswatch "k8s.io/client-go/tools/watch"
|
toolswatch "k8s.io/client-go/tools/watch"
|
||||||
"k8s.io/component-base/configz"
|
"k8s.io/component-base/configz"
|
||||||
@ -74,12 +73,11 @@ var timeoutForNodePodCIDR = 5 * time.Minute
|
|||||||
|
|
||||||
// NewProxyServer returns a new ProxyServer.
|
// NewProxyServer returns a new ProxyServer.
|
||||||
func NewProxyServer(o *Options) (*ProxyServer, error) {
|
func NewProxyServer(o *Options) (*ProxyServer, error) {
|
||||||
return newProxyServer(o.config, o.CleanupAndExit, o.master)
|
return newProxyServer(o.config, o.master)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProxyServer(
|
func newProxyServer(
|
||||||
config *proxyconfigapi.KubeProxyConfiguration,
|
config *proxyconfigapi.KubeProxyConfiguration,
|
||||||
cleanupAndExit bool,
|
|
||||||
master string) (*ProxyServer, error) {
|
master string) (*ProxyServer, error) {
|
||||||
|
|
||||||
if config == nil {
|
if config == nil {
|
||||||
@ -92,34 +90,9 @@ func newProxyServer(
|
|||||||
return nil, fmt.Errorf("unable to register configz: %s", err)
|
return nil, fmt.Errorf("unable to register configz: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var iptInterface utiliptables.Interface
|
|
||||||
var ipvsInterface utilipvs.Interface
|
var ipvsInterface utilipvs.Interface
|
||||||
var kernelHandler ipvs.KernelHandler
|
|
||||||
var ipsetInterface utilipset.Interface
|
var ipsetInterface utilipset.Interface
|
||||||
|
|
||||||
// Create a iptables utils.
|
|
||||||
execer := exec.New()
|
|
||||||
|
|
||||||
kernelHandler = ipvs.NewLinuxKernelHandler()
|
|
||||||
ipsetInterface = utilipset.New(execer)
|
|
||||||
canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler)
|
|
||||||
if string(config.Mode) == proxyModeIPVS && err != nil {
|
|
||||||
klog.ErrorS(err, "Can't use the IPVS proxier")
|
|
||||||
}
|
|
||||||
|
|
||||||
if canUseIPVS {
|
|
||||||
ipvsInterface = utilipvs.New()
|
|
||||||
}
|
|
||||||
|
|
||||||
// We omit creation of pretty much everything if we run in cleanup mode
|
|
||||||
if cleanupAndExit {
|
|
||||||
return &ProxyServer{
|
|
||||||
execer: execer,
|
|
||||||
IpvsInterface: ipvsInterface,
|
|
||||||
IpsetInterface: ipsetInterface,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(config.ShowHiddenMetricsForVersion) > 0 {
|
if len(config.ShowHiddenMetricsForVersion) > 0 {
|
||||||
metrics.SetShowHidden()
|
metrics.SetShowHidden()
|
||||||
}
|
}
|
||||||
@ -156,7 +129,7 @@ func newProxyServer(
|
|||||||
var proxier proxy.Provider
|
var proxier proxy.Provider
|
||||||
var detectLocalMode proxyconfigapi.LocalMode
|
var detectLocalMode proxyconfigapi.LocalMode
|
||||||
|
|
||||||
proxyMode := getProxyMode(string(config.Mode), canUseIPVS, iptables.LinuxKernelCompatTester{})
|
proxyMode := getProxyMode(config.Mode)
|
||||||
detectLocalMode, err = getDetectLocalMode(config)
|
detectLocalMode, err = getDetectLocalMode(config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err)
|
return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err)
|
||||||
@ -178,12 +151,13 @@ func newProxyServer(
|
|||||||
if netutils.IsIPv6(nodeIP) {
|
if netutils.IsIPv6(nodeIP) {
|
||||||
primaryProtocol = utiliptables.ProtocolIPv6
|
primaryProtocol = utiliptables.ProtocolIPv6
|
||||||
}
|
}
|
||||||
iptInterface = utiliptables.New(execer, primaryProtocol)
|
execer := exec.New()
|
||||||
|
iptInterface := utiliptables.New(execer, primaryProtocol)
|
||||||
|
|
||||||
var ipt [2]utiliptables.Interface
|
var ipt [2]utiliptables.Interface
|
||||||
dualStack := true // While we assume that node supports, we do further checks below
|
dualStack := true // While we assume that node supports, we do further checks below
|
||||||
|
|
||||||
if proxyMode != proxyModeUserspace {
|
if proxyMode != proxyconfigapi.ProxyModeUserspace {
|
||||||
// Create iptables handlers for both families, one is already created
|
// Create iptables handlers for both families, one is already created
|
||||||
// Always ordered as IPv4, IPv6
|
// Always ordered as IPv4, IPv6
|
||||||
if primaryProtocol == utiliptables.ProtocolIPv4 {
|
if primaryProtocol == utiliptables.ProtocolIPv4 {
|
||||||
@ -202,7 +176,7 @@ func newProxyServer(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if proxyMode == proxyModeIPTables {
|
if proxyMode == proxyconfigapi.ProxyModeIPTables {
|
||||||
klog.V(0).InfoS("Using iptables Proxier")
|
klog.V(0).InfoS("Using iptables Proxier")
|
||||||
if config.IPTables.MasqueradeBit == nil {
|
if config.IPTables.MasqueradeBit == nil {
|
||||||
// MasqueradeBit must be specified or defaulted.
|
// MasqueradeBit must be specified or defaulted.
|
||||||
@ -265,7 +239,14 @@ func newProxyServer(
|
|||||||
return nil, fmt.Errorf("unable to create proxier: %v", err)
|
return nil, fmt.Errorf("unable to create proxier: %v", err)
|
||||||
}
|
}
|
||||||
proxymetrics.RegisterMetrics()
|
proxymetrics.RegisterMetrics()
|
||||||
} else if proxyMode == proxyModeIPVS {
|
} else if proxyMode == proxyconfigapi.ProxyModeIPVS {
|
||||||
|
kernelHandler := ipvs.NewLinuxKernelHandler()
|
||||||
|
ipsetInterface = utilipset.New(execer)
|
||||||
|
if err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler); err != nil {
|
||||||
|
return nil, fmt.Errorf("can't use the IPVS proxier: %v", err)
|
||||||
|
}
|
||||||
|
ipvsInterface = utilipvs.New()
|
||||||
|
|
||||||
klog.V(0).InfoS("Using ipvs Proxier")
|
klog.V(0).InfoS("Using ipvs Proxier")
|
||||||
if dualStack {
|
if dualStack {
|
||||||
klog.V(0).InfoS("Creating dualStackProxier for ipvs")
|
klog.V(0).InfoS("Creating dualStackProxier for ipvs")
|
||||||
@ -361,7 +342,7 @@ func newProxyServer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
useEndpointSlices := true
|
useEndpointSlices := true
|
||||||
if proxyMode == proxyModeUserspace {
|
if proxyMode == proxyconfigapi.ProxyModeUserspace {
|
||||||
// userspace mode doesn't support endpointslice.
|
// userspace mode doesn't support endpointslice.
|
||||||
useEndpointSlices = false
|
useEndpointSlices = false
|
||||||
}
|
}
|
||||||
@ -566,40 +547,37 @@ func cidrTuple(cidrList string) [2]string {
|
|||||||
return cidrs
|
return cidrs
|
||||||
}
|
}
|
||||||
|
|
||||||
func getProxyMode(proxyMode string, canUseIPVS bool, kcompat iptables.KernelCompatTester) string {
|
func getProxyMode(proxyMode proxyconfigapi.ProxyMode) proxyconfigapi.ProxyMode {
|
||||||
switch proxyMode {
|
if proxyMode == "" {
|
||||||
case proxyModeUserspace:
|
klog.InfoS("Using iptables proxy")
|
||||||
return proxyModeUserspace
|
return proxyconfigapi.ProxyModeIPTables
|
||||||
case proxyModeIPTables:
|
} else {
|
||||||
return tryIPTablesProxy(kcompat)
|
return proxyMode
|
||||||
case proxyModeIPVS:
|
|
||||||
return tryIPVSProxy(canUseIPVS, kcompat)
|
|
||||||
}
|
}
|
||||||
klog.InfoS("Unknown proxy mode, assuming iptables proxy", "proxyMode", proxyMode)
|
|
||||||
return tryIPTablesProxy(kcompat)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func tryIPVSProxy(canUseIPVS bool, kcompat iptables.KernelCompatTester) string {
|
// cleanupAndExit remove iptables rules and ipset/ipvs rules
|
||||||
if canUseIPVS {
|
func cleanupAndExit() error {
|
||||||
return proxyModeIPVS
|
execer := exec.New()
|
||||||
|
|
||||||
|
// cleanup IPv6 and IPv4 iptables rules, regardless of current configuration
|
||||||
|
ipts := []utiliptables.Interface{
|
||||||
|
utiliptables.New(execer, utiliptables.ProtocolIPv4),
|
||||||
|
utiliptables.New(execer, utiliptables.ProtocolIPv6),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to fallback to iptables before falling back to userspace
|
ipsetInterface := utilipset.New(execer)
|
||||||
klog.V(1).InfoS("Can't use ipvs proxier, trying iptables proxier")
|
ipvsInterface := utilipvs.New()
|
||||||
return tryIPTablesProxy(kcompat)
|
|
||||||
}
|
|
||||||
|
|
||||||
func tryIPTablesProxy(kcompat iptables.KernelCompatTester) string {
|
var encounteredError bool
|
||||||
// guaranteed false on error, error only necessary for debugging
|
for _, ipt := range ipts {
|
||||||
useIPTablesProxy, err := iptables.CanUseIPTablesProxier(kcompat)
|
encounteredError = userspace.CleanupLeftovers(ipt) || encounteredError
|
||||||
if err != nil {
|
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
|
||||||
utilruntime.HandleError(fmt.Errorf("can't determine whether to use iptables proxy, using userspace proxier: %v", err))
|
encounteredError = ipvs.CleanupLeftovers(ipvsInterface, ipt, ipsetInterface) || encounteredError
|
||||||
return proxyModeUserspace
|
|
||||||
}
|
}
|
||||||
if useIPTablesProxy {
|
if encounteredError {
|
||||||
return proxyModeIPTables
|
return errors.New("encountered an error while tearing down rules")
|
||||||
}
|
}
|
||||||
// Fallback.
|
|
||||||
klog.V(1).InfoS("Can't use iptables proxy, using userspace proxier")
|
return nil
|
||||||
return proxyModeUserspace
|
|
||||||
}
|
}
|
||||||
|
@ -20,7 +20,6 @@ limitations under the License.
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"net"
|
"net"
|
||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
@ -32,154 +31,11 @@ import (
|
|||||||
clientsetfake "k8s.io/client-go/kubernetes/fake"
|
clientsetfake "k8s.io/client-go/kubernetes/fake"
|
||||||
|
|
||||||
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
|
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
|
||||||
"k8s.io/kubernetes/pkg/proxy/ipvs"
|
|
||||||
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
|
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
|
||||||
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
|
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
|
||||||
utiliptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
|
utiliptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
type fakeIPSetVersioner struct {
|
|
||||||
version string // what to return
|
|
||||||
err error // what to return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fake *fakeIPSetVersioner) GetVersion() (string, error) {
|
|
||||||
return fake.version, fake.err
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeKernelCompatTester struct {
|
|
||||||
ok bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fake *fakeKernelCompatTester) IsCompatible() error {
|
|
||||||
if !fake.ok {
|
|
||||||
return fmt.Errorf("error")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fakeKernelHandler implements KernelHandler.
|
|
||||||
type fakeKernelHandler struct {
|
|
||||||
modules []string
|
|
||||||
kernelVersion string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
|
|
||||||
return fake.modules, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fake *fakeKernelHandler) GetKernelVersion() (string, error) {
|
|
||||||
return fake.kernelVersion, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_getProxyMode(t *testing.T) {
|
|
||||||
var cases = []struct {
|
|
||||||
flag string
|
|
||||||
ipsetVersion string
|
|
||||||
kmods []string
|
|
||||||
kernelVersion string
|
|
||||||
kernelCompat bool
|
|
||||||
ipsetError error
|
|
||||||
expected string
|
|
||||||
scheduler string
|
|
||||||
}{
|
|
||||||
{ // flag says userspace
|
|
||||||
flag: "userspace",
|
|
||||||
expected: proxyModeUserspace,
|
|
||||||
},
|
|
||||||
{ // flag says iptables, kernel not compatible
|
|
||||||
flag: "iptables",
|
|
||||||
kernelCompat: false,
|
|
||||||
expected: proxyModeUserspace,
|
|
||||||
},
|
|
||||||
{ // flag says iptables, kernel is compatible
|
|
||||||
flag: "iptables",
|
|
||||||
kernelCompat: true,
|
|
||||||
expected: proxyModeIPTables,
|
|
||||||
},
|
|
||||||
{ // detect, kernel not compatible
|
|
||||||
flag: "",
|
|
||||||
kernelCompat: false,
|
|
||||||
expected: proxyModeUserspace,
|
|
||||||
},
|
|
||||||
{ // detect, kernel is compatible
|
|
||||||
flag: "",
|
|
||||||
kernelCompat: true,
|
|
||||||
expected: proxyModeIPTables,
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, ipset version ok, kernel modules installed for linux kernel before 4.19
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
|
|
||||||
kernelVersion: "4.18",
|
|
||||||
ipsetVersion: ipvs.MinIPSetCheckVersion,
|
|
||||||
expected: proxyModeIPVS,
|
|
||||||
scheduler: "rr",
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, ipset version ok, kernel modules installed for linux kernel 4.19
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: ipvs.MinIPSetCheckVersion,
|
|
||||||
expected: proxyModeIPVS,
|
|
||||||
scheduler: "rr",
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, ipset version too low, fallback on iptables mode
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: "0.0",
|
|
||||||
kernelCompat: true,
|
|
||||||
expected: proxyModeIPTables,
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, bad ipset version, fallback on iptables mode
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: "a.b.c",
|
|
||||||
kernelCompat: true,
|
|
||||||
expected: proxyModeIPTables,
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, required kernel modules are not installed, fallback on iptables mode
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"foo", "bar", "baz"},
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: ipvs.MinIPSetCheckVersion,
|
|
||||||
kernelCompat: true,
|
|
||||||
expected: proxyModeIPTables,
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, ipset version ok, kernel modules installed for sed scheduler
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_sed"},
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: ipvs.MinIPSetCheckVersion,
|
|
||||||
expected: proxyModeIPVS,
|
|
||||||
scheduler: "sed",
|
|
||||||
},
|
|
||||||
{ // flag says ipvs, kernel modules not installed for sed scheduler, fallback to iptables
|
|
||||||
flag: "ipvs",
|
|
||||||
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: ipvs.MinIPSetCheckVersion,
|
|
||||||
expected: proxyModeIPTables,
|
|
||||||
kernelCompat: true,
|
|
||||||
scheduler: "sed",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for i, c := range cases {
|
|
||||||
kcompater := &fakeKernelCompatTester{c.kernelCompat}
|
|
||||||
ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
|
|
||||||
khandler := &fakeKernelHandler{
|
|
||||||
modules: c.kmods,
|
|
||||||
kernelVersion: c.kernelVersion,
|
|
||||||
}
|
|
||||||
canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver, cases[i].scheduler)
|
|
||||||
r := getProxyMode(c.flag, canUseIPVS, kcompater)
|
|
||||||
if r != c.expected {
|
|
||||||
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_getDetectLocalMode(t *testing.T) {
|
func Test_getDetectLocalMode(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
detectLocal string
|
detectLocal string
|
||||||
|
@ -52,10 +52,10 @@ import (
|
|||||||
|
|
||||||
// NewProxyServer returns a new ProxyServer.
|
// NewProxyServer returns a new ProxyServer.
|
||||||
func NewProxyServer(o *Options) (*ProxyServer, error) {
|
func NewProxyServer(o *Options) (*ProxyServer, error) {
|
||||||
return newProxyServer(o.config, o.CleanupAndExit, o.master)
|
return newProxyServer(o.config, o.master)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, master string) (*ProxyServer, error) {
|
func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, master string) (*ProxyServer, error) {
|
||||||
if config == nil {
|
if config == nil {
|
||||||
return nil, errors.New("config is required")
|
return nil, errors.New("config is required")
|
||||||
}
|
}
|
||||||
@ -66,11 +66,6 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
|
|||||||
return nil, fmt.Errorf("unable to register configz: %s", err)
|
return nil, fmt.Errorf("unable to register configz: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// We omit creation of pretty much everything if we run in cleanup mode
|
|
||||||
if cleanupAndExit {
|
|
||||||
return &ProxyServer{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(config.ShowHiddenMetricsForVersion) > 0 {
|
if len(config.ShowHiddenMetricsForVersion) > 0 {
|
||||||
metrics.SetShowHidden()
|
metrics.SetShowHidden()
|
||||||
}
|
}
|
||||||
@ -107,9 +102,9 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
|
|||||||
}
|
}
|
||||||
|
|
||||||
var proxier proxy.Provider
|
var proxier proxy.Provider
|
||||||
proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{})
|
proxyMode := getProxyMode(config.Mode, winkernel.WindowsKernelCompatTester{})
|
||||||
dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{})
|
dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{})
|
||||||
if proxyMode == proxyModeKernelspace {
|
if proxyMode == proxyconfigapi.ProxyModeKernelspace {
|
||||||
klog.V(0).InfoS("Using Kernelspace Proxier.")
|
klog.V(0).InfoS("Using Kernelspace Proxier.")
|
||||||
if dualStackMode {
|
if dualStackMode {
|
||||||
klog.V(0).InfoS("Creating dualStackProxier for Windows kernel.")
|
klog.V(0).InfoS("Creating dualStackProxier for Windows kernel.")
|
||||||
@ -171,7 +166,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
useEndpointSlices := true
|
useEndpointSlices := true
|
||||||
if proxyMode == proxyModeUserspace {
|
if proxyMode == proxyconfigapi.ProxyModeUserspace {
|
||||||
// userspace mode doesn't support endpointslice.
|
// userspace mode doesn't support endpointslice.
|
||||||
useEndpointSlices = false
|
useEndpointSlices = false
|
||||||
}
|
}
|
||||||
@ -197,18 +192,18 @@ func getDualStackMode(networkname string, compatTester winkernel.StackCompatTest
|
|||||||
return compatTester.DualStackCompatible(networkname)
|
return compatTester.DualStackCompatible(networkname)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getProxyMode(proxyMode string, kcompat winkernel.KernelCompatTester) string {
|
func getProxyMode(proxyMode proxyconfigapi.ProxyMode, kcompat winkernel.KernelCompatTester) proxyconfigapi.ProxyMode {
|
||||||
if proxyMode == proxyModeKernelspace {
|
if proxyMode == proxyconfigapi.ProxyModeKernelspace {
|
||||||
return tryWinKernelSpaceProxy(kcompat)
|
return tryWinKernelSpaceProxy(kcompat)
|
||||||
}
|
}
|
||||||
return proxyModeUserspace
|
return proxyconfigapi.ProxyModeUserspace
|
||||||
}
|
}
|
||||||
|
|
||||||
func detectNumCPU() int {
|
func detectNumCPU() int {
|
||||||
return goruntime.NumCPU()
|
return goruntime.NumCPU()
|
||||||
}
|
}
|
||||||
|
|
||||||
func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string {
|
func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) proxyconfigapi.ProxyMode {
|
||||||
// Check for Windows Kernel Version if we can support Kernel Space proxy
|
// Check for Windows Kernel Version if we can support Kernel Space proxy
|
||||||
// Check for Windows Version
|
// Check for Windows Version
|
||||||
|
|
||||||
@ -216,12 +211,17 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string {
|
|||||||
useWinKernelProxy, err := winkernel.CanUseWinKernelProxier(kcompat)
|
useWinKernelProxy, err := winkernel.CanUseWinKernelProxier(kcompat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.ErrorS(err, "Can't determine whether to use windows kernel proxy, using userspace proxier")
|
klog.ErrorS(err, "Can't determine whether to use windows kernel proxy, using userspace proxier")
|
||||||
return proxyModeUserspace
|
return proxyconfigapi.ProxyModeUserspace
|
||||||
}
|
}
|
||||||
if useWinKernelProxy {
|
if useWinKernelProxy {
|
||||||
return proxyModeKernelspace
|
return proxyconfigapi.ProxyModeKernelspace
|
||||||
}
|
}
|
||||||
// Fallback.
|
// Fallback.
|
||||||
klog.V(1).InfoS("Can't use winkernel proxy, using userspace proxier")
|
klog.V(1).InfoS("Can't use winkernel proxy, using userspace proxier")
|
||||||
return proxyModeUserspace
|
return proxyconfigapi.ProxyModeUserspace
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupAndExit cleans up after a previous proxy run
|
||||||
|
func cleanupAndExit() error {
|
||||||
|
return errors.New("--cleanup-and-exit is not implemented on Windows")
|
||||||
}
|
}
|
||||||
|
@ -85,34 +85,6 @@ const (
|
|||||||
largeClusterEndpointsThreshold = 1000
|
largeClusterEndpointsThreshold = 1000
|
||||||
)
|
)
|
||||||
|
|
||||||
// KernelCompatTester tests whether the required kernel capabilities are
|
|
||||||
// present to run the iptables proxier.
|
|
||||||
type KernelCompatTester interface {
|
|
||||||
IsCompatible() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// CanUseIPTablesProxier returns true if we should use the iptables Proxier
|
|
||||||
// instead of the "classic" userspace Proxier.
|
|
||||||
func CanUseIPTablesProxier(kcompat KernelCompatTester) (bool, error) {
|
|
||||||
if err := kcompat.IsCompatible(); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ KernelCompatTester = LinuxKernelCompatTester{}
|
|
||||||
|
|
||||||
// LinuxKernelCompatTester is the Linux implementation of KernelCompatTester
|
|
||||||
type LinuxKernelCompatTester struct{}
|
|
||||||
|
|
||||||
// IsCompatible checks for the required sysctls. We don't care about the value, just
|
|
||||||
// that it exists. If this Proxier is chosen, we'll initialize it as we
|
|
||||||
// need.
|
|
||||||
func (lkct LinuxKernelCompatTester) IsCompatible() error {
|
|
||||||
_, err := utilsysctl.New().GetSysctl(sysctlRouteLocalnet)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
|
const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
|
||||||
const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
|
const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
|
||||||
|
|
||||||
|
@ -706,25 +706,25 @@ func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) {
|
|||||||
return strings.TrimSpace(string(fileContent)), nil
|
return strings.TrimSpace(string(fileContent)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CanUseIPVSProxier returns true if we can use the ipvs Proxier.
|
// CanUseIPVSProxier checks if we can use the ipvs Proxier.
|
||||||
// This is determined by checking if all the required kernel modules can be loaded. It may
|
// This is determined by checking if all the required kernel modules can be loaded. It may
|
||||||
// return an error if it fails to get the kernel modules information without error, in which
|
// return an error if it fails to get the kernel modules information without error, in which
|
||||||
// case it will also return false.
|
// case it will also return false.
|
||||||
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) (bool, error) {
|
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) error {
|
||||||
mods, err := handle.GetModules()
|
mods, err := handle.GetModules()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
|
return fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
|
||||||
}
|
}
|
||||||
loadModules := sets.NewString()
|
loadModules := sets.NewString()
|
||||||
loadModules.Insert(mods...)
|
loadModules.Insert(mods...)
|
||||||
|
|
||||||
kernelVersionStr, err := handle.GetKernelVersion()
|
kernelVersionStr, err := handle.GetKernelVersion()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("error determining kernel version to find required kernel modules for ipvs support: %v", err)
|
return fmt.Errorf("error determining kernel version to find required kernel modules for ipvs support: %v", err)
|
||||||
}
|
}
|
||||||
kernelVersion, err := version.ParseGeneric(kernelVersionStr)
|
kernelVersion, err := version.ParseGeneric(kernelVersionStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err)
|
return fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err)
|
||||||
}
|
}
|
||||||
mods = utilipvs.GetRequiredIPVSModules(kernelVersion)
|
mods = utilipvs.GetRequiredIPVSModules(kernelVersion)
|
||||||
wantModules := sets.NewString()
|
wantModules := sets.NewString()
|
||||||
@ -751,18 +751,18 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(missingMods) != 0 {
|
if len(missingMods) != 0 {
|
||||||
return false, fmt.Errorf("IPVS proxier will not be used because the following required kernel modules are not loaded: %v", missingMods)
|
return fmt.Errorf("IPVS proxier will not be used because the following required kernel modules are not loaded: %v", missingMods)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check ipset version
|
// Check ipset version
|
||||||
versionString, err := ipsetver.GetVersion()
|
versionString, err := ipsetver.GetVersion()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("error getting ipset version, error: %v", err)
|
return fmt.Errorf("error getting ipset version, error: %v", err)
|
||||||
}
|
}
|
||||||
if !checkMinVersion(versionString) {
|
if !checkMinVersion(versionString) {
|
||||||
return false, fmt.Errorf("ipset version: %s is less than min required version: %s", versionString, MinIPSetCheckVersion)
|
return fmt.Errorf("ipset version: %s is less than min required version: %s", versionString, MinIPSetCheckVersion)
|
||||||
}
|
}
|
||||||
return true, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupIptablesLeftovers removes all iptables rules and chains created by the Proxier
|
// CleanupIptablesLeftovers removes all iptables rules and chains created by the Proxier
|
||||||
|
@ -364,9 +364,9 @@ func TestCanUseIPVSProxier(t *testing.T) {
|
|||||||
for i := range testCases {
|
for i := range testCases {
|
||||||
handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion}
|
handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion}
|
||||||
versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
|
versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
|
||||||
ok, err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler)
|
err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler)
|
||||||
if ok != testCases[i].ok {
|
if (err == nil) != testCases[i].ok {
|
||||||
t.Errorf("Case [%d], expect %v, got %v: err: %v", i, testCases[i].ok, ok, err)
|
t.Errorf("Case [%d], expect %v, got err: %v", i, testCases[i].ok, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user