From e28f3de6568cad6f01bb8ce616bb42291a0d0b3a Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Sun, 30 Apr 2023 09:05:42 +0800 Subject: [PATCH] Revert "proxy startup-time config handling cleanup" --- cmd/kube-proxy/app/server.go | 57 ++++--- cmd/kube-proxy/app/server_others.go | 102 ++++++++---- cmd/kube-proxy/app/server_others_test.go | 156 ++++++++++++------ cmd/kube-proxy/app/server_windows.go | 30 ++-- .../apis/config/validation/validation.go | 18 -- .../apis/config/validation/validation_test.go | 22 --- pkg/proxy/kubemark/hollow_proxy.go | 21 +-- 7 files changed, 233 insertions(+), 173 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 1dfe4830a90..3503f585fe0 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -452,8 +452,6 @@ func (o *Options) ApplyDefaults(in *kubeproxyconfig.KubeProxyConfiguration) (*ku out := internal.(*kubeproxyconfig.KubeProxyConfiguration) - o.platformApplyDefaults(out) - return out, nil } @@ -524,16 +522,21 @@ with the apiserver API to configure the proxy.`, // ProxyServer represents all the parameters required to start the Kubernetes proxy server. All // fields are required. type ProxyServer struct { - Config *kubeproxyconfig.KubeProxyConfiguration - - Client clientset.Interface - Broadcaster events.EventBroadcaster - Recorder events.EventRecorder - Conntracker Conntracker // if nil, ignored - NodeRef *v1.ObjectReference - HealthzServer healthcheck.ProxierHealthUpdater - - Proxier proxy.Provider + Client clientset.Interface + Proxier proxy.Provider + Broadcaster events.EventBroadcaster + Recorder events.EventRecorder + ConntrackConfiguration kubeproxyconfig.KubeProxyConntrackConfiguration + Conntracker Conntracker // if nil, ignored + ProxyMode kubeproxyconfig.ProxyMode + NodeRef *v1.ObjectReference + MetricsBindAddress string + BindAddressHardFail bool + EnableProfiling bool + OOMScoreAdj *int32 + ConfigSyncPeriod time.Duration + HealthzServer healthcheck.ProxierHealthUpdater + localDetectorMode kubeproxyconfig.LocalMode } // createClient creates a kube client from the given config and masterOverride. @@ -642,9 +645,9 @@ func (s *ProxyServer) Run() error { // TODO(vmarmol): Use container config for this. var oomAdjuster *oom.OOMAdjuster - if s.Config.OOMScoreAdj != nil { + if s.OOMScoreAdj != nil { oomAdjuster = oom.NewOOMAdjuster() - if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*s.Config.OOMScoreAdj)); err != nil { + if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*s.OOMScoreAdj)); err != nil { klog.V(2).InfoS("Failed to apply OOMScore", "err", err) } } @@ -657,7 +660,7 @@ func (s *ProxyServer) Run() error { // TODO(thockin): make it possible for healthz and metrics to be on the same port. var errCh chan error - if s.Config.BindAddressHardFail { + if s.BindAddressHardFail { errCh = make(chan error) } @@ -665,12 +668,12 @@ func (s *ProxyServer) Run() error { serveHealthz(s.HealthzServer, errCh) // Start up a metrics server if requested - serveMetrics(s.Config.MetricsBindAddress, s.Config.Mode, s.Config.EnableProfiling, errCh) + serveMetrics(s.MetricsBindAddress, s.ProxyMode, s.EnableProfiling, errCh) // Tune conntrack, if requested // Conntracker is always nil for windows if s.Conntracker != nil { - max, err := getConntrackMax(s.Config.Conntrack) + max, err := getConntrackMax(s.ConntrackConfiguration) if err != nil { return err } @@ -693,15 +696,15 @@ func (s *ProxyServer) Run() error { } } - if s.Config.Conntrack.TCPEstablishedTimeout != nil && s.Config.Conntrack.TCPEstablishedTimeout.Duration > 0 { - timeout := int(s.Config.Conntrack.TCPEstablishedTimeout.Duration / time.Second) + if s.ConntrackConfiguration.TCPEstablishedTimeout != nil && s.ConntrackConfiguration.TCPEstablishedTimeout.Duration > 0 { + timeout := int(s.ConntrackConfiguration.TCPEstablishedTimeout.Duration / time.Second) if err := s.Conntracker.SetTCPEstablishedTimeout(timeout); err != nil { return err } } - if s.Config.Conntrack.TCPCloseWaitTimeout != nil && s.Config.Conntrack.TCPCloseWaitTimeout.Duration > 0 { - timeout := int(s.Config.Conntrack.TCPCloseWaitTimeout.Duration / time.Second) + if s.ConntrackConfiguration.TCPCloseWaitTimeout != nil && s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration > 0 { + timeout := int(s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration / time.Second) if err := s.Conntracker.SetTCPCloseWaitTimeout(timeout); err != nil { return err } @@ -722,7 +725,7 @@ func (s *ProxyServer) Run() error { labelSelector = labelSelector.Add(*noProxyName, *noHeadlessEndpoints) // Make informers that filter out objects that want a non-default service proxy. - informerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.Config.ConfigSyncPeriod.Duration, + informerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.ConfigSyncPeriod, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = labelSelector.String() })) @@ -731,11 +734,11 @@ func (s *ProxyServer) Run() error { // Note: RegisterHandler() calls need to happen before creation of Sources because sources // only notify on changes, and the initial update (on process start) may be lost if no handlers // are registered yet. - serviceConfig := config.NewServiceConfig(informerFactory.Core().V1().Services(), s.Config.ConfigSyncPeriod.Duration) + serviceConfig := config.NewServiceConfig(informerFactory.Core().V1().Services(), s.ConfigSyncPeriod) serviceConfig.RegisterEventHandler(s.Proxier) go serviceConfig.Run(wait.NeverStop) - endpointSliceConfig := config.NewEndpointSliceConfig(informerFactory.Discovery().V1().EndpointSlices(), s.Config.ConfigSyncPeriod.Duration) + endpointSliceConfig := config.NewEndpointSliceConfig(informerFactory.Discovery().V1().EndpointSlices(), s.ConfigSyncPeriod) endpointSliceConfig.RegisterEventHandler(s.Proxier) go endpointSliceConfig.Run(wait.NeverStop) @@ -744,13 +747,13 @@ func (s *ProxyServer) Run() error { informerFactory.Start(wait.NeverStop) // Make an informer that selects for our nodename. - currentNodeInformerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.Config.ConfigSyncPeriod.Duration, + currentNodeInformerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.ConfigSyncPeriod, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", s.NodeRef.Name).String() })) - nodeConfig := config.NewNodeConfig(currentNodeInformerFactory.Core().V1().Nodes(), s.Config.ConfigSyncPeriod.Duration) + nodeConfig := config.NewNodeConfig(currentNodeInformerFactory.Core().V1().Nodes(), s.ConfigSyncPeriod) // https://issues.k8s.io/111321 - if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeNodeCIDR { + if s.localDetectorMode == kubeproxyconfig.LocalModeNodeCIDR { nodeConfig.RegisterEventHandler(&proxy.NodePodCIDRHandler{}) } nodeConfig.RegisterEventHandler(s.Proxier) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index f78ab70792b..c4121feb989 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -70,19 +70,6 @@ import ( // node after it is registered. var timeoutForNodePodCIDR = 5 * time.Minute -func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) { - if config.Mode == "" { - klog.InfoS("Using iptables proxy") - config.Mode = proxyconfigapi.ProxyModeIPTables - } - - if config.DetectLocalMode == "" { - klog.V(4).InfoS("Defaulting detect-local-mode", "localModeClusterCIDR", string(proxyconfigapi.LocalModeClusterCIDR)) - config.DetectLocalMode = proxyconfigapi.LocalModeClusterCIDR - } - klog.V(2).InfoS("DetectLocalMode", "localMode", string(config.DetectLocalMode)) -} - // NewProxyServer returns a new ProxyServer. func NewProxyServer(o *Options) (*ProxyServer, error) { return newProxyServer(o.config, o.master) @@ -92,6 +79,10 @@ func newProxyServer( config *proxyconfigapi.KubeProxyConfiguration, master string) (*ProxyServer, error) { + if config == nil { + return nil, errors.New("config is required") + } + if c, err := configz.New(proxyconfigapi.GroupName); err == nil { c.Set(config) } else { @@ -135,9 +126,16 @@ func newProxyServer( } var proxier proxy.Provider + var detectLocalMode proxyconfigapi.LocalMode + + proxyMode := getProxyMode(config.Mode) + detectLocalMode, err = getDetectLocalMode(config) + if err != nil { + return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err) + } var nodeInfo *v1.Node - if config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { + if detectLocalMode == proxyconfigapi.LocalModeNodeCIDR { klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", hostname) nodeInfo, err = waitForPodCIDR(client, hostname) if err != nil { @@ -146,6 +144,8 @@ func newProxyServer( klog.InfoS("NodeInfo", "podCIDR", nodeInfo.Spec.PodCIDR, "podCIDRs", nodeInfo.Spec.PodCIDRs) } + klog.V(2).InfoS("DetectLocalMode", "localMode", string(detectLocalMode)) + primaryFamily := v1.IPv4Protocol primaryProtocol := utiliptables.ProtocolIPv4 if netutils.IsIPv6(nodeIP) { @@ -186,15 +186,19 @@ func newProxyServer( } } - if config.Mode == proxyconfigapi.ProxyModeIPTables { + if proxyMode == proxyconfigapi.ProxyModeIPTables { klog.InfoS("Using iptables Proxier") + if config.IPTables.MasqueradeBit == nil { + // MasqueradeBit must be specified or defaulted. + return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") + } if dualStack { klog.InfoS("kube-proxy running in dual-stack mode", "ipFamily", iptInterface.Protocol()) klog.InfoS("Creating dualStackProxier for iptables") // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -219,7 +223,7 @@ func newProxyServer( } else { // Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support). var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -248,7 +252,7 @@ func newProxyServer( return nil, fmt.Errorf("unable to create proxier: %v", err) } proxymetrics.RegisterMetrics() - } else if config.Mode == proxyconfigapi.ProxyModeIPVS { + } else if proxyMode == proxyconfigapi.ProxyModeIPVS { kernelHandler := ipvs.NewLinuxKernelHandler() ipsetInterface = utilipset.New(execer) ipvsInterface = utilipvs.New() @@ -264,7 +268,7 @@ func newProxyServer( // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -295,7 +299,7 @@ func newProxyServer( ) } else { var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -333,14 +337,21 @@ func newProxyServer( } return &ProxyServer{ - Config: config, - Client: client, - Proxier: proxier, - Broadcaster: eventBroadcaster, - Recorder: recorder, - Conntracker: &realConntracker{}, - NodeRef: nodeRef, - HealthzServer: healthzServer, + Client: client, + Proxier: proxier, + Broadcaster: eventBroadcaster, + Recorder: recorder, + ConntrackConfiguration: config.Conntrack, + Conntracker: &realConntracker{}, + ProxyMode: proxyMode, + NodeRef: nodeRef, + MetricsBindAddress: config.MetricsBindAddress, + BindAddressHardFail: config.BindAddressHardFail, + EnableProfiling: config.EnableProfiling, + OOMScoreAdj: config.OOMScoreAdj, + ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, + HealthzServer: healthzServer, + localDetectorMode: detectLocalMode, }, nil } @@ -397,11 +408,23 @@ func detectNumCPU() int { return numCPU } +func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) { + mode := config.DetectLocalMode + switch mode { + case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR, proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: + return mode, nil + default: + if strings.TrimSpace(mode.String()) != "" { + return mode, fmt.Errorf("unknown detect-local-mode: %v", mode) + } + klog.V(4).InfoS("Defaulting detect-local-mode", "localModeClusterCIDR", string(proxyconfigapi.LocalModeClusterCIDR)) + return proxyconfigapi.LocalModeClusterCIDR, nil + } +} + func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) { switch mode { case proxyconfigapi.LocalModeClusterCIDR: - // LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed, - // but --cluster-cidr is optional. if len(strings.TrimSpace(config.ClusterCIDR)) == 0 { klog.InfoS("Detect-local-mode set to ClusterCIDR, but no cluster CIDR defined") break @@ -414,8 +437,14 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube } return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) case proxyconfigapi.LocalModeBridgeInterface: + if len(strings.TrimSpace(config.DetectLocal.BridgeInterface)) == 0 { + return nil, fmt.Errorf("Detect-local-mode set to BridgeInterface, but no bridge-interface-name %s is defined", config.DetectLocal.BridgeInterface) + } return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface) case proxyconfigapi.LocalModeInterfaceNamePrefix: + if len(strings.TrimSpace(config.DetectLocal.InterfaceNamePrefix)) == 0 { + return nil, fmt.Errorf("Detect-local-mode set to InterfaceNamePrefix, but no interface-prefix %s is defined", config.DetectLocal.InterfaceNamePrefix) + } return proxyutiliptables.NewDetectLocalByInterfaceNamePrefix(config.DetectLocal.InterfaceNamePrefix) } klog.InfoS("Defaulting to no-op detect-local", "detectLocalMode", string(mode)) @@ -427,8 +456,6 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()} switch mode { case proxyconfigapi.LocalModeClusterCIDR: - // LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed, - // but --cluster-cidr is optional. if len(strings.TrimSpace(config.ClusterCIDR)) == 0 { klog.InfoS("Detect-local-mode set to ClusterCIDR, but no cluster CIDR defined") break @@ -452,7 +479,7 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy } return localDetectors, err case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if nodeInfo == nil || len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { klog.InfoS("No node info available to configure detect-local-mode NodeCIDR") break } @@ -514,6 +541,15 @@ func cidrTuple(cidrList string) [2]string { return cidrs } +func getProxyMode(proxyMode proxyconfigapi.ProxyMode) proxyconfigapi.ProxyMode { + if proxyMode == "" { + klog.InfoS("Using iptables proxy") + return proxyconfigapi.ProxyModeIPTables + } else { + return proxyMode + } +} + // cleanupAndExit remove iptables rules and ipset/ipvs rules func cleanupAndExit() error { execer := exec.New() diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index e6836138c2f..a1f3f691180 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -44,66 +44,54 @@ import ( utiliptablestest "k8s.io/kubernetes/pkg/util/iptables/testing" ) -func Test_platformApplyDefaults(t *testing.T) { - testCases := []struct { - name string - mode proxyconfigapi.ProxyMode - expectedMode proxyconfigapi.ProxyMode - detectLocal proxyconfigapi.LocalMode - expectedDetectLocal proxyconfigapi.LocalMode +func Test_getDetectLocalMode(t *testing.T) { + cases := []struct { + detectLocal string + expected proxyconfigapi.LocalMode + errExpected bool }{ { - name: "defaults", - mode: "", - expectedMode: proxyconfigapi.ProxyModeIPTables, - detectLocal: "", - expectedDetectLocal: proxyconfigapi.LocalModeClusterCIDR, + detectLocal: "", + expected: proxyconfigapi.LocalModeClusterCIDR, + errExpected: false, }, { - name: "explicit", - mode: proxyconfigapi.ProxyModeIPTables, - expectedMode: proxyconfigapi.ProxyModeIPTables, - detectLocal: proxyconfigapi.LocalModeClusterCIDR, - expectedDetectLocal: proxyconfigapi.LocalModeClusterCIDR, + detectLocal: string(proxyconfigapi.LocalModeClusterCIDR), + expected: proxyconfigapi.LocalModeClusterCIDR, + errExpected: false, }, { - name: "override mode", - mode: "ipvs", - expectedMode: proxyconfigapi.ProxyModeIPVS, - detectLocal: "", - expectedDetectLocal: proxyconfigapi.LocalModeClusterCIDR, + detectLocal: string(proxyconfigapi.LocalModeInterfaceNamePrefix), + expected: proxyconfigapi.LocalModeInterfaceNamePrefix, + errExpected: false, }, { - name: "override detect-local", - mode: "", - expectedMode: proxyconfigapi.ProxyModeIPTables, - detectLocal: "NodeCIDR", - expectedDetectLocal: proxyconfigapi.LocalModeNodeCIDR, + detectLocal: string(proxyconfigapi.LocalModeBridgeInterface), + expected: proxyconfigapi.LocalModeBridgeInterface, + errExpected: false, }, { - name: "override both", - mode: "ipvs", - expectedMode: proxyconfigapi.ProxyModeIPVS, - detectLocal: "NodeCIDR", - expectedDetectLocal: proxyconfigapi.LocalModeNodeCIDR, + detectLocal: "abcd", + expected: proxyconfigapi.LocalMode("abcd"), + errExpected: true, }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - options := NewOptions() - config := &proxyconfigapi.KubeProxyConfiguration{ - Mode: tc.mode, - DetectLocalMode: tc.detectLocal, + for i, c := range cases { + proxyConfig := &proxyconfigapi.KubeProxyConfiguration{DetectLocalMode: proxyconfigapi.LocalMode(c.detectLocal)} + r, err := getDetectLocalMode(proxyConfig) + if c.errExpected { + if err == nil { + t.Errorf("Expected error, but did not fail for mode %v", c.detectLocal) } - - options.platformApplyDefaults(config) - if config.Mode != tc.expectedMode { - t.Fatalf("expected mode: %s, but got: %s", tc.expectedMode, config.Mode) - } - if config.DetectLocalMode != tc.expectedDetectLocal { - t.Fatalf("expected detect-local: %s, but got: %s", tc.expectedDetectLocal, config.DetectLocalMode) - } - }) + continue + } + if err != nil { + t.Errorf("Got error parsing mode: %v", err) + continue + } + if r != c.expected { + t.Errorf("Case[%d] Expected %q got %q", i, c.expected, r) + } } } @@ -236,6 +224,20 @@ func Test_getLocalDetector(t *testing.T) { expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/64", utiliptablestest.NewIPv6Fake())), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeClusterCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeClusterCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: nil, + errExpected: true, + }, { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -274,6 +276,22 @@ func Test_getLocalDetector(t *testing.T) { nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + nodeInfo: makeNodeWithPodCIDRs("10.0.0.0"), + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: nil, + nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101"), + errExpected: true, + }, { mode: proxyconfigapi.LocalModeNodeCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -315,6 +333,13 @@ func Test_getLocalDetector(t *testing.T) { expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: ""}, + }, + errExpected: true, + }, { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -332,6 +357,13 @@ func Test_getLocalDetector(t *testing.T) { expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByInterfaceNamePrefix("eth")), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: ""}, + }, + errExpected: true, + }, { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -461,6 +493,22 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { nodeInfo: makeNodeWithPodCIDRs(), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + nodeInfo: nil, + errExpected: false, + }, + // unknown mode, nodeInfo would be nil for these cases + { + mode: proxyconfigapi.LocalMode("abcd"), + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + errExpected: false, + }, // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases { mode: proxyconfigapi.LocalModeBridgeInterface, @@ -472,6 +520,13 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: ""}, + }, + errExpected: true, + }, // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, @@ -483,6 +538,13 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { proxyutiliptables.NewDetectLocalByInterfaceNamePrefix("veth")), errExpected: false, }, + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: ""}, + }, + errExpected: true, + }, } for i, c := range cases { r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index 43ae62600e9..a6ba3eff5ce 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -45,18 +45,16 @@ import ( "k8s.io/kubernetes/pkg/proxy/winkernel" ) -func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) { - if config.Mode == "" { - config.Mode = proxyconfigapi.ProxyModeKernelspace - } -} - // NewProxyServer returns a new ProxyServer. func NewProxyServer(o *Options) (*ProxyServer, error) { return newProxyServer(o.config, o.master) } func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, master string) (*ProxyServer, error) { + if config == nil { + return nil, errors.New("config is required") + } + if c, err := configz.New(proxyconfigapi.GroupName); err == nil { c.Set(config) } else { @@ -105,6 +103,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, master string } var proxier proxy.Provider + proxyMode := proxyconfigapi.ProxyModeKernelspace dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{}) if dualStackMode { klog.InfoS("Creating dualStackProxier for Windows kernel.") @@ -139,13 +138,18 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, master string winkernel.RegisterMetrics() return &ProxyServer{ - Config: config, - Client: client, - Proxier: proxier, - Broadcaster: eventBroadcaster, - Recorder: recorder, - NodeRef: nodeRef, - HealthzServer: healthzServer, + Client: client, + Proxier: proxier, + Broadcaster: eventBroadcaster, + Recorder: recorder, + ProxyMode: proxyMode, + NodeRef: nodeRef, + MetricsBindAddress: config.MetricsBindAddress, + BindAddressHardFail: config.BindAddressHardFail, + EnableProfiling: config.EnableProfiling, + OOMScoreAdj: config.OOMScoreAdj, + ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, + HealthzServer: healthzServer, }, nil } diff --git a/pkg/proxy/apis/config/validation/validation.go b/pkg/proxy/apis/config/validation/validation.go index e487e826037..607a5928aaa 100644 --- a/pkg/proxy/apis/config/validation/validation.go +++ b/pkg/proxy/apis/config/validation/validation.go @@ -95,8 +95,6 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList { allErrs = append(allErrs, validateKubeProxyNodePortAddress(config.NodePortAddresses, newPath.Child("NodePortAddresses"))...) allErrs = append(allErrs, validateShowHiddenMetricsVersion(config.ShowHiddenMetricsForVersion, newPath.Child("ShowHiddenMetricsForVersion"))...) - - allErrs = append(allErrs, validateDetectLocalMode(config.DetectLocalMode, newPath.Child("DetectLocalMode"))...) if config.DetectLocalMode == kubeproxyconfig.LocalModeBridgeInterface { allErrs = append(allErrs, validateInterface(config.DetectLocal.BridgeInterface, newPath.Child("InterfaceName"))...) } @@ -207,22 +205,6 @@ func validateProxyModeWindows(mode kubeproxyconfig.ProxyMode, fldPath *field.Pat return field.ErrorList{field.Invalid(fldPath.Child("ProxyMode"), string(mode), errMsg)} } -func validateDetectLocalMode(mode kubeproxyconfig.LocalMode, fldPath *field.Path) field.ErrorList { - validModes := []string{ - string(kubeproxyconfig.LocalModeClusterCIDR), - string(kubeproxyconfig.LocalModeNodeCIDR), - string(kubeproxyconfig.LocalModeBridgeInterface), - string(kubeproxyconfig.LocalModeInterfaceNamePrefix), - "", - } - - if sets.New(validModes...).Has(string(mode)) { - return nil - } - - return field.ErrorList{field.NotSupported(fldPath, string(mode), validModes)} -} - func validateClientConnectionConfiguration(config componentbaseconfig.ClientConnectionConfiguration, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(config.Burst), fldPath.Child("Burst"))...) diff --git a/pkg/proxy/apis/config/validation/validation_test.go b/pkg/proxy/apis/config/validation/validation_test.go index 04f1931fcb7..eee26449a21 100644 --- a/pkg/proxy/apis/config/validation/validation_test.go +++ b/pkg/proxy/apis/config/validation/validation_test.go @@ -424,28 +424,6 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { }, expectedErrs: field.ErrorList{field.Invalid(newPath.Child("InterfaceName"), "", "must not be empty")}, }, - "invalid DetectLocalMode": { - config: kubeproxyconfig.KubeProxyConfiguration{ - BindAddress: "10.10.12.11", - HealthzBindAddress: "0.0.0.0:12345", - MetricsBindAddress: "127.0.0.1:10249", - ClusterCIDR: "192.168.59.0/24", - ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, - IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ - MasqueradeAll: true, - SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, - MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, - }, - Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ - MaxPerCore: pointer.Int32(1), - Min: pointer.Int32(1), - TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, - TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, - }, - DetectLocalMode: "Guess", - }, - expectedErrs: field.ErrorList{field.NotSupported(newPath.Child("DetectLocalMode"), "Guess", []string{"ClusterCIDR", "NodeCIDR", "BridgeInterface", "InterfaceNamePrefix", ""})}, - }, } for name, testCase := range testCases { diff --git a/pkg/proxy/kubemark/hollow_proxy.go b/pkg/proxy/kubemark/hollow_proxy.go index 1462c30a84f..e87d165b84b 100644 --- a/pkg/proxy/kubemark/hollow_proxy.go +++ b/pkg/proxy/kubemark/hollow_proxy.go @@ -22,7 +22,6 @@ import ( v1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" @@ -30,7 +29,6 @@ import ( utilsysctl "k8s.io/component-helpers/node/util/sysctl" proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" "k8s.io/kubernetes/pkg/proxy" - proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" proxyconfig "k8s.io/kubernetes/pkg/proxy/config" "k8s.io/kubernetes/pkg/proxy/iptables" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" @@ -126,17 +124,14 @@ func NewHollowProxyOrDie( } return &HollowProxy{ ProxyServer: &proxyapp.ProxyServer{ - Config: &proxyconfigapi.KubeProxyConfiguration{ - Mode: proxyconfigapi.ProxyMode("fake"), - ConfigSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - OOMScoreAdj: utilpointer.Int32Ptr(0), - }, - - Client: client, - Proxier: proxier, - Broadcaster: broadcaster, - Recorder: recorder, - NodeRef: nodeRef, + Client: client, + Proxier: proxier, + Broadcaster: broadcaster, + Recorder: recorder, + ProxyMode: "fake", + NodeRef: nodeRef, + OOMScoreAdj: utilpointer.Int32Ptr(0), + ConfigSyncPeriod: 30 * time.Second, }, }, nil }