From a74b9fde3aafbb03dde46b475d9da93568964577 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 30 Jun 2023 11:30:17 -0400 Subject: [PATCH 1/5] Don't pass around full node in proxy constructor, just pass PodCIDRs --- cmd/kube-proxy/app/server_others.go | 41 ++++--- cmd/kube-proxy/app/server_others_test.go | 130 +++++++++++------------ 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 1b14b51a0c2..76d44231757 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -81,15 +81,14 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio var proxier proxy.Provider var err error - var nodeInfo *v1.Node if config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname) - nodeInfo, err = waitForPodCIDR(s.Client, s.Hostname) + node, err := waitForPodCIDR(s.Client, s.Hostname) if err != nil { return nil, err } - s.podCIDRs = nodeInfo.Spec.PodCIDRs - klog.InfoS("NodeInfo", "podCIDR", nodeInfo.Spec.PodCIDR, "podCIDRs", nodeInfo.Spec.PodCIDRs) + s.podCIDRs = node.Spec.PodCIDRs + klog.InfoS("NodeInfo", "podCIDRs", node.Spec.PodCIDRs) } primaryProtocol := utiliptables.ProtocolIPv4 @@ -138,7 +137,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio klog.InfoS("Creating dualStackProxier for iptables") // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -163,7 +162,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio } else { // Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support). var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -205,7 +204,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -236,7 +235,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ) } else { var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -389,7 +388,7 @@ func detectNumCPU() int { return numCPU } -func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) { +func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodePodCIDRs []string) (proxyutiliptables.LocalTrafficDetector, error) { switch mode { case proxyconfigapi.LocalModeClusterCIDR: // LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed, @@ -400,11 +399,11 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube } return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt) case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if len(nodePodCIDRs) == 0 { klog.InfoS("Detect-local-mode set to NodeCIDR, but no PodCIDR defined at node") break } - return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) + return proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt) case proxyconfigapi.LocalModeBridgeInterface: return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface) case proxyconfigapi.LocalModeInterfaceNamePrefix: @@ -414,7 +413,7 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube return proxyutiliptables.NewNoOpLocalDetector(), nil } -func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) { +func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodePodCIDRs []string) ([2]proxyutiliptables.LocalTrafficDetector, error) { var err error localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()} switch mode { @@ -444,32 +443,32 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy } return localDetectors, err case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if len(nodePodCIDRs) == 0 { klog.InfoS("No node info available to configure detect-local-mode NodeCIDR") break } // localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR. // so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1] - if netutils.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1]) + if netutils.IsIPv6CIDRString(nodePodCIDRs[0]) { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[1]) if err != nil { return localDetectors, err } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0]) + if len(nodePodCIDRs) > 1 { + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[0]) } } else { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0]) + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[0]) if err != nil { return localDetectors, err } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1]) + if len(nodePodCIDRs) > 1 { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[1]) } } return localDetectors, err case proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: - localDetector, err := getLocalDetector(mode, config, ipt[0], nodeInfo) + localDetector, err := getLocalDetector(mode, config, ipt[0], nodePodCIDRs) if err == nil { localDetectors[0] = localDetector localDetectors[1] = localDetector diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 06d1a435123..620ad82fd60 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -109,14 +109,14 @@ func Test_platformApplyDefaults(t *testing.T) { func Test_getLocalDetector(t *testing.T) { cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt utiliptables.Interface - expected proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool + mode proxyconfigapi.LocalMode + config *proxyconfigapi.KubeProxyConfiguration + ipt utiliptables.Interface + expected proxyutiliptables.LocalTrafficDetector + nodePodCIDRs []string + errExpected bool }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases + // LocalModeClusterCIDR { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -154,46 +154,46 @@ func Test_getLocalDetector(t *testing.T) { }, // LocalModeNodeCIDR { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewFake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: true, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: nil, + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: true, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: true, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: true, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: utiliptablestest.NewFake(), + expected: proxyutiliptables.NewNoOpLocalDetector(), + nodePodCIDRs: []string{}, + errExpected: false, }, - // unknown mode, nodeInfo would be nil for these cases + // unknown mode { mode: proxyconfigapi.LocalMode("abcd"), config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -201,7 +201,7 @@ func Test_getLocalDetector(t *testing.T) { expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, }, - // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + // LocalModeBridgeInterface { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -218,7 +218,7 @@ func Test_getLocalDetector(t *testing.T) { expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("1234567890123456789")), errExpected: false, }, - // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + // LocalModeInterfaceNamePrefix { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -237,7 +237,7 @@ func Test_getLocalDetector(t *testing.T) { }, } for i, c := range cases { - r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo) + r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Case[%d] Expected error, but succeeded with %v", i, r) @@ -256,14 +256,14 @@ func Test_getLocalDetector(t *testing.T) { func Test_getDualStackLocalDetectorTuple(t *testing.T) { cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt [2]utiliptables.Interface - expected [2]proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool + mode proxyconfigapi.LocalMode + config *proxyconfigapi.KubeProxyConfiguration + ipt [2]utiliptables.Interface + expected [2]proxyutiliptables.LocalTrafficDetector + nodePodCIDRs []string + errExpected bool }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases + // LocalModeClusterCIDR { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, @@ -315,8 +315,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: resolveDualStackLocalDetectors(t)( proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + nodePodCIDRs: []string{"10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -325,8 +325,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: resolveDualStackLocalDetectors(t)( proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"), - errExpected: false, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -335,8 +335,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{ resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -345,18 +345,18 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{ proxyutiliptables.NewNoOpLocalDetector(), resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake()))}, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + nodePodCIDRs: []string{}, + errExpected: false, }, - // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + // LocalModeBridgeInterface { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -367,7 +367,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), errExpected: false, }, - // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + // LocalModeInterfaceNamePrefix { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -380,7 +380,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { }, } for i, c := range cases { - r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) + r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Case[%d] expected error, but succeeded with %q", i, r) From ebb0fdd4f6fd8cdf389e99eceb813a64c8720819 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 30 Jun 2023 12:19:44 -0400 Subject: [PATCH 2/5] Don't explicitly filter config.NodePortAddresses by IP family There's no need to do this any more: proxyutil.NodePortAddresses does it itself. --- cmd/kube-proxy/app/server_others.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 76d44231757..c5961ee035a 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -111,8 +111,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ipt[1] = iptInterface } - nodePortAddresses := config.NodePortAddresses - if !ipt[0].Present() { return nil, fmt.Errorf("iptables is not supported for primary IP family %q", primaryProtocol) } else if !ipt[1].Present() { @@ -125,7 +123,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio badAddrs := npaByFamily[secondaryFamily] if len(badAddrs) > 0 { klog.InfoS("Ignoring --nodeport-addresses of the wrong family", "ipFamily", secondaryFamily, "addresses", badAddrs) - nodePortAddresses = npaByFamily[s.PrimaryIPFamily] } } @@ -157,7 +154,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.NodeIPs, s.Recorder, s.HealthzServer, - nodePortAddresses, + config.NodePortAddresses, ) } else { // Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support). @@ -183,7 +180,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.NodeIPs[s.PrimaryIPFamily], s.Recorder, s.HealthzServer, - nodePortAddresses, + config.NodePortAddresses, ) } @@ -230,7 +227,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.Recorder, s.HealthzServer, config.IPVS.Scheduler, - nodePortAddresses, + config.NodePortAddresses, kernelHandler, ) } else { @@ -262,7 +259,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.Recorder, s.HealthzServer, config.IPVS.Scheduler, - nodePortAddresses, + config.NodePortAddresses, kernelHandler, ) } From 8abfa89e82b242abae47575c8ef6c15a56b516b8 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 30 Jun 2023 17:39:36 -0400 Subject: [PATCH 3/5] Move proxy platformSetup call, and do LocalDetector setup from there --- cmd/kube-proxy/app/server.go | 11 ++++---- cmd/kube-proxy/app/server_others.go | 32 +++++++++++++++++------- cmd/kube-proxy/app/server_others_test.go | 7 +++--- cmd/kube-proxy/app/server_windows.go | 15 +++++++---- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 89468c59175..6b12ab4e6cb 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -567,6 +567,11 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin s.HealthzServer = healthcheck.NewProxierHealthServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, s.Recorder, s.NodeRef) } + err = s.platformSetup() + if err != nil { + return nil, err + } + s.Proxier, err = s.createProxier(config) if err != nil { return nil, err @@ -706,12 +711,6 @@ func (s *ProxyServer) Run() error { // Start up a metrics server if requested serveMetrics(s.Config.MetricsBindAddress, s.Config.Mode, s.Config.EnableProfiling, errCh) - // Do platform-specific setup - err := s.platformSetup() - if err != nil { - return err - } - noProxyName, err := labels.NewRequirement(apis.LabelServiceProxyName, selection.DoesNotExist, nil) if err != nil { return err diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index c5961ee035a..3ad02d7840f 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -63,6 +63,8 @@ import ( // node after it is registered. var timeoutForNodePodCIDR = 5 * time.Minute +// platformApplyDefaults is called after parsing command-line flags and/or reading the +// config file, to apply platform-specific default values to config. func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) { if config.Mode == "" { klog.InfoS("Using iptables proxy") @@ -76,21 +78,34 @@ func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfigur klog.V(2).InfoS("DetectLocalMode", "localMode", string(config.DetectLocalMode)) } -// createProxier creates the proxy.Provider -func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { - var proxier proxy.Provider - var err error - - if config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { +// platformSetup is called after setting up the ProxyServer, but before creating the +// Proxier. It should fill in any platform-specific fields and perform other +// platform-specific setup. +func (s *ProxyServer) platformSetup() error { + if s.Config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname) node, err := waitForPodCIDR(s.Client, s.Hostname) if err != nil { - return nil, err + return err } s.podCIDRs = node.Spec.PodCIDRs klog.InfoS("NodeInfo", "podCIDRs", node.Spec.PodCIDRs) } + err := s.setupConntrack() + if err != nil { + return err + } + + proxymetrics.RegisterMetrics() + return nil +} + +// createProxier creates the proxy.Provider +func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { + var proxier proxy.Provider + var err error + primaryProtocol := utiliptables.ProtocolIPv4 if s.PrimaryIPFamily == v1.IPv6Protocol { primaryProtocol = utiliptables.ProtocolIPv6 @@ -271,7 +286,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio return proxier, nil } -func (s *ProxyServer) platformSetup() error { +func (s *ProxyServer) setupConntrack() error { ct := &realConntracker{} max, err := getConntrackMax(s.Config.Conntrack) @@ -311,7 +326,6 @@ func (s *ProxyServer) platformSetup() error { } } - proxymetrics.RegisterMetrics() return nil } diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 620ad82fd60..9323f8327a1 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -646,7 +646,7 @@ func TestGetConntrackMax(t *testing.T) { } } -func TestProxyServer_createProxier(t *testing.T) { +func TestProxyServer_platformSetup(t *testing.T) { tests := []struct { name string node *v1.Node @@ -683,9 +683,8 @@ func TestProxyServer_createProxier(t *testing.T) { v1.IPv6Protocol: net.IPv6zero, }, } - _, err := s.createProxier(tt.config) - // TODO: mock the exec.Interface to not fail probing iptables - if (err != nil) && !strings.Contains(err.Error(), "iptables is not supported for primary IP family") { + err := s.platformSetup() + if err != nil { t.Errorf("ProxyServer.createProxier() error = %v", err) return } diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index 4c3ef13340f..c076abf90dc 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -36,12 +36,22 @@ import ( "k8s.io/kubernetes/pkg/proxy/winkernel" ) +// platformApplyDefaults is called after parsing command-line flags and/or reading the +// config file, to apply platform-specific default values to config. func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) { if config.Mode == "" { config.Mode = proxyconfigapi.ProxyModeKernelspace } } +// platformSetup is called after setting up the ProxyServer, but before creating the +// Proxier. It should fill in any platform-specific fields and perform other +// platform-specific setup. +func (s *ProxyServer) platformSetup() error { + winkernel.RegisterMetrics() + return nil +} + // createProxier creates the proxy.Provider func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { var healthzPort int @@ -93,11 +103,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio return proxier, nil } -func (s *ProxyServer) platformSetup() error { - winkernel.RegisterMetrics() - return nil -} - func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool { return compatTester.DualStackCompatible(networkname) } From 1f2bf32e95fd42f1972c49238ef163fea125bcdb Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 30 Jun 2023 12:22:19 -0400 Subject: [PATCH 4/5] Figure out single-stack/dual-stack support before creating the proxier Rather than having this as part of createProxier(), explicitly figure out what IP families the proxier can support beforehand, and bail out if this conflicts with the detected IP family. --- cmd/kube-proxy/app/server.go | 13 +++++++++- cmd/kube-proxy/app/server_others.go | 38 +++++++++++++++++++--------- cmd/kube-proxy/app/server_windows.go | 37 ++++++++++++++++----------- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 6b12ab4e6cb..99992b2da2e 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -572,7 +572,18 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin return nil, err } - s.Proxier, err = s.createProxier(config) + ipv4Supported, ipv6Supported, dualStackSupported, err := s.platformCheckSupported() + if err != nil { + return nil, err + } else if (s.PrimaryIPFamily == v1.IPv4Protocol && !ipv4Supported) || (s.PrimaryIPFamily == v1.IPv6Protocol && !ipv6Supported) { + return nil, fmt.Errorf("no support for primary IP family %q", s.PrimaryIPFamily) + } else if dualStackSupported { + klog.InfoS("kube-proxy running in dual-stack mode", "primary ipFamily", s.PrimaryIPFamily) + } else { + klog.InfoS("kube-proxy running in single-stack mode", "ipFamily", s.PrimaryIPFamily) + } + + s.Proxier, err = s.createProxier(config, dualStackSupported) if err != nil { return nil, err } diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 3ad02d7840f..4eee907f2d1 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -101,8 +101,32 @@ func (s *ProxyServer) platformSetup() error { return nil } +// platformCheckSupported is called immediately before creating the Proxier, to check +// what IP families are supported (and whether the configuration is usable at all). +func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) { + execer := exec.New() + ipt := utiliptables.New(execer, utiliptables.ProtocolIPv4) + ipv4Supported = ipt.Present() + ipt = utiliptables.New(execer, utiliptables.ProtocolIPv6) + ipv6Supported = ipt.Present() + + // The Linux proxies can always support dual-stack if they can support both IPv4 + // and IPv6. + dualStackSupported = ipv4Supported && ipv6Supported + + if !ipv4Supported && !ipv6Supported { + err = fmt.Errorf("iptables is not available on this host") + } else if !ipv4Supported { + klog.InfoS("No iptables support for family", "ipFamily", v1.IPv4Protocol) + } else if !ipv6Supported { + klog.InfoS("No iptables support for family", "ipFamily", v1.IPv6Protocol) + } + + return +} + // createProxier creates the proxy.Provider -func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { +func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStack bool) (proxy.Provider, error) { var proxier proxy.Provider var err error @@ -114,7 +138,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio iptInterface := utiliptables.New(execer, primaryProtocol) var ipt [2]utiliptables.Interface - dualStack := true // While we assume that node supports, we do further checks below // Create iptables handlers for both families, one is already created // Always ordered as IPv4, IPv6 @@ -126,12 +149,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ipt[1] = iptInterface } - if !ipt[0].Present() { - return nil, fmt.Errorf("iptables is not supported for primary IP family %q", primaryProtocol) - } else if !ipt[1].Present() { - klog.InfoS("kube-proxy running in single-stack mode: secondary ipFamily is not supported", "ipFamily", ipt[1].Protocol()) - dualStack = false - + if !dualStack { // Validate NodePortAddresses is single-stack npaByFamily := proxyutil.MapCIDRsByIPFamily(config.NodePortAddresses) secondaryFamily := proxyutil.OtherIPFamily(s.PrimaryIPFamily) @@ -145,8 +163,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio klog.InfoS("Using iptables Proxier") if dualStack { - klog.InfoS("kube-proxy running in dual-stack mode", "ipFamily", iptInterface.Protocol()) - klog.InfoS("Creating dualStackProxier for iptables") // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) @@ -212,8 +228,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio klog.InfoS("Using ipvs Proxier") if dualStack { - klog.InfoS("Creating dualStackProxier for ipvs") - // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index c076abf90dc..62adf4a41d1 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -30,7 +30,6 @@ import ( // Enable pprof HTTP handlers. _ "net/http/pprof" - "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" "k8s.io/kubernetes/pkg/proxy/winkernel" @@ -52,26 +51,38 @@ func (s *ProxyServer) platformSetup() error { return nil } +// platformCheckSupported is called immediately before creating the Proxier, to check +// what IP families are supported (and whether the configuration is usable at all). +func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) { + // Check if Kernel proxier can be used at all + _, err = winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{}) + if err != nil { + return false, false, false, err + } + + // winkernel always supports both single-stack IPv4 and single-stack IPv6, but may + // not support dual-stack. + ipv4Supported = true + ipv6Supported = true + + compatTester := winkernel.DualStackCompatTester{} + dualStackSupported = compatTester.DualStackCompatible(s.Config.Winkernel.NetworkName) + + return +} + // createProxier creates the proxy.Provider -func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { +func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStackMode bool) (proxy.Provider, error) { var healthzPort int if len(config.HealthzBindAddress) > 0 { _, port, _ := net.SplitHostPort(config.HealthzBindAddress) healthzPort, _ = strconv.Atoi(port) } - // Check if Kernel Space can be used. - canUseWinKernelProxy, err := winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{}) - if !canUseWinKernelProxy && err != nil { - return nil, err - } - var proxier proxy.Provider + var err error - dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{}) if dualStackMode { - klog.InfoS("Creating dualStackProxier for Windows kernel.") - proxier, err = winkernel.NewDualStackProxier( config.IPTables.SyncPeriod.Duration, config.IPTables.MinSyncPeriod.Duration, @@ -103,10 +114,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio return proxier, nil } -func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool { - return compatTester.DualStackCompatible(networkname) -} - // cleanupAndExit cleans up after a previous proxy run func cleanupAndExit() error { return errors.New("--cleanup-and-exit is not implemented on Windows") From a966d186083d2f511f451eb1f0f2a848b2d8400b Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 30 Jun 2023 12:25:11 -0400 Subject: [PATCH 5/5] Double-check the proxy configuration vs the available IP families --- cmd/kube-proxy/app/server.go | 102 ++++++++++ cmd/kube-proxy/app/server_others.go | 11 - cmd/kube-proxy/app/server_test.go | 299 ++++++++++++++++++++++++++++ 3 files changed, 401 insertions(+), 11 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 99992b2da2e..7fe01c87769 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/healthz" @@ -583,6 +584,14 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin klog.InfoS("kube-proxy running in single-stack mode", "ipFamily", s.PrimaryIPFamily) } + err, fatal := checkIPConfig(s, dualStackSupported) + if err != nil { + if fatal { + return nil, fmt.Errorf("kube-proxy configuration is incorrect: %v", err) + } + klog.ErrorS(err, "Kube-proxy configuration may be incomplete or incorrect") + } + s.Proxier, err = s.createProxier(config, dualStackSupported) if err != nil { return nil, err @@ -591,6 +600,99 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin return s, nil } +// checkIPConfig confirms that s has proper configuration for its primary IP family. +func checkIPConfig(s *ProxyServer, dualStackSupported bool) (error, bool) { + var errors []error + var badFamily netutils.IPFamily + + if s.PrimaryIPFamily == v1.IPv4Protocol { + badFamily = netutils.IPv6 + } else { + badFamily = netutils.IPv4 + } + + var clusterType string + if dualStackSupported { + clusterType = fmt.Sprintf("%s-primary", s.PrimaryIPFamily) + } else { + clusterType = fmt.Sprintf("%s-only", s.PrimaryIPFamily) + } + + // Historically, we did not check most of the config options, so we cannot + // retroactively make IP family mismatches in those options be fatal. When we add + // new options to check here, we should make problems with those options be fatal. + fatal := false + + if s.Config.ClusterCIDR != "" { + clusterCIDRs := strings.Split(s.Config.ClusterCIDR, ",") + if badCIDRs(clusterCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but clusterCIDRs contains only IPv%s addresses", clusterType, badFamily)) + if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeClusterCIDR { + // This has always been a fatal error + fatal = true + } + } + } + + if badCIDRs(s.Config.NodePortAddresses, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but nodePortAddresses contains only IPv%s addresses", clusterType, badFamily)) + } + + if badCIDRs(s.podCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but node.spec.podCIDRs contains only IPv%s addresses", clusterType, badFamily)) + if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeNodeCIDR { + // This has always been a fatal error + fatal = true + } + } + + if netutils.IPFamilyOfString(s.Config.Winkernel.SourceVip) == badFamily { + errors = append(errors, fmt.Errorf("cluster is %s but winkernel.sourceVip is IPv%s", clusterType, badFamily)) + } + + // In some cases, wrong-IP-family is only a problem when the secondary IP family + // isn't present at all. + if !dualStackSupported { + if badCIDRs(s.Config.IPVS.ExcludeCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but ipvs.excludeCIDRs contains only IPv%s addresses", clusterType, badFamily)) + } + + if badBindAddress(s.Config.HealthzBindAddress, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but healthzBindAddress is IPv%s", clusterType, badFamily)) + } + if badBindAddress(s.Config.MetricsBindAddress, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but metricsBindAddress is IPv%s", clusterType, badFamily)) + } + } + + return utilerrors.NewAggregate(errors), fatal +} + +// badCIDRs returns true if cidrs is a non-empty list of CIDRs, all of wrongFamily. +func badCIDRs(cidrs []string, wrongFamily netutils.IPFamily) bool { + if len(cidrs) == 0 { + return false + } + for _, cidr := range cidrs { + if netutils.IPFamilyOfCIDRString(cidr) != wrongFamily { + return false + } + } + return true +} + +// badBindAddress returns true if bindAddress is an "IP:port" string where IP is a +// non-zero IP of wrongFamily. +func badBindAddress(bindAddress string, wrongFamily netutils.IPFamily) bool { + if host, _, _ := net.SplitHostPort(bindAddress); host != "" { + ip := netutils.ParseIPSloppy(host) + if ip != nil && netutils.IPFamilyOf(ip) == wrongFamily && !ip.IsUnspecified() { + return true + } + } + return false +} + // createClient creates a kube client from the given config and masterOverride. // TODO remove masterOverride when CLI flags are removed. func createClient(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, error) { diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 4eee907f2d1..061152a8882 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -50,7 +50,6 @@ import ( utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset" utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util" proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables" "k8s.io/utils/exec" @@ -149,16 +148,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ipt[1] = iptInterface } - if !dualStack { - // Validate NodePortAddresses is single-stack - npaByFamily := proxyutil.MapCIDRsByIPFamily(config.NodePortAddresses) - secondaryFamily := proxyutil.OtherIPFamily(s.PrimaryIPFamily) - badAddrs := npaByFamily[secondaryFamily] - if len(badAddrs) > 0 { - klog.InfoS("Ignoring --nodeport-addresses of the wrong family", "ipFamily", secondaryFamily, "addresses", badAddrs) - } - } - if config.Mode == proxyconfigapi.ProxyModeIPTables { klog.InfoS("Using iptables Proxier") diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 8d5f3241510..2b7709a152c 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -632,3 +632,302 @@ func Test_detectNodeIPs(t *testing.T) { }) } } + +func Test_checkIPConfig(t *testing.T) { + cases := []struct { + name string + proxy *ProxyServer + ssErr bool + dsErr bool + fatal bool + }{ + { + name: "empty config", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{}, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + + { + name: "ok single-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "10.0.0.0/8", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "10.0.0.0/8,fd01:2345::/64", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64,10.0.0.0/8", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + { + name: "wrong-family clusterCIDR when using ClusterCIDR LocalDetector", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64", + DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: true, + }, + + { + name: "ok single-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8", "fd01:2345::/64", "fd01:abcd::/64"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"fd01:2345::/64", "fd01:abcd::/64", "10.0.0.0/8"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8"}, + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + + { + name: "ok single-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"10.0.0.0/8"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"10.0.0.0/8", "fd01:2345::/64"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"fd01:2345::/64", "10.0.0.0/8"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"fd01:2345::/64"}, + }, + ssErr: true, + dsErr: true, + fatal: true, + }, + + { + name: "ok winkernel.sourceVip", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + SourceVip: "10.0.0.1", + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family winkernel.sourceVip", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + SourceVip: "fd01:2345::1", + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + + { + name: "ok IPv4 metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "10.0.0.1:9999", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok IPv6 metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "[fd01:2345::1]:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok unspecified wrong-family metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "0.0.0.0:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "10.0.0.1:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: false, + fatal: false, + }, + + { + name: "ok ipvs.excludeCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: []string{"10.0.0.0/8"}, + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family ipvs.excludeCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: false, + fatal: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err, fatal := checkIPConfig(c.proxy, false) + if err != nil && !c.ssErr { + t.Errorf("unexpected error in single-stack case: %v", err) + } else if err == nil && c.ssErr { + t.Errorf("unexpected lack of error in single-stack case") + } else if fatal != c.fatal { + t.Errorf("expected fatal=%v, got %v", c.fatal, fatal) + } + + err, fatal = checkIPConfig(c.proxy, true) + if err != nil && !c.dsErr { + t.Errorf("unexpected error in dual-stack case: %v", err) + } else if err == nil && c.dsErr { + t.Errorf("unexpected lack of error in dual-stack case") + } else if fatal != c.fatal { + t.Errorf("expected fatal=%v, got %v", c.fatal, fatal) + } + }) + } +}