From a74b9fde3aafbb03dde46b475d9da93568964577 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 30 Jun 2023 11:30:17 -0400 Subject: [PATCH] Don't pass around full node in proxy constructor, just pass PodCIDRs --- cmd/kube-proxy/app/server_others.go | 41 ++++--- cmd/kube-proxy/app/server_others_test.go | 130 +++++++++++------------ 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 1b14b51a0c2..76d44231757 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -81,15 +81,14 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio var proxier proxy.Provider var err error - var nodeInfo *v1.Node if config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname) - nodeInfo, err = waitForPodCIDR(s.Client, s.Hostname) + node, err := waitForPodCIDR(s.Client, s.Hostname) if err != nil { return nil, err } - s.podCIDRs = nodeInfo.Spec.PodCIDRs - klog.InfoS("NodeInfo", "podCIDR", nodeInfo.Spec.PodCIDR, "podCIDRs", nodeInfo.Spec.PodCIDRs) + s.podCIDRs = node.Spec.PodCIDRs + klog.InfoS("NodeInfo", "podCIDRs", node.Spec.PodCIDRs) } primaryProtocol := utiliptables.ProtocolIPv4 @@ -138,7 +137,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio klog.InfoS("Creating dualStackProxier for iptables") // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -163,7 +162,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio } else { // Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support). var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -205,7 +204,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -236,7 +235,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ) } else { var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -389,7 +388,7 @@ func detectNumCPU() int { return numCPU } -func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) { +func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodePodCIDRs []string) (proxyutiliptables.LocalTrafficDetector, error) { switch mode { case proxyconfigapi.LocalModeClusterCIDR: // LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed, @@ -400,11 +399,11 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube } return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt) case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if len(nodePodCIDRs) == 0 { klog.InfoS("Detect-local-mode set to NodeCIDR, but no PodCIDR defined at node") break } - return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) + return proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt) case proxyconfigapi.LocalModeBridgeInterface: return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface) case proxyconfigapi.LocalModeInterfaceNamePrefix: @@ -414,7 +413,7 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube return proxyutiliptables.NewNoOpLocalDetector(), nil } -func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) { +func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodePodCIDRs []string) ([2]proxyutiliptables.LocalTrafficDetector, error) { var err error localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()} switch mode { @@ -444,32 +443,32 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy } return localDetectors, err case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if len(nodePodCIDRs) == 0 { klog.InfoS("No node info available to configure detect-local-mode NodeCIDR") break } // localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR. // so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1] - if netutils.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1]) + if netutils.IsIPv6CIDRString(nodePodCIDRs[0]) { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[1]) if err != nil { return localDetectors, err } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0]) + if len(nodePodCIDRs) > 1 { + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[0]) } } else { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0]) + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[0]) if err != nil { return localDetectors, err } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1]) + if len(nodePodCIDRs) > 1 { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[1]) } } return localDetectors, err case proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: - localDetector, err := getLocalDetector(mode, config, ipt[0], nodeInfo) + localDetector, err := getLocalDetector(mode, config, ipt[0], nodePodCIDRs) if err == nil { localDetectors[0] = localDetector localDetectors[1] = localDetector diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 06d1a435123..620ad82fd60 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -109,14 +109,14 @@ func Test_platformApplyDefaults(t *testing.T) { func Test_getLocalDetector(t *testing.T) { cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt utiliptables.Interface - expected proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool + mode proxyconfigapi.LocalMode + config *proxyconfigapi.KubeProxyConfiguration + ipt utiliptables.Interface + expected proxyutiliptables.LocalTrafficDetector + nodePodCIDRs []string + errExpected bool }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases + // LocalModeClusterCIDR { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -154,46 +154,46 @@ func Test_getLocalDetector(t *testing.T) { }, // LocalModeNodeCIDR { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewFake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: true, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: nil, + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: true, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: true, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: true, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: utiliptablestest.NewFake(), + expected: proxyutiliptables.NewNoOpLocalDetector(), + nodePodCIDRs: []string{}, + errExpected: false, }, - // unknown mode, nodeInfo would be nil for these cases + // unknown mode { mode: proxyconfigapi.LocalMode("abcd"), config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -201,7 +201,7 @@ func Test_getLocalDetector(t *testing.T) { expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, }, - // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + // LocalModeBridgeInterface { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -218,7 +218,7 @@ func Test_getLocalDetector(t *testing.T) { expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("1234567890123456789")), errExpected: false, }, - // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + // LocalModeInterfaceNamePrefix { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -237,7 +237,7 @@ func Test_getLocalDetector(t *testing.T) { }, } for i, c := range cases { - r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo) + r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Case[%d] Expected error, but succeeded with %v", i, r) @@ -256,14 +256,14 @@ func Test_getLocalDetector(t *testing.T) { func Test_getDualStackLocalDetectorTuple(t *testing.T) { cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt [2]utiliptables.Interface - expected [2]proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool + mode proxyconfigapi.LocalMode + config *proxyconfigapi.KubeProxyConfiguration + ipt [2]utiliptables.Interface + expected [2]proxyutiliptables.LocalTrafficDetector + nodePodCIDRs []string + errExpected bool }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases + // LocalModeClusterCIDR { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, @@ -315,8 +315,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: resolveDualStackLocalDetectors(t)( proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + nodePodCIDRs: []string{"10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -325,8 +325,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: resolveDualStackLocalDetectors(t)( proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"), - errExpected: false, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -335,8 +335,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{ resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -345,18 +345,18 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{ proxyutiliptables.NewNoOpLocalDetector(), resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake()))}, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + nodePodCIDRs: []string{}, + errExpected: false, }, - // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + // LocalModeBridgeInterface { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -367,7 +367,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), errExpected: false, }, - // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + // LocalModeInterfaceNamePrefix { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -380,7 +380,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { }, } for i, c := range cases { - r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) + r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Case[%d] expected error, but succeeded with %q", i, r)