From cefd50a753e16ee2753155b65c5d6558020d1280 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 1 Jul 2023 09:11:28 -0400 Subject: [PATCH] Improve the single-stack LocalDetector behavior 1. When bringing up a single-stack kube-proxy in a dual-stack cluster, allow using either the primary or secondary IP family. 2. Since the earlier config-checking code will already have bailed out if the single-stack configuration is unusably broken, we don't need to do that here. Instead, just return a no-op local detector if there are no usable CIDRs of the expected IP family. --- cmd/kube-proxy/app/server_others.go | 37 ++++++++++++------ cmd/kube-proxy/app/server_others_test.go | 49 +++++++++++++++++++----- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 061152a8882..349a8e31132 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -50,6 +50,7 @@ import ( utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset" utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util" proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables" "k8s.io/utils/exec" @@ -179,7 +180,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio } else { // Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support). var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) + localDetector, err = getLocalDetector(s.PrimaryIPFamily, config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -250,7 +251,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ) } else { var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) + localDetector, err = getLocalDetector(s.PrimaryIPFamily, config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -402,28 +403,40 @@ func detectNumCPU() int { return numCPU } -func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodePodCIDRs []string) (proxyutiliptables.LocalTrafficDetector, error) { +func getLocalDetector(ipFamily v1.IPFamily, mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodePodCIDRs []string) (proxyutiliptables.LocalTrafficDetector, error) { switch mode { case proxyconfigapi.LocalModeClusterCIDR: // LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed, // but --cluster-cidr is optional. - if len(strings.TrimSpace(config.ClusterCIDR)) == 0 { + clusterCIDRs := strings.TrimSpace(config.ClusterCIDR) + if len(clusterCIDRs) == 0 { klog.InfoS("Detect-local-mode set to ClusterCIDR, but no cluster CIDR defined") break } - return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt) - case proxyconfigapi.LocalModeNodeCIDR: - if len(nodePodCIDRs) == 0 { - klog.InfoS("Detect-local-mode set to NodeCIDR, but no PodCIDR defined at node") - break + + cidrsByFamily := proxyutil.MapCIDRsByIPFamily(strings.Split(clusterCIDRs, ",")) + if len(cidrsByFamily[ipFamily]) != 0 { + return proxyutiliptables.NewDetectLocalByCIDR(cidrsByFamily[ipFamily][0], ipt) } - return proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt) + + klog.InfoS("Detect-local-mode set to ClusterCIDR, but no cluster CIDR for family", "ipFamily", ipFamily) + + case proxyconfigapi.LocalModeNodeCIDR: + cidrsByFamily := proxyutil.MapCIDRsByIPFamily(nodePodCIDRs) + if len(cidrsByFamily[ipFamily]) != 0 { + return proxyutiliptables.NewDetectLocalByCIDR(cidrsByFamily[ipFamily][0], ipt) + } + + klog.InfoS("Detect-local-mode set to NodeCIDR, but no PodCIDR defined at node for family", "ipFamily", ipFamily) + case proxyconfigapi.LocalModeBridgeInterface: return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface) + case proxyconfigapi.LocalModeInterfaceNamePrefix: return proxyutiliptables.NewDetectLocalByInterfaceNamePrefix(config.DetectLocal.InterfaceNamePrefix) } - klog.InfoS("Defaulting to no-op detect-local", "detectLocalMode", string(mode)) + + klog.InfoS("Defaulting to no-op detect-local") return proxyutiliptables.NewNoOpLocalDetector(), nil } @@ -482,7 +495,7 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy } return localDetectors, err case proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: - localDetector, err := getLocalDetector(mode, config, ipt[0], nodePodCIDRs) + localDetector, err := getLocalDetector("", mode, config, nil, nodePodCIDRs) if err == nil { localDetectors[0] = localDetector localDetectors[1] = localDetector diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index a90d2c36d15..797fcfe0301 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -112,6 +112,7 @@ func Test_getLocalDetector(t *testing.T) { name string mode proxyconfigapi.LocalMode config *proxyconfigapi.KubeProxyConfiguration + family v1.IPFamily ipt utiliptables.Interface expected proxyutiliptables.LocalTrafficDetector nodePodCIDRs []string @@ -122,6 +123,7 @@ func Test_getLocalDetector(t *testing.T) { name: "LocalModeClusterCIDR, IPv4 cluster", mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/14", utiliptablestest.NewFake())), errExpected: false, @@ -130,6 +132,7 @@ func Test_getLocalDetector(t *testing.T) { name: "LocalModeClusterCIDR, IPv6 cluster", mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + family: v1.IPv6Protocol, ipt: utiliptablestest.NewIPv6Fake(), expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/64", utiliptablestest.NewIPv6Fake())), errExpected: false, @@ -138,22 +141,34 @@ func Test_getLocalDetector(t *testing.T) { name: "LocalModeClusterCIDR, IPv6 cluster with IPv6 config", mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + family: v1.IPv6Protocol, ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - errExpected: true, + expected: proxyutiliptables.NewNoOpLocalDetector(), + errExpected: false, }, { name: "LocalModeClusterCIDR, IPv4 cluster with IPv6 config", mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), - expected: nil, - errExpected: true, + expected: proxyutiliptables.NewNoOpLocalDetector(), + errExpected: false, + }, + { + name: "LocalModeClusterCIDR, IPv4 kube-proxy in dual-stack IPv6-primary cluster", + mode: proxyconfigapi.LocalModeClusterCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64,10.0.0.0/14"}, + family: v1.IPv4Protocol, + ipt: utiliptablestest.NewFake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/14", utiliptablestest.NewFake())), + errExpected: false, }, { name: "LocalModeClusterCIDR, no ClusterCIDR", mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, @@ -163,6 +178,7 @@ func Test_getLocalDetector(t *testing.T) { name: "LocalModeNodeCIDR, IPv4 cluster", mode: proxyconfigapi.LocalModeNodeCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), nodePodCIDRs: []string{"10.0.0.0/24"}, @@ -172,6 +188,7 @@ func Test_getLocalDetector(t *testing.T) { name: "LocalModeNodeCIDR, IPv6 cluster", mode: proxyconfigapi.LocalModeNodeCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + family: v1.IPv6Protocol, ipt: utiliptablestest.NewIPv6Fake(), expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, @@ -181,24 +198,37 @@ func Test_getLocalDetector(t *testing.T) { name: "LocalModeNodeCIDR, IPv6 cluster with IPv4 config", mode: proxyconfigapi.LocalModeNodeCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + family: v1.IPv6Protocol, ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, + expected: proxyutiliptables.NewNoOpLocalDetector(), nodePodCIDRs: []string{"10.0.0.0/24"}, - errExpected: true, + errExpected: false, }, { name: "LocalModeNodeCIDR, IPv4 cluster with IPv6 config", mode: proxyconfigapi.LocalModeNodeCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), - expected: nil, + expected: proxyutiliptables.NewNoOpLocalDetector(), nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, - errExpected: true, + errExpected: false, + }, + { + name: "LocalModeNodeCIDR, IPv6 kube-proxy in dual-stack IPv4-primary cluster", + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, + family: v1.IPv6Protocol, + ipt: utiliptablestest.NewIPv6Fake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), + nodePodCIDRs: []string{"10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { name: "LocalModeNodeCIDR, no PodCIDRs", mode: proxyconfigapi.LocalModeNodeCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), expected: proxyutiliptables.NewNoOpLocalDetector(), nodePodCIDRs: []string{}, @@ -209,6 +239,7 @@ func Test_getLocalDetector(t *testing.T) { name: "unknown LocalMode", mode: proxyconfigapi.LocalMode("abcd"), config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + family: v1.IPv4Protocol, ipt: utiliptablestest.NewFake(), expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, @@ -254,7 +285,7 @@ func Test_getLocalDetector(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodePodCIDRs) + r, err := getLocalDetector(c.family, c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Expected error, but succeeded with %v", r)