From e053fdd08ae00d1bff3dd42a2c034f4810f8a55d Mon Sep 17 00:00:00 2001 From: Satyadeep Musuvathy Date: Thu, 5 Mar 2020 17:22:31 -0800 Subject: [PATCH] Add NodeCIDR for detect-local-mode --- cmd/kube-proxy/app/BUILD | 44 +++++++ cmd/kube-proxy/app/server_others.go | 105 ++++++++++++++-- cmd/kube-proxy/app/server_others_test.go | 151 +++++++++++++++++++++-- pkg/proxy/apis/config/types.go | 1 + 4 files changed, 285 insertions(+), 16 deletions(-) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index db32ed09245..0c88dd2cd8d 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -83,6 +83,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -92,6 +95,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -101,6 +107,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -110,6 +119,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -119,6 +131,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -128,6 +143,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -137,6 +155,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -146,6 +167,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -155,6 +179,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -164,6 +191,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -173,6 +203,9 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], @@ -211,66 +244,77 @@ go_test( "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:ios": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", ], "//conditions:default": [], }), diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 15d3a1ae178..026b2585fb9 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -21,17 +21,29 @@ limitations under the License. package app import ( + "context" "errors" "fmt" "net" "strings" + "time" + + "k8s.io/apimachinery/pkg/watch" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" + + "k8s.io/apimachinery/pkg/fields" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" + toolswatch "k8s.io/client-go/tools/watch" "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" @@ -55,6 +67,10 @@ import ( "k8s.io/klog" ) +// timeoutForNodePodCIDR is the time to wait for allocators to assign a PodCIDR to the +// node after it is registered. +var timeoutForNodePodCIDR = 5 * time.Minute + // NewProxyServer returns a new ProxyServer. func NewProxyServer(o *Options) (*ProxyServer, error) { return newProxyServer(o.config, o.CleanupAndExit, o.master) @@ -145,6 +161,16 @@ func newProxyServer( return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err) } + var nodeInfo *v1.Node + if detectLocalMode == proxyconfigapi.LocalModeNodeCIDR { + klog.Infof("Watching for node %s, awaiting podCIDR allocation", hostname) + nodeInfo, err = waitForPodCIDR(client, hostname) + if err != nil { + return nil, err + } + klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs) + } + nodeIP := net.ParseIP(config.BindAddress) if nodeIP.IsUnspecified() { nodeIP = utilnode.GetNodeIP(client, hostname) @@ -179,7 +205,7 @@ func newProxyServer( // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt) + localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -202,7 +228,7 @@ func newProxyServer( ) } else { // Create a single-stack proxier. var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface) + localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -249,7 +275,7 @@ func newProxyServer( // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt) + localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -279,7 +305,7 @@ func newProxyServer( ) } else { var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface) + localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -355,10 +381,44 @@ func newProxyServer( }, nil } +func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, error) { + // since allocators can assign the podCIDR after the node registers, we do a watch here to wait + // for podCIDR to be assigned, instead of assuming that the Get() on startup will have it. + ctx, cancelFunc := context.WithTimeout(context.TODO(), timeoutForNodePodCIDR) + defer cancelFunc() + + fieldSelector := fields.OneTermEqualSelector("metadata.name", nodeName).String() + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) { + options.FieldSelector = fieldSelector + return client.CoreV1().Nodes().List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { + options.FieldSelector = fieldSelector + return client.CoreV1().Nodes().Watch(ctx, options) + }, + } + condition := func(event watch.Event) (bool, error) { + if n, ok := event.Object.(*v1.Node); ok { + return n.Spec.PodCIDR != "" && len(n.Spec.PodCIDRs) > 0, nil + } + return false, fmt.Errorf("event object not of type Node") + } + + evt, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition) + if err != nil { + return nil, fmt.Errorf("timeout waiting for PodCIDR allocation to configure detect-local-mode %v: %v", proxyconfigapi.LocalModeNodeCIDR, err) + } + if n, ok := evt.Object.(*v1.Node); ok { + return n, nil + } + return nil, fmt.Errorf("event object not of type node") +} + func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) { mode := config.DetectLocalMode switch mode { - case proxyconfigapi.LocalModeClusterCIDR: + case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR: return mode, nil default: if strings.TrimSpace(mode.String()) != "" { @@ -369,7 +429,7 @@ func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxycon } } -func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface) (proxyutiliptables.LocalTrafficDetector, error) { +func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) { switch mode { case proxyconfigapi.LocalModeClusterCIDR: if len(strings.TrimSpace(config.ClusterCIDR)) == 0 { @@ -377,12 +437,18 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube break } return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt) + case proxyconfigapi.LocalModeNodeCIDR: + if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + klog.Warning("detect-local-mode set to NodeCIDR, but no PodCIDR defined at node") + break + } + return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) } klog.V(0).Info("detect-local-mode: ", string(mode), " , defaulting to no-op detect-local") return proxyutiliptables.NewNoOpLocalDetector(), nil } -func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface) ([2]proxyutiliptables.LocalTrafficDetector, error) { +func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) { var err error localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()} switch mode { @@ -409,6 +475,31 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(clusterCIDRs[1], ipt[1]) } return localDetectors, err + case proxyconfigapi.LocalModeNodeCIDR: + if nodeInfo == nil || len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + klog.Warning("No node info available to configure detect-local-mode NodeCIDR") + break + } + // localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR. + // so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1] + if utilsnet.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1]) + if err != nil { + return localDetectors, err + } + if len(nodeInfo.Spec.PodCIDRs) > 1 { + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0]) + } + } else { + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0]) + if err != nil { + return localDetectors, err + } + if len(nodeInfo.Spec.PodCIDRs) > 1 { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1]) + } + } + return localDetectors, err default: klog.Warningf("unknown detect-local-mode: %v", mode) } diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 56c66375ffa..b6c382de992 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -23,6 +23,7 @@ import ( "reflect" "testing" + v1 "k8s.io/api/core/v1" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" "k8s.io/kubernetes/pkg/proxy/ipvs" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" @@ -198,8 +199,10 @@ func Test_getLocalDetector(t *testing.T) { config *proxyconfigapi.KubeProxyConfiguration ipt utiliptables.Interface expected proxyutiliptables.LocalTrafficDetector + nodeInfo *v1.Node errExpected bool }{ + // LocalModeClusterCIDR, nodeInfo would be nil for these cases { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -242,13 +245,6 @@ func Test_getLocalDetector(t *testing.T) { expected: nil, errExpected: true, }, - { - mode: proxyconfigapi.LocalMode("abcd"), - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - errExpected: false, - }, { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, @@ -256,9 +252,74 @@ func Test_getLocalDetector(t *testing.T) { expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, }, + // LocalModeNodeCIDR + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewFake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), + nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewIpv6Fake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIpv6Fake())), + nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + nodeInfo: makeNodeWithPodCIDRs("10.0.0.0"), + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101"}, + ipt: utiliptablestest.NewIpv6Fake(), + expected: nil, + nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101"), + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewIpv6Fake(), + expected: nil, + nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: utiliptablestest.NewFake(), + expected: proxyutiliptables.NewNoOpLocalDetector(), + nodeInfo: makeNodeWithPodCIDRs(), + errExpected: false, + }, + // unknown mode, nodeInfo would be nil for these cases + { + mode: proxyconfigapi.LocalMode("abcd"), + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewFake(), + expected: proxyutiliptables.NewNoOpLocalDetector(), + errExpected: false, + }, } for i, c := range cases { - r, err := getLocalDetector(c.mode, c.config, c.ipt) + r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo) if c.errExpected { if err == nil { t.Errorf("Case[%d] Expected error, but succeeded with %v", i, r) @@ -281,8 +342,10 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { config *proxyconfigapi.KubeProxyConfiguration ipt [2]utiliptables.Interface expected [2]proxyutiliptables.LocalTrafficDetector + nodeInfo *v1.Node errExpected bool }{ + // LocalModeClusterCIDR, nodeInfo would be nil for these cases { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, @@ -326,6 +389,64 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, errExpected: false, }, + // LocalModeNodeCIDR + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIpv6Fake()}, + expected: resolveDualStackLocalDetectors(t)( + proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( + proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIpv6Fake())), + nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64,10.0.0.0/14"}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIpv6Fake()}, + expected: resolveDualStackLocalDetectors(t)( + proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( + proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIpv6Fake())), + nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIpv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{ + resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), + proxyutiliptables.NewNoOpLocalDetector()}, + nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIpv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{ + proxyutiliptables.NewNoOpLocalDetector(), + resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIpv6Fake()))}, + nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIpv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + nodeInfo: makeNodeWithPodCIDRs(), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIpv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + nodeInfo: nil, + errExpected: false, + }, + // unknown mode, nodeInfo would be nil for these cases { mode: proxyconfigapi.LocalMode("abcd"), config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, @@ -335,7 +456,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { }, } for i, c := range cases { - r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt) + r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) if c.errExpected { if err == nil { t.Errorf("Case[%d] expected error, but succeeded with %q", i, r) @@ -352,6 +473,18 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { } } +func makeNodeWithPodCIDRs(cidrs ...string) *v1.Node { + if len(cidrs) == 0 { + return &v1.Node{} + } + return &v1.Node{ + Spec: v1.NodeSpec{ + PodCIDR: cidrs[0], + PodCIDRs: cidrs, + }, + } +} + func resolveLocalDetector(t *testing.T) func(proxyutiliptables.LocalTrafficDetector, error) proxyutiliptables.LocalTrafficDetector { return func(localDetector proxyutiliptables.LocalTrafficDetector, err error) proxyutiliptables.LocalTrafficDetector { t.Helper() diff --git a/pkg/proxy/apis/config/types.go b/pkg/proxy/apis/config/types.go index a5c578a71ce..c500d5ee75f 100644 --- a/pkg/proxy/apis/config/types.go +++ b/pkg/proxy/apis/config/types.go @@ -196,6 +196,7 @@ type LocalMode string // Currently supported modes for LocalMode const ( LocalModeClusterCIDR LocalMode = "ClusterCIDR" + LocalModeNodeCIDR LocalMode = "NodeCIDR" ) // IPVSSchedulerMethod is the algorithm for allocating TCP connections and