From d5d9327351de99e6068f62efb388a6ede38d944e Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Mon, 12 Apr 2021 17:42:07 -0700 Subject: [PATCH] Only use dualstack if the node and config supports it --- cmd/kube-proxy/app/server.go | 37 +++++++ cmd/kube-proxy/app/server_others.go | 33 ------- cmd/kube-proxy/app/server_windows.go | 33 ++----- pkg/proxy/winkernel/proxier.go | 139 ++++++++++++++++++++++----- pkg/proxy/winkernel/proxier_test.go | 52 ++++++++++ 5 files changed, 215 insertions(+), 79 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 06a5ad19e89..3d66f2546a6 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -22,11 +22,14 @@ import ( "errors" "fmt" "io/ioutil" + "net" "net/http" "os" "strings" "time" + utilnode "k8s.io/kubernetes/pkg/util/node" + "github.com/fsnotify/fsnotify" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -83,6 +86,7 @@ import ( utilipvs "k8s.io/kubernetes/pkg/util/ipvs" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/utils/exec" + utilsnet "k8s.io/utils/net" utilpointer "k8s.io/utils/pointer" ) @@ -817,3 +821,36 @@ func (s *ProxyServer) CleanupAndExit() error { return nil } + +// detectNodeIP returns the nodeIP used by the proxier +// The order of precedence is: +// 1. config.bindAddress if bindAddress is not 0.0.0.0 or :: +// 2. the primary IP from the Node object, if set +// 3. if no IP is found it defaults to 127.0.0.1 and IPv4 +func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP { + nodeIP := net.ParseIP(bindAddress) + if nodeIP.IsUnspecified() { + nodeIP = utilnode.GetNodeIP(client, hostname) + } + if nodeIP == nil { + klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag") + nodeIP = net.ParseIP("127.0.0.1") + } + return nodeIP +} + +// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6) +// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address +// will have "any" address (0.0.0.0 or ::) inserted. +func nodeIPTuple(bindAddress string) [2]net.IP { + nodes := [2]net.IP{net.IPv4zero, net.IPv6zero} + + adr := net.ParseIP(bindAddress) + if utilsnet.IsIPv6(adr) { + nodes[1] = adr + } else { + nodes[0] = adr + } + + return nodes +} diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 77efde0280c..1d9a4f617f9 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -428,23 +428,6 @@ func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, erro return nil, fmt.Errorf("event object not of type node") } -// detectNodeIP returns the nodeIP used by the proxier -// The order of precedence is: -// 1. config.bindAddress if bindAddress is not 0.0.0.0 or :: -// 2. the primary IP from the Node object, if set -// 3. if no IP is found it defaults to 127.0.0.1 and IPv4 -func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP { - nodeIP := net.ParseIP(bindAddress) - if nodeIP.IsUnspecified() { - nodeIP = utilnode.GetNodeIP(client, hostname) - } - if nodeIP == nil { - klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag") - nodeIP = net.ParseIP("127.0.0.1") - } - return nodeIP -} - func detectNumCPU() int { // try get numCPU from /sys firstly due to a known issue (https://github.com/kubernetes/kubernetes/issues/99225) _, numCPU, err := machine.GetTopology(sysfs.NewRealSysFs()) @@ -570,22 +553,6 @@ func cidrTuple(cidrList string) [2]string { return cidrs } -// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6) -// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address -// will have "any" address (0.0.0.0 or ::) inserted. -func nodeIPTuple(bindAddress string) [2]net.IP { - nodes := [2]net.IP{net.IPv4zero, net.IPv6zero} - - adr := net.ParseIP(bindAddress) - if utilsnet.IsIPv6(adr) { - nodes[1] = adr - } else { - nodes[0] = adr - } - - return nodes -} - func getProxyMode(proxyMode string, canUseIPVS bool, kcompat iptables.KernelCompatTester) string { switch proxyMode { case proxyModeUserspace: diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index fd0c2f25bf1..7a9ee8fd36e 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -32,12 +32,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/events" "k8s.io/component-base/configz" "k8s.io/component-base/metrics" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" @@ -47,7 +45,6 @@ import ( utilnetsh "k8s.io/kubernetes/pkg/util/netsh" utilnode "k8s.io/kubernetes/pkg/util/node" "k8s.io/utils/exec" - utilsnet "k8s.io/utils/net" ) // NewProxyServer returns a new ProxyServer. @@ -85,6 +82,9 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi if err != nil { return nil, err } + nodeIP := detectNodeIP(client, hostname, config.BindAddress) + klog.InfoS("Detected node IP", "IP", nodeIP.String()) + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()}) recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, "kube-proxy") @@ -101,12 +101,11 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi } var proxier proxy.Provider - proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{}) + dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{}) if proxyMode == proxyModeKernelspace { klog.V(0).InfoS("Using Kernelspace Proxier.") - isIPv6DualStackEnabled := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) - if isIPv6DualStackEnabled { + if dualStackMode { klog.V(0).InfoS("Creating dualStackProxier for Windows kernel.") proxier, err = winkernel.NewDualStackProxier( @@ -130,7 +129,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi int(*config.IPTables.MasqueradeBit), config.ClusterCIDR, hostname, - utilnode.GetNodeIP(client, hostname), + nodeIP, recorder, healthzServer, config.Winkernel, @@ -183,6 +182,10 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi }, nil } +func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool { + return compatTester.DualStackCompatible(networkname) +} + func getProxyMode(proxyMode string, kcompat winkernel.KernelCompatTester) string { if proxyMode == proxyModeKernelspace { return tryWinKernelSpaceProxy(kcompat) @@ -211,19 +214,3 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string { klog.V(1).InfoS("Can't use winkernel proxy, using userspace proxier") return proxyModeUserspace } - -// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6) -// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address -// will have "any" address (0.0.0.0 or ::) inserted. -func nodeIPTuple(bindAddress string) [2]net.IP { - nodes := [2]net.IP{net.IPv4zero, net.IPv6zero} - - adr := net.ParseIP(bindAddress) - if utilsnet.IsIPv6(adr) { - nodes[1] = adr - } else { - nodes[0] = adr - } - - return nodes -} diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index f212f7a6d45..e49fd514e88 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -31,13 +31,14 @@ import ( "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" - discovery "k8s.io/api/discovery/v1" - "github.com/davecgh/go-spew/spew" v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + apiutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/events" @@ -137,6 +138,101 @@ type remoteSubnetInfo struct { const NETWORK_TYPE_OVERLAY = "overlay" +func newHostNetworkService() (HostNetworkService, hcn.SupportedFeatures) { + var hns HostNetworkService + hns = hnsV1{} + supportedFeatures := hcn.GetSupportedFeatures() + if supportedFeatures.Api.V2 { + hns = hnsV2{} + } + + return hns, supportedFeatures +} + +func getNetworkName(hnsNetworkName string) (string, error) { + if len(hnsNetworkName) == 0 { + klog.V(3).InfoS("network-name flag not set. Checking environment variable") + hnsNetworkName = os.Getenv("KUBE_NETWORK") + if len(hnsNetworkName) == 0 { + return "", fmt.Errorf("Environment variable KUBE_NETWORK and network-flag not initialized") + } + } + return hnsNetworkName, nil +} + +func getNetworkInfo(hns HostNetworkService, hnsNetworkName string) (*hnsNetworkInfo, error) { + hnsNetworkInfo, err := hns.getNetworkByName(hnsNetworkName) + for err != nil { + klog.ErrorS(err, "Unable to find HNS Network specified. Please check network name and CNI deployment", "hnsNetworkName", hnsNetworkName) + time.Sleep(1 * time.Second) + hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName) + } + return hnsNetworkInfo, err +} + +func isOverlay(hnsNetworkInfo *hnsNetworkInfo) bool { + return strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) +} + +// StackCompatTester tests whether the required kernel and network are dualstack capable +type StackCompatTester interface { + DualStackCompatible(networkName string) bool +} + +type DualStackCompatTester struct{} + +func (t DualStackCompatTester) DualStackCompatible(networkName string) bool { + dualStackFeatureEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.IPv6DualStack) + if !dualStackFeatureEnabled { + return false + } + + globals, err := hcn.GetGlobals() + if err != nil { + klog.ErrorS(err, "Unable to determine networking stack version. Falling back to single-stack") + return false + } + + if !kernelSupportsDualstack(globals.Version) { + klog.InfoS("This version of Windows does not support dual-stack. Falling back to single-stack") + return false + } + + // check if network is using overlay + hns, _ := newHostNetworkService() + networkName, err = getNetworkName(networkName) + if err != nil { + klog.ErrorS(err, "unable to determine dual-stack status %v. Falling back to single-stack") + return false + } + networkInfo, err := getNetworkInfo(hns, networkName) + if err != nil { + klog.ErrorS(err, "unable to determine dual-stack status %v. Falling back to single-stack") + return false + } + + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WinOverlay) && isOverlay(networkInfo) { + // Overlay (VXLAN) networks on Windows do not support dual-stack networking today + klog.InfoS("Winoverlay does not support dual-stack. Falling back to single-stack") + return false + } + + return true +} + +// The hcsshim version logic has a bug that did not calculate the versioning of DualStack correctly. +// DualStack is supported in WS 2004+ (10.0.19041+) where HCN component version is 11.10+ +// https://github.com/microsoft/hcsshim/pull/1003#issuecomment-827930358 +func kernelSupportsDualstack(currentVersion hcn.Version) bool { + hnsVersion := fmt.Sprintf("%d.%d.0", currentVersion.Major, currentVersion.Minor) + v, err := version.ParseSemantic(hnsVersion) + if err != nil { + return false + } + + return v.AtLeast(version.MustParseSemantic("11.10.0")) +} + func Log(v interface{}, message string, level klog.Level) { klog.V(level).InfoS("%s", message, "spewConfig", spewSdump(v)) } @@ -546,36 +642,24 @@ func NewProxier( } serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder) - var hns HostNetworkService - hns = hnsV1{} - supportedFeatures := hcn.GetSupportedFeatures() - if supportedFeatures.Api.V2 { - hns = hnsV2{} - } - - hnsNetworkName := config.NetworkName - if len(hnsNetworkName) == 0 { - klog.V(3).InfoS("network-name flag not set. Checking environment variable") - hnsNetworkName = os.Getenv("KUBE_NETWORK") - if len(hnsNetworkName) == 0 { - return nil, fmt.Errorf("Environment variable KUBE_NETWORK and network-flag not initialized") - } + hns, supportedFeatures := newHostNetworkService() + hnsNetworkName, err := getNetworkName(config.NetworkName) + if err != nil { + return nil, err } klog.V(3).InfoS("Cleaning up old HNS policy lists") deleteAllHnsLoadBalancerPolicy() // Get HNS network information - hnsNetworkInfo, err := hns.getNetworkByName(hnsNetworkName) - for err != nil { - klog.ErrorS(err, "Unable to find HNS Network specified. Please check network name and CNI deployment", "hnsNetworkName", hnsNetworkName) - time.Sleep(1 * time.Second) - hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName) + hnsNetworkInfo, err := getNetworkInfo(hns, hnsNetworkName) + if err != nil { + return nil, err } // Network could have been detected before Remote Subnet Routes are applied or ManagementIP is updated // Sleep and update the network to include new information - if strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) { + if isOverlay(hnsNetworkInfo) { time.Sleep(10 * time.Second) hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName) if err != nil { @@ -595,7 +679,7 @@ func NewProxier( var sourceVip string var hostMac string - if strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) { + if isOverlay(hnsNetworkInfo) { if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WinOverlay) { return nil, fmt.Errorf("WinOverlay feature gate not enabled") } @@ -608,6 +692,15 @@ func NewProxier( return nil, fmt.Errorf("source-vip flag not set") } + if nodeIP.IsUnspecified() { + // attempt to get the correct ip address + klog.V(2).InfoS("node ip was unspecified. Attempting to find node ip") + nodeIP, err = apiutil.ResolveBindAddress(nodeIP) + if err != nil { + klog.InfoS("failed to find an ip. You may need set the --bind-address flag", "err", err) + } + } + interfaces, _ := net.Interfaces() //TODO create interfaces for _, inter := range interfaces { addresses, _ := inter.Addrs() diff --git a/pkg/proxy/winkernel/proxier_test.go b/pkg/proxy/winkernel/proxier_test.go index 177711076ec..86b435b01ef 100644 --- a/pkg/proxy/winkernel/proxier_test.go +++ b/pkg/proxy/winkernel/proxier_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/hcn" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -928,3 +929,54 @@ func makeTestEndpointSlice(namespace, name string, sliceNum int, epsFunc func(*d epsFunc(eps) return eps } + +func Test_kernelSupportsDualstack(t *testing.T) { + tests := []struct { + currentVersion hcn.Version + name string + want bool + }{ + { + hcn.Version{Major: 10, Minor: 10}, + "Less than minimal should not be supported", + false, + }, + { + hcn.Version{Major: 9, Minor: 11}, + "Less than minimal should not be supported", + false, + }, + { + hcn.Version{Major: 11, Minor: 1}, + "Less than minimal should not be supported", + false, + }, + { + hcn.Version{Major: 11, Minor: 10}, + "Current version should be supported", + true, + }, + { + hcn.Version{Major: 11, Minor: 11}, + "Greater than minimal version should be supported", + true, + }, + { + hcn.Version{Major: 12, Minor: 1}, + "Greater than minimal version should be supported", + true, + }, + { + hcn.Version{Major: 12, Minor: 12}, + "Greater than minimal should be supported", + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := kernelSupportsDualstack(tt.currentVersion); got != tt.want { + t.Errorf("kernelSupportsDualstack on version %v: got %v, want %v", tt.currentVersion, got, tt.want) + } + }) + } +}