Only use dualstack if the node and config supports it

cmd/kube-proxy/app/server.go

@@ -22,11 +22,14 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
+	utilnode "k8s.io/kubernetes/pkg/util/node"
+
 	"github.com/fsnotify/fsnotify"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -83,6 +86,7 @@ import (
 	utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
 	"k8s.io/kubernetes/pkg/util/oom"
 	"k8s.io/utils/exec"
+	utilsnet "k8s.io/utils/net"
 	utilpointer "k8s.io/utils/pointer"
 )
 
@@ -817,3 +821,36 @@ func (s *ProxyServer) CleanupAndExit() error {
 
 	return nil
 }
+
+// detectNodeIP returns the nodeIP used by the proxier
+// The order of precedence is:
+// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
+// 2. the primary IP from the Node object, if set
+// 3. if no IP is found it defaults to 127.0.0.1 and IPv4
+func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP {
+	nodeIP := net.ParseIP(bindAddress)
+	if nodeIP.IsUnspecified() {
+		nodeIP = utilnode.GetNodeIP(client, hostname)
+	}
+	if nodeIP == nil {
+		klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
+		nodeIP = net.ParseIP("127.0.0.1")
+	}
+	return nodeIP
+}
+
+// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
+// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
+// will have "any" address (0.0.0.0 or ::) inserted.
+func nodeIPTuple(bindAddress string) [2]net.IP {
+	nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
+
+	adr := net.ParseIP(bindAddress)
+	if utilsnet.IsIPv6(adr) {
+		nodes[1] = adr
+	} else {
+		nodes[0] = adr
+	}
+
+	return nodes
+}

cmd/kube-proxy/app/server_others.go

@@ -428,23 +428,6 @@ func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, erro
 	return nil, fmt.Errorf("event object not of type node")
 }
 
-// detectNodeIP returns the nodeIP used by the proxier
-// The order of precedence is:
-// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
-// 2. the primary IP from the Node object, if set
-// 3. if no IP is found it defaults to 127.0.0.1 and IPv4
-func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP {
-	nodeIP := net.ParseIP(bindAddress)
-	if nodeIP.IsUnspecified() {
-		nodeIP = utilnode.GetNodeIP(client, hostname)
-	}
-	if nodeIP == nil {
-		klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
-		nodeIP = net.ParseIP("127.0.0.1")
-	}
-	return nodeIP
-}
-
 func detectNumCPU() int {
 	// try get numCPU from /sys firstly due to a known issue (https://github.com/kubernetes/kubernetes/issues/99225)
 	_, numCPU, err := machine.GetTopology(sysfs.NewRealSysFs())
@@ -570,22 +553,6 @@ func cidrTuple(cidrList string) [2]string {
 	return cidrs
 }
 
-// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
-// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
-// will have "any" address (0.0.0.0 or ::) inserted.
-func nodeIPTuple(bindAddress string) [2]net.IP {
-	nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
-
-	adr := net.ParseIP(bindAddress)
-	if utilsnet.IsIPv6(adr) {
-		nodes[1] = adr
-	} else {
-		nodes[0] = adr
-	}
-
-	return nodes
-}
-
 func getProxyMode(proxyMode string, canUseIPVS bool, kcompat iptables.KernelCompatTester) string {
 	switch proxyMode {
 	case proxyModeUserspace:

cmd/kube-proxy/app/server_windows.go

@@ -32,12 +32,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/events"
 	"k8s.io/component-base/configz"
 	"k8s.io/component-base/metrics"
 	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
 	proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme"
@@ -47,7 +45,6 @@ import (
 	utilnetsh "k8s.io/kubernetes/pkg/util/netsh"
 	utilnode "k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/utils/exec"
-	utilsnet "k8s.io/utils/net"
 )
 
 // NewProxyServer returns a new ProxyServer.
@@ -85,6 +82,9 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 	if err != nil {
 		return nil, err
 	}
+	nodeIP := detectNodeIP(client, hostname, config.BindAddress)
+	klog.InfoS("Detected node IP", "IP", nodeIP.String())
+
 	eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
 	recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, "kube-proxy")
 
@@ -101,12 +101,11 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 	}
 
 	var proxier proxy.Provider
-
 	proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{})
+	dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{})
 	if proxyMode == proxyModeKernelspace {
 		klog.V(0).InfoS("Using Kernelspace Proxier.")
-		isIPv6DualStackEnabled := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack)
-		if isIPv6DualStackEnabled {
+		if dualStackMode {
 			klog.V(0).InfoS("Creating dualStackProxier for Windows kernel.")
 
 			proxier, err = winkernel.NewDualStackProxier(
@@ -130,7 +129,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 				int(*config.IPTables.MasqueradeBit),
 				config.ClusterCIDR,
 				hostname,
-				utilnode.GetNodeIP(client, hostname),
+				nodeIP,
 				recorder,
 				healthzServer,
 				config.Winkernel,
@@ -183,6 +182,10 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 	}, nil
 }
 
+func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool {
+	return compatTester.DualStackCompatible(networkname)
+}
+
 func getProxyMode(proxyMode string, kcompat winkernel.KernelCompatTester) string {
 	if proxyMode == proxyModeKernelspace {
 		return tryWinKernelSpaceProxy(kcompat)
@@ -211,19 +214,3 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string {
 	klog.V(1).InfoS("Can't use winkernel proxy, using userspace proxier")
 	return proxyModeUserspace
 }
-
-// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
-// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
-// will have "any" address (0.0.0.0 or ::) inserted.
-func nodeIPTuple(bindAddress string) [2]net.IP {
-	nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
-
-	adr := net.ParseIP(bindAddress)
-	if utilsnet.IsIPv6(adr) {
-		nodes[1] = adr
-	} else {
-		nodes[0] = adr
-	}
-
-	return nodes
-}

pkg/proxy/winkernel/proxier.go

@@ -31,13 +31,14 @@ import (
 
 	"github.com/Microsoft/hcsshim"
 	"github.com/Microsoft/hcsshim/hcn"
-	discovery "k8s.io/api/discovery/v1"
-
 	"github.com/davecgh/go-spew/spew"
 	v1 "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	apiutil "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/events"
@@ -137,6 +138,101 @@ type remoteSubnetInfo struct {
 
 const NETWORK_TYPE_OVERLAY = "overlay"
 
+func newHostNetworkService() (HostNetworkService, hcn.SupportedFeatures) {
+	var hns HostNetworkService
+	hns = hnsV1{}
+	supportedFeatures := hcn.GetSupportedFeatures()
+	if supportedFeatures.Api.V2 {
+		hns = hnsV2{}
+	}
+
+	return hns, supportedFeatures
+}
+
+func getNetworkName(hnsNetworkName string) (string, error) {
+	if len(hnsNetworkName) == 0 {
+		klog.V(3).InfoS("network-name flag not set. Checking environment variable")
+		hnsNetworkName = os.Getenv("KUBE_NETWORK")
+		if len(hnsNetworkName) == 0 {
+			return "", fmt.Errorf("Environment variable KUBE_NETWORK and network-flag not initialized")
+		}
+	}
+	return hnsNetworkName, nil
+}
+
+func getNetworkInfo(hns HostNetworkService, hnsNetworkName string) (*hnsNetworkInfo, error) {
+	hnsNetworkInfo, err := hns.getNetworkByName(hnsNetworkName)
+	for err != nil {
+		klog.ErrorS(err, "Unable to find HNS Network specified. Please check network name and CNI deployment", "hnsNetworkName", hnsNetworkName)
+		time.Sleep(1 * time.Second)
+		hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName)
+	}
+	return hnsNetworkInfo, err
+}
+
+func isOverlay(hnsNetworkInfo *hnsNetworkInfo) bool {
+	return strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY)
+}
+
+// StackCompatTester tests whether the required kernel and network are dualstack capable
+type StackCompatTester interface {
+	DualStackCompatible(networkName string) bool
+}
+
+type DualStackCompatTester struct{}
+
+func (t DualStackCompatTester) DualStackCompatible(networkName string) bool {
+	dualStackFeatureEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.IPv6DualStack)
+	if !dualStackFeatureEnabled {
+		return false
+	}
+
+	globals, err := hcn.GetGlobals()
+	if err != nil {
+		klog.ErrorS(err, "Unable to determine networking stack version. Falling back to single-stack")
+		return false
+	}
+
+	if !kernelSupportsDualstack(globals.Version) {
+		klog.InfoS("This version of Windows does not support dual-stack. Falling back to single-stack")
+		return false
+	}
+
+	// check if network is using overlay
+	hns, _ := newHostNetworkService()
+	networkName, err = getNetworkName(networkName)
+	if err != nil {
+		klog.ErrorS(err, "unable to determine dual-stack status %v. Falling back to single-stack")
+		return false
+	}
+	networkInfo, err := getNetworkInfo(hns, networkName)
+	if err != nil {
+		klog.ErrorS(err, "unable to determine dual-stack status %v. Falling back to single-stack")
+		return false
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WinOverlay) && isOverlay(networkInfo) {
+		// Overlay (VXLAN) networks on Windows do not support dual-stack networking today
+		klog.InfoS("Winoverlay does not support dual-stack. Falling back to single-stack")
+		return false
+	}
+
+	return true
+}
+
+// The hcsshim version logic has a bug that did not calculate the versioning of DualStack correctly.
+// DualStack is supported in WS 2004+ (10.0.19041+) where HCN component version is 11.10+
+// https://github.com/microsoft/hcsshim/pull/1003#issuecomment-827930358
+func kernelSupportsDualstack(currentVersion hcn.Version) bool {
+	hnsVersion := fmt.Sprintf("%d.%d.0", currentVersion.Major, currentVersion.Minor)
+	v, err := version.ParseSemantic(hnsVersion)
+	if err != nil {
+		return false
+	}
+
+	return v.AtLeast(version.MustParseSemantic("11.10.0"))
+}
+
 func Log(v interface{}, message string, level klog.Level) {
 	klog.V(level).InfoS("%s", message, "spewConfig", spewSdump(v))
 }
@@ -546,36 +642,24 @@ func NewProxier(
 	}
 
 	serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder)
-	var hns HostNetworkService
-	hns = hnsV1{}
-	supportedFeatures := hcn.GetSupportedFeatures()
-	if supportedFeatures.Api.V2 {
-		hns = hnsV2{}
-	}
-
-	hnsNetworkName := config.NetworkName
-	if len(hnsNetworkName) == 0 {
-		klog.V(3).InfoS("network-name flag not set. Checking environment variable")
-		hnsNetworkName = os.Getenv("KUBE_NETWORK")
-		if len(hnsNetworkName) == 0 {
-			return nil, fmt.Errorf("Environment variable KUBE_NETWORK and network-flag not initialized")
-		}
+	hns, supportedFeatures := newHostNetworkService()
+	hnsNetworkName, err := getNetworkName(config.NetworkName)
+	if err != nil {
+		return nil, err
 	}
 
 	klog.V(3).InfoS("Cleaning up old HNS policy lists")
 	deleteAllHnsLoadBalancerPolicy()
 
 	// Get HNS network information
-	hnsNetworkInfo, err := hns.getNetworkByName(hnsNetworkName)
-	for err != nil {
-		klog.ErrorS(err, "Unable to find HNS Network specified. Please check network name and CNI deployment", "hnsNetworkName", hnsNetworkName)
-		time.Sleep(1 * time.Second)
-		hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName)
+	hnsNetworkInfo, err := getNetworkInfo(hns, hnsNetworkName)
+	if err != nil {
+		return nil, err
 	}
 
 	// Network could have been detected before Remote Subnet Routes are applied or ManagementIP is updated
 	// Sleep and update the network to include new information
-	if strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) {
+	if isOverlay(hnsNetworkInfo) {
 		time.Sleep(10 * time.Second)
 		hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName)
 		if err != nil {
@@ -595,7 +679,7 @@ func NewProxier(
 
 	var sourceVip string
 	var hostMac string
-	if strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) {
+	if isOverlay(hnsNetworkInfo) {
 		if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WinOverlay) {
 			return nil, fmt.Errorf("WinOverlay feature gate not enabled")
 		}
@@ -608,6 +692,15 @@ func NewProxier(
 			return nil, fmt.Errorf("source-vip flag not set")
 		}
 
+		if nodeIP.IsUnspecified() {
+			// attempt to get the correct ip address
+			klog.V(2).InfoS("node ip was unspecified.  Attempting to find node ip")
+			nodeIP, err = apiutil.ResolveBindAddress(nodeIP)
+			if err != nil {
+				klog.InfoS("failed to find an ip. You may need set the --bind-address flag", "err", err)
+			}
+		}
+
 		interfaces, _ := net.Interfaces() //TODO create interfaces
 		for _, inter := range interfaces {
 			addresses, _ := inter.Addrs()

pkg/proxy/winkernel/proxier_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/Microsoft/hcsshim/hcn"
 	v1 "k8s.io/api/core/v1"
 	discovery "k8s.io/api/discovery/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -928,3 +929,54 @@ func makeTestEndpointSlice(namespace, name string, sliceNum int, epsFunc func(*d
 	epsFunc(eps)
 	return eps
 }
+
+func Test_kernelSupportsDualstack(t *testing.T) {
+	tests := []struct {
+		currentVersion hcn.Version
+		name           string
+		want           bool
+	}{
+		{
+			hcn.Version{Major: 10, Minor: 10},
+			"Less than minimal should not be supported",
+			false,
+		},
+		{
+			hcn.Version{Major: 9, Minor: 11},
+			"Less than minimal should not be supported",
+			false,
+		},
+		{
+			hcn.Version{Major: 11, Minor: 1},
+			"Less than minimal should not be supported",
+			false,
+		},
+		{
+			hcn.Version{Major: 11, Minor: 10},
+			"Current version should be supported",
+			true,
+		},
+		{
+			hcn.Version{Major: 11, Minor: 11},
+			"Greater than minimal version should be supported",
+			true,
+		},
+		{
+			hcn.Version{Major: 12, Minor: 1},
+			"Greater than minimal version should be supported",
+			true,
+		},
+		{
+			hcn.Version{Major: 12, Minor: 12},
+			"Greater than minimal should be supported",
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := kernelSupportsDualstack(tt.currentVersion); got != tt.want {
+				t.Errorf("kernelSupportsDualstack on version %v: got %v, want %v", tt.currentVersion, got, tt.want)
+			}
+		})
+	}
+}