Merge pull request #90853 from kumarvin123/feature/IPv6DualStackProxy

KubeProxy and DockerShim changes for Ipv6 dual stack support on Windows

cmd/kube-proxy/app/BUILD

@@ -218,6 +218,7 @@ go_library(
             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
             "//staging/src/k8s.io/component-base/metrics:go_default_library",
+            "//vendor/k8s.io/utils/net:go_default_library",
         ],
         "//conditions:default": [],
     }),

cmd/kube-proxy/app/server_windows.go

@@ -28,12 +28,15 @@ import (
 	// Enable pprof HTTP handlers.
 	_ "net/http/pprof"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/component-base/configz"
 	"k8s.io/component-base/metrics"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
 	proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme"
@@ -43,8 +46,7 @@ import (
 	utilnetsh "k8s.io/kubernetes/pkg/util/netsh"
 	utilnode "k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/utils/exec"
-
+	utilsnet "k8s.io/utils/net"
-	"k8s.io/klog/v2"
 )
 
 // NewProxyServer returns a new ProxyServer.
@@ -102,6 +104,24 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 	proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{})
 	if proxyMode == proxyModeKernelspace {
 		klog.V(0).Info("Using Kernelspace Proxier.")
+		isIPv6DualStackEnabled := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack)
+		if isIPv6DualStackEnabled {
+			klog.V(0).Info("creating dualStackProxier for Windows kernel.")
+
+			proxier, err = winkernel.NewDualStackProxier(
+				config.IPTables.SyncPeriod.Duration,
+				config.IPTables.MinSyncPeriod.Duration,
+				config.IPTables.MasqueradeAll,
+				int(*config.IPTables.MasqueradeBit),
+				config.ClusterCIDR,
+				hostname,
+				nodeIPTuple(config.BindAddress),
+				recorder,
+				healthzServer,
+				config.Winkernel,
+			)
+		} else {
+
 			proxier, err = winkernel.NewProxier(
 				config.IPTables.SyncPeriod.Duration,
 				config.IPTables.MinSyncPeriod.Duration,
@@ -114,6 +134,9 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 				healthzServer,
 				config.Winkernel,
 			)
+
+		}
+
 		if err != nil {
 			return nil, fmt.Errorf("unable to create proxier: %v", err)
 		}
@@ -181,3 +204,19 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string {
 	klog.V(1).Infof("Can't use winkernel proxy, using userspace proxier")
 	return proxyModeUserspace
 }
+
+// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
+// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
+// will have "any" address (0.0.0.0 or ::) inserted.
+func nodeIPTuple(bindAddress string) [2]net.IP {
+	nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
+
+	adr := net.ParseIP(bindAddress)
+	if utilsnet.IsIPv6(adr) {
+		nodes[1] = adr
+	} else {
+		nodes[0] = adr
+	}
+
+	return nodes
+}

pkg/kubelet/dockershim/network/cni/cni_windows.go

@@ -21,13 +21,13 @@ package cni
 import (
 	"context"
 	"fmt"
-	"time"
-
 	cniTypes020 "github.com/containernetworking/cni/pkg/types/020"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	"k8s.io/klog/v2"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/dockershim/network"
+	"net"
+	"time"
 )
 
 func getLoNetwork(binDirs []string) *cniNetwork {
@@ -67,7 +67,14 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin
 		klog.Errorf("error while cni parsing result: %s", err)
 		return nil, err
 	}
-	return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil
+
+	var list = []net.IP{result020.IP4.IP.IP}
+
+	if result020.IP6 != nil {
+		list = append(list, result020.IP6.IP.IP)
+	}
+
+	return &network.PodNetworkStatus{IP: result020.IP4.IP.IP, IPs: list}, nil
 }
 
 // buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig.

pkg/proxy/winkernel/BUILD

@@ -21,6 +21,7 @@ go_library(
             "//pkg/proxy/apis/config:go_default_library",
             "//pkg/proxy/config:go_default_library",
             "//pkg/proxy/healthcheck:go_default_library",
+            "//pkg/proxy/metaproxier:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/async:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
@@ -35,6 +36,7 @@ go_library(
             "//vendor/github.com/Microsoft/hcsshim/hcn:go_default_library",
             "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
             "//vendor/k8s.io/klog/v2:go_default_library",
+            "//vendor/k8s.io/utils/net:go_default_library",
         ],
         "//conditions:default": [],
     }),

pkg/proxy/winkernel/hnsV2.go

@@ -30,6 +30,11 @@ import (
 
 type hnsV2 struct{}
 
+var (
+	// LoadBalancerFlagsIPv6 enables IPV6.
+	LoadBalancerFlagsIPv6 hcn.LoadBalancerFlags = 2
+)
+
 func (hns hnsV2) getNetworkByName(name string) (*hnsNetworkInfo, error) {
 	hnsnetwork, err := hcn.GetNetworkByName(name)
 	if err != nil {
@@ -90,10 +95,14 @@ func (hns hnsV2) getEndpointByIpAddress(ip string, networkName string) (*endpoin
 		equal := false
 		if endpoint.IpConfigurations != nil && len(endpoint.IpConfigurations) > 0 {
 			equal = endpoint.IpConfigurations[0].IpAddress == ip
+
+			if !equal && len(endpoint.IpConfigurations) > 1 {
+				equal = endpoint.IpConfigurations[1].IpAddress == ip
+			}
 		}
 		if equal && strings.EqualFold(endpoint.HostComputeNetwork, hnsnetwork.Id) {
 			return &endpointsInfo{
-				ip:         endpoint.IpConfigurations[0].IpAddress,
+				ip:         ip,
 				isLocal:    uint32(endpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote
 				macAddress: endpoint.MacAddress,
 				hnsID:      endpoint.Id,
@@ -232,6 +241,10 @@ func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFl
 		lbFlags |= hcn.LoadBalancerFlagsDSR
 	}
 
+	if flags.isIPv6 {
+		lbFlags |= LoadBalancerFlagsIPv6
+	}
+
 	lbDistributionType := hcn.LoadBalancerDistributionNone
 
 	if flags.sessionAffinity {

pkg/proxy/winkernel/proxier.go

@@ -48,8 +48,10 @@ import (
 	"k8s.io/kubernetes/pkg/proxy/apis/config"
 	proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
+	"k8s.io/kubernetes/pkg/proxy/metaproxier"
 	"k8s.io/kubernetes/pkg/proxy/metrics"
 	"k8s.io/kubernetes/pkg/util/async"
+	utilnet "k8s.io/utils/net"
 )
 
 // KernelCompatTester tests whether the required kernel capabilities are
@@ -101,6 +103,7 @@ type loadBalancerFlags struct {
 	useMUX          bool
 	preserveDIP     bool
 	sessionAffinity bool
+	isIPv6          bool
 }
 
 // internal struct for string service information
@@ -163,13 +166,17 @@ type endpointsInfo struct {
 	hns             HostNetworkService
 }
 
-//Uses mac prefix and IPv4 address to return a mac address
+//Uses mac prefix and IP address to return a mac address
 //This ensures mac addresses are unique for proper load balancing
-//Does not support IPv6 and returns a dummy mac
+//There is a possibility of MAC collisions but this Mac address is used for remote endpoints only
+//and not sent on the wire.
 func conjureMac(macPrefix string, ip net.IP) string {
 	if ip4 := ip.To4(); ip4 != nil {
 		a, b, c, d := ip4[0], ip4[1], ip4[2], ip4[3]
 		return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", macPrefix, a, b, c, d)
+	} else if ip6 := ip.To16(); ip6 != nil {
+		a, b, c, d := ip6[15], ip6[14], ip6[13], ip6[12]
+		return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", macPrefix, a, b, c, d)
 	}
 	return "02-11-22-33-44-55"
 }
@@ -502,6 +509,7 @@ type Proxier struct {
 	// with some partial data after kube-proxy restart.
 	endpointsSynced bool
 	servicesSynced  bool
+	isIPv6Mode      bool
 	initialized     int32
 	syncRunner      *async.BoundedFrequencyRunner // governs calls to syncProxyRules
 
@@ -664,6 +672,8 @@ func NewProxier(
 		}
 	}
 
+	isIPv6 := utilnet.IsIPv6(nodeIP)
+
 	proxier := &Proxier{
 		portsMap:            make(map[localPort]closeable),
 		serviceMap:          make(proxyServiceMap),
@@ -685,6 +695,7 @@ func NewProxier(
 		hostMac:             hostMac,
 		isDSR:               isDSR,
 		supportedFeatures:   supportedFeatures,
+		isIPv6Mode:          isIPv6,
 	}
 
 	burstSyncs := 2
@@ -694,6 +705,38 @@ func NewProxier(
 
 }
 
+func NewDualStackProxier(
+	syncPeriod time.Duration,
+	minSyncPeriod time.Duration,
+	masqueradeAll bool,
+	masqueradeBit int,
+	clusterCIDR string,
+	hostname string,
+	nodeIP [2]net.IP,
+	recorder record.EventRecorder,
+	healthzServer healthcheck.ProxierHealthUpdater,
+	config config.KubeProxyWinkernelConfiguration,
+) (proxy.Provider, error) {
+
+	// Create an ipv4 instance of the single-stack proxier
+	ipv4Proxier, err := NewProxier(syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit,
+		clusterCIDR, hostname, nodeIP[0], recorder, healthzServer, config)
+
+	if err != nil {
+		return nil, fmt.Errorf("unable to create ipv4 proxier: %v, hostname: %s, clusterCIDR : %s, nodeIP:%v", err, hostname, clusterCIDR, nodeIP[0])
+	}
+
+	ipv6Proxier, err := NewProxier(syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit,
+		clusterCIDR, hostname, nodeIP[1], recorder, healthzServer, config)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create ipv6 proxier: %v, hostname: %s, clusterCIDR : %s, nodeIP:%v", err, hostname, clusterCIDR, nodeIP[1])
+	}
+
+	// Return a meta-proxier that dispatch calls between the two
+	// single-stack proxier instances
+	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
+}
+
 // CleanupLeftovers removes all hns rules created by the Proxier
 // It returns true if an error was encountered. Errors are logged.
 func CleanupLeftovers() (encounteredError bool) {
@@ -1275,7 +1318,7 @@ func (proxier *Proxier) syncProxyRules() {
 
 		hnsLoadBalancer, err := hns.getLoadBalancer(
 			hnsEndpoints,
-			loadBalancerFlags{isDSR: proxier.isDSR, sessionAffinity: sessionAffinityClientIP},
+			loadBalancerFlags{isDSR: proxier.isDSR, isIPv6: proxier.isIPv6Mode, sessionAffinity: sessionAffinityClientIP},
 			sourceVip,
 			svcInfo.clusterIP.String(),
 			Enum(svcInfo.protocol),
@@ -1300,7 +1343,7 @@ func (proxier *Proxier) syncProxyRules() {
 			}
 			hnsLoadBalancer, err := hns.getLoadBalancer(
 				nodePortEndpoints,
-				loadBalancerFlags{localRoutedVIP: true, sessionAffinity: sessionAffinityClientIP},
+				loadBalancerFlags{localRoutedVIP: true, sessionAffinity: sessionAffinityClientIP, isIPv6: proxier.isIPv6Mode},
 				sourceVip,
 				"",
 				Enum(svcInfo.protocol),
@@ -1321,7 +1364,7 @@ func (proxier *Proxier) syncProxyRules() {
 			// Try loading existing policies, if already available
 			hnsLoadBalancer, err = hns.getLoadBalancer(
 				hnsEndpoints,
-				loadBalancerFlags{sessionAffinity: sessionAffinityClientIP},
+				loadBalancerFlags{sessionAffinity: sessionAffinityClientIP, isIPv6: proxier.isIPv6Mode},
 				sourceVip,
 				externalIP.ip,
 				Enum(svcInfo.protocol),
@@ -1344,7 +1387,7 @@ func (proxier *Proxier) syncProxyRules() {
 			}
 			hnsLoadBalancer, err := hns.getLoadBalancer(
 				lbIngressEndpoints,
-				loadBalancerFlags{isDSR: svcInfo.preserveDIP || proxier.isDSR, useMUX: svcInfo.preserveDIP, preserveDIP: svcInfo.preserveDIP, sessionAffinity: sessionAffinityClientIP},
+				loadBalancerFlags{isDSR: svcInfo.preserveDIP || proxier.isDSR, useMUX: svcInfo.preserveDIP, preserveDIP: svcInfo.preserveDIP, sessionAffinity: sessionAffinityClientIP, isIPv6: proxier.isIPv6Mode},
 				sourceVip,
 				lbIngressIP.ip,
 				Enum(svcInfo.protocol),