diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 7f519212c3e..17b7e64faf1 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -218,6 +218,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], "//conditions:default": [], }), diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index 9212a5e1d72..66ed52ccd59 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -28,12 +28,15 @@ import ( // Enable pprof HTTP handlers. _ "net/http/pprof" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" "k8s.io/component-base/configz" "k8s.io/component-base/metrics" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" @@ -43,8 +46,7 @@ import ( utilnetsh "k8s.io/kubernetes/pkg/util/netsh" utilnode "k8s.io/kubernetes/pkg/util/node" "k8s.io/utils/exec" - - "k8s.io/klog/v2" + utilsnet "k8s.io/utils/net" ) // NewProxyServer returns a new ProxyServer. @@ -102,18 +104,39 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{}) if proxyMode == proxyModeKernelspace { klog.V(0).Info("Using Kernelspace Proxier.") - proxier, err = winkernel.NewProxier( - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - config.ClusterCIDR, - hostname, - utilnode.GetNodeIP(client, hostname), - recorder, - healthzServer, - config.Winkernel, - ) + isIPv6DualStackEnabled := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) + if isIPv6DualStackEnabled { + klog.V(0).Info("creating dualStackProxier for Windows kernel.") + + proxier, err = winkernel.NewDualStackProxier( + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, + config.IPTables.MasqueradeAll, + int(*config.IPTables.MasqueradeBit), + config.ClusterCIDR, + hostname, + nodeIPTuple(config.BindAddress), + recorder, + healthzServer, + config.Winkernel, + ) + } else { + + proxier, err = winkernel.NewProxier( + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, + config.IPTables.MasqueradeAll, + int(*config.IPTables.MasqueradeBit), + config.ClusterCIDR, + hostname, + utilnode.GetNodeIP(client, hostname), + recorder, + healthzServer, + config.Winkernel, + ) + + } + if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -181,3 +204,19 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string { klog.V(1).Infof("Can't use winkernel proxy, using userspace proxier") return proxyModeUserspace } + +// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6) +// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address +// will have "any" address (0.0.0.0 or ::) inserted. +func nodeIPTuple(bindAddress string) [2]net.IP { + nodes := [2]net.IP{net.IPv4zero, net.IPv6zero} + + adr := net.ParseIP(bindAddress) + if utilsnet.IsIPv6(adr) { + nodes[1] = adr + } else { + nodes[0] = adr + } + + return nodes +} diff --git a/pkg/kubelet/dockershim/network/cni/cni_windows.go b/pkg/kubelet/dockershim/network/cni/cni_windows.go index b326906d4e7..e76f69816c9 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_windows.go +++ b/pkg/kubelet/dockershim/network/cni/cni_windows.go @@ -21,13 +21,13 @@ package cni import ( "context" "fmt" - "time" - cniTypes020 "github.com/containernetworking/cni/pkg/types/020" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/klog/v2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" + "net" + "time" ) func getLoNetwork(binDirs []string) *cniNetwork { @@ -67,7 +67,14 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin klog.Errorf("error while cni parsing result: %s", err) return nil, err } - return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil + + var list = []net.IP{result020.IP4.IP.IP} + + if result020.IP6 != nil { + list = append(list, result020.IP6.IP.IP) + } + + return &network.PodNetworkStatus{IP: result020.IP4.IP.IP, IPs: list}, nil } // buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig. diff --git a/pkg/proxy/winkernel/BUILD b/pkg/proxy/winkernel/BUILD index 54249093e13..fae9fc2011e 100644 --- a/pkg/proxy/winkernel/BUILD +++ b/pkg/proxy/winkernel/BUILD @@ -21,6 +21,7 @@ go_library( "//pkg/proxy/apis/config:go_default_library", "//pkg/proxy/config:go_default_library", "//pkg/proxy/healthcheck:go_default_library", + "//pkg/proxy/metaproxier:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/async:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -35,6 +36,7 @@ go_library( "//vendor/github.com/Microsoft/hcsshim/hcn:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/proxy/winkernel/hnsV2.go b/pkg/proxy/winkernel/hnsV2.go index 62053f122bf..54c6b92a01c 100644 --- a/pkg/proxy/winkernel/hnsV2.go +++ b/pkg/proxy/winkernel/hnsV2.go @@ -30,6 +30,11 @@ import ( type hnsV2 struct{} +var ( + // LoadBalancerFlagsIPv6 enables IPV6. + LoadBalancerFlagsIPv6 hcn.LoadBalancerFlags = 2 +) + func (hns hnsV2) getNetworkByName(name string) (*hnsNetworkInfo, error) { hnsnetwork, err := hcn.GetNetworkByName(name) if err != nil { @@ -90,10 +95,14 @@ func (hns hnsV2) getEndpointByIpAddress(ip string, networkName string) (*endpoin equal := false if endpoint.IpConfigurations != nil && len(endpoint.IpConfigurations) > 0 { equal = endpoint.IpConfigurations[0].IpAddress == ip + + if !equal && len(endpoint.IpConfigurations) > 1 { + equal = endpoint.IpConfigurations[1].IpAddress == ip + } } if equal && strings.EqualFold(endpoint.HostComputeNetwork, hnsnetwork.Id) { return &endpointsInfo{ - ip: endpoint.IpConfigurations[0].IpAddress, + ip: ip, isLocal: uint32(endpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote macAddress: endpoint.MacAddress, hnsID: endpoint.Id, @@ -232,6 +241,10 @@ func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFl lbFlags |= hcn.LoadBalancerFlagsDSR } + if flags.isIPv6 { + lbFlags |= LoadBalancerFlagsIPv6 + } + lbDistributionType := hcn.LoadBalancerDistributionNone if flags.sessionAffinity { diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index 0a9b680bf7b..4c1cae087af 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -48,8 +48,10 @@ import ( "k8s.io/kubernetes/pkg/proxy/apis/config" proxyconfig "k8s.io/kubernetes/pkg/proxy/config" "k8s.io/kubernetes/pkg/proxy/healthcheck" + "k8s.io/kubernetes/pkg/proxy/metaproxier" "k8s.io/kubernetes/pkg/proxy/metrics" "k8s.io/kubernetes/pkg/util/async" + utilnet "k8s.io/utils/net" ) // KernelCompatTester tests whether the required kernel capabilities are @@ -101,6 +103,7 @@ type loadBalancerFlags struct { useMUX bool preserveDIP bool sessionAffinity bool + isIPv6 bool } // internal struct for string service information @@ -163,13 +166,17 @@ type endpointsInfo struct { hns HostNetworkService } -//Uses mac prefix and IPv4 address to return a mac address +//Uses mac prefix and IP address to return a mac address //This ensures mac addresses are unique for proper load balancing -//Does not support IPv6 and returns a dummy mac +//There is a possibility of MAC collisions but this Mac address is used for remote endpoints only +//and not sent on the wire. func conjureMac(macPrefix string, ip net.IP) string { if ip4 := ip.To4(); ip4 != nil { a, b, c, d := ip4[0], ip4[1], ip4[2], ip4[3] return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", macPrefix, a, b, c, d) + } else if ip6 := ip.To16(); ip6 != nil { + a, b, c, d := ip6[15], ip6[14], ip6[13], ip6[12] + return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", macPrefix, a, b, c, d) } return "02-11-22-33-44-55" } @@ -502,6 +509,7 @@ type Proxier struct { // with some partial data after kube-proxy restart. endpointsSynced bool servicesSynced bool + isIPv6Mode bool initialized int32 syncRunner *async.BoundedFrequencyRunner // governs calls to syncProxyRules @@ -664,6 +672,8 @@ func NewProxier( } } + isIPv6 := utilnet.IsIPv6(nodeIP) + proxier := &Proxier{ portsMap: make(map[localPort]closeable), serviceMap: make(proxyServiceMap), @@ -685,6 +695,7 @@ func NewProxier( hostMac: hostMac, isDSR: isDSR, supportedFeatures: supportedFeatures, + isIPv6Mode: isIPv6, } burstSyncs := 2 @@ -694,6 +705,38 @@ func NewProxier( } +func NewDualStackProxier( + syncPeriod time.Duration, + minSyncPeriod time.Duration, + masqueradeAll bool, + masqueradeBit int, + clusterCIDR string, + hostname string, + nodeIP [2]net.IP, + recorder record.EventRecorder, + healthzServer healthcheck.ProxierHealthUpdater, + config config.KubeProxyWinkernelConfiguration, +) (proxy.Provider, error) { + + // Create an ipv4 instance of the single-stack proxier + ipv4Proxier, err := NewProxier(syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, + clusterCIDR, hostname, nodeIP[0], recorder, healthzServer, config) + + if err != nil { + return nil, fmt.Errorf("unable to create ipv4 proxier: %v, hostname: %s, clusterCIDR : %s, nodeIP:%v", err, hostname, clusterCIDR, nodeIP[0]) + } + + ipv6Proxier, err := NewProxier(syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, + clusterCIDR, hostname, nodeIP[1], recorder, healthzServer, config) + if err != nil { + return nil, fmt.Errorf("unable to create ipv6 proxier: %v, hostname: %s, clusterCIDR : %s, nodeIP:%v", err, hostname, clusterCIDR, nodeIP[1]) + } + + // Return a meta-proxier that dispatch calls between the two + // single-stack proxier instances + return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil +} + // CleanupLeftovers removes all hns rules created by the Proxier // It returns true if an error was encountered. Errors are logged. func CleanupLeftovers() (encounteredError bool) { @@ -1275,7 +1318,7 @@ func (proxier *Proxier) syncProxyRules() { hnsLoadBalancer, err := hns.getLoadBalancer( hnsEndpoints, - loadBalancerFlags{isDSR: proxier.isDSR, sessionAffinity: sessionAffinityClientIP}, + loadBalancerFlags{isDSR: proxier.isDSR, isIPv6: proxier.isIPv6Mode, sessionAffinity: sessionAffinityClientIP}, sourceVip, svcInfo.clusterIP.String(), Enum(svcInfo.protocol), @@ -1300,7 +1343,7 @@ func (proxier *Proxier) syncProxyRules() { } hnsLoadBalancer, err := hns.getLoadBalancer( nodePortEndpoints, - loadBalancerFlags{localRoutedVIP: true, sessionAffinity: sessionAffinityClientIP}, + loadBalancerFlags{localRoutedVIP: true, sessionAffinity: sessionAffinityClientIP, isIPv6: proxier.isIPv6Mode}, sourceVip, "", Enum(svcInfo.protocol), @@ -1321,7 +1364,7 @@ func (proxier *Proxier) syncProxyRules() { // Try loading existing policies, if already available hnsLoadBalancer, err = hns.getLoadBalancer( hnsEndpoints, - loadBalancerFlags{sessionAffinity: sessionAffinityClientIP}, + loadBalancerFlags{sessionAffinity: sessionAffinityClientIP, isIPv6: proxier.isIPv6Mode}, sourceVip, externalIP.ip, Enum(svcInfo.protocol), @@ -1344,7 +1387,7 @@ func (proxier *Proxier) syncProxyRules() { } hnsLoadBalancer, err := hns.getLoadBalancer( lbIngressEndpoints, - loadBalancerFlags{isDSR: svcInfo.preserveDIP || proxier.isDSR, useMUX: svcInfo.preserveDIP, preserveDIP: svcInfo.preserveDIP, sessionAffinity: sessionAffinityClientIP}, + loadBalancerFlags{isDSR: svcInfo.preserveDIP || proxier.isDSR, useMUX: svcInfo.preserveDIP, preserveDIP: svcInfo.preserveDIP, sessionAffinity: sessionAffinityClientIP, isIPv6: proxier.isIPv6Mode}, sourceVip, lbIngressIP.ip, Enum(svcInfo.protocol),