Only use dualstack if the node and config supports it

This commit is contained in:
James Sturtevant 2021-04-12 17:42:07 -07:00
parent b765496650
commit d5d9327351
5 changed files with 215 additions and 79 deletions

View File

@ -22,11 +22,14 @@ import (
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"strings"
"time"
utilnode "k8s.io/kubernetes/pkg/util/node"
"github.com/fsnotify/fsnotify"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -83,6 +86,7 @@ import (
utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
"k8s.io/kubernetes/pkg/util/oom"
"k8s.io/utils/exec"
utilsnet "k8s.io/utils/net"
utilpointer "k8s.io/utils/pointer"
)
@ -817,3 +821,36 @@ func (s *ProxyServer) CleanupAndExit() error {
return nil
}
// detectNodeIP returns the nodeIP used by the proxier
// The order of precedence is:
// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
// 2. the primary IP from the Node object, if set
// 3. if no IP is found it defaults to 127.0.0.1 and IPv4
func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP {
nodeIP := net.ParseIP(bindAddress)
if nodeIP.IsUnspecified() {
nodeIP = utilnode.GetNodeIP(client, hostname)
}
if nodeIP == nil {
klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
nodeIP = net.ParseIP("127.0.0.1")
}
return nodeIP
}
// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
// will have "any" address (0.0.0.0 or ::) inserted.
func nodeIPTuple(bindAddress string) [2]net.IP {
nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
adr := net.ParseIP(bindAddress)
if utilsnet.IsIPv6(adr) {
nodes[1] = adr
} else {
nodes[0] = adr
}
return nodes
}

View File

@ -428,23 +428,6 @@ func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, erro
return nil, fmt.Errorf("event object not of type node")
}
// detectNodeIP returns the nodeIP used by the proxier
// The order of precedence is:
// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
// 2. the primary IP from the Node object, if set
// 3. if no IP is found it defaults to 127.0.0.1 and IPv4
func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP {
nodeIP := net.ParseIP(bindAddress)
if nodeIP.IsUnspecified() {
nodeIP = utilnode.GetNodeIP(client, hostname)
}
if nodeIP == nil {
klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
nodeIP = net.ParseIP("127.0.0.1")
}
return nodeIP
}
func detectNumCPU() int {
// try get numCPU from /sys firstly due to a known issue (https://github.com/kubernetes/kubernetes/issues/99225)
_, numCPU, err := machine.GetTopology(sysfs.NewRealSysFs())
@ -570,22 +553,6 @@ func cidrTuple(cidrList string) [2]string {
return cidrs
}
// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
// will have "any" address (0.0.0.0 or ::) inserted.
func nodeIPTuple(bindAddress string) [2]net.IP {
nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
adr := net.ParseIP(bindAddress)
if utilsnet.IsIPv6(adr) {
nodes[1] = adr
} else {
nodes[0] = adr
}
return nodes
}
func getProxyMode(proxyMode string, canUseIPVS bool, kcompat iptables.KernelCompatTester) string {
switch proxyMode {
case proxyModeUserspace:

View File

@ -32,12 +32,10 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/events"
"k8s.io/component-base/configz"
"k8s.io/component-base/metrics"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme"
@ -47,7 +45,6 @@ import (
utilnetsh "k8s.io/kubernetes/pkg/util/netsh"
utilnode "k8s.io/kubernetes/pkg/util/node"
"k8s.io/utils/exec"
utilsnet "k8s.io/utils/net"
)
// NewProxyServer returns a new ProxyServer.
@ -85,6 +82,9 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
if err != nil {
return nil, err
}
nodeIP := detectNodeIP(client, hostname, config.BindAddress)
klog.InfoS("Detected node IP", "IP", nodeIP.String())
eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, "kube-proxy")
@ -101,12 +101,11 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
}
var proxier proxy.Provider
proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{})
dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{})
if proxyMode == proxyModeKernelspace {
klog.V(0).InfoS("Using Kernelspace Proxier.")
isIPv6DualStackEnabled := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack)
if isIPv6DualStackEnabled {
if dualStackMode {
klog.V(0).InfoS("Creating dualStackProxier for Windows kernel.")
proxier, err = winkernel.NewDualStackProxier(
@ -130,7 +129,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
int(*config.IPTables.MasqueradeBit),
config.ClusterCIDR,
hostname,
utilnode.GetNodeIP(client, hostname),
nodeIP,
recorder,
healthzServer,
config.Winkernel,
@ -183,6 +182,10 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
}, nil
}
func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool {
return compatTester.DualStackCompatible(networkname)
}
func getProxyMode(proxyMode string, kcompat winkernel.KernelCompatTester) string {
if proxyMode == proxyModeKernelspace {
return tryWinKernelSpaceProxy(kcompat)
@ -211,19 +214,3 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string {
klog.V(1).InfoS("Can't use winkernel proxy, using userspace proxier")
return proxyModeUserspace
}
// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6)
// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address
// will have "any" address (0.0.0.0 or ::) inserted.
func nodeIPTuple(bindAddress string) [2]net.IP {
nodes := [2]net.IP{net.IPv4zero, net.IPv6zero}
adr := net.ParseIP(bindAddress)
if utilsnet.IsIPv6(adr) {
nodes[1] = adr
} else {
nodes[0] = adr
}
return nodes
}

View File

@ -31,13 +31,14 @@ import (
"github.com/Microsoft/hcsshim"
"github.com/Microsoft/hcsshim/hcn"
discovery "k8s.io/api/discovery/v1"
"github.com/davecgh/go-spew/spew"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
apiutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/events"
@ -137,6 +138,101 @@ type remoteSubnetInfo struct {
const NETWORK_TYPE_OVERLAY = "overlay"
func newHostNetworkService() (HostNetworkService, hcn.SupportedFeatures) {
var hns HostNetworkService
hns = hnsV1{}
supportedFeatures := hcn.GetSupportedFeatures()
if supportedFeatures.Api.V2 {
hns = hnsV2{}
}
return hns, supportedFeatures
}
func getNetworkName(hnsNetworkName string) (string, error) {
if len(hnsNetworkName) == 0 {
klog.V(3).InfoS("network-name flag not set. Checking environment variable")
hnsNetworkName = os.Getenv("KUBE_NETWORK")
if len(hnsNetworkName) == 0 {
return "", fmt.Errorf("Environment variable KUBE_NETWORK and network-flag not initialized")
}
}
return hnsNetworkName, nil
}
func getNetworkInfo(hns HostNetworkService, hnsNetworkName string) (*hnsNetworkInfo, error) {
hnsNetworkInfo, err := hns.getNetworkByName(hnsNetworkName)
for err != nil {
klog.ErrorS(err, "Unable to find HNS Network specified. Please check network name and CNI deployment", "hnsNetworkName", hnsNetworkName)
time.Sleep(1 * time.Second)
hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName)
}
return hnsNetworkInfo, err
}
func isOverlay(hnsNetworkInfo *hnsNetworkInfo) bool {
return strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY)
}
// StackCompatTester tests whether the required kernel and network are dualstack capable
type StackCompatTester interface {
DualStackCompatible(networkName string) bool
}
type DualStackCompatTester struct{}
func (t DualStackCompatTester) DualStackCompatible(networkName string) bool {
dualStackFeatureEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.IPv6DualStack)
if !dualStackFeatureEnabled {
return false
}
globals, err := hcn.GetGlobals()
if err != nil {
klog.ErrorS(err, "Unable to determine networking stack version. Falling back to single-stack")
return false
}
if !kernelSupportsDualstack(globals.Version) {
klog.InfoS("This version of Windows does not support dual-stack. Falling back to single-stack")
return false
}
// check if network is using overlay
hns, _ := newHostNetworkService()
networkName, err = getNetworkName(networkName)
if err != nil {
klog.ErrorS(err, "unable to determine dual-stack status %v. Falling back to single-stack")
return false
}
networkInfo, err := getNetworkInfo(hns, networkName)
if err != nil {
klog.ErrorS(err, "unable to determine dual-stack status %v. Falling back to single-stack")
return false
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WinOverlay) && isOverlay(networkInfo) {
// Overlay (VXLAN) networks on Windows do not support dual-stack networking today
klog.InfoS("Winoverlay does not support dual-stack. Falling back to single-stack")
return false
}
return true
}
// The hcsshim version logic has a bug that did not calculate the versioning of DualStack correctly.
// DualStack is supported in WS 2004+ (10.0.19041+) where HCN component version is 11.10+
// https://github.com/microsoft/hcsshim/pull/1003#issuecomment-827930358
func kernelSupportsDualstack(currentVersion hcn.Version) bool {
hnsVersion := fmt.Sprintf("%d.%d.0", currentVersion.Major, currentVersion.Minor)
v, err := version.ParseSemantic(hnsVersion)
if err != nil {
return false
}
return v.AtLeast(version.MustParseSemantic("11.10.0"))
}
func Log(v interface{}, message string, level klog.Level) {
klog.V(level).InfoS("%s", message, "spewConfig", spewSdump(v))
}
@ -546,36 +642,24 @@ func NewProxier(
}
serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder)
var hns HostNetworkService
hns = hnsV1{}
supportedFeatures := hcn.GetSupportedFeatures()
if supportedFeatures.Api.V2 {
hns = hnsV2{}
}
hnsNetworkName := config.NetworkName
if len(hnsNetworkName) == 0 {
klog.V(3).InfoS("network-name flag not set. Checking environment variable")
hnsNetworkName = os.Getenv("KUBE_NETWORK")
if len(hnsNetworkName) == 0 {
return nil, fmt.Errorf("Environment variable KUBE_NETWORK and network-flag not initialized")
}
hns, supportedFeatures := newHostNetworkService()
hnsNetworkName, err := getNetworkName(config.NetworkName)
if err != nil {
return nil, err
}
klog.V(3).InfoS("Cleaning up old HNS policy lists")
deleteAllHnsLoadBalancerPolicy()
// Get HNS network information
hnsNetworkInfo, err := hns.getNetworkByName(hnsNetworkName)
for err != nil {
klog.ErrorS(err, "Unable to find HNS Network specified. Please check network name and CNI deployment", "hnsNetworkName", hnsNetworkName)
time.Sleep(1 * time.Second)
hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName)
hnsNetworkInfo, err := getNetworkInfo(hns, hnsNetworkName)
if err != nil {
return nil, err
}
// Network could have been detected before Remote Subnet Routes are applied or ManagementIP is updated
// Sleep and update the network to include new information
if strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) {
if isOverlay(hnsNetworkInfo) {
time.Sleep(10 * time.Second)
hnsNetworkInfo, err = hns.getNetworkByName(hnsNetworkName)
if err != nil {
@ -595,7 +679,7 @@ func NewProxier(
var sourceVip string
var hostMac string
if strings.EqualFold(hnsNetworkInfo.networkType, NETWORK_TYPE_OVERLAY) {
if isOverlay(hnsNetworkInfo) {
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WinOverlay) {
return nil, fmt.Errorf("WinOverlay feature gate not enabled")
}
@ -608,6 +692,15 @@ func NewProxier(
return nil, fmt.Errorf("source-vip flag not set")
}
if nodeIP.IsUnspecified() {
// attempt to get the correct ip address
klog.V(2).InfoS("node ip was unspecified. Attempting to find node ip")
nodeIP, err = apiutil.ResolveBindAddress(nodeIP)
if err != nil {
klog.InfoS("failed to find an ip. You may need set the --bind-address flag", "err", err)
}
}
interfaces, _ := net.Interfaces() //TODO create interfaces
for _, inter := range interfaces {
addresses, _ := inter.Addrs()

View File

@ -25,6 +25,7 @@ import (
"testing"
"time"
"github.com/Microsoft/hcsshim/hcn"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -928,3 +929,54 @@ func makeTestEndpointSlice(namespace, name string, sliceNum int, epsFunc func(*d
epsFunc(eps)
return eps
}
func Test_kernelSupportsDualstack(t *testing.T) {
tests := []struct {
currentVersion hcn.Version
name string
want bool
}{
{
hcn.Version{Major: 10, Minor: 10},
"Less than minimal should not be supported",
false,
},
{
hcn.Version{Major: 9, Minor: 11},
"Less than minimal should not be supported",
false,
},
{
hcn.Version{Major: 11, Minor: 1},
"Less than minimal should not be supported",
false,
},
{
hcn.Version{Major: 11, Minor: 10},
"Current version should be supported",
true,
},
{
hcn.Version{Major: 11, Minor: 11},
"Greater than minimal version should be supported",
true,
},
{
hcn.Version{Major: 12, Minor: 1},
"Greater than minimal version should be supported",
true,
},
{
hcn.Version{Major: 12, Minor: 12},
"Greater than minimal should be supported",
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := kernelSupportsDualstack(tt.currentVersion); got != tt.want {
t.Errorf("kernelSupportsDualstack on version %v: got %v, want %v", tt.currentVersion, got, tt.want)
}
})
}
}