diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 89468c59175..7fe01c87769 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/healthz" @@ -567,7 +568,31 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin s.HealthzServer = healthcheck.NewProxierHealthServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, s.Recorder, s.NodeRef) } - s.Proxier, err = s.createProxier(config) + err = s.platformSetup() + if err != nil { + return nil, err + } + + ipv4Supported, ipv6Supported, dualStackSupported, err := s.platformCheckSupported() + if err != nil { + return nil, err + } else if (s.PrimaryIPFamily == v1.IPv4Protocol && !ipv4Supported) || (s.PrimaryIPFamily == v1.IPv6Protocol && !ipv6Supported) { + return nil, fmt.Errorf("no support for primary IP family %q", s.PrimaryIPFamily) + } else if dualStackSupported { + klog.InfoS("kube-proxy running in dual-stack mode", "primary ipFamily", s.PrimaryIPFamily) + } else { + klog.InfoS("kube-proxy running in single-stack mode", "ipFamily", s.PrimaryIPFamily) + } + + err, fatal := checkIPConfig(s, dualStackSupported) + if err != nil { + if fatal { + return nil, fmt.Errorf("kube-proxy configuration is incorrect: %v", err) + } + klog.ErrorS(err, "Kube-proxy configuration may be incomplete or incorrect") + } + + s.Proxier, err = s.createProxier(config, dualStackSupported) if err != nil { return nil, err } @@ -575,6 +600,99 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin return s, nil } +// checkIPConfig confirms that s has proper configuration for its primary IP family. +func checkIPConfig(s *ProxyServer, dualStackSupported bool) (error, bool) { + var errors []error + var badFamily netutils.IPFamily + + if s.PrimaryIPFamily == v1.IPv4Protocol { + badFamily = netutils.IPv6 + } else { + badFamily = netutils.IPv4 + } + + var clusterType string + if dualStackSupported { + clusterType = fmt.Sprintf("%s-primary", s.PrimaryIPFamily) + } else { + clusterType = fmt.Sprintf("%s-only", s.PrimaryIPFamily) + } + + // Historically, we did not check most of the config options, so we cannot + // retroactively make IP family mismatches in those options be fatal. When we add + // new options to check here, we should make problems with those options be fatal. + fatal := false + + if s.Config.ClusterCIDR != "" { + clusterCIDRs := strings.Split(s.Config.ClusterCIDR, ",") + if badCIDRs(clusterCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but clusterCIDRs contains only IPv%s addresses", clusterType, badFamily)) + if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeClusterCIDR { + // This has always been a fatal error + fatal = true + } + } + } + + if badCIDRs(s.Config.NodePortAddresses, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but nodePortAddresses contains only IPv%s addresses", clusterType, badFamily)) + } + + if badCIDRs(s.podCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but node.spec.podCIDRs contains only IPv%s addresses", clusterType, badFamily)) + if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeNodeCIDR { + // This has always been a fatal error + fatal = true + } + } + + if netutils.IPFamilyOfString(s.Config.Winkernel.SourceVip) == badFamily { + errors = append(errors, fmt.Errorf("cluster is %s but winkernel.sourceVip is IPv%s", clusterType, badFamily)) + } + + // In some cases, wrong-IP-family is only a problem when the secondary IP family + // isn't present at all. + if !dualStackSupported { + if badCIDRs(s.Config.IPVS.ExcludeCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but ipvs.excludeCIDRs contains only IPv%s addresses", clusterType, badFamily)) + } + + if badBindAddress(s.Config.HealthzBindAddress, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but healthzBindAddress is IPv%s", clusterType, badFamily)) + } + if badBindAddress(s.Config.MetricsBindAddress, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but metricsBindAddress is IPv%s", clusterType, badFamily)) + } + } + + return utilerrors.NewAggregate(errors), fatal +} + +// badCIDRs returns true if cidrs is a non-empty list of CIDRs, all of wrongFamily. +func badCIDRs(cidrs []string, wrongFamily netutils.IPFamily) bool { + if len(cidrs) == 0 { + return false + } + for _, cidr := range cidrs { + if netutils.IPFamilyOfCIDRString(cidr) != wrongFamily { + return false + } + } + return true +} + +// badBindAddress returns true if bindAddress is an "IP:port" string where IP is a +// non-zero IP of wrongFamily. +func badBindAddress(bindAddress string, wrongFamily netutils.IPFamily) bool { + if host, _, _ := net.SplitHostPort(bindAddress); host != "" { + ip := netutils.ParseIPSloppy(host) + if ip != nil && netutils.IPFamilyOf(ip) == wrongFamily && !ip.IsUnspecified() { + return true + } + } + return false +} + // createClient creates a kube client from the given config and masterOverride. // TODO remove masterOverride when CLI flags are removed. func createClient(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, error) { @@ -706,12 +824,6 @@ func (s *ProxyServer) Run() error { // Start up a metrics server if requested serveMetrics(s.Config.MetricsBindAddress, s.Config.Mode, s.Config.EnableProfiling, errCh) - // Do platform-specific setup - err := s.platformSetup() - if err != nil { - return err - } - noProxyName, err := labels.NewRequirement(apis.LabelServiceProxyName, selection.DoesNotExist, nil) if err != nil { return err diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 1b14b51a0c2..061152a8882 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -50,7 +50,6 @@ import ( utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset" utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util" proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables" "k8s.io/utils/exec" @@ -63,6 +62,8 @@ import ( // node after it is registered. var timeoutForNodePodCIDR = 5 * time.Minute +// platformApplyDefaults is called after parsing command-line flags and/or reading the +// config file, to apply platform-specific default values to config. func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) { if config.Mode == "" { klog.InfoS("Using iptables proxy") @@ -76,22 +77,58 @@ func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfigur klog.V(2).InfoS("DetectLocalMode", "localMode", string(config.DetectLocalMode)) } +// platformSetup is called after setting up the ProxyServer, but before creating the +// Proxier. It should fill in any platform-specific fields and perform other +// platform-specific setup. +func (s *ProxyServer) platformSetup() error { + if s.Config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { + klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname) + node, err := waitForPodCIDR(s.Client, s.Hostname) + if err != nil { + return err + } + s.podCIDRs = node.Spec.PodCIDRs + klog.InfoS("NodeInfo", "podCIDRs", node.Spec.PodCIDRs) + } + + err := s.setupConntrack() + if err != nil { + return err + } + + proxymetrics.RegisterMetrics() + return nil +} + +// platformCheckSupported is called immediately before creating the Proxier, to check +// what IP families are supported (and whether the configuration is usable at all). +func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) { + execer := exec.New() + ipt := utiliptables.New(execer, utiliptables.ProtocolIPv4) + ipv4Supported = ipt.Present() + ipt = utiliptables.New(execer, utiliptables.ProtocolIPv6) + ipv6Supported = ipt.Present() + + // The Linux proxies can always support dual-stack if they can support both IPv4 + // and IPv6. + dualStackSupported = ipv4Supported && ipv6Supported + + if !ipv4Supported && !ipv6Supported { + err = fmt.Errorf("iptables is not available on this host") + } else if !ipv4Supported { + klog.InfoS("No iptables support for family", "ipFamily", v1.IPv4Protocol) + } else if !ipv6Supported { + klog.InfoS("No iptables support for family", "ipFamily", v1.IPv6Protocol) + } + + return +} + // createProxier creates the proxy.Provider -func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { +func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStack bool) (proxy.Provider, error) { var proxier proxy.Provider var err error - var nodeInfo *v1.Node - if config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR { - klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname) - nodeInfo, err = waitForPodCIDR(s.Client, s.Hostname) - if err != nil { - return nil, err - } - s.podCIDRs = nodeInfo.Spec.PodCIDRs - klog.InfoS("NodeInfo", "podCIDR", nodeInfo.Spec.PodCIDR, "podCIDRs", nodeInfo.Spec.PodCIDRs) - } - primaryProtocol := utiliptables.ProtocolIPv4 if s.PrimaryIPFamily == v1.IPv6Protocol { primaryProtocol = utiliptables.ProtocolIPv6 @@ -100,7 +137,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio iptInterface := utiliptables.New(execer, primaryProtocol) var ipt [2]utiliptables.Interface - dualStack := true // While we assume that node supports, we do further checks below // Create iptables handlers for both families, one is already created // Always ordered as IPv4, IPv6 @@ -112,33 +148,13 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ipt[1] = iptInterface } - nodePortAddresses := config.NodePortAddresses - - if !ipt[0].Present() { - return nil, fmt.Errorf("iptables is not supported for primary IP family %q", primaryProtocol) - } else if !ipt[1].Present() { - klog.InfoS("kube-proxy running in single-stack mode: secondary ipFamily is not supported", "ipFamily", ipt[1].Protocol()) - dualStack = false - - // Validate NodePortAddresses is single-stack - npaByFamily := proxyutil.MapCIDRsByIPFamily(config.NodePortAddresses) - secondaryFamily := proxyutil.OtherIPFamily(s.PrimaryIPFamily) - badAddrs := npaByFamily[secondaryFamily] - if len(badAddrs) > 0 { - klog.InfoS("Ignoring --nodeport-addresses of the wrong family", "ipFamily", secondaryFamily, "addresses", badAddrs) - nodePortAddresses = npaByFamily[s.PrimaryIPFamily] - } - } - if config.Mode == proxyconfigapi.ProxyModeIPTables { klog.InfoS("Using iptables Proxier") if dualStack { - klog.InfoS("kube-proxy running in dual-stack mode", "ipFamily", iptInterface.Protocol()) - klog.InfoS("Creating dualStackProxier for iptables") // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -158,12 +174,12 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.NodeIPs, s.Recorder, s.HealthzServer, - nodePortAddresses, + config.NodePortAddresses, ) } else { // Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support). var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -184,7 +200,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.NodeIPs[s.PrimaryIPFamily], s.Recorder, s.HealthzServer, - nodePortAddresses, + config.NodePortAddresses, ) } @@ -201,11 +217,9 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio klog.InfoS("Using ipvs Proxier") if dualStack { - klog.InfoS("Creating dualStackProxier for ipvs") - // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo) + localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -231,12 +245,12 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.Recorder, s.HealthzServer, config.IPVS.Scheduler, - nodePortAddresses, + config.NodePortAddresses, kernelHandler, ) } else { var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo) + localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } @@ -263,7 +277,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio s.Recorder, s.HealthzServer, config.IPVS.Scheduler, - nodePortAddresses, + config.NodePortAddresses, kernelHandler, ) } @@ -275,7 +289,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio return proxier, nil } -func (s *ProxyServer) platformSetup() error { +func (s *ProxyServer) setupConntrack() error { ct := &realConntracker{} max, err := getConntrackMax(s.Config.Conntrack) @@ -315,7 +329,6 @@ func (s *ProxyServer) platformSetup() error { } } - proxymetrics.RegisterMetrics() return nil } @@ -389,7 +402,7 @@ func detectNumCPU() int { return numCPU } -func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) { +func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodePodCIDRs []string) (proxyutiliptables.LocalTrafficDetector, error) { switch mode { case proxyconfigapi.LocalModeClusterCIDR: // LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed, @@ -400,11 +413,11 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube } return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt) case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if len(nodePodCIDRs) == 0 { klog.InfoS("Detect-local-mode set to NodeCIDR, but no PodCIDR defined at node") break } - return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) + return proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt) case proxyconfigapi.LocalModeBridgeInterface: return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface) case proxyconfigapi.LocalModeInterfaceNamePrefix: @@ -414,7 +427,7 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube return proxyutiliptables.NewNoOpLocalDetector(), nil } -func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) { +func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodePodCIDRs []string) ([2]proxyutiliptables.LocalTrafficDetector, error) { var err error localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()} switch mode { @@ -444,32 +457,32 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy } return localDetectors, err case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { + if len(nodePodCIDRs) == 0 { klog.InfoS("No node info available to configure detect-local-mode NodeCIDR") break } // localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR. // so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1] - if netutils.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1]) + if netutils.IsIPv6CIDRString(nodePodCIDRs[0]) { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[1]) if err != nil { return localDetectors, err } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0]) + if len(nodePodCIDRs) > 1 { + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[0]) } } else { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0]) + localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[0]) if err != nil { return localDetectors, err } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1]) + if len(nodePodCIDRs) > 1 { + localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[1]) } } return localDetectors, err case proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: - localDetector, err := getLocalDetector(mode, config, ipt[0], nodeInfo) + localDetector, err := getLocalDetector(mode, config, ipt[0], nodePodCIDRs) if err == nil { localDetectors[0] = localDetector localDetectors[1] = localDetector diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 06d1a435123..9323f8327a1 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -109,14 +109,14 @@ func Test_platformApplyDefaults(t *testing.T) { func Test_getLocalDetector(t *testing.T) { cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt utiliptables.Interface - expected proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool + mode proxyconfigapi.LocalMode + config *proxyconfigapi.KubeProxyConfiguration + ipt utiliptables.Interface + expected proxyutiliptables.LocalTrafficDetector + nodePodCIDRs []string + errExpected bool }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases + // LocalModeClusterCIDR { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -154,46 +154,46 @@ func Test_getLocalDetector(t *testing.T) { }, // LocalModeNodeCIDR { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewFake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: true, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, + ipt: utiliptablestest.NewIPv6Fake(), + expected: nil, + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: true, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: true, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, + ipt: utiliptablestest.NewFake(), + expected: nil, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: true, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: utiliptablestest.NewFake(), + expected: proxyutiliptables.NewNoOpLocalDetector(), + nodePodCIDRs: []string{}, + errExpected: false, }, - // unknown mode, nodeInfo would be nil for these cases + // unknown mode { mode: proxyconfigapi.LocalMode("abcd"), config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, @@ -201,7 +201,7 @@ func Test_getLocalDetector(t *testing.T) { expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, }, - // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + // LocalModeBridgeInterface { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -218,7 +218,7 @@ func Test_getLocalDetector(t *testing.T) { expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("1234567890123456789")), errExpected: false, }, - // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + // LocalModeInterfaceNamePrefix { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -237,7 +237,7 @@ func Test_getLocalDetector(t *testing.T) { }, } for i, c := range cases { - r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo) + r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Case[%d] Expected error, but succeeded with %v", i, r) @@ -256,14 +256,14 @@ func Test_getLocalDetector(t *testing.T) { func Test_getDualStackLocalDetectorTuple(t *testing.T) { cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt [2]utiliptables.Interface - expected [2]proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool + mode proxyconfigapi.LocalMode + config *proxyconfigapi.KubeProxyConfiguration + ipt [2]utiliptables.Interface + expected [2]proxyutiliptables.LocalTrafficDetector + nodePodCIDRs []string + errExpected bool }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases + // LocalModeClusterCIDR { mode: proxyconfigapi.LocalModeClusterCIDR, config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, @@ -315,8 +315,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: resolveDualStackLocalDetectors(t)( proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + nodePodCIDRs: []string{"10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -325,8 +325,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: resolveDualStackLocalDetectors(t)( proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"), - errExpected: false, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -335,8 +335,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{ resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, + nodePodCIDRs: []string{"10.0.0.0/24"}, + errExpected: false, }, { mode: proxyconfigapi.LocalModeNodeCIDR, @@ -345,18 +345,18 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{ proxyutiliptables.NewNoOpLocalDetector(), resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake()))}, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, + nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"}, + errExpected: false, }, { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, + mode: proxyconfigapi.LocalModeNodeCIDR, + config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, + ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, + expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, + nodePodCIDRs: []string{}, + errExpected: false, }, - // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + // LocalModeBridgeInterface { mode: proxyconfigapi.LocalModeBridgeInterface, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -367,7 +367,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), errExpected: false, }, - // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + // LocalModeInterfaceNamePrefix { mode: proxyconfigapi.LocalModeInterfaceNamePrefix, config: &proxyconfigapi.KubeProxyConfiguration{ @@ -380,7 +380,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { }, } for i, c := range cases { - r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) + r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodePodCIDRs) if c.errExpected { if err == nil { t.Errorf("Case[%d] expected error, but succeeded with %q", i, r) @@ -646,7 +646,7 @@ func TestGetConntrackMax(t *testing.T) { } } -func TestProxyServer_createProxier(t *testing.T) { +func TestProxyServer_platformSetup(t *testing.T) { tests := []struct { name string node *v1.Node @@ -683,9 +683,8 @@ func TestProxyServer_createProxier(t *testing.T) { v1.IPv6Protocol: net.IPv6zero, }, } - _, err := s.createProxier(tt.config) - // TODO: mock the exec.Interface to not fail probing iptables - if (err != nil) && !strings.Contains(err.Error(), "iptables is not supported for primary IP family") { + err := s.platformSetup() + if err != nil { t.Errorf("ProxyServer.createProxier() error = %v", err) return } diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 8d5f3241510..2b7709a152c 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -632,3 +632,302 @@ func Test_detectNodeIPs(t *testing.T) { }) } } + +func Test_checkIPConfig(t *testing.T) { + cases := []struct { + name string + proxy *ProxyServer + ssErr bool + dsErr bool + fatal bool + }{ + { + name: "empty config", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{}, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + + { + name: "ok single-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "10.0.0.0/8", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "10.0.0.0/8,fd01:2345::/64", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64,10.0.0.0/8", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + { + name: "wrong-family clusterCIDR when using ClusterCIDR LocalDetector", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64", + DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: true, + }, + + { + name: "ok single-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8", "fd01:2345::/64", "fd01:abcd::/64"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"fd01:2345::/64", "fd01:abcd::/64", "10.0.0.0/8"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8"}, + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + + { + name: "ok single-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"10.0.0.0/8"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"10.0.0.0/8", "fd01:2345::/64"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"fd01:2345::/64", "10.0.0.0/8"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"fd01:2345::/64"}, + }, + ssErr: true, + dsErr: true, + fatal: true, + }, + + { + name: "ok winkernel.sourceVip", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + SourceVip: "10.0.0.1", + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family winkernel.sourceVip", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + SourceVip: "fd01:2345::1", + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + + { + name: "ok IPv4 metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "10.0.0.1:9999", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok IPv6 metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "[fd01:2345::1]:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok unspecified wrong-family metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "0.0.0.0:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "10.0.0.1:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: false, + fatal: false, + }, + + { + name: "ok ipvs.excludeCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: []string{"10.0.0.0/8"}, + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family ipvs.excludeCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: false, + fatal: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err, fatal := checkIPConfig(c.proxy, false) + if err != nil && !c.ssErr { + t.Errorf("unexpected error in single-stack case: %v", err) + } else if err == nil && c.ssErr { + t.Errorf("unexpected lack of error in single-stack case") + } else if fatal != c.fatal { + t.Errorf("expected fatal=%v, got %v", c.fatal, fatal) + } + + err, fatal = checkIPConfig(c.proxy, true) + if err != nil && !c.dsErr { + t.Errorf("unexpected error in dual-stack case: %v", err) + } else if err == nil && c.dsErr { + t.Errorf("unexpected lack of error in dual-stack case") + } else if fatal != c.fatal { + t.Errorf("expected fatal=%v, got %v", c.fatal, fatal) + } + }) + } +} diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index 4c3ef13340f..62adf4a41d1 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -30,38 +30,59 @@ import ( // Enable pprof HTTP handlers. _ "net/http/pprof" - "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" "k8s.io/kubernetes/pkg/proxy/winkernel" ) +// platformApplyDefaults is called after parsing command-line flags and/or reading the +// config file, to apply platform-specific default values to config. func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) { if config.Mode == "" { config.Mode = proxyconfigapi.ProxyModeKernelspace } } +// platformSetup is called after setting up the ProxyServer, but before creating the +// Proxier. It should fill in any platform-specific fields and perform other +// platform-specific setup. +func (s *ProxyServer) platformSetup() error { + winkernel.RegisterMetrics() + return nil +} + +// platformCheckSupported is called immediately before creating the Proxier, to check +// what IP families are supported (and whether the configuration is usable at all). +func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) { + // Check if Kernel proxier can be used at all + _, err = winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{}) + if err != nil { + return false, false, false, err + } + + // winkernel always supports both single-stack IPv4 and single-stack IPv6, but may + // not support dual-stack. + ipv4Supported = true + ipv6Supported = true + + compatTester := winkernel.DualStackCompatTester{} + dualStackSupported = compatTester.DualStackCompatible(s.Config.Winkernel.NetworkName) + + return +} + // createProxier creates the proxy.Provider -func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) { +func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStackMode bool) (proxy.Provider, error) { var healthzPort int if len(config.HealthzBindAddress) > 0 { _, port, _ := net.SplitHostPort(config.HealthzBindAddress) healthzPort, _ = strconv.Atoi(port) } - // Check if Kernel Space can be used. - canUseWinKernelProxy, err := winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{}) - if !canUseWinKernelProxy && err != nil { - return nil, err - } - var proxier proxy.Provider + var err error - dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{}) if dualStackMode { - klog.InfoS("Creating dualStackProxier for Windows kernel.") - proxier, err = winkernel.NewDualStackProxier( config.IPTables.SyncPeriod.Duration, config.IPTables.MinSyncPeriod.Duration, @@ -93,15 +114,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio return proxier, nil } -func (s *ProxyServer) platformSetup() error { - winkernel.RegisterMetrics() - return nil -} - -func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool { - return compatTester.DualStackCompatible(networkname) -} - // cleanupAndExit cleans up after a previous proxy run func cleanupAndExit() error { return errors.New("--cleanup-and-exit is not implemented on Windows")