github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-06 18:54:06 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #119003 from danwinship/proxy-single-dual

Browse Source 
do a better job of validating IP family of kube-proxy config
This commit is contained in:
Kubernetes Prow Robot 2023-07-04 02:16:53 -07:00 committed by GitHub
commit 75889ecec5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 590 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
cmd/kube-proxy/app/server.go
View File

@ -41,6 +41,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
@ -567,7 +568,31 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin
s.HealthzServer = healthcheck.NewProxierHealthServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, s.Recorder, s.NodeRef)
}
s.Proxier, err = s.createProxier(config)
err = s.platformSetup()
if err != nil {
return nil, err
}
ipv4Supported, ipv6Supported, dualStackSupported, err := s.platformCheckSupported()
if err != nil {
return nil, err
} else if (s.PrimaryIPFamily == v1.IPv4Protocol && !ipv4Supported) || (s.PrimaryIPFamily == v1.IPv6Protocol && !ipv6Supported) {
return nil, fmt.Errorf("no support for primary IP family %q", s.PrimaryIPFamily)
} else if dualStackSupported {
klog.InfoS("kube-proxy running in dual-stack mode", "primary ipFamily", s.PrimaryIPFamily)
} else {
klog.InfoS("kube-proxy running in single-stack mode", "ipFamily", s.PrimaryIPFamily)
}
err, fatal := checkIPConfig(s, dualStackSupported)
if err != nil {
if fatal {
return nil, fmt.Errorf("kube-proxy configuration is incorrect: %v", err)
}
klog.ErrorS(err, "Kube-proxy configuration may be incomplete or incorrect")
}
s.Proxier, err = s.createProxier(config, dualStackSupported)
if err != nil {
return nil, err
}
@ -575,6 +600,99 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin
return s, nil
}
// checkIPConfig confirms that s has proper configuration for its primary IP family.
func checkIPConfig(s *ProxyServer, dualStackSupported bool) (error, bool) {
var errors []error
var badFamily netutils.IPFamily
if s.PrimaryIPFamily == v1.IPv4Protocol {
badFamily = netutils.IPv6
} else {
badFamily = netutils.IPv4
}
var clusterType string
if dualStackSupported {
clusterType = fmt.Sprintf("%s-primary", s.PrimaryIPFamily)
} else {
clusterType = fmt.Sprintf("%s-only", s.PrimaryIPFamily)
}
// Historically, we did not check most of the config options, so we cannot
// retroactively make IP family mismatches in those options be fatal. When we add
// new options to check here, we should make problems with those options be fatal.
fatal := false
if s.Config.ClusterCIDR != "" {
clusterCIDRs := strings.Split(s.Config.ClusterCIDR, ",")
if badCIDRs(clusterCIDRs, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but clusterCIDRs contains only IPv%s addresses", clusterType, badFamily))
if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeClusterCIDR {
// This has always been a fatal error
fatal = true
}
}
}
if badCIDRs(s.Config.NodePortAddresses, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but nodePortAddresses contains only IPv%s addresses", clusterType, badFamily))
}
if badCIDRs(s.podCIDRs, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but node.spec.podCIDRs contains only IPv%s addresses", clusterType, badFamily))
if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeNodeCIDR {
// This has always been a fatal error
fatal = true
}
}
if netutils.IPFamilyOfString(s.Config.Winkernel.SourceVip) == badFamily {
errors = append(errors, fmt.Errorf("cluster is %s but winkernel.sourceVip is IPv%s", clusterType, badFamily))
}
// In some cases, wrong-IP-family is only a problem when the secondary IP family
// isn't present at all.
if !dualStackSupported {
if badCIDRs(s.Config.IPVS.ExcludeCIDRs, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but ipvs.excludeCIDRs contains only IPv%s addresses", clusterType, badFamily))
}
if badBindAddress(s.Config.HealthzBindAddress, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but healthzBindAddress is IPv%s", clusterType, badFamily))
}
if badBindAddress(s.Config.MetricsBindAddress, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but metricsBindAddress is IPv%s", clusterType, badFamily))
}
}
return utilerrors.NewAggregate(errors), fatal
}
// badCIDRs returns true if cidrs is a non-empty list of CIDRs, all of wrongFamily.
func badCIDRs(cidrs []string, wrongFamily netutils.IPFamily) bool {
if len(cidrs) == 0 {
return false
}
for _, cidr := range cidrs {
if netutils.IPFamilyOfCIDRString(cidr) != wrongFamily {
return false
}
}
return true
}
// badBindAddress returns true if bindAddress is an "IP:port" string where IP is a
// non-zero IP of wrongFamily.
func badBindAddress(bindAddress string, wrongFamily netutils.IPFamily) bool {
if host, _, _ := net.SplitHostPort(bindAddress); host != "" {
ip := netutils.ParseIPSloppy(host)
if ip != nil && netutils.IPFamilyOf(ip) == wrongFamily && !ip.IsUnspecified() {
return true
}
}
return false
}
// createClient creates a kube client from the given config and masterOverride.
// TODO remove masterOverride when CLI flags are removed.
func createClient(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, error) {
@ -706,12 +824,6 @@ func (s *ProxyServer) Run() error {
// Start up a metrics server if requested
serveMetrics(s.Config.MetricsBindAddress, s.Config.Mode, s.Config.EnableProfiling, errCh)
// Do platform-specific setup
err := s.platformSetup()
if err != nil {
return err
}
noProxyName, err := labels.NewRequirement(apis.LabelServiceProxyName, selection.DoesNotExist, nil)
if err != nil {
return err

131
cmd/kube-proxy/app/server_others.go
View File

@ -50,7 +50,6 @@ import (
utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics"
proxyutil "k8s.io/kubernetes/pkg/proxy/util"
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
"k8s.io/utils/exec"
@ -63,6 +62,8 @@ import (
// node after it is registered.
var timeoutForNodePodCIDR = 5 * time.Minute
// platformApplyDefaults is called after parsing command-line flags and/or reading the
// config file, to apply platform-specific default values to config.
func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) {
if config.Mode == "" {
klog.InfoS("Using iptables proxy")
@ -76,22 +77,58 @@ func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfigur
klog.V(2).InfoS("DetectLocalMode", "localMode", string(config.DetectLocalMode))
}
// platformSetup is called after setting up the ProxyServer, but before creating the
// Proxier. It should fill in any platform-specific fields and perform other
// platform-specific setup.
func (s *ProxyServer) platformSetup() error {
if s.Config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR {
klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname)
node, err := waitForPodCIDR(s.Client, s.Hostname)
if err != nil {
return err
}
s.podCIDRs = node.Spec.PodCIDRs
klog.InfoS("NodeInfo", "podCIDRs", node.Spec.PodCIDRs)
}
err := s.setupConntrack()
if err != nil {
return err
}
proxymetrics.RegisterMetrics()
return nil
}
// platformCheckSupported is called immediately before creating the Proxier, to check
// what IP families are supported (and whether the configuration is usable at all).
func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) {
execer := exec.New()
ipt := utiliptables.New(execer, utiliptables.ProtocolIPv4)
ipv4Supported = ipt.Present()
ipt = utiliptables.New(execer, utiliptables.ProtocolIPv6)
ipv6Supported = ipt.Present()
// The Linux proxies can always support dual-stack if they can support both IPv4
// and IPv6.
dualStackSupported = ipv4Supported && ipv6Supported
if !ipv4Supported && !ipv6Supported {
err = fmt.Errorf("iptables is not available on this host")
} else if !ipv4Supported {
klog.InfoS("No iptables support for family", "ipFamily", v1.IPv4Protocol)
} else if !ipv6Supported {
klog.InfoS("No iptables support for family", "ipFamily", v1.IPv6Protocol)
}
return
}
// createProxier creates the proxy.Provider
func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) {
func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStack bool) (proxy.Provider, error) {
var proxier proxy.Provider
var err error
var nodeInfo *v1.Node
if config.DetectLocalMode == proxyconfigapi.LocalModeNodeCIDR {
klog.InfoS("Watching for node, awaiting podCIDR allocation", "hostname", s.Hostname)
nodeInfo, err = waitForPodCIDR(s.Client, s.Hostname)
if err != nil {
return nil, err
}
s.podCIDRs = nodeInfo.Spec.PodCIDRs
klog.InfoS("NodeInfo", "podCIDR", nodeInfo.Spec.PodCIDR, "podCIDRs", nodeInfo.Spec.PodCIDRs)
}
primaryProtocol := utiliptables.ProtocolIPv4
if s.PrimaryIPFamily == v1.IPv6Protocol {
primaryProtocol = utiliptables.ProtocolIPv6
@ -100,7 +137,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
iptInterface := utiliptables.New(execer, primaryProtocol)
var ipt [2]utiliptables.Interface
dualStack := true // While we assume that node supports, we do further checks below
// Create iptables handlers for both families, one is already created
// Always ordered as IPv4, IPv6
@ -112,33 +148,13 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
ipt[1] = iptInterface
}
nodePortAddresses := config.NodePortAddresses
if !ipt[0].Present() {
return nil, fmt.Errorf("iptables is not supported for primary IP family %q", primaryProtocol)
} else if !ipt[1].Present() {
klog.InfoS("kube-proxy running in single-stack mode: secondary ipFamily is not supported", "ipFamily", ipt[1].Protocol())
dualStack = false
// Validate NodePortAddresses is single-stack
npaByFamily := proxyutil.MapCIDRsByIPFamily(config.NodePortAddresses)
secondaryFamily := proxyutil.OtherIPFamily(s.PrimaryIPFamily)
badAddrs := npaByFamily[secondaryFamily]
if len(badAddrs) > 0 {
klog.InfoS("Ignoring --nodeport-addresses of the wrong family", "ipFamily", secondaryFamily, "addresses", badAddrs)
nodePortAddresses = npaByFamily[s.PrimaryIPFamily]
}
}
if config.Mode == proxyconfigapi.ProxyModeIPTables {
klog.InfoS("Using iptables Proxier")
if dualStack {
klog.InfoS("kube-proxy running in dual-stack mode", "ipFamily", iptInterface.Protocol())
klog.InfoS("Creating dualStackProxier for iptables")
// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo)
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
@ -158,12 +174,12 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
s.NodeIPs,
s.Recorder,
s.HealthzServer,
nodePortAddresses,
config.NodePortAddresses,
)
} else {
// Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support).
var localDetector proxyutiliptables.LocalTrafficDetector
localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo)
localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
@ -184,7 +200,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
s.NodeIPs[s.PrimaryIPFamily],
s.Recorder,
s.HealthzServer,
nodePortAddresses,
config.NodePortAddresses,
)
}
@ -201,11 +217,9 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
klog.InfoS("Using ipvs Proxier")
if dualStack {
klog.InfoS("Creating dualStackProxier for ipvs")
// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, nodeInfo)
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, ipt, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
@ -231,12 +245,12 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
s.Recorder,
s.HealthzServer,
config.IPVS.Scheduler,
nodePortAddresses,
config.NodePortAddresses,
kernelHandler,
)
} else {
var localDetector proxyutiliptables.LocalTrafficDetector
localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, nodeInfo)
localDetector, err = getLocalDetector(config.DetectLocalMode, config, iptInterface, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
@ -263,7 +277,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
s.Recorder,
s.HealthzServer,
config.IPVS.Scheduler,
nodePortAddresses,
config.NodePortAddresses,
kernelHandler,
)
}
@ -275,7 +289,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
return proxier, nil
}
func (s *ProxyServer) platformSetup() error {
func (s *ProxyServer) setupConntrack() error {
ct := &realConntracker{}
max, err := getConntrackMax(s.Config.Conntrack)
@ -315,7 +329,6 @@ func (s *ProxyServer) platformSetup() error {
}
}
proxymetrics.RegisterMetrics()
return nil
}
@ -389,7 +402,7 @@ func detectNumCPU() int {
return numCPU
}
func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) {
func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodePodCIDRs []string) (proxyutiliptables.LocalTrafficDetector, error) {
switch mode {
case proxyconfigapi.LocalModeClusterCIDR:
// LocalModeClusterCIDR is the default if --detect-local-mode wasn't passed,
@ -400,11 +413,11 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube
}
return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt)
case proxyconfigapi.LocalModeNodeCIDR:
if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 {
if len(nodePodCIDRs) == 0 {
klog.InfoS("Detect-local-mode set to NodeCIDR, but no PodCIDR defined at node")
break
}
return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt)
return proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt)
case proxyconfigapi.LocalModeBridgeInterface:
return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface)
case proxyconfigapi.LocalModeInterfaceNamePrefix:
@ -414,7 +427,7 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube
return proxyutiliptables.NewNoOpLocalDetector(), nil
}
func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) {
func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodePodCIDRs []string) ([2]proxyutiliptables.LocalTrafficDetector, error) {
var err error
localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}
switch mode {
@ -444,32 +457,32 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy
}
return localDetectors, err
case proxyconfigapi.LocalModeNodeCIDR:
if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 {
if len(nodePodCIDRs) == 0 {
klog.InfoS("No node info available to configure detect-local-mode NodeCIDR")
break
}
// localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR.
// so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1]
if netutils.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) {
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1])
if netutils.IsIPv6CIDRString(nodePodCIDRs[0]) {
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[1])
if err != nil {
return localDetectors, err
}
if len(nodeInfo.Spec.PodCIDRs) > 1 {
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0])
if len(nodePodCIDRs) > 1 {
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[0])
}
} else {
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0])
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[0], ipt[0])
if err != nil {
return localDetectors, err
}
if len(nodeInfo.Spec.PodCIDRs) > 1 {
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1])
if len(nodePodCIDRs) > 1 {
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodePodCIDRs[1], ipt[1])
}
}
return localDetectors, err
case proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix:
localDetector, err := getLocalDetector(mode, config, ipt[0], nodeInfo)
localDetector, err := getLocalDetector(mode, config, ipt[0], nodePodCIDRs)
if err == nil {
localDetectors[0] = localDetector
localDetectors[1] = localDetector

137
cmd/kube-proxy/app/server_others_test.go
View File

@ -109,14 +109,14 @@ func Test_platformApplyDefaults(t *testing.T) {
func Test_getLocalDetector(t *testing.T) {
cases := []struct {
mode proxyconfigapi.LocalMode
config *proxyconfigapi.KubeProxyConfiguration
ipt utiliptables.Interface
expected proxyutiliptables.LocalTrafficDetector
nodeInfo *v1.Node
errExpected bool
mode proxyconfigapi.LocalMode
config *proxyconfigapi.KubeProxyConfiguration
ipt utiliptables.Interface
expected proxyutiliptables.LocalTrafficDetector
nodePodCIDRs []string
errExpected bool
}{
// LocalModeClusterCIDR, nodeInfo would be nil for these cases
// LocalModeClusterCIDR
{
mode: proxyconfigapi.LocalModeClusterCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"},
@ -154,46 +154,46 @@ func Test_getLocalDetector(t *testing.T) {
},
// LocalModeNodeCIDR
{
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"},
ipt: utiliptablestest.NewFake(),
expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())),
nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"),
errExpected: false,
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"},
ipt: utiliptablestest.NewFake(),
expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())),
nodePodCIDRs: []string{"10.0.0.0/24"},
errExpected: false,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"},
ipt: utiliptablestest.NewIPv6Fake(),
expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())),
nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"),
errExpected: false,
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"},
ipt: utiliptablestest.NewIPv6Fake(),
expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())),
nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"},
errExpected: false,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"},
ipt: utiliptablestest.NewIPv6Fake(),
expected: nil,
nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"),
errExpected: true,
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"},
ipt: utiliptablestest.NewIPv6Fake(),
expected: nil,
nodePodCIDRs: []string{"10.0.0.0/24"},
errExpected: true,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"},
ipt: utiliptablestest.NewFake(),
expected: nil,
nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"),
errExpected: true,
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"},
ipt: utiliptablestest.NewFake(),
expected: nil,
nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"},
errExpected: true,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""},
ipt: utiliptablestest.NewFake(),
expected: proxyutiliptables.NewNoOpLocalDetector(),
nodeInfo: makeNodeWithPodCIDRs(),
errExpected: false,
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""},
ipt: utiliptablestest.NewFake(),
expected: proxyutiliptables.NewNoOpLocalDetector(),
nodePodCIDRs: []string{},
errExpected: false,
},
// unknown mode, nodeInfo would be nil for these cases
// unknown mode
{
mode: proxyconfigapi.LocalMode("abcd"),
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"},
@ -201,7 +201,7 @@ func Test_getLocalDetector(t *testing.T) {
expected: proxyutiliptables.NewNoOpLocalDetector(),
errExpected: false,
},
// LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases
// LocalModeBridgeInterface
{
mode: proxyconfigapi.LocalModeBridgeInterface,
config: &proxyconfigapi.KubeProxyConfiguration{
@ -218,7 +218,7 @@ func Test_getLocalDetector(t *testing.T) {
expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("1234567890123456789")),
errExpected: false,
},
// LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases
// LocalModeInterfaceNamePrefix
{
mode: proxyconfigapi.LocalModeInterfaceNamePrefix,
config: &proxyconfigapi.KubeProxyConfiguration{
@ -237,7 +237,7 @@ func Test_getLocalDetector(t *testing.T) {
},
}
for i, c := range cases {
r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo)
r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodePodCIDRs)
if c.errExpected {
if err == nil {
t.Errorf("Case[%d] Expected error, but succeeded with %v", i, r)
@ -256,14 +256,14 @@ func Test_getLocalDetector(t *testing.T) {
func Test_getDualStackLocalDetectorTuple(t *testing.T) {
cases := []struct {
mode proxyconfigapi.LocalMode
config *proxyconfigapi.KubeProxyConfiguration
ipt [2]utiliptables.Interface
expected [2]proxyutiliptables.LocalTrafficDetector
nodeInfo *v1.Node
errExpected bool
mode proxyconfigapi.LocalMode
config *proxyconfigapi.KubeProxyConfiguration
ipt [2]utiliptables.Interface
expected [2]proxyutiliptables.LocalTrafficDetector
nodePodCIDRs []string
errExpected bool
}{
// LocalModeClusterCIDR, nodeInfo would be nil for these cases
// LocalModeClusterCIDR
{
mode: proxyconfigapi.LocalModeClusterCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"},
@ -315,8 +315,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
expected: resolveDualStackLocalDetectors(t)(
proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))(
proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())),
nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"),
errExpected: false,
nodePodCIDRs: []string{"10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"},
errExpected: false,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
@ -325,8 +325,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
expected: resolveDualStackLocalDetectors(t)(
proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))(
proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())),
nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"),
errExpected: false,
nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"},
errExpected: false,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
@ -335,8 +335,8 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
expected: [2]proxyutiliptables.LocalTrafficDetector{
resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())),
proxyutiliptables.NewNoOpLocalDetector()},
nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"),
errExpected: false,
nodePodCIDRs: []string{"10.0.0.0/24"},
errExpected: false,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
@ -345,18 +345,18 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
expected: [2]proxyutiliptables.LocalTrafficDetector{
proxyutiliptables.NewNoOpLocalDetector(),
resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake()))},
nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"),
errExpected: false,
nodePodCIDRs: []string{"2002::1234:abcd:ffff:c0a8:101/96"},
errExpected: false,
},
{
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""},
ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()},
expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()},
nodeInfo: makeNodeWithPodCIDRs(),
errExpected: false,
mode: proxyconfigapi.LocalModeNodeCIDR,
config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""},
ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()},
expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()},
nodePodCIDRs: []string{},
errExpected: false,
},
// LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases
// LocalModeBridgeInterface
{
mode: proxyconfigapi.LocalModeBridgeInterface,
config: &proxyconfigapi.KubeProxyConfiguration{
@ -367,7 +367,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
proxyutiliptables.NewDetectLocalByBridgeInterface("eth")),
errExpected: false,
},
// LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases
// LocalModeInterfaceNamePrefix
{
mode: proxyconfigapi.LocalModeInterfaceNamePrefix,
config: &proxyconfigapi.KubeProxyConfiguration{
@ -380,7 +380,7 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
},
}
for i, c := range cases {
r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo)
r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodePodCIDRs)
if c.errExpected {
if err == nil {
t.Errorf("Case[%d] expected error, but succeeded with %q", i, r)
@ -646,7 +646,7 @@ func TestGetConntrackMax(t *testing.T) {
}
}
func TestProxyServer_createProxier(t *testing.T) {
func TestProxyServer_platformSetup(t *testing.T) {
tests := []struct {
name string
node *v1.Node
@ -683,9 +683,8 @@ func TestProxyServer_createProxier(t *testing.T) {
v1.IPv6Protocol: net.IPv6zero,
},
}
_, err := s.createProxier(tt.config)
// TODO: mock the exec.Interface to not fail probing iptables
if (err != nil) && !strings.Contains(err.Error(), "iptables is not supported for primary IP family") {
err := s.platformSetup()
if err != nil {
t.Errorf("ProxyServer.createProxier() error = %v", err)
return
}

299
cmd/kube-proxy/app/server_test.go
View File

@ -632,3 +632,302 @@ func Test_detectNodeIPs(t *testing.T) {
})
}
}
func Test_checkIPConfig(t *testing.T) {
cases := []struct {
name string
proxy *ProxyServer
ssErr bool
dsErr bool
fatal bool
}{
{
name: "empty config",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok single-stack clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "10.0.0.0/8",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok dual-stack clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "10.0.0.0/8,fd01:2345::/64",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok reversed dual-stack clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "fd01:2345::/64,10.0.0.0/8",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong-family clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "fd01:2345::/64",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: true,
dsErr: true,
fatal: false,
},
{
name: "wrong-family clusterCIDR when using ClusterCIDR LocalDetector",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "fd01:2345::/64",
DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: true,
dsErr: true,
fatal: true,
},
{
name: "ok single-stack nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"10.0.0.0/8", "192.168.0.0/24"},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok dual-stack nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"10.0.0.0/8", "fd01:2345::/64", "fd01:abcd::/64"},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok reversed dual-stack nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"fd01:2345::/64", "fd01:abcd::/64", "10.0.0.0/8"},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong-family nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"10.0.0.0/8"},
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: true,
dsErr: true,
fatal: false,
},
{
name: "ok single-stack node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"10.0.0.0/8"},
},
ssErr: false,
dsErr: false,
},
{
name: "ok dual-stack node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"10.0.0.0/8", "fd01:2345::/64"},
},
ssErr: false,
dsErr: false,
},
{
name: "ok reversed dual-stack node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"fd01:2345::/64", "10.0.0.0/8"},
},
ssErr: false,
dsErr: false,
},
{
name: "wrong-family node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"fd01:2345::/64"},
},
ssErr: true,
dsErr: true,
fatal: true,
},
{
name: "ok winkernel.sourceVip",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{
SourceVip: "10.0.0.1",
},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong family winkernel.sourceVip",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{
SourceVip: "fd01:2345::1",
},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: true,
dsErr: true,
fatal: false,
},
{
name: "ok IPv4 metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "10.0.0.1:9999",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok IPv6 metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "[fd01:2345::1]:9999",
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok unspecified wrong-family metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "0.0.0.0:9999",
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong family metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "10.0.0.1:9999",
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: true,
dsErr: false,
fatal: false,
},
{
name: "ok ipvs.excludeCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
ExcludeCIDRs: []string{"10.0.0.0/8"},
},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong family ipvs.excludeCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
ExcludeCIDRs: []string{"10.0.0.0/8", "192.168.0.0/24"},
},
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: true,
dsErr: false,
fatal: false,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
err, fatal := checkIPConfig(c.proxy, false)
if err != nil && !c.ssErr {
t.Errorf("unexpected error in single-stack case: %v", err)
} else if err == nil && c.ssErr {
t.Errorf("unexpected lack of error in single-stack case")
} else if fatal != c.fatal {
t.Errorf("expected fatal=%v, got %v", c.fatal, fatal)
}
err, fatal = checkIPConfig(c.proxy, true)
if err != nil && !c.dsErr {
t.Errorf("unexpected error in dual-stack case: %v", err)
} else if err == nil && c.dsErr {
t.Errorf("unexpected lack of error in dual-stack case")
} else if fatal != c.fatal {
t.Errorf("expected fatal=%v, got %v", c.fatal, fatal)
}
})
}
}

52
cmd/kube-proxy/app/server_windows.go
View File

@ -30,38 +30,59 @@ import (
// Enable pprof HTTP handlers.
_ "net/http/pprof"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/proxy"
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/proxy/winkernel"
)
// platformApplyDefaults is called after parsing command-line flags and/or reading the
// config file, to apply platform-specific default values to config.
func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfiguration) {
if config.Mode == "" {
config.Mode = proxyconfigapi.ProxyModeKernelspace
}
}
// platformSetup is called after setting up the ProxyServer, but before creating the
// Proxier. It should fill in any platform-specific fields and perform other
// platform-specific setup.
func (s *ProxyServer) platformSetup() error {
winkernel.RegisterMetrics()
return nil
}
// platformCheckSupported is called immediately before creating the Proxier, to check
// what IP families are supported (and whether the configuration is usable at all).
func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) {
// Check if Kernel proxier can be used at all
_, err = winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{})
if err != nil {
return false, false, false, err
}
// winkernel always supports both single-stack IPv4 and single-stack IPv6, but may
// not support dual-stack.
ipv4Supported = true
ipv6Supported = true
compatTester := winkernel.DualStackCompatTester{}
dualStackSupported = compatTester.DualStackCompatible(s.Config.Winkernel.NetworkName)
return
}
// createProxier creates the proxy.Provider
func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration) (proxy.Provider, error) {
func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStackMode bool) (proxy.Provider, error) {
var healthzPort int
if len(config.HealthzBindAddress) > 0 {
_, port, _ := net.SplitHostPort(config.HealthzBindAddress)
healthzPort, _ = strconv.Atoi(port)
}
// Check if Kernel Space can be used.
canUseWinKernelProxy, err := winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{})
if !canUseWinKernelProxy && err != nil {
return nil, err
}
var proxier proxy.Provider
var err error
dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{})
if dualStackMode {
klog.InfoS("Creating dualStackProxier for Windows kernel.")
proxier, err = winkernel.NewDualStackProxier(
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
@ -93,15 +114,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
return proxier, nil
}
func (s *ProxyServer) platformSetup() error {
winkernel.RegisterMetrics()
return nil
}
func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool {
return compatTester.DualStackCompatible(networkname)
}
// cleanupAndExit cleans up after a previous proxy run
func cleanupAndExit() error {
return errors.New("--cleanup-and-exit is not implemented on Windows")