diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 99992b2da2e..7fe01c87769 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/healthz" @@ -583,6 +584,14 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin klog.InfoS("kube-proxy running in single-stack mode", "ipFamily", s.PrimaryIPFamily) } + err, fatal := checkIPConfig(s, dualStackSupported) + if err != nil { + if fatal { + return nil, fmt.Errorf("kube-proxy configuration is incorrect: %v", err) + } + klog.ErrorS(err, "Kube-proxy configuration may be incomplete or incorrect") + } + s.Proxier, err = s.createProxier(config, dualStackSupported) if err != nil { return nil, err @@ -591,6 +600,99 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin return s, nil } +// checkIPConfig confirms that s has proper configuration for its primary IP family. +func checkIPConfig(s *ProxyServer, dualStackSupported bool) (error, bool) { + var errors []error + var badFamily netutils.IPFamily + + if s.PrimaryIPFamily == v1.IPv4Protocol { + badFamily = netutils.IPv6 + } else { + badFamily = netutils.IPv4 + } + + var clusterType string + if dualStackSupported { + clusterType = fmt.Sprintf("%s-primary", s.PrimaryIPFamily) + } else { + clusterType = fmt.Sprintf("%s-only", s.PrimaryIPFamily) + } + + // Historically, we did not check most of the config options, so we cannot + // retroactively make IP family mismatches in those options be fatal. When we add + // new options to check here, we should make problems with those options be fatal. + fatal := false + + if s.Config.ClusterCIDR != "" { + clusterCIDRs := strings.Split(s.Config.ClusterCIDR, ",") + if badCIDRs(clusterCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but clusterCIDRs contains only IPv%s addresses", clusterType, badFamily)) + if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeClusterCIDR { + // This has always been a fatal error + fatal = true + } + } + } + + if badCIDRs(s.Config.NodePortAddresses, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but nodePortAddresses contains only IPv%s addresses", clusterType, badFamily)) + } + + if badCIDRs(s.podCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but node.spec.podCIDRs contains only IPv%s addresses", clusterType, badFamily)) + if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeNodeCIDR { + // This has always been a fatal error + fatal = true + } + } + + if netutils.IPFamilyOfString(s.Config.Winkernel.SourceVip) == badFamily { + errors = append(errors, fmt.Errorf("cluster is %s but winkernel.sourceVip is IPv%s", clusterType, badFamily)) + } + + // In some cases, wrong-IP-family is only a problem when the secondary IP family + // isn't present at all. + if !dualStackSupported { + if badCIDRs(s.Config.IPVS.ExcludeCIDRs, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but ipvs.excludeCIDRs contains only IPv%s addresses", clusterType, badFamily)) + } + + if badBindAddress(s.Config.HealthzBindAddress, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but healthzBindAddress is IPv%s", clusterType, badFamily)) + } + if badBindAddress(s.Config.MetricsBindAddress, badFamily) { + errors = append(errors, fmt.Errorf("cluster is %s but metricsBindAddress is IPv%s", clusterType, badFamily)) + } + } + + return utilerrors.NewAggregate(errors), fatal +} + +// badCIDRs returns true if cidrs is a non-empty list of CIDRs, all of wrongFamily. +func badCIDRs(cidrs []string, wrongFamily netutils.IPFamily) bool { + if len(cidrs) == 0 { + return false + } + for _, cidr := range cidrs { + if netutils.IPFamilyOfCIDRString(cidr) != wrongFamily { + return false + } + } + return true +} + +// badBindAddress returns true if bindAddress is an "IP:port" string where IP is a +// non-zero IP of wrongFamily. +func badBindAddress(bindAddress string, wrongFamily netutils.IPFamily) bool { + if host, _, _ := net.SplitHostPort(bindAddress); host != "" { + ip := netutils.ParseIPSloppy(host) + if ip != nil && netutils.IPFamilyOf(ip) == wrongFamily && !ip.IsUnspecified() { + return true + } + } + return false +} + // createClient creates a kube client from the given config and masterOverride. // TODO remove masterOverride when CLI flags are removed. func createClient(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, error) { diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 4eee907f2d1..061152a8882 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -50,7 +50,6 @@ import ( utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset" utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util" proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables" "k8s.io/utils/exec" @@ -149,16 +148,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio ipt[1] = iptInterface } - if !dualStack { - // Validate NodePortAddresses is single-stack - npaByFamily := proxyutil.MapCIDRsByIPFamily(config.NodePortAddresses) - secondaryFamily := proxyutil.OtherIPFamily(s.PrimaryIPFamily) - badAddrs := npaByFamily[secondaryFamily] - if len(badAddrs) > 0 { - klog.InfoS("Ignoring --nodeport-addresses of the wrong family", "ipFamily", secondaryFamily, "addresses", badAddrs) - } - } - if config.Mode == proxyconfigapi.ProxyModeIPTables { klog.InfoS("Using iptables Proxier") diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 8d5f3241510..2b7709a152c 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -632,3 +632,302 @@ func Test_detectNodeIPs(t *testing.T) { }) } } + +func Test_checkIPConfig(t *testing.T) { + cases := []struct { + name string + proxy *ProxyServer + ssErr bool + dsErr bool + fatal bool + }{ + { + name: "empty config", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{}, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + + { + name: "ok single-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "10.0.0.0/8", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "10.0.0.0/8,fd01:2345::/64", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64,10.0.0.0/8", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family clusterCIDR", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + { + name: "wrong-family clusterCIDR when using ClusterCIDR LocalDetector", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + ClusterCIDR: "fd01:2345::/64", + DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: true, + }, + + { + name: "ok single-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8", "fd01:2345::/64", "fd01:abcd::/64"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"fd01:2345::/64", "fd01:abcd::/64", "10.0.0.0/8"}, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family nodePortAddresses", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + NodePortAddresses: []string{"10.0.0.0/8"}, + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + + { + name: "ok single-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"10.0.0.0/8"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok dual-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"10.0.0.0/8", "fd01:2345::/64"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok reversed dual-stack node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"fd01:2345::/64", "10.0.0.0/8"}, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong-family node.spec.podCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR, + }, + PrimaryIPFamily: v1.IPv4Protocol, + podCIDRs: []string{"fd01:2345::/64"}, + }, + ssErr: true, + dsErr: true, + fatal: true, + }, + + { + name: "ok winkernel.sourceVip", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + SourceVip: "10.0.0.1", + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family winkernel.sourceVip", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + SourceVip: "fd01:2345::1", + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: true, + dsErr: true, + fatal: false, + }, + + { + name: "ok IPv4 metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "10.0.0.1:9999", + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok IPv6 metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "[fd01:2345::1]:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "ok unspecified wrong-family metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "0.0.0.0:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family metricsBindAddress", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + MetricsBindAddress: "10.0.0.1:9999", + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: false, + fatal: false, + }, + + { + name: "ok ipvs.excludeCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: []string{"10.0.0.0/8"}, + }, + }, + PrimaryIPFamily: v1.IPv4Protocol, + }, + ssErr: false, + dsErr: false, + }, + { + name: "wrong family ipvs.excludeCIDRs", + proxy: &ProxyServer{ + Config: &kubeproxyconfig.KubeProxyConfiguration{ + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + }, + PrimaryIPFamily: v1.IPv6Protocol, + }, + ssErr: true, + dsErr: false, + fatal: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err, fatal := checkIPConfig(c.proxy, false) + if err != nil && !c.ssErr { + t.Errorf("unexpected error in single-stack case: %v", err) + } else if err == nil && c.ssErr { + t.Errorf("unexpected lack of error in single-stack case") + } else if fatal != c.fatal { + t.Errorf("expected fatal=%v, got %v", c.fatal, fatal) + } + + err, fatal = checkIPConfig(c.proxy, true) + if err != nil && !c.dsErr { + t.Errorf("unexpected error in dual-stack case: %v", err) + } else if err == nil && c.dsErr { + t.Errorf("unexpected lack of error in dual-stack case") + } else if fatal != c.fatal { + t.Errorf("expected fatal=%v, got %v", c.fatal, fatal) + } + }) + } +}