diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index e82bb4d2407..af396548427 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getAPIServerCommand(cfg, k8sVersion), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), - LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), Resources: staticpodutil.ComponentResources("250m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)), @@ -87,7 +87,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getControllerManagerCommand(cfg, k8sVersion), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), - LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP), Resources: staticpodutil.ComponentResources("200m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)), @@ -96,7 +96,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getSchedulerCommand(cfg), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), - LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP), Resources: staticpodutil.ComponentResources("100m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler)), diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index d635c5a464d..522e6e5dae9 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -59,7 +59,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, "", cfg.Etcd.Image), // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, - LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP), }, etcdMounts) } diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD index 6400a61d1e2..343f457a60c 100644 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ b/cmd/kubeadm/app/util/staticpod/BUILD @@ -12,6 +12,8 @@ go_test( importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod", library = ":go_default_library", deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", @@ -23,6 +25,7 @@ go_library( srcs = ["utils.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod", deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//pkg/kubelet/types:go_default_library", diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 0a7607d5daf..43c92e09890 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -19,17 +19,34 @@ package staticpod import ( "fmt" "io/ioutil" + "net" + "net/url" "os" + "strings" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) +const ( + // kubeControllerManagerAddressArg represents the address argument of the kube-controller-manager configuration. + kubeControllerManagerAddressArg = "address" + + // kubeSchedulerAddressArg represents the address argument of the kube-scheduler configuration. + kubeSchedulerAddressArg = "address" + + // etcdListenClientURLsArg represents the listen-client-urls argument of the etcd configuration. + etcdListenClientURLsArg = "listen-client-urls" +) + // ComponentPod returns a Pod object from the container and volume specifications func ComponentPod(container v1.Container, volumes map[string]v1.Volume) v1.Pod { return v1.Pod{ @@ -63,12 +80,11 @@ func ComponentResources(cpu string) v1.ResourceRequirements { } // ComponentProbe is a helper function building a ready v1.Probe object from some simple parameters -func ComponentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe { +func ComponentProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, port int, path string, scheme v1.URIScheme) *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ - // Host has to be set to "127.0.0.1" here due to that our static Pods are on the host's network - Host: "127.0.0.1", + Host: GetProbeAddress(cfg, componentName), Path: path, Port: intstr.FromInt(port), Scheme: scheme, @@ -162,3 +178,59 @@ func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error { return nil } + +// GetProbeAddress returns an IP address or 127.0.0.1 to use for liveness probes +// in static pod manifests. +func GetProbeAddress(cfg *kubeadmapi.MasterConfiguration, componentName string) string { + switch { + case componentName == kubeadmconstants.KubeAPIServer: + if cfg.API.AdvertiseAddress != "" { + return cfg.API.AdvertiseAddress + } + case componentName == kubeadmconstants.KubeControllerManager: + if addr, exists := cfg.ControllerManagerExtraArgs[kubeControllerManagerAddressArg]; exists { + return addr + } + case componentName == kubeadmconstants.KubeScheduler: + if addr, exists := cfg.SchedulerExtraArgs[kubeSchedulerAddressArg]; exists { + return addr + } + case componentName == kubeadmconstants.Etcd: + if cfg.Etcd.ExtraArgs != nil { + if arg, exists := cfg.Etcd.ExtraArgs[etcdListenClientURLsArg]; exists { + // Use the first url in the listen-client-urls if multiple url's are specified. + if strings.ContainsAny(arg, ",") { + arg = strings.Split(arg, ",")[0] + } + parsedURL, err := url.Parse(arg) + if err != nil || parsedURL.Hostname() == "" { + break + } + // Return the IP if the URL contains an address instead of a name. + if ip := net.ParseIP(parsedURL.Hostname()); ip != nil { + return ip.String() + } + // Use the local resolver to try resolving the name within the URL. + // If the name can not be resolved, return an IPv4 loopback address. + // Otherwise, select the first valid IPv4 address. + // If the name does not resolve to an IPv4 address, select the first valid IPv6 address. + addrs, err := net.LookupIP(parsedURL.Hostname()) + if err != nil { + break + } + var ip net.IP + for _, addr := range addrs { + if addr.To4() != nil { + ip = addr + break + } + if addr.To16() != nil && ip == nil { + ip = addr + } + } + return ip.String() + } + } + } + return "127.0.0.1" +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 0dd73b978c3..5b681335841 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -24,6 +24,9 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) func TestComponentResources(t *testing.T) { @@ -37,43 +40,174 @@ func TestComponentResources(t *testing.T) { func TestComponentProbe(t *testing.T) { var tests = []struct { - port int - path string - scheme v1.URIScheme + name string + cfg *kubeadmapi.MasterConfiguration + component string + port int + path string + scheme v1.URIScheme + expected string }{ { - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, + name: "default apiserver advertise address with http", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "", + }, + }, + component: kubeadmconstants.KubeAPIServer, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "127.0.0.1", }, { - port: 2, - path: "bar", - scheme: v1.URISchemeHTTPS, + name: "default apiserver advertise address with https", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "", + }, + }, + component: kubeadmconstants.KubeAPIServer, + port: 2, + path: "bar", + scheme: v1.URISchemeHTTPS, + expected: "127.0.0.1", + }, + { + name: "valid ipv4 apiserver advertise address with http", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + }, + }, + component: kubeadmconstants.KubeAPIServer, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "1.2.3.4", + }, + { + name: "valid ipv6 apiserver advertise address with http", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "2001:db8::1", + }, + }, + component: kubeadmconstants.KubeAPIServer, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "2001:db8::1", + }, + { + name: "valid IPv4 controller-manager probe", + cfg: &kubeadmapi.MasterConfiguration{ + ControllerManagerExtraArgs: map[string]string{"address": "1.2.3.4"}, + }, + component: kubeadmconstants.KubeControllerManager, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "1.2.3.4", + }, + { + name: "valid IPv6 controller-manager probe", + cfg: &kubeadmapi.MasterConfiguration{ + ControllerManagerExtraArgs: map[string]string{"address": "2001:db8::1"}, + }, + component: kubeadmconstants.KubeControllerManager, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "2001:db8::1", + }, + { + name: "valid IPv4 scheduler probe", + cfg: &kubeadmapi.MasterConfiguration{ + SchedulerExtraArgs: map[string]string{"address": "1.2.3.4"}, + }, + component: kubeadmconstants.KubeScheduler, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "1.2.3.4", + }, + { + name: "valid IPv6 scheduler probe", + cfg: &kubeadmapi.MasterConfiguration{ + SchedulerExtraArgs: map[string]string{"address": "2001:db8::1"}, + }, + component: kubeadmconstants.KubeScheduler, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "2001:db8::1", + }, + { + name: "valid etcd probe using listen-client-urls IPv4 addresses", + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"}, + }, + }, + component: kubeadmconstants.Etcd, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "1.2.3.4", + }, + { + name: "valid etcd probe using listen-client-urls IPv6 addresses", + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"}, + }, + }, + component: kubeadmconstants.Etcd, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "2001:db8::1", + }, + { + name: "valid IPv4 etcd probe using hostname for listen-client-urls", + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://localhost:2379"}, + }, + }, + component: kubeadmconstants.Etcd, + port: 1, + path: "foo", + scheme: v1.URISchemeHTTP, + expected: "127.0.0.1", }, } for _, rt := range tests { - actual := ComponentProbe(rt.port, rt.path, rt.scheme) + actual := ComponentProbe(rt.cfg, rt.component, rt.port, rt.path, rt.scheme) + if actual.Handler.HTTPGet.Host != rt.expected { + t.Errorf("%s test case failed:\n\texpected: %s\n\t actual: %s", + rt.name, rt.expected, + actual.Handler.HTTPGet.Host) + } if actual.Handler.HTTPGet.Port != intstr.FromInt(rt.port) { - t.Errorf( - "failed componentProbe:\n\texpected: %v\n\t actual: %v", - rt.port, - actual.Handler.HTTPGet.Port, - ) + t.Errorf("%s test case failed:\n\texpected: %v\n\t actual: %v", + rt.name, rt.port, + actual.Handler.HTTPGet.Port) } if actual.Handler.HTTPGet.Path != rt.path { - t.Errorf( - "failed componentProbe:\n\texpected: %s\n\t actual: %s", - rt.path, - actual.Handler.HTTPGet.Path, - ) + t.Errorf("%s test case failed:\n\texpected: %s\n\t actual: %s", + rt.name, rt.path, + actual.Handler.HTTPGet.Path) } if actual.Handler.HTTPGet.Scheme != rt.scheme { - t.Errorf( - "failed componentProbe:\n\texpected: %v\n\t actual: %v", - rt.scheme, - actual.Handler.HTTPGet.Scheme, - ) + t.Errorf("%s test case failed:\n\texpected: %v\n\t actual: %v", + rt.name, rt.scheme, + actual.Handler.HTTPGet.Scheme) } } }