Merge pull request #53484 from danehans/kubeadm_probe

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adds Support for Configurable Kubeadm Probes.

**What this PR does / why we need it**:
Allows kubeadm liveness probes to be configurable using extra args. Needed to provide deployment flexibility where services are not bound to `127.0.0.1`.

**Which issue this PR fixes**: fixes https://github.com/kubernetes/kubeadm/issues/473

**Special notes for your reviewer**:
Needed for IPv6 support.
/sig network
/area ipv6

**Release note**:
```release-note
```

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
 			Image:         images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
 			Command:       getAPIServerCommand(cfg, k8sVersion),
 			VolumeMounts:  staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
-			LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
+			LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
 			Resources:     staticpodutil.ComponentResources("250m"),
 			Env:           getProxyEnvVars(),
 		}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)),
@@ -87,7 +87,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
 			Image:         images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
 			Command:       getControllerManagerCommand(cfg, k8sVersion),
 			VolumeMounts:  staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
-			LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP),
+			LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP),
 			Resources:     staticpodutil.ComponentResources("200m"),
 			Env:           getProxyEnvVars(),
 		}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)),
@@ -96,7 +96,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
 			Image:         images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
 			Command:       getSchedulerCommand(cfg),
 			VolumeMounts:  staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
-			LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP),
+			LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP),
 			Resources:     staticpodutil.ComponentResources("100m"),
 			Env:           getProxyEnvVars(),
 		}, mounts.GetVolumes(kubeadmconstants.KubeScheduler)),

cmd/kubeadm/app/phases/etcd/local.go

@@ -59,7 +59,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
 		Image:   images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, "", cfg.Etcd.Image),
 		// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
 		VolumeMounts:  []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)},
-		LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP),
+		LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP),
 	}, etcdMounts)
 }
 

cmd/kubeadm/app/util/staticpod/BUILD

@@ -12,6 +12,8 @@ go_test(
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod",
     library = ":go_default_library",
     deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
@@ -23,6 +25,7 @@ go_library(
     srcs = ["utils.go"],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod",
     deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//pkg/kubelet/types:go_default_library",

cmd/kubeadm/app/util/staticpod/utils.go

@@ -19,17 +19,34 @@ package staticpod
 import (
 	"fmt"
 	"io/ioutil"
+	"net"
+	"net/url"
 	"os"
+	"strings"
 
 	"k8s.io/api/core/v1"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
+const (
+	// kubeControllerManagerAddressArg represents the address argument of the kube-controller-manager configuration.
+	kubeControllerManagerAddressArg = "address"
+
+	// kubeSchedulerAddressArg represents the address argument of the kube-scheduler configuration.
+	kubeSchedulerAddressArg = "address"
+
+	// etcdListenClientURLsArg represents the listen-client-urls argument of the etcd configuration.
+	etcdListenClientURLsArg = "listen-client-urls"
+)
+
 // ComponentPod returns a Pod object from the container and volume specifications
 func ComponentPod(container v1.Container, volumes map[string]v1.Volume) v1.Pod {
 	return v1.Pod{
@@ -63,12 +80,11 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
 }
 
 // ComponentProbe is a helper function building a ready v1.Probe object from some simple parameters
-func ComponentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe {
+func ComponentProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, port int, path string, scheme v1.URIScheme) *v1.Probe {
 	return &v1.Probe{
 		Handler: v1.Handler{
 			HTTPGet: &v1.HTTPGetAction{
-				// Host has to be set to "127.0.0.1" here due to that our static Pods are on the host's network
+				Host:   GetProbeAddress(cfg, componentName),
-				Host:   "127.0.0.1",
 				Path:   path,
 				Port:   intstr.FromInt(port),
 				Scheme: scheme,
@@ -162,3 +178,59 @@ func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {
 
 	return nil
 }
+
+// GetProbeAddress returns an IP address or 127.0.0.1 to use for liveness probes
+// in static pod manifests.
+func GetProbeAddress(cfg *kubeadmapi.MasterConfiguration, componentName string) string {
+	switch {
+	case componentName == kubeadmconstants.KubeAPIServer:
+		if cfg.API.AdvertiseAddress != "" {
+			return cfg.API.AdvertiseAddress
+		}
+	case componentName == kubeadmconstants.KubeControllerManager:
+		if addr, exists := cfg.ControllerManagerExtraArgs[kubeControllerManagerAddressArg]; exists {
+			return addr
+		}
+	case componentName == kubeadmconstants.KubeScheduler:
+		if addr, exists := cfg.SchedulerExtraArgs[kubeSchedulerAddressArg]; exists {
+			return addr
+		}
+	case componentName == kubeadmconstants.Etcd:
+		if cfg.Etcd.ExtraArgs != nil {
+			if arg, exists := cfg.Etcd.ExtraArgs[etcdListenClientURLsArg]; exists {
+				// Use the first url in the listen-client-urls if multiple url's are specified.
+				if strings.ContainsAny(arg, ",") {
+					arg = strings.Split(arg, ",")[0]
+				}
+				parsedURL, err := url.Parse(arg)
+				if err != nil || parsedURL.Hostname() == "" {
+					break
+				}
+				// Return the IP if the URL contains an address instead of a name.
+				if ip := net.ParseIP(parsedURL.Hostname()); ip != nil {
+					return ip.String()
+				}
+				// Use the local resolver to try resolving the name within the URL.
+				// If the name can not be resolved, return an IPv4 loopback address.
+				// Otherwise, select the first valid IPv4 address.
+				// If the name does not resolve to an IPv4 address, select the first valid IPv6 address.
+				addrs, err := net.LookupIP(parsedURL.Hostname())
+				if err != nil {
+					break
+				}
+				var ip net.IP
+				for _, addr := range addrs {
+					if addr.To4() != nil {
+						ip = addr
+						break
+					}
+					if addr.To16() != nil && ip == nil {
+						ip = addr
+					}
+				}
+				return ip.String()
+			}
+		}
+	}
+	return "127.0.0.1"
+}

cmd/kubeadm/app/util/staticpod/utils_test.go

@@ -24,6 +24,9 @@ import (
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 func TestComponentResources(t *testing.T) {
@@ -37,43 +40,174 @@ func TestComponentResources(t *testing.T) {
 
 func TestComponentProbe(t *testing.T) {
 	var tests = []struct {
-		port   int
+		name      string
-		path   string
+		cfg       *kubeadmapi.MasterConfiguration
-		scheme v1.URIScheme
+		component string
+		port      int
+		path      string
+		scheme    v1.URIScheme
+		expected  string
 	}{
 		{
-			port:   1,
+			name: "default apiserver advertise address with http",
-			path:   "foo",
+			cfg: &kubeadmapi.MasterConfiguration{
-			scheme: v1.URISchemeHTTP,
+				API: kubeadmapi.API{
+					AdvertiseAddress: "",
+				},
+			},
+			component: kubeadmconstants.KubeAPIServer,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "127.0.0.1",
 		},
 		{
-			port:   2,
+			name: "default apiserver advertise address with https",
-			path:   "bar",
+			cfg: &kubeadmapi.MasterConfiguration{
-			scheme: v1.URISchemeHTTPS,
+				API: kubeadmapi.API{
+					AdvertiseAddress: "",
+				},
+			},
+			component: kubeadmconstants.KubeAPIServer,
+			port:      2,
+			path:      "bar",
+			scheme:    v1.URISchemeHTTPS,
+			expected:  "127.0.0.1",
+		},
+		{
+			name: "valid ipv4 apiserver advertise address with http",
+			cfg: &kubeadmapi.MasterConfiguration{
+				API: kubeadmapi.API{
+					AdvertiseAddress: "1.2.3.4",
+				},
+			},
+			component: kubeadmconstants.KubeAPIServer,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "1.2.3.4",
+		},
+		{
+			name: "valid ipv6 apiserver advertise address with http",
+			cfg: &kubeadmapi.MasterConfiguration{
+				API: kubeadmapi.API{
+					AdvertiseAddress: "2001:db8::1",
+				},
+			},
+			component: kubeadmconstants.KubeAPIServer,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "2001:db8::1",
+		},
+		{
+			name: "valid IPv4 controller-manager probe",
+			cfg: &kubeadmapi.MasterConfiguration{
+				ControllerManagerExtraArgs: map[string]string{"address": "1.2.3.4"},
+			},
+			component: kubeadmconstants.KubeControllerManager,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "1.2.3.4",
+		},
+		{
+			name: "valid IPv6 controller-manager probe",
+			cfg: &kubeadmapi.MasterConfiguration{
+				ControllerManagerExtraArgs: map[string]string{"address": "2001:db8::1"},
+			},
+			component: kubeadmconstants.KubeControllerManager,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "2001:db8::1",
+		},
+		{
+			name: "valid IPv4 scheduler probe",
+			cfg: &kubeadmapi.MasterConfiguration{
+				SchedulerExtraArgs: map[string]string{"address": "1.2.3.4"},
+			},
+			component: kubeadmconstants.KubeScheduler,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "1.2.3.4",
+		},
+		{
+			name: "valid IPv6 scheduler probe",
+			cfg: &kubeadmapi.MasterConfiguration{
+				SchedulerExtraArgs: map[string]string{"address": "2001:db8::1"},
+			},
+			component: kubeadmconstants.KubeScheduler,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "2001:db8::1",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls IPv4 addresses",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					ExtraArgs: map[string]string{
+						"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "1.2.3.4",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls IPv6 addresses",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					ExtraArgs: map[string]string{
+						"listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "2001:db8::1",
+		},
+		{
+			name: "valid IPv4 etcd probe using hostname for listen-client-urls",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					ExtraArgs: map[string]string{
+						"listen-client-urls": "http://localhost:2379"},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			path:      "foo",
+			scheme:    v1.URISchemeHTTP,
+			expected:  "127.0.0.1",
 		},
 	}
 	for _, rt := range tests {
-		actual := ComponentProbe(rt.port, rt.path, rt.scheme)
+		actual := ComponentProbe(rt.cfg, rt.component, rt.port, rt.path, rt.scheme)
+		if actual.Handler.HTTPGet.Host != rt.expected {
+			t.Errorf("%s test case failed:\n\texpected: %s\n\t  actual: %s",
+				rt.name, rt.expected,
+				actual.Handler.HTTPGet.Host)
+		}
 		if actual.Handler.HTTPGet.Port != intstr.FromInt(rt.port) {
-			t.Errorf(
+			t.Errorf("%s test case failed:\n\texpected: %v\n\t  actual: %v",
-				"failed componentProbe:\n\texpected: %v\n\t  actual: %v",
+				rt.name, rt.port,
-				rt.port,
+				actual.Handler.HTTPGet.Port)
-				actual.Handler.HTTPGet.Port,
-			)
 		}
 		if actual.Handler.HTTPGet.Path != rt.path {
-			t.Errorf(
+			t.Errorf("%s test case failed:\n\texpected: %s\n\t  actual: %s",
-				"failed componentProbe:\n\texpected: %s\n\t  actual: %s",
+				rt.name, rt.path,
-				rt.path,
+				actual.Handler.HTTPGet.Path)
-				actual.Handler.HTTPGet.Path,
-			)
 		}
 		if actual.Handler.HTTPGet.Scheme != rt.scheme {
-			t.Errorf(
+			t.Errorf("%s test case failed:\n\texpected: %v\n\t  actual: %v",
-				"failed componentProbe:\n\texpected: %v\n\t  actual: %v",
+				rt.name, rt.scheme,
-				rt.scheme,
+				actual.Handler.HTTPGet.Scheme)
-				actual.Handler.HTTPGet.Scheme,
-			)
 		}
 	}
 }