github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 23:37:01 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #53484 from danehans/kubeadm_probe

Browse Source 
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adds Support for Configurable Kubeadm Probes.

**What this PR does / why we need it**:
Allows kubeadm liveness probes to be configurable using extra args. Needed to provide deployment flexibility where services are not bound to `127.0.0.1`.

**Which issue this PR fixes**: fixes https://github.com/kubernetes/kubeadm/issues/473

**Special notes for your reviewer**:
Needed for IPv6 support.
/sig network
/area ipv6

**Release note**:
```release-note
```
This commit is contained in:
Kubernetes Submit Queue 2017-11-04 14:04:27 -07:00 committed by GitHub
commit 3e245366f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 241 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/kubeadm/app/phases/controlplane/manifests.go
View File

@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
Command: getAPIServerCommand(cfg, k8sVersion),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
Resources: staticpodutil.ComponentResources("250m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)),
@ -87,7 +87,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
Command: getControllerManagerCommand(cfg, k8sVersion),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP),
Resources: staticpodutil.ComponentResources("200m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)),
@ -96,7 +96,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
Command: getSchedulerCommand(cfg),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP),
Resources: staticpodutil.ComponentResources("100m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler)),

2
cmd/kubeadm/app/phases/etcd/local.go
View File

@ -59,7 +59,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, "", cfg.Etcd.Image),
// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)},
LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP),
}, etcdMounts)
}

3
cmd/kubeadm/app/util/staticpod/BUILD
View File

@ -12,6 +12,8 @@ go_test(
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod",
library = ":go_default_library",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
@ -23,6 +25,7 @@ go_library(
srcs = ["utils.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/kubelet/types:go_default_library",

78
cmd/kubeadm/app/util/staticpod/utils.go
View File

@ -19,17 +19,34 @@ package staticpod
import (
"fmt"
"io/ioutil"
"net"
"net/url"
"os"
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)
const (
// kubeControllerManagerAddressArg represents the address argument of the kube-controller-manager configuration.
kubeControllerManagerAddressArg = "address"
// kubeSchedulerAddressArg represents the address argument of the kube-scheduler configuration.
kubeSchedulerAddressArg = "address"
// etcdListenClientURLsArg represents the listen-client-urls argument of the etcd configuration.
etcdListenClientURLsArg = "listen-client-urls"
)
// ComponentPod returns a Pod object from the container and volume specifications
func ComponentPod(container v1.Container, volumes map[string]v1.Volume) v1.Pod {
return v1.Pod{
@ -63,12 +80,11 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
}
// ComponentProbe is a helper function building a ready v1.Probe object from some simple parameters
func ComponentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe {
func ComponentProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, port int, path string, scheme v1.URIScheme) *v1.Probe {
return &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
// Host has to be set to "127.0.0.1" here due to that our static Pods are on the host's network
Host: "127.0.0.1",
Host: GetProbeAddress(cfg, componentName),
Path: path,
Port: intstr.FromInt(port),
Scheme: scheme,
@ -162,3 +178,59 @@ func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {
return nil
}
// GetProbeAddress returns an IP address or 127.0.0.1 to use for liveness probes
// in static pod manifests.
func GetProbeAddress(cfg *kubeadmapi.MasterConfiguration, componentName string) string {
switch {
case componentName == kubeadmconstants.KubeAPIServer:
if cfg.API.AdvertiseAddress != "" {
return cfg.API.AdvertiseAddress
}
case componentName == kubeadmconstants.KubeControllerManager:
if addr, exists := cfg.ControllerManagerExtraArgs[kubeControllerManagerAddressArg]; exists {
return addr
}
case componentName == kubeadmconstants.KubeScheduler:
if addr, exists := cfg.SchedulerExtraArgs[kubeSchedulerAddressArg]; exists {
return addr
}
case componentName == kubeadmconstants.Etcd:
if cfg.Etcd.ExtraArgs != nil {
if arg, exists := cfg.Etcd.ExtraArgs[etcdListenClientURLsArg]; exists {
// Use the first url in the listen-client-urls if multiple url's are specified.
if strings.ContainsAny(arg, ",") {
arg = strings.Split(arg, ",")[0]
}
parsedURL, err := url.Parse(arg)
if err != nil || parsedURL.Hostname() == "" {
break
}
// Return the IP if the URL contains an address instead of a name.
if ip := net.ParseIP(parsedURL.Hostname()); ip != nil {
return ip.String()
}
// Use the local resolver to try resolving the name within the URL.
// If the name can not be resolved, return an IPv4 loopback address.
// Otherwise, select the first valid IPv4 address.
// If the name does not resolve to an IPv4 address, select the first valid IPv6 address.
addrs, err := net.LookupIP(parsedURL.Hostname())
if err != nil {
break
}
var ip net.IP
for _, addr := range addrs {
if addr.To4() != nil {
ip = addr
break
}
if addr.To16() != nil && ip == nil {
ip = addr
}
}
return ip.String()
}
}
}
return "127.0.0.1"
}

166
cmd/kubeadm/app/util/staticpod/utils_test.go
View File

@ -24,6 +24,9 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
func TestComponentResources(t *testing.T) {
@ -37,43 +40,174 @@ func TestComponentResources(t *testing.T) {
func TestComponentProbe(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.MasterConfiguration
component string
port int
path string
scheme v1.URIScheme
expected string
}{
{
name: "default apiserver advertise address with http",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{
AdvertiseAddress: "",
},
},
component: kubeadmconstants.KubeAPIServer,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "127.0.0.1",
},
{
name: "default apiserver advertise address with https",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{
AdvertiseAddress: "",
},
},
component: kubeadmconstants.KubeAPIServer,
port: 2,
path: "bar",
scheme: v1.URISchemeHTTPS,
expected: "127.0.0.1",
},
{
name: "valid ipv4 apiserver advertise address with http",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{
AdvertiseAddress: "1.2.3.4",
},
},
component: kubeadmconstants.KubeAPIServer,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "1.2.3.4",
},
{
name: "valid ipv6 apiserver advertise address with http",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{
AdvertiseAddress: "2001:db8::1",
},
},
component: kubeadmconstants.KubeAPIServer,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "2001:db8::1",
},
{
name: "valid IPv4 controller-manager probe",
cfg: &kubeadmapi.MasterConfiguration{
ControllerManagerExtraArgs: map[string]string{"address": "1.2.3.4"},
},
component: kubeadmconstants.KubeControllerManager,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "1.2.3.4",
},
{
name: "valid IPv6 controller-manager probe",
cfg: &kubeadmapi.MasterConfiguration{
ControllerManagerExtraArgs: map[string]string{"address": "2001:db8::1"},
},
component: kubeadmconstants.KubeControllerManager,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "2001:db8::1",
},
{
name: "valid IPv4 scheduler probe",
cfg: &kubeadmapi.MasterConfiguration{
SchedulerExtraArgs: map[string]string{"address": "1.2.3.4"},
},
component: kubeadmconstants.KubeScheduler,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "1.2.3.4",
},
{
name: "valid IPv6 scheduler probe",
cfg: &kubeadmapi.MasterConfiguration{
SchedulerExtraArgs: map[string]string{"address": "2001:db8::1"},
},
component: kubeadmconstants.KubeScheduler,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "2001:db8::1",
},
{
name: "valid etcd probe using listen-client-urls IPv4 addresses",
cfg: &kubeadmapi.MasterConfiguration{
Etcd: kubeadmapi.Etcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
},
},
component: kubeadmconstants.Etcd,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "1.2.3.4",
},
{
name: "valid etcd probe using listen-client-urls IPv6 addresses",
cfg: &kubeadmapi.MasterConfiguration{
Etcd: kubeadmapi.Etcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"},
},
},
component: kubeadmconstants.Etcd,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "2001:db8::1",
},
{
name: "valid IPv4 etcd probe using hostname for listen-client-urls",
cfg: &kubeadmapi.MasterConfiguration{
Etcd: kubeadmapi.Etcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://localhost:2379"},
},
},
component: kubeadmconstants.Etcd,
port: 1,
path: "foo",
scheme: v1.URISchemeHTTP,
expected: "127.0.0.1",
},
}
for _, rt := range tests {
actual := ComponentProbe(rt.port, rt.path, rt.scheme)
actual := ComponentProbe(rt.cfg, rt.component, rt.port, rt.path, rt.scheme)
if actual.Handler.HTTPGet.Host != rt.expected {
t.Errorf("%s test case failed:\n\texpected: %s\n\t actual: %s",
rt.name, rt.expected,
actual.Handler.HTTPGet.Host)
}
if actual.Handler.HTTPGet.Port != intstr.FromInt(rt.port) {
t.Errorf(
"failed componentProbe:\n\texpected: %v\n\t actual: %v",
rt.port,
actual.Handler.HTTPGet.Port,
)
t.Errorf("%s test case failed:\n\texpected: %v\n\t actual: %v",
rt.name, rt.port,
actual.Handler.HTTPGet.Port)
}
if actual.Handler.HTTPGet.Path != rt.path {
t.Errorf(
"failed componentProbe:\n\texpected: %s\n\t actual: %s",
rt.path,
actual.Handler.HTTPGet.Path,
)
t.Errorf("%s test case failed:\n\texpected: %s\n\t actual: %s",
rt.name, rt.path,
actual.Handler.HTTPGet.Path)
}
if actual.Handler.HTTPGet.Scheme != rt.scheme {
t.Errorf(
"failed componentProbe:\n\texpected: %v\n\t actual: %v",
rt.scheme,
actual.Handler.HTTPGet.Scheme,
)
t.Errorf("%s test case failed:\n\texpected: %v\n\t actual: %v",
rt.name, rt.scheme,
actual.Handler.HTTPGet.Scheme)
}
}
}