github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-14 21:53:52 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #81385 from neolit123/etcd-probe

Browse Source 
kubeadm: use etcd's /health endpoint for it's liveness probe
This commit is contained in:
Kubernetes Prow Robot
2019-08-16 14:06:09 -07:00
committed by GitHub
parent 8f0d626228 99b64f1e8e
commit 9e60bedebe
8 changed files with 112 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kubeadm/app/util/staticpod/BUILD
View File

@@ -12,6 +12,7 @@ go_test(
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -31,6 +32,7 @@ go_library(
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)

120
cmd/kubeadm/app/util/staticpod/utils.go
View File

@@ -20,7 +20,6 @@ import (
"bytes"
"fmt"
"io/ioutil"
"net"
"net/url"
"os"
"sort"
@@ -31,6 +30,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -43,9 +43,6 @@ const (
// kubeSchedulerAddressArg represents the address argument of the kube-scheduler configuration.
kubeSchedulerAddressArg = "address"
// etcdListenClientURLsArg represents the listen-client-urls argument of the etcd configuration.
etcdListenClientURLsArg = "listen-client-urls"
)
// ComponentPod returns a Pod object from the container and volume specifications
@@ -80,24 +77,6 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
}
}
// EtcdProbe is a helper function for building a shell-based, etcdctl v1.Probe object to healthcheck etcd
func EtcdProbe(cfg *kubeadmapi.Etcd, port int, certsDir string, CACertName string, CertName string, KeyName string) *v1.Probe {
tlsFlags := fmt.Sprintf("--cacert=%[1]s/%[2]s --cert=%[1]s/%[3]s --key=%[1]s/%[4]s", certsDir, CACertName, CertName, KeyName)
// etcd pod is alive if a linearizable get succeeds.
cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://[%s]:%d %s get foo", GetEtcdProbeAddress(cfg), port, tlsFlags)
return &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
Command: []string{"/bin/sh", "-ec", cmd},
},
},
InitialDelaySeconds: 15,
TimeoutSeconds: 15,
FailureThreshold: 8,
}
}
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
return v1.Volume{
@@ -238,6 +217,23 @@ func ReadStaticPodFromDisk(manifestPath string) (*v1.Pod, error) {
return pod, nil
}
// LivenessProbe creates a Probe object with a HTTPGet handler
func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
return &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Host: host,
Path: path,
Port: intstr.FromInt(port),
Scheme: scheme,
},
},
InitialDelaySeconds: 15,
TimeoutSeconds: 15,
FailureThreshold: 8,
}
}
// GetAPIServerProbeAddress returns the probe address for the API server
func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string {
// In the case of a self-hosted deployment, the initial host on which kubeadm --init is run,
@@ -270,51 +266,43 @@ func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
return "127.0.0.1"
}
// GetEtcdProbeAddress returns the etcd probe address
func GetEtcdProbeAddress(cfg *kubeadmapi.Etcd) string {
if cfg.Local != nil && cfg.Local.ExtraArgs != nil {
if arg, exists := cfg.Local.ExtraArgs[etcdListenClientURLsArg]; exists {
// Use the first url in the listen-client-urls if multiple url's are specified.
if strings.ContainsAny(arg, ",") {
arg = strings.Split(arg, ",")[0]
}
parsedURL, err := url.Parse(arg)
if err != nil || parsedURL.Hostname() == "" {
return "127.0.0.1"
}
// Return the IP if the URL contains an address instead of a name.
if ip := net.ParseIP(parsedURL.Hostname()); ip != nil {
// etcdctl doesn't support auto-converting zero addresses into loopback addresses
if ip.Equal(net.IPv4zero) {
return "127.0.0.1"
}
if ip.Equal(net.IPv6zero) {
return net.IPv6loopback.String()
}
return ip.String()
}
// Use the local resolver to try resolving the name within the URL.
// If the name can not be resolved, return an IPv4 loopback address.
// Otherwise, select the first valid IPv4 address.
// If the name does not resolve to an IPv4 address, select the first valid IPv6 address.
addrs, err := net.LookupIP(parsedURL.Hostname())
if err != nil {
return "127.0.0.1"
}
var ip net.IP
for _, addr := range addrs {
if addr.To4() != nil {
ip = addr
break
}
if addr.To16() != nil && ip == nil {
ip = addr
}
}
return ip.String()
}
// GetEtcdProbeEndpoint takes a kubeadm Etcd configuration object and attempts to parse
// the first URL in the listen-metrics-urls argument, returning an etcd probe hostname,
// port and scheme
func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd) (string, int, v1.URIScheme) {
localhost := "127.0.0.1"
if cfg.Local == nil || cfg.Local.ExtraArgs == nil {
return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
}
return "127.0.0.1"
if arg, exists := cfg.Local.ExtraArgs["listen-metrics-urls"]; exists {
// Use the first url in the listen-metrics-urls if multiple URL's are specified.
arg = strings.Split(arg, ",")[0]
parsedURL, err := url.Parse(arg)
if err != nil {
return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
}
// Parse scheme
scheme := v1.URISchemeHTTP
if parsedURL.Scheme == "https" {
scheme = v1.URISchemeHTTPS
}
// Parse hostname
hostname := parsedURL.Hostname()
if len(hostname) == 0 {
hostname = localhost
}
// Parse port
port := kubeadmconstants.EtcdMetricsPort
portStr := parsedURL.Port()
if len(portStr) != 0 {
p, err := util.ParsePort(portStr)
if err == nil {
port = p
}
}
return hostname, port, scheme
}
return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
}
// ManifestFilesAreEqual compares 2 files. It returns true if their contents are equal, false otherwise

131
cmd/kubeadm/app/util/staticpod/utils_test.go
View File

@@ -30,6 +30,7 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
)
@@ -114,130 +115,86 @@ func TestGetControllerManagerProbeAddress(t *testing.T) {
}
}
func TestEtcdProbe(t *testing.T) {
func TestGetEtcdProbeEndpoint(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.Etcd
port int
certsDir string
cacert string
cert string
key string
expected string
name string
cfg *kubeadmapi.Etcd
expectedHostname string
expectedPort int
expectedScheme v1.URIScheme
}{
{
name: "valid etcd probe using listen-client-urls IPv4 addresses",
name: "etcd probe URL from two URLs",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
"listen-metrics-urls": "https://1.2.3.4:1234,https://4.3.2.1:2381"},
},
},
port: 1,
certsDir: "secretsA",
cacert: "ca1",
cert: "cert1",
key: "key1",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[1.2.3.4]:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo",
expectedHostname: "1.2.3.4",
expectedPort: 1234,
expectedScheme: v1.URISchemeHTTPS,
},
{
name: "valid etcd probe using listen-client-urls unspecified IPv6 address",
name: "etcd probe URL with HTTP scheme",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://[0:0:0:0:0:0:0:0]:2379"},
"listen-metrics-urls": "http://1.2.3.4:1234"},
},
},
port: 1,
certsDir: "secretsB",
cacert: "ca2",
cert: "cert2",
key: "key2",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
expectedHostname: "1.2.3.4",
expectedPort: 1234,
expectedScheme: v1.URISchemeHTTP,
},
{
name: "valid etcd probe using listen-client-urls unspecified IPv6 address 2",
name: "etcd probe URL without scheme should result in defaults",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://[::0:0]:2379"},
"listen-metrics-urls": "1.2.3.4"},
},
},
port: 1,
certsDir: "secretsB",
cacert: "ca2",
cert: "cert2",
key: "key2",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
expectedHostname: "127.0.0.1",
expectedPort: kubeadmconstants.EtcdMetricsPort,
expectedScheme: v1.URISchemeHTTP,
},
{
name: "valid etcd probe using listen-client-urls unspecified IPv6 address 3",
name: "etcd probe URL without port",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://[::]:2379"},
"listen-metrics-urls": "https://1.2.3.4"},
},
},
port: 1,
certsDir: "secretsB",
cacert: "ca2",
cert: "cert2",
key: "key2",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
expectedHostname: "1.2.3.4",
expectedPort: kubeadmconstants.EtcdMetricsPort,
expectedScheme: v1.URISchemeHTTPS,
},
{
name: "valid etcd probe using listen-client-urls unspecified IPv4 address",
name: "etcd probe URL from defaults",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
},
Local: &kubeadmapi.LocalEtcd{},
},
port: 1,
certsDir: "secretsA",
cacert: "ca1",
cert: "cert1",
key: "key1",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[1.2.3.4]:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo",
},
{
name: "valid etcd probe using listen-client-urls IPv6 addresses",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"},
},
},
port: 1,
certsDir: "secretsB",
cacert: "ca2",
cert: "cert2",
key: "key2",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[2001:db8::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
},
{
name: "valid IPv4 etcd probe using hostname for listen-client-urls",
cfg: &kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ExtraArgs: map[string]string{
"listen-client-urls": "http://localhost:2379"},
},
},
port: 1,
certsDir: "secretsC",
cacert: "ca3",
cert: "cert3",
key: "key3",
expected: "ETCDCTL_API=3 etcdctl --endpoints=https://[127.0.0.1]:1 --cacert=secretsC/ca3 --cert=secretsC/cert3 --key=secretsC/key3 get foo",
expectedHostname: "127.0.0.1",
expectedPort: kubeadmconstants.EtcdMetricsPort,
expectedScheme: v1.URISchemeHTTP,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
actual := EtcdProbe(rt.cfg, rt.port, rt.certsDir, rt.cacert, rt.cert, rt.key)
if actual.Handler.Exec.Command[2] != rt.expected {
t.Errorf("%s test case failed:\n\texpected: %s\n\t actual: %s",
rt.name, rt.expected,
actual.Handler.Exec.Command[2])
hostname, port, scheme := GetEtcdProbeEndpoint(rt.cfg)
if hostname != rt.expectedHostname {
t.Errorf("%q test case failed:\n\texpected hostname: %s\n\tgot: %s",
rt.name, rt.expectedHostname, hostname)
}
if port != rt.expectedPort {
t.Errorf("%q test case failed:\n\texpected port: %d\n\tgot: %d",
rt.name, rt.expectedPort, port)
}
if scheme != rt.expectedScheme {
t.Errorf("%q test case failed:\n\texpected scheme: %v\n\tgot: %v",
rt.name, rt.expectedScheme, scheme)
}
})
}