From 7dc6c94395f0d2b6b03dff75e17fc82e43603308 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 20 Dec 2019 22:03:49 +0100 Subject: [PATCH 1/2] kubeadm: use bind-address instead of address kubeadm removed the deprecated "--address" flag for controller-manager and scheduler in favor of "--bind-address" We should use bind-address to configure the manifest probe addresses. --- cmd/kubeadm/app/util/staticpod/utils.go | 12 ++++++------ cmd/kubeadm/app/util/staticpod/utils_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 242e89bcb06..16a1a4cc5ae 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -38,11 +38,11 @@ import ( ) const ( - // kubeControllerManagerAddressArg represents the address argument of the kube-controller-manager configuration. - kubeControllerManagerAddressArg = "address" + // kubeControllerManagerBindAddressArg represents the bind-address argument of the kube-controller-manager configuration. + kubeControllerManagerBindAddressArg = "bind-address" - // kubeSchedulerAddressArg represents the address argument of the kube-scheduler configuration. - kubeSchedulerAddressArg = "address" + // kubeSchedulerBindAddressArg represents the bind-address argument of the kube-scheduler configuration. + kubeSchedulerBindAddressArg = "bind-address" ) // ComponentPod returns a Pod object from the container and volume specifications @@ -252,7 +252,7 @@ func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string { // GetControllerManagerProbeAddress returns the kubernetes controller manager probe address func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { - if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerAddressArg]; exists { + if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerBindAddressArg]; exists { return addr } return "127.0.0.1" @@ -260,7 +260,7 @@ func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) stri // GetSchedulerProbeAddress returns the kubernetes scheduler probe address func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { - if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerAddressArg]; exists { + if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerBindAddressArg]; exists { return addr } return "127.0.0.1" diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 22276d52ead..2d6fc867721 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -104,7 +104,7 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ ExtraArgs: map[string]string{ - kubeControllerManagerAddressArg: "10.10.10.10", + kubeControllerManagerBindAddressArg: "10.10.10.10", }, }, }, @@ -115,7 +115,7 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ ExtraArgs: map[string]string{ - kubeControllerManagerAddressArg: "2001:abcd:bcda::1", + kubeControllerManagerBindAddressArg: "2001:abcd:bcda::1", }, }, }, From c0bc36c12e17fec422dcf6ae431f1cdb55c2d82b Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 20 Dec 2019 22:42:37 +0100 Subject: [PATCH 2/2] kubeadm: probe address for unspecified ips kubeadm deploys the apiserver, controller-manager and the scheduler using liveness probes. The bind-address option is used to configure the probe address, in case this is configured with an unspecified address, the probe will fail. When using an unspecified address the probe host field is left empty, otherwise the bind-address is used. --- cmd/kubeadm/app/util/staticpod/utils.go | 19 +++- cmd/kubeadm/app/util/staticpod/utils_test.go | 106 +++++++++++++++++++ 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 16a1a4cc5ae..56ef74fce3d 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -244,7 +244,7 @@ func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string { // probes do not support the Downward API we cannot dynamically set the advertise address to // the node's IP. The only option then is to use localhost. if endpoint != nil && endpoint.AdvertiseAddress != "" { - return endpoint.AdvertiseAddress + return getProbeAddress(endpoint.AdvertiseAddress) } return "127.0.0.1" @@ -253,7 +253,7 @@ func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string { // GetControllerManagerProbeAddress returns the kubernetes controller manager probe address func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerBindAddressArg]; exists { - return addr + return getProbeAddress(addr) } return "127.0.0.1" } @@ -261,7 +261,7 @@ func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) stri // GetSchedulerProbeAddress returns the kubernetes scheduler probe address func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerBindAddressArg]; exists { - return addr + return getProbeAddress(addr) } return "127.0.0.1" } @@ -321,3 +321,16 @@ func ManifestFilesAreEqual(path1, path2 string) (bool, error) { return bytes.Equal(content1, content2), nil } + +// getProbeAddress returns a valid probe address. +// Kubeadm uses the bind-address to configure the probe address. It's common to use the +// unspecified address "0.0.0.0" or "::" as bind-address when we want to listen in all interfaces, +// however this address can't be used as probe #86504. +// If the address is an unspecified address getProbeAddress returns empty, +// that means that kubelet will use the PodIP as probe address. +func getProbeAddress(addr string) string { + if addr == "0.0.0.0" || addr == "::" { + return "" + } + return addr +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 2d6fc867721..b262cb91b07 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -72,6 +72,20 @@ func TestGetAPIServerProbeAddress(t *testing.T) { }, expected: "2001:abcd:bcda::1", }, + { + desc: "filled in 0.0.0.0 AdvertiseAddress endpoint returns empty", + endpoint: &kubeadmapi.APIEndpoint{ + AdvertiseAddress: "0.0.0.0", + }, + expected: "", + }, + { + desc: "filled in :: AdvertiseAddress endpoint returns empty", + endpoint: &kubeadmapi.APIEndpoint{ + AdvertiseAddress: "::", + }, + expected: "", + }, } for _, test := range tests { @@ -121,6 +135,28 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { }, expected: "2001:abcd:bcda::1", }, + { + desc: "setting controller manager extra address arg to 0.0.0.0 returns empty", + cfg: &kubeadmapi.ClusterConfiguration{ + ControllerManager: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeControllerManagerBindAddressArg: "0.0.0.0", + }, + }, + }, + expected: "", + }, + { + desc: "setting controller manager extra ipv6 address arg to :: returns empty", + cfg: &kubeadmapi.ClusterConfiguration{ + ControllerManager: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeControllerManagerBindAddressArg: "::", + }, + }, + }, + expected: "", + }, } for _, test := range tests { @@ -133,6 +169,76 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { } } +func TestGetSchedulerProbeAddress(t *testing.T) { + tests := []struct { + desc string + cfg *kubeadmapi.ClusterConfiguration + expected string + }{ + { + desc: "no scheduler extra args leads to 127.0.0.1 being used", + cfg: &kubeadmapi.ClusterConfiguration{ + Scheduler: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{}, + }, + }, + expected: "127.0.0.1", + }, + { + desc: "setting scheduler extra address arg to something acknowledges it", + cfg: &kubeadmapi.ClusterConfiguration{ + Scheduler: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeSchedulerBindAddressArg: "10.10.10.10", + }, + }, + }, + expected: "10.10.10.10", + }, + { + desc: "setting scheduler extra ipv6 address arg to something acknowledges it", + cfg: &kubeadmapi.ClusterConfiguration{ + Scheduler: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeSchedulerBindAddressArg: "2001:abcd:bcda::1", + }, + }, + }, + expected: "2001:abcd:bcda::1", + }, + { + desc: "setting scheduler extra ipv6 address arg to 0.0.0.0 returns empty", + cfg: &kubeadmapi.ClusterConfiguration{ + Scheduler: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeSchedulerBindAddressArg: "0.0.0.0", + }, + }, + }, + expected: "", + }, + { + desc: "setting scheduler extra ipv6 address arg to :: returns empty", + cfg: &kubeadmapi.ClusterConfiguration{ + Scheduler: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeSchedulerBindAddressArg: "::", + }, + }, + }, + expected: "", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + actual := GetSchedulerProbeAddress(test.cfg) + if actual != test.expected { + t.Errorf("Unexpected result from GetSchedulerProbeAddress:\n\texpected: %s\n\tactual: %s", test.expected, actual) + } + }) + } +} func TestGetEtcdProbeEndpoint(t *testing.T) { var tests = []struct { name string