From 335a3e9efbc8316cd030a42645d79e2a97bf2655 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 29 Nov 2019 07:22:54 +0100 Subject: [PATCH] kubeadm: use correct IP family for etcd localhost kubeadm always use the IPv4 localhost address by defaultA for etcd The probe hostname is obtained before the generation of the etcd parameters, so it can't detect the right IP familiy for the host of the probe. This causes that with IPv6 clusters doesn't work because the probe uses the IPv4 localhost address. This patchs configures the right localhost address based on the used AdvertiseAddress IP family. --- .../app/phases/controlplane/manifests.go | 7 +- .../app/phases/controlplane/manifests_test.go | 10 +-- cmd/kubeadm/app/phases/etcd/BUILD | 1 + cmd/kubeadm/app/phases/etcd/local.go | 15 +++- cmd/kubeadm/app/phases/etcd/local_test.go | 4 +- cmd/kubeadm/app/util/staticpod/utils.go | 5 +- cmd/kubeadm/app/util/staticpod/utils_test.go | 88 ++++++++++++++++++- 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 7afe84a5290..a3f77f6515d 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -162,7 +162,12 @@ func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint } } else { // Default to etcd static pod on localhost - defaultArguments["etcd-servers"] = fmt.Sprintf("https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort) + // localhost IP family should be the same that the AdvertiseAddress + etcdLocalhostAddress := "127.0.0.1" + if utilsnet.IsIPv6String(localAPIEndpoint.AdvertiseAddress) { + etcdLocalhostAddress = "::1" + } + defaultArguments["etcd-servers"] = fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort))) defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName) defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName) defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 5f78aa29106..9ae2a536498 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -266,7 +266,7 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), + fmt.Sprintf("--etcd-servers=https://[::1]:%d", kubeadmconstants.EtcdListenClientPort), "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", @@ -278,7 +278,7 @@ func TestGetAPIServerCommand(t *testing.T) { Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, Etcd: kubeadmapi.Etcd{ External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, + Endpoints: []string{"https://[2001:abcd:bcda::1]:2379", "https://[2001:abcd:bcda::2]:2379"}, CAFile: "fuz", CertFile: "fiz", KeyFile: "faz", @@ -311,7 +311,7 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=https://8.6.4.1:2379,https://8.6.4.2:2379", + "--etcd-servers=https://[2001:abcd:bcda::1]:2379,https://[2001:abcd:bcda::2]:2379", "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", @@ -323,7 +323,7 @@ func TestGetAPIServerCommand(t *testing.T) { Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, Etcd: kubeadmapi.Etcd{ External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"}, + Endpoints: []string{"http://[::1]:2379", "http://[::1]:2380"}, }, }, CertificatesDir: testCertsDir, @@ -353,7 +353,7 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:2380", + "--etcd-servers=http://[::1]:2379,http://[::1]:2380", }, }, { diff --git a/cmd/kubeadm/app/phases/etcd/BUILD b/cmd/kubeadm/app/phases/etcd/BUILD index 5da59b22551..5df019b65e1 100644 --- a/cmd/kubeadm/app/phases/etcd/BUILD +++ b/cmd/kubeadm/app/phases/etcd/BUILD @@ -35,6 +35,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 3dffd842f54..05fc6543967 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -18,7 +18,9 @@ package etcd import ( "fmt" + "net" "path/filepath" + "strconv" "strings" "time" @@ -33,6 +35,7 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" + utilsnet "k8s.io/utils/net" ) const ( @@ -176,7 +179,8 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType), certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType), } - probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd) + // probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress + probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress)) return staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.Etcd, Command: getEtcdCommand(cfg, endpoint, nodeName, initialCluster), @@ -193,9 +197,14 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A // getEtcdCommand builds the right etcd command from the given config object func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string { + // localhost IP family should be the same that the AdvertiseAddress + etcdLocalhostAddress := "127.0.0.1" + if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) { + etcdLocalhostAddress = "::1" + } defaultArguments := map[string]string{ "name": nodeName, - "listen-client-urls": fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP("127.0.0.1"), etcdutil.GetClientURL(endpoint)), + "listen-client-urls": fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)), "advertise-client-urls": etcdutil.GetClientURL(endpoint), "listen-peer-urls": etcdutil.GetPeerURL(endpoint), "initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint), @@ -209,7 +218,7 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), "peer-client-cert-auth": "true", "snapshot-count": "10000", - "listen-metrics-urls": fmt.Sprintf("http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort), + "listen-metrics-urls": fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))), } if len(initialCluster) == 0 { diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 70d036df608..b2c0dd0ccdc 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -266,8 +266,8 @@ func TestGetEtcdCommand(t *testing.T) { expected: []string{ "etcd", "--name=foo", - fmt.Sprintf("--listen-client-urls=https://127.0.0.1:%d,https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort), + fmt.Sprintf("--listen-client-urls=https://[::1]:%d,https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort), + fmt.Sprintf("--listen-metrics-urls=http://[::1]:%d", kubeadmconstants.EtcdMetricsPort), fmt.Sprintf("--advertise-client-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort), fmt.Sprintf("--listen-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort), fmt.Sprintf("--initial-advertise-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort), diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index e661558a8a0..242e89bcb06 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -269,8 +269,11 @@ func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { // GetEtcdProbeEndpoint takes a kubeadm Etcd configuration object and attempts to parse // the first URL in the listen-metrics-urls argument, returning an etcd probe hostname, // port and scheme -func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd) (string, int, v1.URIScheme) { +func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd, isIPv6 bool) (string, int, v1.URIScheme) { localhost := "127.0.0.1" + if isIPv6 { + localhost = "::1" + } if cfg.Local == nil || cfg.Local.ExtraArgs == nil { return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP } diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index ac6a1f757a3..22276d52ead 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -65,6 +65,13 @@ func TestGetAPIServerProbeAddress(t *testing.T) { }, expected: "10.10.10.10", }, + { + desc: "filled in ipv6 AdvertiseAddress endpoint returns it", + endpoint: &kubeadmapi.APIEndpoint{ + AdvertiseAddress: "2001:abcd:bcda::1", + }, + expected: "2001:abcd:bcda::1", + }, } for _, test := range tests { @@ -103,6 +110,17 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { }, expected: "10.10.10.10", }, + { + desc: "setting controller manager extra ipv6 address arg to something acknowledges it", + cfg: &kubeadmapi.ClusterConfiguration{ + ControllerManager: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + kubeControllerManagerAddressArg: "2001:abcd:bcda::1", + }, + }, + }, + expected: "2001:abcd:bcda::1", + }, } for _, test := range tests { @@ -119,6 +137,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { var tests = []struct { name string cfg *kubeadmapi.Etcd + isIPv6 bool expectedHostname string expectedPort int expectedScheme v1.URIScheme @@ -131,6 +150,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { "listen-metrics-urls": "https://1.2.3.4:1234,https://4.3.2.1:2381"}, }, }, + isIPv6: false, expectedHostname: "1.2.3.4", expectedPort: 1234, expectedScheme: v1.URISchemeHTTPS, @@ -143,6 +163,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { "listen-metrics-urls": "http://1.2.3.4:1234"}, }, }, + isIPv6: false, expectedHostname: "1.2.3.4", expectedPort: 1234, expectedScheme: v1.URISchemeHTTP, @@ -155,6 +176,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { "listen-metrics-urls": "1.2.3.4"}, }, }, + isIPv6: false, expectedHostname: "127.0.0.1", expectedPort: kubeadmconstants.EtcdMetricsPort, expectedScheme: v1.URISchemeHTTP, @@ -167,23 +189,87 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { "listen-metrics-urls": "https://1.2.3.4"}, }, }, + isIPv6: false, expectedHostname: "1.2.3.4", expectedPort: kubeadmconstants.EtcdMetricsPort, expectedScheme: v1.URISchemeHTTPS, }, + { + name: "etcd probe URL from two IPv6 URLs", + cfg: &kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-metrics-urls": "https://[2001:abcd:bcda::1]:1234,https://[2001:abcd:bcda::2]:2381"}, + }, + }, + isIPv6: true, + expectedHostname: "2001:abcd:bcda::1", + expectedPort: 1234, + expectedScheme: v1.URISchemeHTTPS, + }, + { + name: "etcd probe localhost IPv6 URL with HTTP scheme", + cfg: &kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-metrics-urls": "http://[::1]:1234"}, + }, + }, + isIPv6: true, + expectedHostname: "::1", + expectedPort: 1234, + expectedScheme: v1.URISchemeHTTP, + }, + { + name: "etcd probe IPv6 URL with HTTP scheme", + cfg: &kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-metrics-urls": "http://[2001:abcd:bcda::1]:1234"}, + }, + }, + isIPv6: true, + expectedHostname: "2001:abcd:bcda::1", + expectedPort: 1234, + expectedScheme: v1.URISchemeHTTP, + }, + { + name: "etcd probe IPv6 URL without port", + cfg: &kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-metrics-urls": "https://[2001:abcd:bcda::1]"}, + }, + }, + isIPv6: true, + expectedHostname: "2001:abcd:bcda::1", + expectedPort: kubeadmconstants.EtcdMetricsPort, + expectedScheme: v1.URISchemeHTTPS, + }, { name: "etcd probe URL from defaults", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{}, }, + isIPv6: false, expectedHostname: "127.0.0.1", expectedPort: kubeadmconstants.EtcdMetricsPort, expectedScheme: v1.URISchemeHTTP, }, + { + name: "etcd probe URL from defaults if IPv6", + cfg: &kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{}, + }, + isIPv6: true, + expectedHostname: "::1", + expectedPort: kubeadmconstants.EtcdMetricsPort, + expectedScheme: v1.URISchemeHTTP, + }, } for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { - hostname, port, scheme := GetEtcdProbeEndpoint(rt.cfg) + hostname, port, scheme := GetEtcdProbeEndpoint(rt.cfg, rt.isIPv6) if hostname != rt.expectedHostname { t.Errorf("%q test case failed:\n\texpected hostname: %s\n\tgot: %s", rt.name, rt.expectedHostname, hostname)