From d9441906c4155173ce1a75421d8fcd1d2f79c471 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Tue, 1 Sep 2020 17:07:30 +0300 Subject: [PATCH] kubeadm: make the scheduler and KCM connect to local endpoint Pinning the kube-controller-manager and kube-scheduler kubeconfig files to point to the control-plane-endpoint can be problematic during immutable upgrades if one of these components ends up contacting an N-1 kube-apiserver: https://kubernetes.io/docs/setup/release/version-skew-policy/#kube-controller-manager-kube-scheduler-and-cloud-controller-manager For example, the components can send a request for a non-existing API version. Instead of using the CPE for these components, use the LocalAPIEndpoint. This guarantees that the components would talk to the local kube-apiserver, which should be the same version, unless the user explicitly patched manifests. --- .../app/phases/kubeconfig/kubeconfig.go | 15 ++-- .../app/phases/kubeconfig/kubeconfig_test.go | 18 ++++- cmd/kubeadm/app/util/endpoint.go | 70 +++++++++++++------ 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index dbc2aed5b16..785376b3a9a 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -362,32 +362,37 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu } func getKubeConfigSpecsBase(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) { - apiServer, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) + controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) if err != nil { return nil, err } + localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) + if err != nil { + return nil, err + } + return map[string]*kubeConfigSpec{ kubeadmconstants.AdminKubeConfigFileName: { - APIServer: apiServer, + APIServer: controlPlaneEndpoint, ClientName: "kubernetes-admin", ClientCertAuth: &clientCertAuth{ Organizations: []string{kubeadmconstants.SystemPrivilegedGroup}, }, }, kubeadmconstants.KubeletKubeConfigFileName: { - APIServer: apiServer, + APIServer: controlPlaneEndpoint, ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name), ClientCertAuth: &clientCertAuth{ Organizations: []string{kubeadmconstants.NodesGroup}, }, }, kubeadmconstants.ControllerManagerKubeConfigFileName: { - APIServer: apiServer, + APIServer: localAPIEndpoint, ClientName: kubeadmconstants.ControllerManagerUser, ClientCertAuth: &clientCertAuth{}, }, kubeadmconstants.SchedulerKubeConfigFileName: { - APIServer: apiServer, + APIServer: localAPIEndpoint, ClientName: kubeadmconstants.SchedulerUser, ClientCertAuth: &clientCertAuth{}, }, diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go index 40ad4a79794..ede3bf5ada3 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -167,8 +167,22 @@ func TestGetKubeConfigSpecs(t *testing.T) { if err != nil { t.Error(err) } - if spec.APIServer != controlPlaneEndpoint { - t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer endpoint into spec for %s", assertion.kubeConfigFile) + localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) + if err != nil { + t.Error(err) + } + + switch assertion.kubeConfigFile { + case kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.KubeletKubeConfigFileName: + if spec.APIServer != controlPlaneEndpoint { + t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s", + assertion.kubeConfigFile, controlPlaneEndpoint, spec.APIServer) + } + case kubeadmconstants.ControllerManagerKubeConfigFileName, kubeadmconstants.SchedulerKubeConfigFileName: + if spec.APIServer != localAPIEndpoint { + t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s", + assertion.kubeConfigFile, localAPIEndpoint, spec.APIServer) + } } // Asserts CA certs and CA keys loaded into specs diff --git a/cmd/kubeadm/app/util/endpoint.go b/cmd/kubeadm/app/util/endpoint.go index 305f9e880c1..d11060e5e26 100644 --- a/cmd/kubeadm/app/util/endpoint.go +++ b/cmd/kubeadm/app/util/endpoint.go @@ -34,22 +34,10 @@ import ( // - if the controlPlaneEndpoint is defined but without a port number, use the controlPlaneEndpoint + localEndpoint.BindPort is used. // - Otherwise, in case the controlPlaneEndpoint is not defined, use the localEndpoint.AdvertiseAddress + the localEndpoint.BindPort. func GetControlPlaneEndpoint(controlPlaneEndpoint string, localEndpoint *kubeadmapi.APIEndpoint) (string, error) { - // parse the bind port - bindPortString := strconv.Itoa(int(localEndpoint.BindPort)) - if _, err := ParsePort(bindPortString); err != nil { - return "", errors.Wrapf(err, "invalid value %q given for api.bindPort", localEndpoint.BindPort) - } - - // parse the AdvertiseAddress - var ip = net.ParseIP(localEndpoint.AdvertiseAddress) - if ip == nil { - return "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", localEndpoint.AdvertiseAddress) - } - - // set the control-plane url using localEndpoint.AdvertiseAddress + the localEndpoint.BindPort - controlPlaneURL := &url.URL{ - Scheme: "https", - Host: net.JoinHostPort(ip.String(), bindPortString), + // get the URL of the local endpoint + localAPIEndpoint, err := GetLocalAPIEndpoint(localEndpoint) + if err != nil { + return "", err } // if the controlplane endpoint is defined @@ -62,22 +50,32 @@ func GetControlPlaneEndpoint(controlPlaneEndpoint string, localEndpoint *kubeadm } // if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport + localEndpointPort := strconv.Itoa(int(localEndpoint.BindPort)) if port != "" { - if port != bindPortString { + if port != localEndpointPort { fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address") } } else { - port = bindPortString + port = localEndpointPort } // overrides the control-plane url using the controlPlaneAddress (and eventually the bindport) - controlPlaneURL = &url.URL{ - Scheme: "https", - Host: net.JoinHostPort(host, port), - } + return formatURL(host, port).String(), nil } - return controlPlaneURL.String(), nil + return localAPIEndpoint, nil +} + +// GetLocalAPIEndpoint parses an APIEndpoint and returns it as a string, +// or returns and error in case it cannot be parsed. +func GetLocalAPIEndpoint(localEndpoint *kubeadmapi.APIEndpoint) (string, error) { + // get the URL of the local endpoint + localEndpointIP, localEndpointPort, err := parseAPIEndpoint(localEndpoint) + if err != nil { + return "", err + } + url := formatURL(localEndpointIP.String(), localEndpointPort) + return url.String(), nil } // ParseHostPort parses a network address of the form "host:port", "ipv4:port", "[ipv6]:port" into host and port; @@ -123,3 +121,29 @@ func ParsePort(port string) (int, error) { return 0, errors.New("port must be a valid number between 1 and 65535, inclusive") } + +// parseAPIEndpoint parses an APIEndpoint and returns the AdvertiseAddress as net.IP and the BindPort as string. +// If the BindPort or AdvertiseAddress are invalid it returns an error. +func parseAPIEndpoint(localEndpoint *kubeadmapi.APIEndpoint) (net.IP, string, error) { + // parse the bind port + bindPortString := strconv.Itoa(int(localEndpoint.BindPort)) + if _, err := ParsePort(bindPortString); err != nil { + return nil, "", errors.Wrapf(err, "invalid value %q given for api.bindPort", localEndpoint.BindPort) + } + + // parse the AdvertiseAddress + var ip = net.ParseIP(localEndpoint.AdvertiseAddress) + if ip == nil { + return nil, "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", localEndpoint.AdvertiseAddress) + } + + return ip, bindPortString, nil +} + +// formatURL takes a host and a port string and creates a net.URL using https scheme +func formatURL(host, port string) *url.URL { + return &url.URL{ + Scheme: "https", + Host: net.JoinHostPort(host, port), + } +}