From fcefbf70163b3094107d192e6403bc8687868d90 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Mon, 12 Mar 2018 21:04:28 +0200 Subject: [PATCH] Secure kubelet port access --- cluster/cluster.go | 1 + cluster/plan.go | 4 ++++ services/healthcheck.go | 27 +++++++++++++++++++++++---- services/kubeapi.go | 2 +- services/kubecontroller.go | 2 +- services/kubelet.go | 5 +++-- services/kubeproxy.go | 2 +- services/scheduler.go | 2 +- services/workerplane.go | 9 +++++---- 9 files changed, 40 insertions(+), 14 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index d311f2c7..cab32aea 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -100,6 +100,7 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context) error { c.PrivateRegistriesMap, processMap, kubeletProcessHostMap, + c.Certificates, ); err != nil { return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err) } diff --git a/cluster/plan.go b/cluster/plan.go index c8a646a8..d3532d05 100644 --- a/cluster/plan.go +++ b/cluster/plan.go @@ -90,6 +90,8 @@ func (c *Cluster) BuildKubeAPIProcess() v3.Process { "--client-ca-file=" + pki.GetCertPath(pki.CACertName), "--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName), "--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName), + "--kubelet-client-certificate=" + pki.GetCertPath(pki.KubeAPICertName), + "--kubelet-client-key=" + pki.GetKeyPath(pki.KubeAPICertName), "--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName), } args := []string{ @@ -201,6 +203,8 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host) v3.Process { "--allow-privileged=true", "--cloud-provider=", "--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName), + "--client-ca-file=" + pki.GetCertPath(pki.CACertName), + "--anonymous-auth=false", "--volume-plugin-dir=/var/lib/kubelet/volumeplugins", "--require-kubeconfig=True", "--fail-swap-on=" + strconv.FormatBool(c.Services.Kubelet.FailSwapOn), diff --git a/services/healthcheck.go b/services/healthcheck.go index dcd333a8..ee97d6a9 100644 --- a/services/healthcheck.go +++ b/services/healthcheck.go @@ -12,7 +12,9 @@ import ( "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" + "github.com/rancher/rke/pki" "github.com/sirupsen/logrus" + "k8s.io/client-go/util/cert" ) const ( @@ -22,13 +24,23 @@ const ( HTTPSProtoPrefix = "https://" ) -func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string) error { +func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string, certMap map[string]pki.CertificatePKI) error { log.Infof(ctx, "[healthcheck] Start Healthcheck on service [%s] on host [%s]", serviceName, host.Address) + var x509Pair tls.Certificate + port, err := getPortFromURL(url) if err != nil { return err } - client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory) + if serviceName == KubeletContainerName { + certificate := cert.EncodeCertPEM(certMap[pki.KubeNodeCertName].Certificate) + key := cert.EncodePrivateKeyPEM(certMap[pki.KubeNodeCertName].Key) + x509Pair, err = tls.X509KeyPair(certificate, key) + if err != nil { + return err + } + } + client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory, &x509Pair) if err != nil { return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address) } @@ -44,7 +56,7 @@ func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, l return fmt.Errorf("Failed to verify healthcheck: %v", err) } -func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory hosts.DialerFactory) (*http.Client, error) { +func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory hosts.DialerFactory, x509KeyPair *tls.Certificate) (*http.Client, error) { host.LocalConnPort = port var factory hosts.DialerFactory if localConnDialerFactory == nil { @@ -56,10 +68,17 @@ func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory if err != nil { return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", host.Address, err) } + tlsConfig := &tls.Config{InsecureSkipVerify: true} + if x509KeyPair != nil { + tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{*x509KeyPair}, + } + } return &http.Client{ Transport: &http.Transport{ Dial: dialer, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: tlsConfig, }, }, nil } diff --git a/services/kubeapi.go b/services/kubeapi.go index 497fa7f1..58ab1259 100644 --- a/services/kubeapi.go +++ b/services/kubeapi.go @@ -14,7 +14,7 @@ func runKubeAPI(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole, prsMap); err != nil { return err } - return runHealthcheck(ctx, host, KubeAPIContainerName, df, healthCheckURL) + return runHealthcheck(ctx, host, KubeAPIContainerName, df, healthCheckURL, nil) } func removeKubeAPI(ctx context.Context, host *hosts.Host) error { diff --git a/services/kubecontroller.go b/services/kubecontroller.go index 395a43f4..a2cca2bf 100644 --- a/services/kubecontroller.go +++ b/services/kubecontroller.go @@ -13,7 +13,7 @@ func runKubeController(ctx context.Context, host *hosts.Host, df hosts.DialerFac if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeControllerContainerName, host.Address, ControlRole, prsMap); err != nil { return err } - return runHealthcheck(ctx, host, KubeControllerContainerName, df, healthCheckURL) + return runHealthcheck(ctx, host, KubeControllerContainerName, df, healthCheckURL, nil) } func removeKubeController(ctx context.Context, host *hosts.Host) error { diff --git a/services/kubelet.go b/services/kubelet.go index 4036264f..3eddd875 100644 --- a/services/kubelet.go +++ b/services/kubelet.go @@ -5,15 +5,16 @@ import ( "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" + "github.com/rancher/rke/pki" "github.com/rancher/types/apis/management.cattle.io/v3" ) -func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeletProcess v3.Process) error { +func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeletProcess v3.Process, certMap map[string]pki.CertificatePKI) error { imageCfg, hostCfg, healthCheckURL := GetProcessConfig(kubeletProcess) if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeletContainerName, host.Address, WorkerRole, prsMap); err != nil { return err } - return runHealthcheck(ctx, host, KubeletContainerName, df, healthCheckURL) + return runHealthcheck(ctx, host, KubeletContainerName, df, healthCheckURL, certMap) } func removeKubelet(ctx context.Context, host *hosts.Host) error { diff --git a/services/kubeproxy.go b/services/kubeproxy.go index cfd27e0d..42a9efb0 100644 --- a/services/kubeproxy.go +++ b/services/kubeproxy.go @@ -13,7 +13,7 @@ func runKubeproxy(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeproxyContainerName, host.Address, WorkerRole, prsMap); err != nil { return err } - return runHealthcheck(ctx, host, KubeproxyContainerName, df, healthCheckURL) + return runHealthcheck(ctx, host, KubeproxyContainerName, df, healthCheckURL, nil) } func removeKubeproxy(ctx context.Context, host *hosts.Host) error { diff --git a/services/scheduler.go b/services/scheduler.go index f6eada2a..0086e9f4 100644 --- a/services/scheduler.go +++ b/services/scheduler.go @@ -13,7 +13,7 @@ func runScheduler(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, SchedulerContainerName, host.Address, ControlRole, prsMap); err != nil { return err } - return runHealthcheck(ctx, host, SchedulerContainerName, df, healthCheckURL) + return runHealthcheck(ctx, host, SchedulerContainerName, df, healthCheckURL, nil) } func removeScheduler(ctx context.Context, host *hosts.Host) error { diff --git a/services/workerplane.go b/services/workerplane.go index 89fa2294..4c354e69 100644 --- a/services/workerplane.go +++ b/services/workerplane.go @@ -5,6 +5,7 @@ import ( "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" + "github.com/rancher/rke/pki" "github.com/rancher/types/apis/management.cattle.io/v3" "golang.org/x/sync/errgroup" ) @@ -13,7 +14,7 @@ const ( unschedulableEtcdTaint = "node-role.kubernetes.io/etcd=true:NoExecute" ) -func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process) error { +func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process, certMap map[string]pki.CertificatePKI) error { log.Infof(ctx, "[%s] Building up Worker Plane..", WorkerRole) var errgrp errgroup.Group for _, host := range allHosts { @@ -26,7 +27,7 @@ func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialer hostProcessMap := copyProcessMap(processMap) errgrp.Go(func() error { hostProcessMap[KubeletContainerName] = kubeletProcessHostMap[runHost] - return doDeployWorkerPlane(ctx, runHost, localConnDialerFactory, prsMap, hostProcessMap) + return doDeployWorkerPlane(ctx, runHost, localConnDialerFactory, prsMap, hostProcessMap, certMap) }) } if err := errgrp.Wait(); err != nil { @@ -65,7 +66,7 @@ func RemoveWorkerPlane(ctx context.Context, workerHosts []*hosts.Host, force boo func doDeployWorkerPlane(ctx context.Context, host *hosts.Host, localConnDialerFactory hosts.DialerFactory, - prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process) error { + prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, certMap map[string]pki.CertificatePKI) error { // run nginx proxy if !host.IsControl { if err := runNginxProxy(ctx, host, prsMap, processMap[NginxProxyContainerName]); err != nil { @@ -77,7 +78,7 @@ func doDeployWorkerPlane(ctx context.Context, host *hosts.Host, return err } // run kubelet - if err := runKubelet(ctx, host, localConnDialerFactory, prsMap, processMap[KubeletContainerName]); err != nil { + if err := runKubelet(ctx, host, localConnDialerFactory, prsMap, processMap[KubeletContainerName], certMap); err != nil { return err } return runKubeproxy(ctx, host, localConnDialerFactory, prsMap, processMap[KubeproxyContainerName])