mirror of
https://github.com/rancher/rke.git
synced 2025-08-11 11:42:44 +00:00
Merge pull request #410 from galal-hussein/secure_kubelet
Secure kubelet port access
This commit is contained in:
commit
bc05bc2dcc
@ -100,6 +100,7 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context) error {
|
|||||||
c.PrivateRegistriesMap,
|
c.PrivateRegistriesMap,
|
||||||
processMap,
|
processMap,
|
||||||
kubeletProcessHostMap,
|
kubeletProcessHostMap,
|
||||||
|
c.Certificates,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err)
|
return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -90,6 +90,8 @@ func (c *Cluster) BuildKubeAPIProcess() v3.Process {
|
|||||||
"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
|
"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
|
||||||
"--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName),
|
"--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName),
|
||||||
"--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
|
"--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
|
||||||
|
"--kubelet-client-certificate=" + pki.GetCertPath(pki.KubeAPICertName),
|
||||||
|
"--kubelet-client-key=" + pki.GetKeyPath(pki.KubeAPICertName),
|
||||||
"--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
|
"--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
|
||||||
}
|
}
|
||||||
args := []string{
|
args := []string{
|
||||||
@ -201,6 +203,8 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host) v3.Process {
|
|||||||
"--allow-privileged=true",
|
"--allow-privileged=true",
|
||||||
"--cloud-provider=",
|
"--cloud-provider=",
|
||||||
"--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName),
|
"--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName),
|
||||||
|
"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
|
||||||
|
"--anonymous-auth=false",
|
||||||
"--volume-plugin-dir=/var/lib/kubelet/volumeplugins",
|
"--volume-plugin-dir=/var/lib/kubelet/volumeplugins",
|
||||||
"--require-kubeconfig=True",
|
"--require-kubeconfig=True",
|
||||||
"--fail-swap-on=" + strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
|
"--fail-swap-on=" + strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
|
||||||
|
@ -12,7 +12,9 @@ import (
|
|||||||
|
|
||||||
"github.com/rancher/rke/hosts"
|
"github.com/rancher/rke/hosts"
|
||||||
"github.com/rancher/rke/log"
|
"github.com/rancher/rke/log"
|
||||||
|
"github.com/rancher/rke/pki"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
"k8s.io/client-go/util/cert"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -22,13 +24,23 @@ const (
|
|||||||
HTTPSProtoPrefix = "https://"
|
HTTPSProtoPrefix = "https://"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string) error {
|
func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string, certMap map[string]pki.CertificatePKI) error {
|
||||||
log.Infof(ctx, "[healthcheck] Start Healthcheck on service [%s] on host [%s]", serviceName, host.Address)
|
log.Infof(ctx, "[healthcheck] Start Healthcheck on service [%s] on host [%s]", serviceName, host.Address)
|
||||||
|
var x509Pair tls.Certificate
|
||||||
|
|
||||||
port, err := getPortFromURL(url)
|
port, err := getPortFromURL(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory)
|
if serviceName == KubeletContainerName {
|
||||||
|
certificate := cert.EncodeCertPEM(certMap[pki.KubeNodeCertName].Certificate)
|
||||||
|
key := cert.EncodePrivateKeyPEM(certMap[pki.KubeNodeCertName].Key)
|
||||||
|
x509Pair, err = tls.X509KeyPair(certificate, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory, &x509Pair)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address)
|
return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address)
|
||||||
}
|
}
|
||||||
@ -44,7 +56,7 @@ func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, l
|
|||||||
return fmt.Errorf("Failed to verify healthcheck: %v", err)
|
return fmt.Errorf("Failed to verify healthcheck: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory hosts.DialerFactory) (*http.Client, error) {
|
func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory hosts.DialerFactory, x509KeyPair *tls.Certificate) (*http.Client, error) {
|
||||||
host.LocalConnPort = port
|
host.LocalConnPort = port
|
||||||
var factory hosts.DialerFactory
|
var factory hosts.DialerFactory
|
||||||
if localConnDialerFactory == nil {
|
if localConnDialerFactory == nil {
|
||||||
@ -56,10 +68,17 @@ func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", host.Address, err)
|
return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", host.Address, err)
|
||||||
}
|
}
|
||||||
|
tlsConfig := &tls.Config{InsecureSkipVerify: true}
|
||||||
|
if x509KeyPair != nil {
|
||||||
|
tlsConfig = &tls.Config{
|
||||||
|
InsecureSkipVerify: true,
|
||||||
|
Certificates: []tls.Certificate{*x509KeyPair},
|
||||||
|
}
|
||||||
|
}
|
||||||
return &http.Client{
|
return &http.Client{
|
||||||
Transport: &http.Transport{
|
Transport: &http.Transport{
|
||||||
Dial: dialer,
|
Dial: dialer,
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
TLSClientConfig: tlsConfig,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
@ -14,7 +14,7 @@ func runKubeAPI(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p
|
|||||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole, prsMap); err != nil {
|
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole, prsMap); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runHealthcheck(ctx, host, KubeAPIContainerName, df, healthCheckURL)
|
return runHealthcheck(ctx, host, KubeAPIContainerName, df, healthCheckURL, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeKubeAPI(ctx context.Context, host *hosts.Host) error {
|
func removeKubeAPI(ctx context.Context, host *hosts.Host) error {
|
||||||
|
@ -13,7 +13,7 @@ func runKubeController(ctx context.Context, host *hosts.Host, df hosts.DialerFac
|
|||||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeControllerContainerName, host.Address, ControlRole, prsMap); err != nil {
|
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeControllerContainerName, host.Address, ControlRole, prsMap); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runHealthcheck(ctx, host, KubeControllerContainerName, df, healthCheckURL)
|
return runHealthcheck(ctx, host, KubeControllerContainerName, df, healthCheckURL, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeKubeController(ctx context.Context, host *hosts.Host) error {
|
func removeKubeController(ctx context.Context, host *hosts.Host) error {
|
||||||
|
@ -5,15 +5,16 @@ import (
|
|||||||
|
|
||||||
"github.com/rancher/rke/docker"
|
"github.com/rancher/rke/docker"
|
||||||
"github.com/rancher/rke/hosts"
|
"github.com/rancher/rke/hosts"
|
||||||
|
"github.com/rancher/rke/pki"
|
||||||
"github.com/rancher/types/apis/management.cattle.io/v3"
|
"github.com/rancher/types/apis/management.cattle.io/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeletProcess v3.Process) error {
|
func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeletProcess v3.Process, certMap map[string]pki.CertificatePKI) error {
|
||||||
imageCfg, hostCfg, healthCheckURL := GetProcessConfig(kubeletProcess)
|
imageCfg, hostCfg, healthCheckURL := GetProcessConfig(kubeletProcess)
|
||||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeletContainerName, host.Address, WorkerRole, prsMap); err != nil {
|
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeletContainerName, host.Address, WorkerRole, prsMap); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runHealthcheck(ctx, host, KubeletContainerName, df, healthCheckURL)
|
return runHealthcheck(ctx, host, KubeletContainerName, df, healthCheckURL, certMap)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeKubelet(ctx context.Context, host *hosts.Host) error {
|
func removeKubelet(ctx context.Context, host *hosts.Host) error {
|
||||||
|
@ -13,7 +13,7 @@ func runKubeproxy(ctx context.Context, host *hosts.Host, df hosts.DialerFactory,
|
|||||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeproxyContainerName, host.Address, WorkerRole, prsMap); err != nil {
|
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeproxyContainerName, host.Address, WorkerRole, prsMap); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runHealthcheck(ctx, host, KubeproxyContainerName, df, healthCheckURL)
|
return runHealthcheck(ctx, host, KubeproxyContainerName, df, healthCheckURL, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeKubeproxy(ctx context.Context, host *hosts.Host) error {
|
func removeKubeproxy(ctx context.Context, host *hosts.Host) error {
|
||||||
|
@ -13,7 +13,7 @@ func runScheduler(ctx context.Context, host *hosts.Host, df hosts.DialerFactory,
|
|||||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, SchedulerContainerName, host.Address, ControlRole, prsMap); err != nil {
|
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, SchedulerContainerName, host.Address, ControlRole, prsMap); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runHealthcheck(ctx, host, SchedulerContainerName, df, healthCheckURL)
|
return runHealthcheck(ctx, host, SchedulerContainerName, df, healthCheckURL, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeScheduler(ctx context.Context, host *hosts.Host) error {
|
func removeScheduler(ctx context.Context, host *hosts.Host) error {
|
||||||
|
@ -5,6 +5,7 @@ import (
|
|||||||
|
|
||||||
"github.com/rancher/rke/hosts"
|
"github.com/rancher/rke/hosts"
|
||||||
"github.com/rancher/rke/log"
|
"github.com/rancher/rke/log"
|
||||||
|
"github.com/rancher/rke/pki"
|
||||||
"github.com/rancher/types/apis/management.cattle.io/v3"
|
"github.com/rancher/types/apis/management.cattle.io/v3"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
@ -13,7 +14,7 @@ const (
|
|||||||
unschedulableEtcdTaint = "node-role.kubernetes.io/etcd=true:NoExecute"
|
unschedulableEtcdTaint = "node-role.kubernetes.io/etcd=true:NoExecute"
|
||||||
)
|
)
|
||||||
|
|
||||||
func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process) error {
|
func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process, certMap map[string]pki.CertificatePKI) error {
|
||||||
log.Infof(ctx, "[%s] Building up Worker Plane..", WorkerRole)
|
log.Infof(ctx, "[%s] Building up Worker Plane..", WorkerRole)
|
||||||
var errgrp errgroup.Group
|
var errgrp errgroup.Group
|
||||||
for _, host := range allHosts {
|
for _, host := range allHosts {
|
||||||
@ -26,7 +27,7 @@ func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialer
|
|||||||
hostProcessMap := copyProcessMap(processMap)
|
hostProcessMap := copyProcessMap(processMap)
|
||||||
errgrp.Go(func() error {
|
errgrp.Go(func() error {
|
||||||
hostProcessMap[KubeletContainerName] = kubeletProcessHostMap[runHost]
|
hostProcessMap[KubeletContainerName] = kubeletProcessHostMap[runHost]
|
||||||
return doDeployWorkerPlane(ctx, runHost, localConnDialerFactory, prsMap, hostProcessMap)
|
return doDeployWorkerPlane(ctx, runHost, localConnDialerFactory, prsMap, hostProcessMap, certMap)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if err := errgrp.Wait(); err != nil {
|
if err := errgrp.Wait(); err != nil {
|
||||||
@ -65,7 +66,7 @@ func RemoveWorkerPlane(ctx context.Context, workerHosts []*hosts.Host, force boo
|
|||||||
|
|
||||||
func doDeployWorkerPlane(ctx context.Context, host *hosts.Host,
|
func doDeployWorkerPlane(ctx context.Context, host *hosts.Host,
|
||||||
localConnDialerFactory hosts.DialerFactory,
|
localConnDialerFactory hosts.DialerFactory,
|
||||||
prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process) error {
|
prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, certMap map[string]pki.CertificatePKI) error {
|
||||||
// run nginx proxy
|
// run nginx proxy
|
||||||
if !host.IsControl {
|
if !host.IsControl {
|
||||||
if err := runNginxProxy(ctx, host, prsMap, processMap[NginxProxyContainerName]); err != nil {
|
if err := runNginxProxy(ctx, host, prsMap, processMap[NginxProxyContainerName]); err != nil {
|
||||||
@ -77,7 +78,7 @@ func doDeployWorkerPlane(ctx context.Context, host *hosts.Host,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// run kubelet
|
// run kubelet
|
||||||
if err := runKubelet(ctx, host, localConnDialerFactory, prsMap, processMap[KubeletContainerName]); err != nil {
|
if err := runKubelet(ctx, host, localConnDialerFactory, prsMap, processMap[KubeletContainerName], certMap); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runKubeproxy(ctx, host, localConnDialerFactory, prsMap, processMap[KubeproxyContainerName])
|
return runKubeproxy(ctx, host, localConnDialerFactory, prsMap, processMap[KubeproxyContainerName])
|
||||||
|
Loading…
Reference in New Issue
Block a user