rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-08-31 22:46:25 +00:00
Code Issues Releases Wiki Activity

External etcd

Browse Source
This commit is contained in:
galal-hussein
2018-02-14 22:58:35 +02:00
parent e75df68c35
commit c2c1804500
15 changed files with 205 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
README.md
View File

@@ -222,6 +222,32 @@ ingress:
RKE will deploy Nginx Ingress controller as a DaemonSet with `hostnetwork: true`, so ports `80`, and `443` will be opened on each node where the controller is deployed.
## External etcd
RKE supports using external etcd instead of deploying etcd servers, to enable external etcd the following parameters should be populated:
```
services:
etcd:
path: /etcdcluster
external_urls:
- https://etcd-example.com:2379
ca_cert: |-
-----BEGIN CERTIFICATE-----
xxxxxxxxxx
-----END CERTIFICATE-----
cert: |-
-----BEGIN CERTIFICATE-----
xxxxxxxxxx
-----END CERTIFICATE-----
key: |-
-----BEGIN PRIVATE KEY-----
xxxxxxxxxx
-----END PRIVATE KEY-----
```
Note that RKE only supports connecting to TLS enabled etcd setup, user can enable multiple endpoints in the `external_urls` field. RKE will not accept having external urls and nodes with `etcd` role at the same time, user should only specify either etcd role for servers or external etcd but not both.
## Operating Systems Notes
### Atomic OS

17
cluster.yml
View File

@@ -68,9 +68,24 @@ nodes:
services:
etcd:
# if external etcd is used
# path: /etcdcluster
# external_urls:
# - https://etcd-example.com:2379
# ca_cert: |-
# -----BEGIN CERTIFICATE-----
# xxxxxxxxxx
# -----END CERTIFICATE-----
# cert: |-
# -----BEGIN CERTIFICATE-----
# xxxxxxxxxx
# -----END CERTIFICATE-----
# key: |-
# -----BEGIN PRIVATE KEY-----
# xxxxxxxxxx
# -----END PRIVATE KEY-----
kube-api:
service_cluster_ip_range: 10.233.0.0/18
pod_security_policy: false
extra_args:

27
cluster/certificates.go
View File

@@ -22,26 +22,26 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
if currentCluster != nil {
kubeCluster.Certificates = currentCluster.Certificates
} else {
log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.EtcdHosts[0].Address)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts[0], kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
if err != nil {
return err
}
if kubeCluster.Certificates != nil {
log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.EtcdHosts[0].Address)
log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
return nil
}
log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.EtcdHosts[0].Address)
log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
kubeCluster.Certificates, err = pki.GenerateRKECerts(ctx, kubeCluster.RancherKubernetesEngineConfig, kubeCluster.LocalKubeConfigPath, "")
if err != nil {
return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err)
}
log.Infof(ctx, "[certificates] Temporarily saving certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil {
log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.ControlPlaneHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil {
return err
}
log.Infof(ctx, "[certificates] Saved certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
log.Infof(ctx, "[certificates] Saved certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
}
}
return nil
@@ -123,11 +123,14 @@ func saveCertToKubernetes(kubeClient *kubernetes.Clientset, crtName string, crt
timeout := make(chan bool, 1)
// build secret Data
secretData := map[string][]byte{
"Certificate": cert.EncodeCertPEM(crt.Certificate),
"Key": cert.EncodePrivateKeyPEM(crt.Key),
"EnvName": []byte(crt.EnvName),
"KeyEnvName": []byte(crt.KeyEnvName),
secretData := make(map[string][]byte)
if crt.Certificate != nil {
secretData["Certificate"] = cert.EncodeCertPEM(crt.Certificate)
secretData["EnvName"] = []byte(crt.EnvName)
}
if crt.Key != nil {
secretData["Key"] = cert.EncodePrivateKeyPEM(crt.Key)
secretData["KeyEnvName"] = []byte(crt.KeyEnvName)
}
if len(crt.Config) > 0 {
secretData["ConfigEnvName"] = []byte(crt.ConfigEnvName)

6
cluster/cluster.go
View File

@@ -55,10 +55,14 @@ const (
func (c *Cluster) DeployControlPlane(ctx context.Context) error {
// Deploy Etcd Plane
etcdProcessHostMap := c.getEtcdProcessHostMap(nil)
if len(c.Services.Etcd.ExternalURLs) > 0 {
log.Infof(ctx, "[etcd] External etcd connection string has been specified, skipping etcd plane")
} else {
if err := services.RunEtcdPlane(ctx, c.EtcdHosts, etcdProcessHostMap, c.LocalConnDialerFactory, c.PrivateRegistriesMap); err != nil {
return fmt.Errorf("[etcd] Failed to bring up Etcd Plane: %v", err)
}
}
// Deploy Control plane
processMap := map[string]v3.Process{
services.SidekickContainerName: c.BuildSidecarProcess(),

2
cluster/hosts.go
View File

@@ -21,7 +21,7 @@ const (
func (c *Cluster) TunnelHosts(ctx context.Context, local bool) error {
if local {
if err := c.EtcdHosts[0].TunnelUpLocal(ctx); err != nil {
if err := c.ControlPlaneHosts[0].TunnelUpLocal(ctx); err != nil {
return fmt.Errorf("Failed to connect to docker for local host [%s]: %v", c.EtcdHosts[0].Address, err)
}
return nil

39
cluster/network.go
View File

@@ -73,11 +73,15 @@ const (
APIRoot = "APIRoot"
// kubernetes client certificates and kubeconfig paths
ClientCert = "ClientCert"
EtcdClientCert = "EtcdClientCert"
EtcdClientKey = "EtcdClientKey"
EtcdClientCA = "EtcdClientCA"
EtcdClientCertPath = "EtcdClientCertPath"
EtcdClientKeyPath = "EtcdClientKeyPath"
EtcdClientCAPath = "EtcdClientCAPath"
ClientCertPath = "ClientCertPath"
ClientKey = "ClientKey"
ClientKeyPath = "ClientKeyPath"
ClientCA = "ClientCA"
ClientCAPath = "ClientCAPath"
KubeCfg = "KubeCfg"
@@ -144,19 +148,28 @@ func (c *Cluster) doFlannelDeploy(ctx context.Context) error {
}
func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
clientCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.KubeNodeCertName].Certificate))
clientkey := b64.StdEncoding.EncodeToString(cert.EncodePrivateKeyPEM(c.Certificates[pki.KubeNodeCertName].Key))
etcdEndpoints := services.GetEtcdConnString(c.EtcdHosts)
etcdClientCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.KubeNodeCertName].Certificate))
etcdClientkey := b64.StdEncoding.EncodeToString(cert.EncodePrivateKeyPEM(c.Certificates[pki.KubeNodeCertName].Key))
etcdCaCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.CACertName].Certificate))
clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
caCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.CACertName].Certificate))
// handling external etcd
if len(c.Services.Etcd.ExternalURLs) > 0 {
etcdClientCert = b64.StdEncoding.EncodeToString([]byte(c.Services.Etcd.Cert))
etcdClientkey = b64.StdEncoding.EncodeToString([]byte(c.Services.Etcd.Key))
etcdCaCert = b64.StdEncoding.EncodeToString([]byte(c.Services.Etcd.CACert))
etcdEndpoints = strings.Join(c.Services.Etcd.ExternalURLs, ",")
}
calicoConfig := map[string]string{
EtcdEndpoints: services.GetEtcdConnString(c.EtcdHosts),
EtcdEndpoints: etcdEndpoints,
APIRoot: "https://127.0.0.1:6443",
ClientCert: clientCert,
ClientCertPath: pki.GetCertPath(pki.KubeNodeCertName),
ClientKey: clientkey,
ClientKeyPath: pki.GetKeyPath(pki.KubeNodeCertName),
ClientCA: caCert,
ClientCAPath: pki.GetCertPath(pki.CACertName),
EtcdClientCA: etcdCaCert,
EtcdClientCert: etcdClientCert,
EtcdClientKey: etcdClientkey,
EtcdClientKeyPath: pki.GetKeyPath(pki.EtcdClientCertName),
EtcdClientCertPath: pki.GetCertPath(pki.EtcdClientCertName),
EtcdClientCAPath: pki.GetCertPath(pki.EtcdClientCACertName),
KubeCfg: clientConfig,
ClusterCIDR: c.ClusterCIDR,
CNIImage: c.SystemImages.CalicoCNI,

32
cluster/plan.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strconv"
"strings"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/pki"
@@ -11,6 +12,10 @@ import (
"github.com/rancher/types/apis/management.cattle.io/v3"
)
const (
EtcdPathPrefix = "/registry"
)
func GeneratePlan(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig) (v3.RKEPlan, error) {
clusterPlan := v3.RKEPlan{}
myCluster, _ := ParseCluster(ctx, rkeConfig, "", "", nil, nil)
@@ -55,8 +60,20 @@ func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts
}
func (c *Cluster) BuildKubeAPIProcess() v3.Process {
etcdConnString := services.GetEtcdConnString(c.EtcdHosts)
args := []string{}
// check if external etcd is used
etcdConnectionString := services.GetEtcdConnString(c.EtcdHosts)
etcdPathPrefix := EtcdPathPrefix
etcdClientCert := pki.GetCertPath(pki.KubeNodeCertName)
etcdClientKey := pki.GetKeyPath(pki.KubeNodeCertName)
etcdCAClientCert := pki.GetCertPath(pki.CACertName)
if len(c.Services.Etcd.ExternalURLs) > 0 {
etcdConnectionString = strings.Join(c.Services.Etcd.ExternalURLs, ",")
etcdPathPrefix = c.Services.Etcd.Path
etcdClientCert = pki.GetCertPath(pki.EtcdClientCertName)
etcdClientKey = pki.GetKeyPath(pki.EtcdClientCertName)
etcdCAClientCert = pki.GetCertPath(pki.EtcdClientCACertName)
}
Command := []string{
"/opt/rke/entrypoint.sh",
"kube-apiserver",
@@ -76,11 +93,14 @@ func (c *Cluster) BuildKubeAPIProcess() v3.Process {
"--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName),
"--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
"--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
"--etcd-cafile=" + pki.GetCertPath(pki.CACertName),
"--etcd-certfile=" + pki.GetCertPath(pki.KubeAPICertName),
"--etcd-keyfile=" + pki.GetKeyPath(pki.KubeAPICertName),
}
args = append(args, "--etcd-servers="+etcdConnString)
args := []string{
"--etcd-cafile=" + etcdCAClientCert,
"--etcd-certfile=" + etcdClientCert,
"--etcd-keyfile=" + etcdClientKey,
"--etcd-servers=" + etcdConnectionString,
"--etcd-prefix=" + etcdPathPrefix,
}
if c.Authorization.Mode == services.RBACAuthorizationMode {
args = append(args, "--authorization-mode=RBAC")

10
cluster/remove.go
View File

@@ -10,6 +10,10 @@ import (
)
func (c *Cluster) ClusterRemove(ctx context.Context) error {
externalEtcd := false
if len(c.Services.Etcd.ExternalURLs) > 0 {
externalEtcd = true
}
// Remove Worker Plane
if err := services.RemoveWorkerPlane(ctx, c.WorkerHosts, true); err != nil {
return err
@@ -26,7 +30,7 @@ func (c *Cluster) ClusterRemove(ctx context.Context) error {
}
// Clean up all hosts
if err := cleanUpHosts(ctx, c.ControlPlaneHosts, c.WorkerHosts, c.EtcdHosts, c.SystemImages.Alpine, c.PrivateRegistriesMap); err != nil {
if err := cleanUpHosts(ctx, c.ControlPlaneHosts, c.WorkerHosts, c.EtcdHosts, c.SystemImages.Alpine, c.PrivateRegistriesMap, externalEtcd); err != nil {
return err
}
@@ -34,14 +38,14 @@ func (c *Cluster) ClusterRemove(ctx context.Context) error {
return nil
}
func cleanUpHosts(ctx context.Context, cpHosts, workerHosts, etcdHosts []*hosts.Host, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
func cleanUpHosts(ctx context.Context, cpHosts, workerHosts, etcdHosts []*hosts.Host, cleanerImage string, prsMap map[string]v3.PrivateRegistry, externalEtcd bool) error {
allHosts := []*hosts.Host{}
allHosts = append(allHosts, cpHosts...)
allHosts = append(allHosts, workerHosts...)
allHosts = append(allHosts, etcdHosts...)
for _, host := range allHosts {
if err := host.CleanUpAll(ctx, cleanerImage, prsMap); err != nil {
if err := host.CleanUpAll(ctx, cleanerImage, prsMap, externalEtcd); err != nil {
return err
}
}

20
cluster/validation.go
View File

@@ -12,9 +12,12 @@ func (c *Cluster) ValidateCluster() error {
if len(c.ControlPlaneHosts) == 0 {
return fmt.Errorf("Cluster must have at least one control plane host")
}
if len(c.EtcdHosts) == 0 {
if len(c.EtcdHosts) == 0 && len(c.Services.Etcd.ExternalURLs) == 0 {
return fmt.Errorf("Cluster must have at least one etcd plane host")
}
if len(c.EtcdHosts) > 0 && len(c.Services.Etcd.ExternalURLs) > 0 {
return fmt.Errorf("Cluster can't have both internal and external etcd")
}
// validate hosts options
if err := validateHostsOptions(c); err != nil {
@@ -94,6 +97,21 @@ func validateServicesOptions(c *Cluster) error {
return fmt.Errorf("%s can't be empty", strings.Join(strings.Split(optionName, "_"), " "))
}
}
// Validate external etcd information
if len(c.Services.Etcd.ExternalURLs) > 0 {
if len(c.Services.Etcd.CACert) == 0 {
return fmt.Errorf("External CA Certificate for etcd can't be empty")
}
if len(c.Services.Etcd.Cert) == 0 {
return fmt.Errorf("External Client Certificate for etcd can't be empty")
}
if len(c.Services.Etcd.Key) == 0 {
return fmt.Errorf("External Client Key for etcd can't be empty")
}
if len(c.Services.Etcd.Path) == 0 {
return fmt.Errorf("External etcd path can't be empty")
}
}
return nil
}

6
hosts/hosts.go
View File

@@ -41,16 +41,18 @@ const (
CleanerContainerName = "kube-cleaner"
)
func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry, externalEtcd bool) error {
log.Infof(ctx, "[hosts] Cleaning up host [%s]", h.Address)
toCleanPaths := []string{
ToCleanEtcdDir,
ToCleanSSLDir,
ToCleanCNIConf,
ToCleanCNIBin,
ToCleanCalicoRun,
ToCleanTempCertPath,
}
if externalEtcd {
toCleanPaths = append(toCleanPaths, ToCleanEtcdDir)
}
return h.CleanUp(ctx, toCleanPaths, cleanerImage, prsMap)
}

2
pki/constants.go
View File

@@ -16,6 +16,8 @@ const (
KubeProxyCertName = "kube-proxy"
KubeNodeCertName = "kube-node"
EtcdCertName = "kube-etcd"
EtcdClientCACertName = "kube-etcd-client-ca"
EtcdClientCertName = "kube-etcd-client"
KubeNodeCommonName = "system:node"
KubeNodeOrganizationName = "system:nodes"

18
pki/pki.go
View File

@@ -111,9 +111,27 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
kubeAdminCertObj.Config = kubeAdminConfig
kubeAdminCertObj.ConfigPath = localKubeConfigPath
kubeAdminCertObj.ConfigEnvName = ""
certs[KubeAdminCertName] = kubeAdminCertObj
// generate etcd certificate and key
if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 {
clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert))
if err != nil {
return nil, err
}
clientKey, err := cert.ParsePrivateKeyPEM([]byte(rkeConfig.Services.Etcd.Key))
if err != nil {
return nil, err
}
certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey))
caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert))
if err != nil {
return nil, err
}
certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil)
}
etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP)
for _, host := range etcdHosts {

24
pki/util.go
View File

@@ -107,11 +107,14 @@ func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceI
}
func (c *CertificatePKI) ToEnv() []string {
env := []string{
c.CertToEnv(),
c.KeyToEnv(),
env := []string{}
if c.Key != nil {
env = append(env, c.KeyToEnv())
}
if c.Config != "" {
if c.Certificate != nil {
env = append(env, c.CertToEnv())
}
if c.Config != "" && c.ConfigEnvName != "" {
env = append(env, c.ConfigToEnv())
}
return env
@@ -218,6 +221,8 @@ func getControlCertKeys() []string {
KubeSchedulerCertName,
KubeProxyCertName,
KubeNodeCertName,
EtcdClientCertName,
EtcdClientCACertName,
}
}
@@ -226,6 +231,8 @@ func getWorkerCertKeys() []string {
CACertName,
KubeProxyCertName,
KubeNodeCertName,
EtcdClientCertName,
EtcdClientCACertName,
}
}
@@ -267,3 +274,12 @@ func GetLocalKubeConfig(configPath, configDir string) string {
baseDir += "/"
return fmt.Sprintf("%s%s%s", baseDir, KubeAdminConfigPrefix, fileName)
}
func strCrtToEnv(crtName, crt string) string {
return fmt.Sprintf("%s=%s", getEnvFromName(crtName), crt)
}
func strKeyToEnv(crtName, key string) string {
envName := getEnvFromName(crtName)
return fmt.Sprintf("%s=%s", getKeyEnvFromEnv(envName), key)
}

1
services/kubeapi.go
View File

@@ -9,6 +9,7 @@ import (
)
func runKubeAPI(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeAPIProcess v3.Process) error {
imageCfg, hostCfg, healthCheckURL := getProcessConfig(kubeAPIProcess)
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole, prsMap); err != nil {
return err

18
templates/calico.go
View File

@@ -104,9 +104,9 @@ data:
{
"type": "calico",
"etcd_endpoints": "{{.EtcdEndpoints}}",
"etcd_key_file": "{{.ClientKeyPath}}",
"etcd_cert_file": "{{.ClientCertPath}}",
"etcd_ca_cert_file": "{{.ClientCAPath}}",
"etcd_key_file": "{{.EtcdClientKeyPath}}",
"etcd_cert_file": "{{.EtcdClientCertPath}}",
"etcd_ca_cert_file": "{{.EtcdClientCAPath}}",
"log_level": "info",
"mtu": 1500,
"ipam": {
@@ -114,10 +114,8 @@ data:
},
"policy": {
"type": "k8s",
"k8s_api_root": "{{.APIRoot}}",
"k8s_client_certificate": "{{.ClientCertPath}}",
"k8s_client_key": "{{.ClientKeyPath}}",
"k8s_certificate_authority": "{{.ClientCAPath}}"
"k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
"k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
},
"kubernetes": {
"kubeconfig": "{{.KubeCfg}}"
@@ -144,9 +142,9 @@ metadata:
name: calico-etcd-secrets
namespace: kube-system
data:
etcd-key: {{.ClientKey}}
etcd-cert: {{.ClientCert}}
etcd-ca: {{.ClientCA}}
etcd-key: {{.EtcdClientKey}}
etcd-cert: {{.EtcdClientCert}}
etcd-ca: {{.EtcdClientCA}}
---