diff --git a/cluster/certificates.go b/cluster/certificates.go index 31a95959..75b43683 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/rancher/rke/hosts" "github.com/rancher/rke/k8s" "github.com/rancher/rke/log" "github.com/rancher/rke/pki" @@ -21,7 +22,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust kubeCluster.Certificates = currentCluster.Certificates } else { log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.EtcdHosts[0].Address) - kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath) + kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath) if err != nil { return err } @@ -33,7 +34,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust kubeCluster.Certificates, err = pki.StartCertificatesGeneration(ctx, kubeCluster.ControlPlaneHosts, - kubeCluster.WorkerHosts, + kubeCluster.EtcdHosts, kubeCluster.ClusterDomain, kubeCluster.LocalKubeConfigPath, kubeCluster.KubernetesServiceIP) @@ -41,7 +42,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err) } log.Infof(ctx, "[certificates] Temporarily saving certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address) - if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage]); err != nil { + if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage], pki.TempCertPath); err != nil { return err } log.Infof(ctx, "[certificates] Saved certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address) @@ -56,32 +57,31 @@ func regenerateAPICertificate(c *Cluster, certificates map[string]pki.Certificat caCrt := certificates[pki.CACertName].Certificate caKey := certificates[pki.CACertName].Key kubeAPIKey := certificates[pki.KubeAPICertName].Key - kubeAPICert, err := pki.GenerateCertWithKey(pki.KubeAPICertName, kubeAPIKey, caCrt, caKey, kubeAPIAltNames) + kubeAPICert, _, err := pki.GenerateSignedCertAndKey(caCrt, caKey, true, pki.KubeAPICertName, kubeAPIAltNames, kubeAPIKey, nil) if err != nil { return nil, err } - certificates[pki.KubeAPICertName] = pki.CertificatePKI{ - Certificate: kubeAPICert, - Key: kubeAPIKey, - Config: certificates[pki.KubeAPICertName].Config, - EnvName: certificates[pki.KubeAPICertName].EnvName, - ConfigEnvName: certificates[pki.KubeAPICertName].ConfigEnvName, - KeyEnvName: certificates[pki.KubeAPICertName].KeyEnvName, - } + certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey) return certificates, nil } -func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset) (map[string]pki.CertificatePKI, error) { +func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcdHosts []*hosts.Host) (map[string]pki.CertificatePKI, error) { log.Infof(ctx, "[certificates] Getting Cluster certificates from Kubernetes") certificatesNames := []string{ pki.CACertName, pki.KubeAPICertName, - pki.KubeNodeName, - pki.KubeProxyName, - pki.KubeControllerName, - pki.KubeSchedulerName, - pki.KubeAdminCommonName, + pki.KubeNodeCertName, + pki.KubeProxyCertName, + pki.KubeControllerCertName, + pki.KubeSchedulerCertName, + pki.KubeAdminCertName, } + + for _, etcdHost := range etcdHosts { + etcdName := pki.GetEtcdCrtName(etcdHost.InternalAddress) + certificatesNames = append(certificatesNames, etcdName) + } + certMap := make(map[string]pki.CertificatePKI) for _, certName := range certificatesNames { secret, err := k8s.GetSecret(kubeClient, certName) diff --git a/cluster/cluster.go b/cluster/cluster.go index 9931a4e8..e92fc6db 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -221,7 +221,7 @@ func GetLocalKubeConfig(configPath, configDir string) string { func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error { log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config") var workingConfig, newConfig string - currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCommonName] + currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName] caCrt := kubeCluster.Certificates[pki.CACertName].Certificate for _, cpHost := range kubeCluster.ControlPlaneHosts { if (currentKubeConfig == pki.CertificatePKI{}) { @@ -232,7 +232,7 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error { caData := string(cert.EncodeCertPEM(caCrt)) crtData := string(cert.EncodeCertPEM(currentKubeConfig.Certificate)) keyData := string(cert.EncodePrivateKeyPEM(currentKubeConfig.Key)) - newConfig = pki.GetKubeConfigX509WithData(kubeURL, pki.KubeAdminCommonName, caData, crtData, keyData) + newConfig = pki.GetKubeConfigX509WithData(kubeURL, pki.KubeAdminCertName, caData, crtData, keyData) } if err := pki.DeployAdminConfig(ctx, newConfig, kubeCluster.LocalKubeConfigPath); err != nil { return fmt.Errorf("Failed to redeploy local admin config with new host") @@ -244,7 +244,7 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error { } } currentKubeConfig.Config = workingConfig - kubeCluster.Certificates[pki.KubeAdminCommonName] = currentKubeConfig + kubeCluster.Certificates[pki.KubeAdminCertName] = currentKubeConfig return nil } @@ -271,7 +271,7 @@ func getLocalAdminConfigWithNewAddress(localConfigPath, cpAddress string) string config.Host = fmt.Sprintf("https://%s:6443", cpAddress) return pki.GetKubeConfigX509WithData( "https://"+cpAddress+":6443", - pki.KubeAdminCommonName, + pki.KubeAdminCertName, string(config.CAData), string(config.CertData), string(config.KeyData)) diff --git a/cluster/hosts.go b/cluster/hosts.go index 5a4d861a..5a4f7349 100644 --- a/cluster/hosts.go +++ b/cluster/hosts.go @@ -72,22 +72,18 @@ func (c *Cluster) InvertIndexHosts() error { func (c *Cluster) SetUpHosts(ctx context.Context) error { if c.Authentication.Strategy == X509AuthenticationProvider { log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes") - err := pki.DeployCertificatesOnMasters(ctx, c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage]) - if err != nil { + if err := pki.DeployCertificatesOnMasters(ctx, c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil { return err } - err = pki.DeployCertificatesOnWorkers(ctx, c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage]) - if err != nil { + if err := pki.DeployCertificatesOnWorkers(ctx, c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil { return err } - // Deploying worker certs on etcd hosts as well - err = pki.DeployCertificatesOnWorkers(ctx, c.EtcdHosts, c.Certificates, c.SystemImages[CertDownloaderImage]) - if err != nil { + // Deploying etcd certificates + if err := pki.DeployCertificatesOnEtcd(ctx, c.EtcdHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil { return err } - err = pki.DeployAdminConfig(ctx, c.Certificates[pki.KubeAdminCommonName].Config, c.LocalKubeConfigPath) - if err != nil { + if err := pki.DeployAdminConfig(ctx, c.Certificates[pki.KubeAdminCertName].Config, c.LocalKubeConfigPath); err != nil { return err } log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes") diff --git a/cluster/network.go b/cluster/network.go index b4277796..b75ba35a 100644 --- a/cluster/network.go +++ b/cluster/network.go @@ -8,6 +8,8 @@ import ( "net" "strings" + b64 "encoding/base64" + "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/rancher/rke/docker" @@ -18,6 +20,7 @@ import ( "github.com/rancher/rke/templates" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + "k8s.io/client-go/util/cert" ) const ( @@ -66,10 +69,14 @@ const ( APIRoot = "APIRoot" // kubernetes client certificates and kubeconfig paths - ClientCert = "ClientCert" - ClientKey = "ClientKey" - ClientCA = "ClientCA" - KubeCfg = "KubeCfg" + ClientCert = "ClientCert" + ClientCertPath = "ClientCertPath" + ClientKey = "ClientKey" + ClientKeyPath = "ClientKeyPath" + ClientCA = "ClientCA" + ClientCAPath = "ClientCAPath" + + KubeCfg = "KubeCfg" ClusterCIDR = "ClusterCIDR" // Images key names @@ -120,13 +127,20 @@ func (c *Cluster) doFlannelDeploy(ctx context.Context) error { } func (c *Cluster) doCalicoDeploy(ctx context.Context) error { + clientCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.KubeNodeCertName].Certificate)) + clientkey := b64.StdEncoding.EncodeToString(cert.EncodePrivateKeyPEM(c.Certificates[pki.KubeNodeCertName].Key)) + clientConfig := pki.GetConfigPath(pki.KubeNodeCertName) + caCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.CACertName].Certificate)) calicoConfig := map[string]string{ EtcdEndpoints: services.GetEtcdConnString(c.EtcdHosts), APIRoot: "https://127.0.0.1:6443", - ClientCert: pki.KubeNodeCertPath, - ClientKey: pki.KubeNodeKeyPath, - ClientCA: pki.CACertPath, - KubeCfg: pki.KubeNodeConfigPath, + ClientCert: clientCert, + ClientCertPath: pki.GetCertPath(pki.KubeNodeCertName), + ClientKey: clientkey, + ClientKeyPath: pki.GetKeyPath(pki.KubeNodeCertName), + ClientCA: caCert, + ClientCAPath: pki.GetCertPath(pki.CACertName), + KubeCfg: clientConfig, ClusterCIDR: c.ClusterCIDR, CNIImage: c.Network.Options[CalicoCNIImage], NodeImage: c.Network.Options[CalicoNodeImage], @@ -143,12 +157,13 @@ func (c *Cluster) doCalicoDeploy(ctx context.Context) error { } func (c *Cluster) doCanalDeploy(ctx context.Context) error { + clientConfig := pki.GetConfigPath(pki.KubeNodeCertName) canalConfig := map[string]string{ - ClientCert: pki.KubeNodeCertPath, + ClientCertPath: pki.GetCertPath(pki.KubeNodeCertName), APIRoot: "https://127.0.0.1:6443", - ClientKey: pki.KubeNodeKeyPath, - ClientCA: pki.CACertPath, - KubeCfg: pki.KubeNodeConfigPath, + ClientKeyPath: pki.GetKeyPath(pki.KubeNodeCertName), + ClientCAPath: pki.GetCertPath(pki.CACertName), + KubeCfg: clientConfig, ClusterCIDR: c.ClusterCIDR, NodeImage: c.Network.Options[CanalNodeImage], CNIImage: c.Network.Options[CanalCNIImage], diff --git a/cluster/reconcile.go b/cluster/reconcile.go index 7cf62095..d3c952a8 100644 --- a/cluster/reconcile.go +++ b/cluster/reconcile.go @@ -7,9 +7,11 @@ import ( "github.com/rancher/rke/hosts" "github.com/rancher/rke/k8s" "github.com/rancher/rke/log" + "github.com/rancher/rke/pki" "github.com/rancher/rke/services" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/cert" ) const ( @@ -155,10 +157,14 @@ func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd b } func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, kubeClient *kubernetes.Clientset) error { - log.Infof(ctx, "[reconcile] Check etcd hosts to be deleted") + logrus.Infof("[reconcile] Check etcd hosts to be deleted") + // get tls for the first current etcd host + clientCert := cert.EncodeCertPEM(currentCluster.Certificates[pki.KubeNodeCertName].Certificate) + clientkey := cert.EncodePrivateKeyPEM(currentCluster.Certificates[pki.KubeNodeCertName].Key) + etcdToDelete := hosts.GetToDeleteHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts) for _, etcdHost := range etcdToDelete { - if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil { + if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil { log.Warnf(ctx, "[reconcile] %v", err) continue } @@ -174,15 +180,34 @@ func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, ku } log.Infof(ctx, "[reconcile] Check etcd hosts to be added") etcdToAdd := hosts.GetToAddHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts) + crtMap := currentCluster.Certificates + var err error for _, etcdHost := range etcdToAdd { etcdHost.ToAddEtcdMember = true + // Generate new certificate for the new etcd member + crtMap, err = pki.RegenerateEtcdCertificate( + ctx, + crtMap, + etcdHost, + kubeCluster.EtcdHosts, + kubeCluster.ClusterDomain, + kubeCluster.KubernetesServiceIP) + if err != nil { + return err + } } + currentCluster.Certificates = crtMap for _, etcdHost := range etcdToAdd { - if err := services.AddEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil { + // deploy certificates on new etcd host + if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts, etcdHost, currentCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage], pki.CertPathPrefix); err != nil { + return err + } + + if err := services.AddEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil { return err } etcdHost.ToAddEtcdMember = false - if err := services.ReloadEtcdCluster(ctx, kubeCluster.EtcdHosts, kubeCluster.Services.Etcd, currentCluster.LocalConnDialerFactory); err != nil { + if err := services.ReloadEtcdCluster(ctx, kubeCluster.EtcdHosts, kubeCluster.Services.Etcd, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil { return err } } diff --git a/cluster/state.go b/cluster/state.go index bb3cf1b7..1bbdcf0b 100644 --- a/cluster/state.go +++ b/cluster/state.go @@ -58,7 +58,10 @@ func (c *Cluster) GetClusterState(ctx context.Context) (*Cluster, error) { currentCluster = getStateFromKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath) // Get previous kubernetes certificates if currentCluster != nil { - currentCluster.Certificates, err = getClusterCerts(ctx, c.KubeClient) + if err := currentCluster.InvertIndexHosts(); err != nil { + return nil, fmt.Errorf("Failed to classify hosts from fetched cluster: %v", err) + } + currentCluster.Certificates, err = getClusterCerts(ctx, c.KubeClient, currentCluster.EtcdHosts) currentCluster.DockerDialerFactory = c.DockerDialerFactory if err != nil { return nil, fmt.Errorf("Failed to Get Kubernetes certificates: %v", err) @@ -66,9 +69,6 @@ func (c *Cluster) GetClusterState(ctx context.Context) (*Cluster, error) { // setting cluster defaults for the fetched cluster as well currentCluster.setClusterDefaults(ctx) - if err := currentCluster.InvertIndexHosts(); err != nil { - return nil, fmt.Errorf("Failed to classify hosts from fetched cluster: %v", err) - } currentCluster.Certificates, err = regenerateAPICertificate(c, currentCluster.Certificates) if err != nil { return nil, fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err) diff --git a/cmd/up.go b/cmd/up.go index 63905aec..ed10bd17 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -110,8 +110,8 @@ func ClusterUp( APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443") caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate)) - clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCommonName].Certificate)) - clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCommonName].Key)) + clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate)) + clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key)) log.Infof(ctx, "Finished building Kubernetes cluster successfully") return APIURL, caCrt, clientCert, clientKey, nil diff --git a/pki/constants.go b/pki/constants.go index 92f022a2..6160421c 100644 --- a/pki/constants.go +++ b/pki/constants.go @@ -1,65 +1,25 @@ package pki const ( + CertPathPrefix = "/etc/kubernetes/ssl/" CertificatesServiceName = "certificates" CrtDownloaderContainer = "cert-deployer" CertFetcherContainer = "cert-fetcher" CertificatesSecretName = "k8s-certs" TempCertPath = "/etc/kubernetes/.tmp/" - CACertName = "kube-ca" - CACertENVName = "KUBE_CA" - CAKeyENVName = "KUBE_CA_KEY" - CACertPath = "/etc/kubernetes/ssl/kube-ca.pem" - CAKeyPath = "/etc/kubernetes/ssl/kube-ca-key.pem" + CACertName = "kube-ca" + KubeAPICertName = "kube-apiserver" + KubeControllerCertName = "kube-controller-manager" + KubeSchedulerCertName = "kube-scheduler" + KubeProxyCertName = "kube-proxy" + KubeNodeCertName = "kube-node" + EtcdCertName = "kube-etcd" - KubeAPICertName = "kube-apiserver" - KubeAPICertENVName = "KUBE_API" - KubeAPIKeyENVName = "KUBE_API_KEY" - KubeAPICertPath = "/etc/kubernetes/ssl/kube-api.pem" - KubeAPIKeyPath = "/etc/kubernetes/ssl/kube-api-key.pem" - - KubeControllerName = "kube-controller-manager" - KubeControllerCommonName = "system:kube-controller-manager" - KubeControllerCertENVName = "KUBE_CONTROLLER_MANAGER" - KubeControllerKeyENVName = "KUBE_CONTROLLER_MANAGER_KEY" - KubeControllerConfigENVName = "KUBECFG_CONTROLLER_MANAGER" - KubeControllerCertPath = "/etc/kubernetes/ssl/kube-controller-manager.pem" - KubeControllerKeyPath = "/etc/kubernetes/ssl/kube-controller-manager-key.pem" - KubeControllerConfigPath = "/etc/kubernetes/ssl/kubecfg-controller-manager.yaml" - - KubeSchedulerName = "kube-scheduler" - KubeSchedulerCommonName = "system:kube-scheduler" - KubeSchedulerCertENVName = "KUBE_SCHEDULER" - KubeSchedulerKeyENVName = "KUBE_SCHEDULER_KEY" - KubeSchedulerConfigENVName = "KUBECFG_SCHEDULER" - KubeSchedulerCertPath = "/etc/kubernetes/ssl/kube-scheduler.pem" - KubeSchedulerKeyPath = "/etc/kubernetes/ssl/kube-scheduler-key.pem" - KubeSchedulerConfigPath = "/etc/kubernetes/ssl/kubecfg-scheduler.yaml" - - KubeProxyName = "kube-proxy" - KubeProxyCommonName = "system:kube-proxy" - KubeProxyCertENVName = "KUBE_PROXY" - KubeProxyKeyENVName = "KUBE_PROXY_KEY" - KubeProxyConfigENVName = "KUBECFG_KUBE_PROXY" - KubeProxyCertPath = "/etc/kubernetes/ssl/kube-proxy.pem" - KubeProxyKeyPath = "/etc/kubernetes/ssl/kube-proxy-key.pem" - KubeProxyConfigPath = "/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml" - - KubeNodeName = "kube-node" KubeNodeCommonName = "system:node" KubeNodeOrganizationName = "system:nodes" - KubeNodeCertENVName = "KUBE_NODE" - KubeNodeKeyENVName = "KUBE_NODE_KEY" - KubeNodeConfigENVName = "KUBECFG_KUBE_NODE" - KubeNodeCertPath = "/etc/kubernetes/ssl/kube-node.pem" - KubeNodeKeyPath = "/etc/kubernetes/ssl/kube-node-key.pem" - KubeNodeConfigPath = "/etc/kubernetes/ssl/kubecfg-kube-node.yaml" - KubeAdminCommonName = "kube-admin" + KubeAdminCertName = "kube-admin" KubeAdminOrganizationName = "system:masters" KubeAdminConfigPrefix = ".kube_config_" - KubeAdminConfigENVName = "KUBECFG_ADMIN" - KubeAdminCertEnvName = "KUBE_ADMIN" - KubeAdminKeyEnvName = "KUBE_ADMIN_KEY" ) diff --git a/pki/deploy.go b/pki/deploy.go index d70876ba..c0227cd2 100644 --- a/pki/deploy.go +++ b/pki/deploy.go @@ -24,10 +24,10 @@ func DeployCertificatesOnMasters(ctx context.Context, cpHosts []*hosts.Host, crt crtList := []string{ CACertName, KubeAPICertName, - KubeControllerName, - KubeSchedulerName, - KubeProxyName, - KubeNodeName, + KubeControllerCertName, + KubeSchedulerCertName, + KubeProxyCertName, + KubeNodeCertName, } env := []string{} for _, crtName := range crtList { @@ -48,8 +48,8 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host, // list of certificates that should be deployed on the workers crtList := []string{ CACertName, - KubeProxyName, - KubeNodeName, + KubeProxyCertName, + KubeNodeCertName, } env := []string{} for _, crtName := range crtList { @@ -66,6 +66,31 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host, return nil } +func DeployCertificatesOnEtcd(ctx context.Context, etcdHosts []*hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error { + // list of certificates that should be deployed on the etcd + crtList := []string{ + CACertName, + KubeProxyCertName, + KubeNodeCertName, + } + for _, host := range etcdHosts { + crtList = append(crtList, GetEtcdCrtName(host.InternalAddress)) + } + env := []string{} + for _, crtName := range crtList { + c := crtMap[crtName] + env = append(env, c.ToEnv()...) + } + + for i := range etcdHosts { + err := doRunDeployer(ctx, etcdHosts[i], env, certDownloaderImage) + if err != nil { + return err + } + } + return nil +} + func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string, certDownloaderImage string) error { // remove existing container. Only way it's still here is if previous deployment failed isRunning := false @@ -135,19 +160,22 @@ func RemoveAdminConfig(ctx context.Context, localConfigPath string) { log.Infof(ctx, "Local admin Kubeconfig removed successfully") } -func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error { +func DeployCertificatesOnHost(ctx context.Context, extraHosts []*hosts.Host, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage, certPath string) error { crtList := []string{ CACertName, KubeAPICertName, - KubeControllerName, - KubeSchedulerName, - KubeProxyName, - KubeNodeName, - KubeAdminCommonName, + KubeControllerCertName, + KubeSchedulerCertName, + KubeProxyCertName, + KubeNodeCertName, + KubeAdminCertName, + } + for _, host := range extraHosts { + // Deploy etcd certificates + crtList = append(crtList, GetEtcdCrtName(host.InternalAddress)) } - env := []string{ - "CRTS_DEPLOY_PATH=" + TempCertPath, + "CRTS_DEPLOY_PATH=" + certPath, } for _, crtName := range crtList { c := crtMap[crtName] @@ -157,24 +185,27 @@ func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[ return doRunDeployer(ctx, host, env, certDownloaderImage) } -func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) { +func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) { // rebuilding the certificates. This should look better after refactoring pki tmpCerts := make(map[string]CertificatePKI) - certEnvMap := map[string][]string{ - CACertName: []string{CACertPath, CAKeyPath}, - KubeAPICertName: []string{KubeAPICertPath, KubeAPIKeyPath}, - KubeControllerName: []string{KubeControllerCertPath, KubeControllerKeyPath, KubeControllerConfigPath}, - KubeSchedulerName: []string{KubeSchedulerCertPath, KubeSchedulerKeyPath, KubeSchedulerConfigPath}, - KubeProxyName: []string{KubeProxyCertPath, KubeProxyKeyPath, KubeProxyConfigPath}, - KubeNodeName: []string{KubeNodeCertPath, KubeNodeKeyPath, KubeNodeConfigPath}, - KubeAdminCommonName: []string{"kube-admin.pem", "kube-admin-key.pem", "kubecfg-admin.yaml"}, + crtList := map[string]bool{ + CACertName: false, + KubeAPICertName: false, + KubeControllerCertName: true, + KubeSchedulerCertName: true, + KubeProxyCertName: true, + KubeNodeCertName: true, + KubeAdminCertName: true, + } + for _, etcdHost := range extraHosts { + // Fetch etcd certificates + crtList[GetEtcdCrtName(etcdHost.InternalAddress)] = false } - // get files from hosts - for crtName, certEnv := range certEnvMap { + for certName, config := range crtList { certificate := CertificatePKI{} - crt, err := fetchFileFromHost(ctx, getTempPath(certEnv[0]), image, host) + crt, err := fetchFileFromHost(ctx, GetCertTempPath(certName), image, host) if err != nil { if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "Could not find the file") { @@ -182,10 +213,10 @@ func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, loc } return nil, err } - key, err := fetchFileFromHost(ctx, getTempPath(certEnv[1]), image, host) + key, err := fetchFileFromHost(ctx, GetKeyTempPath(certName), image, host) - if len(certEnv) > 2 { - config, err := fetchFileFromHost(ctx, getTempPath(certEnv[2]), image, host) + if config { + config, err := fetchFileFromHost(ctx, GetConfigTempPath(certName), image, host) if err != nil { return nil, err } @@ -201,14 +232,14 @@ func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, loc } certificate.Certificate = parsedCert[0] certificate.Key = parsedKey.(*rsa.PrivateKey) - tmpCerts[crtName] = certificate - logrus.Debugf("[certificates] Recovered certificate: %s", crtName) + tmpCerts[certName] = certificate + logrus.Debugf("[certificates] Recovered certificate: %s", certName) } if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CertFetcherContainer); err != nil { return nil, err } - return populateCertMap(tmpCerts, localConfigPath), nil + return populateCertMap(tmpCerts, localConfigPath, extraHosts), nil } @@ -244,96 +275,31 @@ func getTempPath(s string) string { return TempCertPath + path.Base(s) } -func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string) map[string]CertificatePKI { +func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI { certs := make(map[string]CertificatePKI) - //CACert - certs[CACertName] = CertificatePKI{ - Certificate: tmpCerts[CACertName].Certificate, - Key: tmpCerts[CACertName].Key, - Name: CACertName, - EnvName: CACertENVName, - KeyEnvName: CAKeyENVName, - Path: CACertPath, - KeyPath: CAKeyPath, - } - //KubeAPI - certs[KubeAPICertName] = CertificatePKI{ - Certificate: tmpCerts[KubeAPICertName].Certificate, - Key: tmpCerts[KubeAPICertName].Key, - Name: KubeAPICertName, - EnvName: KubeAPICertENVName, - KeyEnvName: KubeAPIKeyENVName, - Path: KubeAPICertPath, - KeyPath: KubeAPIKeyPath, - } - //kubeController - certs[KubeControllerName] = CertificatePKI{ - Certificate: tmpCerts[KubeControllerName].Certificate, - Key: tmpCerts[KubeControllerName].Key, - Config: tmpCerts[KubeControllerName].Config, - Name: KubeControllerName, - CommonName: KubeControllerCommonName, - EnvName: KubeControllerCertENVName, - KeyEnvName: KubeControllerKeyENVName, - Path: KubeControllerCertPath, - KeyPath: KubeControllerKeyPath, - ConfigEnvName: KubeControllerConfigENVName, - ConfigPath: KubeControllerConfigPath, - } - //KubeScheduler - certs[KubeSchedulerName] = CertificatePKI{ - Certificate: tmpCerts[KubeSchedulerName].Certificate, - Key: tmpCerts[KubeSchedulerName].Key, - Config: tmpCerts[KubeSchedulerName].Config, - Name: KubeSchedulerName, - CommonName: KubeSchedulerCommonName, - EnvName: KubeSchedulerCertENVName, - KeyEnvName: KubeSchedulerKeyENVName, - Path: KubeSchedulerCertPath, - KeyPath: KubeSchedulerKeyPath, - ConfigEnvName: KubeSchedulerConfigENVName, - ConfigPath: KubeSchedulerConfigPath, - } + // CACert + certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key) + // KubeAPI + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key) + // kubeController + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key) + // KubeScheduler + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key) // KubeProxy - certs[KubeProxyName] = CertificatePKI{ - Certificate: tmpCerts[KubeProxyName].Certificate, - Key: tmpCerts[KubeProxyName].Key, - Config: tmpCerts[KubeProxyName].Config, - Name: KubeProxyName, - CommonName: KubeProxyCommonName, - EnvName: KubeProxyCertENVName, - Path: KubeProxyCertPath, - KeyEnvName: KubeProxyKeyENVName, - KeyPath: KubeProxyKeyPath, - ConfigEnvName: KubeProxyConfigENVName, - ConfigPath: KubeProxyConfigPath, - } + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key) // KubeNode - certs[KubeNodeName] = CertificatePKI{ - Certificate: tmpCerts[KubeNodeName].Certificate, - Key: tmpCerts[KubeNodeName].Key, - Config: tmpCerts[KubeNodeName].Config, - Name: KubeNodeName, - CommonName: KubeNodeCommonName, - OUName: KubeNodeOrganizationName, - EnvName: KubeNodeCertENVName, - KeyEnvName: KubeNodeKeyENVName, - Path: KubeNodeCertPath, - KeyPath: KubeNodeKeyPath, - ConfigEnvName: KubeNodeConfigENVName, - ConfigPath: KubeNodeCommonName, + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key) + // KubeAdmin + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key) + kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config + kubeAdminCertObj.ConfigPath = localConfigPath + certs[KubeAdminCertName] = kubeAdminCertObj + // etcd + for _, host := range extraHosts { + etcdName := GetEtcdCrtName(host.InternalAddress) + etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) } - certs[KubeAdminCommonName] = CertificatePKI{ - Certificate: tmpCerts[KubeAdminCommonName].Certificate, - Key: tmpCerts[KubeAdminCommonName].Key, - Config: tmpCerts[KubeAdminCommonName].Config, - CommonName: KubeAdminCommonName, - OUName: KubeAdminOrganizationName, - ConfigEnvName: KubeAdminConfigENVName, - ConfigPath: localConfigPath, - EnvName: KubeAdminCertEnvName, - KeyEnvName: KubeAdminKeyEnvName, - } return certs } diff --git a/pki/pki.go b/pki/pki.go index 39dd3c8f..3e7a3206 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -4,7 +4,6 @@ import ( "context" "crypto/rsa" "crypto/x509" - "fmt" "net" "github.com/rancher/rke/hosts" @@ -29,16 +28,16 @@ type CertificatePKI struct { } // StartCertificatesGeneration ... -func StartCertificatesGeneration(ctx context.Context, cpHosts []*hosts.Host, workerHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) { - log.Infof(ctx, "[certificates] Generating kubernetes certificates") - certs, err := generateCerts(ctx, cpHosts, clusterDomain, localConfigPath, KubernetesServiceIP) +func StartCertificatesGeneration(ctx context.Context, cpHosts, etcdHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) { + logrus.Infof("[certificates] Generating kubernetes certificates") + certs, err := generateCerts(ctx, cpHosts, etcdHosts, clusterDomain, localConfigPath, KubernetesServiceIP) if err != nil { return nil, err } return certs, nil } -func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) { +func generateCerts(ctx context.Context, cpHosts, etcdHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) { certs := make(map[string]CertificatePKI) // generate CA certificate and key log.Infof(ctx, "[certificates] Generating CA kubernetes certificates") @@ -46,270 +45,100 @@ func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, lo if err != nil { return nil, err } - logrus.Debugf("[certificates] CA Certificate: %s", string(cert.EncodeCertPEM(caCrt))) - certs[CACertName] = CertificatePKI{ - Certificate: caCrt, - Key: caKey, - Name: CACertName, - EnvName: CACertENVName, - KeyEnvName: CAKeyENVName, - Path: CACertPath, - KeyPath: CAKeyPath, - } + certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey) // generate API certificate and key log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates") kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, KubernetesServiceIP) - kubeAPICrt, kubeAPIKey, err := GenerateKubeAPICertAndKey(caCrt, caKey, kubeAPIAltNames) + kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, nil, nil) if err != nil { return nil, err } - logrus.Debugf("[certificates] Kube API Certificate: %s", string(cert.EncodeCertPEM(kubeAPICrt))) - certs[KubeAPICertName] = CertificatePKI{ - Certificate: kubeAPICrt, - Key: kubeAPIKey, - Name: KubeAPICertName, - EnvName: KubeAPICertENVName, - KeyEnvName: KubeAPIKeyENVName, - Path: KubeAPICertPath, - KeyPath: KubeAPIKeyPath, - } + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey) // generate Kube controller-manager certificate and key log.Infof(ctx, "[certificates] Generating Kube Controller certificates") - kubeControllerCrt, kubeControllerKey, err := generateClientCertAndKey(caCrt, caKey, KubeControllerCommonName, []string{}) + kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil) if err != nil { return nil, err } - logrus.Debugf("[certificates] Kube Controller Certificate: %s", string(cert.EncodeCertPEM(kubeControllerCrt))) - certs[KubeControllerName] = CertificatePKI{ - Certificate: kubeControllerCrt, - Key: kubeControllerKey, - Config: getKubeConfigX509("https://127.0.0.1:6443", KubeControllerName, CACertPath, KubeControllerCertPath, KubeControllerKeyPath), - Name: KubeControllerName, - CommonName: KubeControllerCommonName, - EnvName: KubeControllerCertENVName, - KeyEnvName: KubeControllerKeyENVName, - Path: KubeControllerCertPath, - KeyPath: KubeControllerKeyPath, - ConfigEnvName: KubeControllerConfigENVName, - ConfigPath: KubeControllerConfigPath, - } + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey) // generate Kube scheduler certificate and key log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates") - kubeSchedulerCrt, kubeSchedulerKey, err := generateClientCertAndKey(caCrt, caKey, KubeSchedulerCommonName, []string{}) + kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil) if err != nil { return nil, err } - logrus.Debugf("[certificates] Kube Scheduler Certificate: %s", string(cert.EncodeCertPEM(kubeSchedulerCrt))) - certs[KubeSchedulerName] = CertificatePKI{ - Certificate: kubeSchedulerCrt, - Key: kubeSchedulerKey, - Config: getKubeConfigX509("https://127.0.0.1:6443", KubeSchedulerName, CACertPath, KubeSchedulerCertPath, KubeSchedulerKeyPath), - Name: KubeSchedulerName, - CommonName: KubeSchedulerCommonName, - EnvName: KubeSchedulerCertENVName, - KeyEnvName: KubeSchedulerKeyENVName, - Path: KubeSchedulerCertPath, - KeyPath: KubeSchedulerKeyPath, - ConfigEnvName: KubeSchedulerConfigENVName, - ConfigPath: KubeSchedulerConfigPath, - } + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey) // generate Kube Proxy certificate and key log.Infof(ctx, "[certificates] Generating Kube Proxy certificates") - kubeProxyCrt, kubeProxyKey, err := generateClientCertAndKey(caCrt, caKey, KubeProxyCommonName, []string{}) + kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil) if err != nil { return nil, err } - logrus.Debugf("[certificates] Kube Proxy Certificate: %s", string(cert.EncodeCertPEM(kubeProxyCrt))) - certs[KubeProxyName] = CertificatePKI{ - Certificate: kubeProxyCrt, - Key: kubeProxyKey, - Config: getKubeConfigX509("https://127.0.0.1:6443", KubeProxyName, CACertPath, KubeProxyCertPath, KubeProxyKeyPath), - Name: KubeProxyName, - CommonName: KubeProxyCommonName, - EnvName: KubeProxyCertENVName, - Path: KubeProxyCertPath, - KeyEnvName: KubeProxyKeyENVName, - KeyPath: KubeProxyKeyPath, - ConfigEnvName: KubeProxyConfigENVName, - ConfigPath: KubeProxyConfigPath, - } + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey) // generate Kubelet certificate and key log.Infof(ctx, "[certificates] Generating Node certificate") - nodeCrt, nodeKey, err := generateClientCertAndKey(caCrt, caKey, KubeNodeCommonName, []string{KubeNodeOrganizationName}) + nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName}) if err != nil { return nil, err } - logrus.Debugf("[certificates] Node Certificate: %s", string(cert.EncodeCertPEM(kubeProxyCrt))) - certs[KubeNodeName] = CertificatePKI{ - Certificate: nodeCrt, - Key: nodeKey, - Config: getKubeConfigX509("https://127.0.0.1:6443", KubeNodeName, CACertPath, KubeNodeCertPath, KubeNodeKeyPath), - Name: KubeNodeName, - CommonName: KubeNodeCommonName, - OUName: KubeNodeOrganizationName, - EnvName: KubeNodeCertENVName, - KeyEnvName: KubeNodeKeyENVName, - Path: KubeNodeCertPath, - KeyPath: KubeNodeKeyPath, - ConfigEnvName: KubeNodeConfigENVName, - ConfigPath: KubeNodeCommonName, - } - log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig") - kubeAdminCrt, kubeAdminKey, err := generateClientCertAndKey(caCrt, caKey, KubeAdminCommonName, []string{KubeAdminOrganizationName}) + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey) + + // generate Admin certificate and key + logrus.Infof("[certificates] Generating admin certificates and kubeconfig") + kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName}) if err != nil { return nil, err } - logrus.Debugf("[certificates] Admin Certificate: %s", string(cert.EncodeCertPEM(kubeAdminCrt))) - certs[KubeAdminCommonName] = CertificatePKI{ - Certificate: kubeAdminCrt, - Key: kubeAdminKey, - Config: GetKubeConfigX509WithData( - "https://"+cpHosts[0].Address+":6443", - KubeAdminCommonName, - string(cert.EncodeCertPEM(caCrt)), - string(cert.EncodeCertPEM(kubeAdminCrt)), - string(cert.EncodePrivateKeyPEM(kubeAdminKey))), - CommonName: KubeAdminCommonName, - OUName: KubeAdminOrganizationName, - ConfigEnvName: KubeAdminConfigENVName, - ConfigPath: localConfigPath, - EnvName: KubeAdminCertEnvName, - KeyEnvName: KubeAdminKeyEnvName, + kubeAdminConfig := GetKubeConfigX509WithData( + "https://"+cpHosts[0].Address+":6443", + KubeAdminCertName, + string(cert.EncodeCertPEM(caCrt)), + string(cert.EncodeCertPEM(kubeAdminCrt)), + string(cert.EncodePrivateKeyPEM(kubeAdminKey))) + + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey) + kubeAdminCertObj.Config = kubeAdminConfig + kubeAdminCertObj.ConfigPath = localConfigPath + certs[KubeAdminCertName] = kubeAdminCertObj + + etcdAltNames := GetAltNames(etcdHosts, clusterDomain, KubernetesServiceIP) + for _, host := range etcdHosts { + logrus.Infof("[certificates] Generating etcd-%s certificate and key", host.InternalAddress) + etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil) + if err != nil { + return nil, err + } + etcdName := GetEtcdCrtName(host.InternalAddress) + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) } + return certs, nil } -func generateClientCertAndKey(caCrt *x509.Certificate, caKey *rsa.PrivateKey, commonName string, orgs []string) (*x509.Certificate, *rsa.PrivateKey, error) { - rootKey, err := cert.NewPrivateKey() +func RegenerateEtcdCertificate( + ctx context.Context, + crtMap map[string]CertificatePKI, + etcdHost *hosts.Host, + etcdHosts []*hosts.Host, + clusterDomain string, + KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) { + + log.Infof(ctx, "[certificates] Regenerating new etcd-%s certificate and key", etcdHost.InternalAddress) + caCrt := crtMap[CACertName].Certificate + caKey := crtMap[CACertName].Key + etcdAltNames := GetAltNames(etcdHosts, clusterDomain, KubernetesServiceIP) + + etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil) if err != nil { - return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err) + return nil, err } - caConfig := cert.Config{ - CommonName: commonName, - Organization: orgs, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - } - clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey) - if err != nil { - return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err) - } - - return clientCert, rootKey, nil -} - -func GenerateKubeAPICertAndKey(caCrt *x509.Certificate, caKey *rsa.PrivateKey, altNames *cert.AltNames) (*x509.Certificate, *rsa.PrivateKey, error) { - rootKey, err := cert.NewPrivateKey() - if err != nil { - return nil, nil, fmt.Errorf("Failed to generate private key for kube-apiserver certificate: %v", err) - } - caConfig := cert.Config{ - CommonName: KubeAPICertName, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - AltNames: *altNames, - } - kubeCACert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey) - if err != nil { - return nil, nil, fmt.Errorf("Failed to generate kube-apiserver certificate: %v", err) - } - - return kubeCACert, rootKey, nil -} - -func GenerateCertWithKey(commonName string, key *rsa.PrivateKey, caCrt *x509.Certificate, caKey *rsa.PrivateKey, altNames *cert.AltNames) (*x509.Certificate, error) { - caConfig := cert.Config{ - CommonName: commonName, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, - x509.ExtKeyUsageServerAuth}, - AltNames: *altNames, - } - cert, err := cert.NewSignedCert(caConfig, key, caCrt, caKey) - if err != nil { - return nil, fmt.Errorf("Failed to generate certificate with existing key: %v", err) - } - return cert, nil -} - -func generateCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) { - rootKey, err := cert.NewPrivateKey() - if err != nil { - return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err) - } - caConfig := cert.Config{ - CommonName: CACertName, - } - kubeCACert, err := cert.NewSelfSignedCACert(caConfig, rootKey) - if err != nil { - return nil, nil, fmt.Errorf("Failed to generate CA certificate: %v", err) - } - - return kubeCACert, rootKey, nil -} - -func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceIP net.IP) *cert.AltNames { - ips := []net.IP{} - dnsNames := []string{} - for _, host := range cpHosts { - // Check if node address is a valid IP - if nodeIP := net.ParseIP(host.Address); nodeIP != nil { - ips = append(ips, nodeIP) - } else { - dnsNames = append(dnsNames, host.Address) - } - - // Check if node internal address is a valid IP - if len(host.InternalAddress) != 0 && host.InternalAddress != host.Address { - if internalIP := net.ParseIP(host.InternalAddress); internalIP != nil { - ips = append(ips, internalIP) - } else { - dnsNames = append(dnsNames, host.InternalAddress) - } - } - // Add hostname to the ALT dns names - if len(host.HostnameOverride) != 0 && host.HostnameOverride != host.Address { - dnsNames = append(dnsNames, host.HostnameOverride) - } - } - ips = append(ips, net.ParseIP("127.0.0.1")) - ips = append(ips, KubernetesServiceIP) - dnsNames = append(dnsNames, []string{ - "localhost", - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc." + clusterDomain, - }...) - return &cert.AltNames{ - IPs: ips, - DNSNames: dnsNames, - } -} - -func (c *CertificatePKI) ToEnv() []string { - env := []string{ - c.CertToEnv(), - c.KeyToEnv(), - } - if c.Config != "" { - env = append(env, c.ConfigToEnv()) - } - return env -} - -func (c *CertificatePKI) CertToEnv() string { - encodedCrt := cert.EncodeCertPEM(c.Certificate) - return fmt.Sprintf("%s=%s", c.EnvName, string(encodedCrt)) -} - -func (c *CertificatePKI) KeyToEnv() string { - encodedKey := cert.EncodePrivateKeyPEM(c.Key) - return fmt.Sprintf("%s=%s", c.KeyEnvName, string(encodedKey)) -} - -func (c *CertificatePKI) ConfigToEnv() string { - return fmt.Sprintf("%s=%s", c.ConfigEnvName, c.Config) + etcdName := GetEtcdCrtName(etcdHost.InternalAddress) + crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) + log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress) + return crtMap, nil } diff --git a/pki/pki_test.go b/pki/pki_test.go index e8c9522c..4e2da1d1 100644 --- a/pki/pki_test.go +++ b/pki/pki_test.go @@ -38,11 +38,11 @@ func TestPKI(t *testing.T) { certificatesToVerify := []string{ KubeAPICertName, - KubeNodeName, - KubeProxyName, - KubeControllerName, - KubeSchedulerName, - KubeAdminCommonName, + KubeNodeCertName, + KubeProxyCertName, + KubeControllerCertName, + KubeSchedulerCertName, + KubeAdminCertName, } opts := x509.VerifyOptions{ Roots: roots, diff --git a/pki/util.go b/pki/util.go new file mode 100644 index 00000000..e9e4acdf --- /dev/null +++ b/pki/util.go @@ -0,0 +1,209 @@ +package pki + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + "net" + "strings" + + "github.com/rancher/rke/hosts" + "k8s.io/client-go/util/cert" +) + +func GenerateSignedCertAndKey( + caCrt *x509.Certificate, + caKey *rsa.PrivateKey, + serverCrt bool, + commonName string, + altNames *cert.AltNames, + reusedKey *rsa.PrivateKey, + orgs []string) (*x509.Certificate, *rsa.PrivateKey, error) { + // Generate a generic signed certificate + var rootKey *rsa.PrivateKey + var err error + rootKey = reusedKey + if reusedKey == nil { + rootKey, err = cert.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err) + } + } + usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + if serverCrt { + usages = append(usages, x509.ExtKeyUsageServerAuth) + } + if altNames == nil { + altNames = &cert.AltNames{} + } + caConfig := cert.Config{ + CommonName: commonName, + Organization: orgs, + Usages: usages, + AltNames: *altNames, + } + clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey) + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err) + } + return clientCert, rootKey, nil +} + +func generateCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) { + rootKey, err := cert.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err) + } + caConfig := cert.Config{ + CommonName: CACertName, + } + kubeCACert, err := cert.NewSelfSignedCACert(caConfig, rootKey) + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate CA certificate: %v", err) + } + + return kubeCACert, rootKey, nil +} + +func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceIP net.IP) *cert.AltNames { + ips := []net.IP{} + dnsNames := []string{} + for _, host := range cpHosts { + // Check if node address is a valid IP + if nodeIP := net.ParseIP(host.Address); nodeIP != nil { + ips = append(ips, nodeIP) + } else { + dnsNames = append(dnsNames, host.Address) + } + + // Check if node internal address is a valid IP + if len(host.InternalAddress) != 0 && host.InternalAddress != host.Address { + if internalIP := net.ParseIP(host.InternalAddress); internalIP != nil { + ips = append(ips, internalIP) + } else { + dnsNames = append(dnsNames, host.InternalAddress) + } + } + // Add hostname to the ALT dns names + if len(host.HostnameOverride) != 0 && host.HostnameOverride != host.Address { + dnsNames = append(dnsNames, host.HostnameOverride) + } + } + ips = append(ips, net.ParseIP("127.0.0.1")) + ips = append(ips, KubernetesServiceIP) + dnsNames = append(dnsNames, []string{ + "localhost", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc." + clusterDomain, + }...) + return &cert.AltNames{ + IPs: ips, + DNSNames: dnsNames, + } +} + +func (c *CertificatePKI) ToEnv() []string { + env := []string{ + c.CertToEnv(), + c.KeyToEnv(), + } + if c.Config != "" { + env = append(env, c.ConfigToEnv()) + } + return env +} + +func (c *CertificatePKI) CertToEnv() string { + encodedCrt := cert.EncodeCertPEM(c.Certificate) + return fmt.Sprintf("%s=%s", c.EnvName, string(encodedCrt)) +} + +func (c *CertificatePKI) KeyToEnv() string { + encodedKey := cert.EncodePrivateKeyPEM(c.Key) + return fmt.Sprintf("%s=%s", c.KeyEnvName, string(encodedKey)) +} + +func (c *CertificatePKI) ConfigToEnv() string { + return fmt.Sprintf("%s=%s", c.ConfigEnvName, c.Config) +} + +func getEnvFromName(name string) string { + return strings.Replace(strings.ToUpper(name), "-", "_", -1) +} + +func getKeyEnvFromEnv(env string) string { + return fmt.Sprintf("%s_KEY", env) +} + +func getConfigEnvFromEnv(env string) string { + return fmt.Sprintf("KUBECFG_%s", env) +} + +func GetEtcdCrtName(address string) string { + newAddress := strings.Replace(address, ".", "-", -1) + return fmt.Sprintf("%s-%s", EtcdCertName, newAddress) +} + +func GetCertPath(name string) string { + return fmt.Sprintf("%s%s.pem", CertPathPrefix, name) +} + +func GetKeyPath(name string) string { + return fmt.Sprintf("%s%s-key.pem", CertPathPrefix, name) +} + +func GetConfigPath(name string) string { + return fmt.Sprintf("%skubecfg-%s.yaml", CertPathPrefix, name) +} + +func GetCertTempPath(name string) string { + return fmt.Sprintf("%s%s.pem", TempCertPath, name) +} + +func GetKeyTempPath(name string) string { + return fmt.Sprintf("%s%s-key.pem", TempCertPath, name) +} + +func GetConfigTempPath(name string) string { + return fmt.Sprintf("%skubecfg-%s.yaml", TempCertPath, name) +} + +func ToCertObject(componentName, commonName, ouName string, cert *x509.Certificate, key *rsa.PrivateKey) CertificatePKI { + var config, configPath, configEnvName string + if len(commonName) == 0 { + commonName = getDefaultCN(componentName) + } + + envName := getEnvFromName(componentName) + keyEnvName := getKeyEnvFromEnv(envName) + caCertPath := GetCertPath(CACertName) + path := GetCertPath(componentName) + keyPath := GetKeyPath(componentName) + + if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) { + config = getKubeConfigX509("https://127.0.0.1:6443", componentName, caCertPath, path, keyPath) + configPath = GetConfigPath(componentName) + configEnvName = getConfigEnvFromEnv(envName) + } + + return CertificatePKI{ + Certificate: cert, + Key: key, + Config: config, + Name: componentName, + CommonName: commonName, + OUName: ouName, + EnvName: envName, + KeyEnvName: keyEnvName, + ConfigEnvName: configEnvName, + Path: path, + KeyPath: keyPath, + ConfigPath: configPath, + } +} + +func getDefaultCN(name string) string { + return fmt.Sprintf("system:%s", name) +} diff --git a/services/etcd.go b/services/etcd.go index ef1c2139..743d06e6 100644 --- a/services/etcd.go +++ b/services/etcd.go @@ -11,6 +11,7 @@ import ( "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" + "github.com/rancher/rke/pki" "github.com/rancher/types/apis/management.cattle.io/v3" "github.com/sirupsen/logrus" ) @@ -19,7 +20,8 @@ func RunEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.E log.Infof(ctx, "[%s] Building up Etcd Plane..", ETCDRole) initCluster := getEtcdInitialCluster(etcdHosts) for _, host := range etcdHosts { - imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster) + nodeName := pki.GetEtcdCrtName(host.InternalAddress) + imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster, nodeName) err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole) if err != nil { return err @@ -57,7 +59,7 @@ func RemoveEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, force bool) e return nil } -func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster string) (*container.Config, *container.HostConfig) { +func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster, nodeName string) (*container.Config, *container.HostConfig) { clusterState := "new" if host.ExistingEtcdCluster { clusterState = "existing" @@ -67,18 +69,29 @@ func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster s Cmd: []string{"/usr/local/bin/etcd", "--name=etcd-" + host.HostnameOverride, "--data-dir=/etcd-data", - "--advertise-client-urls=http://" + host.InternalAddress + ":2379,http://" + host.InternalAddress + ":4001", - "--listen-client-urls=http://0.0.0.0:2379", - "--initial-advertise-peer-urls=http://" + host.InternalAddress + ":2380", - "--listen-peer-urls=http://0.0.0.0:2380", + "--advertise-client-urls=https://" + host.InternalAddress + ":2379,https://" + host.InternalAddress + ":4001", + "--listen-client-urls=https://0.0.0.0:2379", + "--initial-advertise-peer-urls=https://" + host.InternalAddress + ":2380", + "--listen-peer-urls=https://0.0.0.0:2380", "--initial-cluster-token=etcd-cluster-1", "--initial-cluster=" + initCluster, - "--initial-cluster-state=" + clusterState}, + "--initial-cluster-state=" + clusterState, + "--peer-client-cert-auth", + "--client-cert-auth", + "--trusted-ca-file=" + pki.GetCertPath(pki.CACertName), + "--peer-trusted-ca-file=" + pki.GetCertPath(pki.CACertName), + "--cert-file=" + pki.GetCertPath(nodeName), + "--key-file=" + pki.GetKeyPath(nodeName), + "--peer-cert-file=" + pki.GetCertPath(nodeName), + "--peer-key-file=" + pki.GetKeyPath(nodeName), + }, } hostCfg := &container.HostConfig{ RestartPolicy: container.RestartPolicy{Name: "always"}, Binds: []string{ - "/var/lib/etcd:/etcd-data"}, + "/var/lib/etcd:/etcd-data", + "/etc/kubernetes:/etc/kubernetes", + }, NetworkMode: "host", } for arg, value := range etcdService.ExtraArgs { @@ -89,12 +102,12 @@ func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster s return imageCfg, hostCfg } -func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory) error { +func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error { log.Infof(ctx, "[add/%s] Adding member [etcd-%s] to etcd cluster", ETCDRole, etcdHost.HostnameOverride) - peerURL := fmt.Sprintf("http://%s:2380", etcdHost.InternalAddress) + peerURL := fmt.Sprintf("https://%s:2380", etcdHost.InternalAddress) added := false for _, host := range etcdHosts { - etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory) + etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory, cert, key) if err != nil { logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err) continue @@ -114,12 +127,12 @@ func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts return nil } -func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory) error { +func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error { log.Infof(ctx, "[remove/%s] Removing member [etcd-%s] from etcd cluster", ETCDRole, etcdHost.HostnameOverride) var mID string removed := false for _, host := range etcdHosts { - etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory) + etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory, cert, key) if err != nil { logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err) continue @@ -150,7 +163,7 @@ func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*ho return nil } -func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.ETCDService, localConnDialerFactory hosts.DialerFactory) error { +func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.ETCDService, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error { readyEtcdHosts := []*hosts.Host{} for _, host := range etcdHosts { if !host.ToAddEtcdMember { @@ -160,16 +173,20 @@ func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService } initCluster := getEtcdInitialCluster(readyEtcdHosts) for _, host := range readyEtcdHosts { - imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster) + imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster, pki.GetEtcdCrtName(host.InternalAddress)) if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole); err != nil { return err } } time.Sleep(10 * time.Second) + var healthy bool for _, host := range readyEtcdHosts { - if healthy := isEtcdHealthy(ctx, localConnDialerFactory, host); healthy { + if healthy = isEtcdHealthy(ctx, localConnDialerFactory, host, cert, key); healthy { break } } + if !healthy { + return fmt.Errorf("[etcd] Etcd Cluster is not healthy") + } return nil } diff --git a/services/etcd_test.go b/services/etcd_test.go index 28e001b1..9639fdd6 100644 --- a/services/etcd_test.go +++ b/services/etcd_test.go @@ -5,11 +5,12 @@ import ( "testing" "github.com/rancher/rke/hosts" + "github.com/rancher/rke/pki" "github.com/rancher/types/apis/management.cattle.io/v3" ) const ( - TestInitEtcdClusterString = "etcd-etcd1=http://1.1.1.1:2380,etcd-etcd2=http://2.2.2.2:2380" + TestInitEtcdClusterString = "etcd-etcd1=https://1.1.1.1:2380,etcd-etcd2=https://2.2.2.2:2380" TestEtcdImage = "etcd/etcdImage:latest" TestEtcdNamePrefix = "--name=etcd-" TestEtcdVolumeBind = "/var/lib/etcd:/etcd-data" @@ -46,7 +47,8 @@ func TestEtcdConfig(t *testing.T) { assertEqual(t, initCluster, TestInitEtcdClusterString, "") for _, host := range etcdHosts { - imageCfg, hostCfg := buildEtcdConfig(host, etcdService, TestInitEtcdClusterString) + nodeName := pki.GetEtcdCrtName(host.InternalAddress) + imageCfg, hostCfg := buildEtcdConfig(host, etcdService, TestInitEtcdClusterString, nodeName) assertEqual(t, isStringInSlice(TestEtcdNamePrefix+host.HostnameOverride, imageCfg.Cmd), true, fmt.Sprintf("Failed to find [%s] in Etcd command", TestEtcdNamePrefix+host.HostnameOverride)) assertEqual(t, TestEtcdImage, imageCfg.Image, diff --git a/services/etcd_util.go b/services/etcd_util.go index c6c1b83a..536048c7 100644 --- a/services/etcd_util.go +++ b/services/etcd_util.go @@ -2,6 +2,7 @@ package services import ( "context" + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -14,36 +15,48 @@ import ( "github.com/sirupsen/logrus" ) -func getEtcdClient(ctx context.Context, etcdHost *hosts.Host, localConnDialerFactory hosts.DialerFactory) (etcdclient.Client, error) { +func getEtcdClient(ctx context.Context, etcdHost *hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) (etcdclient.Client, error) { dialer, err := getEtcdDialer(localConnDialerFactory, etcdHost) if err != nil { return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", etcdHost.Address, err) } + tlsConfig, err := getEtcdTLSConfig(cert, key) + if err != nil { + return nil, err + } var DefaultEtcdTransport etcdclient.CancelableTransport = &http.Transport{ - Dial: dialer, + Dial: dialer, + TLSClientConfig: tlsConfig, + TLSHandshakeTimeout: 10 * time.Second, } cfg := etcdclient.Config{ - Endpoints: []string{"http://127.0.0.1:2379"}, + Endpoints: []string{"https://127.0.0.1:2379"}, Transport: DefaultEtcdTransport, } return etcdclient.New(cfg) } -func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFactory, host *hosts.Host) bool { +func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFactory, host *hosts.Host, cert, key []byte) bool { logrus.Debugf("[etcd] Check etcd cluster health") for i := 0; i < 3; i++ { dialer, err := getEtcdDialer(localConnDialerFactory, host) if err != nil { - logrus.Debugf("Failed to create a dialer for host [%s]: %v", host.Address, err) - time.Sleep(5 * time.Second) - continue + return false } + tlsConfig, err := getEtcdTLSConfig(cert, key) + if err != nil { + logrus.Debugf("[etcd] Failed to create etcd tls config for host [%s]: %v", host.Address, err) + return false + } + hc := http.Client{ Transport: &http.Transport{ - Dial: dialer, + Dial: dialer, + TLSClientConfig: tlsConfig, + TLSHandshakeTimeout: 10 * time.Second, }, } healthy, err := getHealthEtcd(hc, host) @@ -62,7 +75,7 @@ func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFacto func getHealthEtcd(hc http.Client, host *hosts.Host) (string, error) { healthy := struct{ Health string }{} - resp, err := hc.Get("http://127.0.0.1:2379/health") + resp, err := hc.Get("https://127.0.0.1:2379/health") if err != nil { return healthy.Health, fmt.Errorf("Failed to get /health for host [%s]: %v", host.Address, err) } @@ -80,7 +93,7 @@ func getHealthEtcd(hc http.Client, host *hosts.Host) (string, error) { func getEtcdInitialCluster(hosts []*hosts.Host) string { initialCluster := "" for i, host := range hosts { - initialCluster += fmt.Sprintf("etcd-%s=http://%s:2380", host.HostnameOverride, host.InternalAddress) + initialCluster += fmt.Sprintf("etcd-%s=https://%s:2380", host.HostnameOverride, host.InternalAddress) if i < (len(hosts) - 1) { initialCluster += "," } @@ -102,10 +115,27 @@ func getEtcdDialer(localConnDialerFactory hosts.DialerFactory, etcdHost *hosts.H func GetEtcdConnString(hosts []*hosts.Host) string { connString := "" for i, host := range hosts { - connString += "http://" + host.InternalAddress + ":2379" + connString += "https://" + host.InternalAddress + ":2379" if i < (len(hosts) - 1) { connString += "," } } return connString } + +func getEtcdTLSConfig(certificate, key []byte) (*tls.Config, error) { + // get tls config + x509Pair, err := tls.X509KeyPair([]byte(certificate), []byte(key)) + if err != nil { + return nil, err + + } + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{x509Pair}, + } + if err != nil { + return nil, err + } + return tlsConfig, nil +} diff --git a/services/kubeapi.go b/services/kubeapi.go index e62a2f67..a859a23d 100644 --- a/services/kubeapi.go +++ b/services/kubeapi.go @@ -41,10 +41,13 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd "--runtime-config=batch/v2alpha1", "--runtime-config=authentication.k8s.io/v1beta1=true", "--storage-backend=etcd3", - "--client-ca-file=" + pki.CACertPath, - "--tls-cert-file=" + pki.KubeAPICertPath, - "--tls-private-key-file=" + pki.KubeAPIKeyPath, - "--service-account-key-file=" + pki.KubeAPIKeyPath}, + "--client-ca-file=" + pki.GetCertPath(pki.CACertName), + "--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName), + "--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName), + "--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName), + "--etcd-cafile=" + pki.GetCertPath(pki.CACertName), + "--etcd-certfile=" + pki.GetCertPath(pki.KubeAPICertName), + "--etcd-keyfile=" + pki.GetKeyPath(pki.KubeAPICertName)}, } imageCfg.Cmd = append(imageCfg.Cmd, "--etcd-servers="+etcdConnString) diff --git a/services/kubeapi_test.go b/services/kubeapi_test.go index 5a889ecf..8e48124d 100644 --- a/services/kubeapi_test.go +++ b/services/kubeapi_test.go @@ -9,7 +9,7 @@ import ( ) const ( - TestEtcdConnString = "http://1.1.1.1:2379,http://2.2.2.2:2379" + TestEtcdConnString = "https://1.1.1.1:2379,https://2.2.2.2:2379" TestKubeAPIImage = "rancher/k8s:latest" TestInsecureBindAddress = "--insecure-bind-address=127.0.0.1" TestKubeAPIVolumeBind = "/etc/kubernetes:/etc/kubernetes" diff --git a/services/kubecontroller.go b/services/kubecontroller.go index 237100d5..1d81bb3d 100644 --- a/services/kubecontroller.go +++ b/services/kubecontroller.go @@ -31,7 +31,7 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService, a "--address=0.0.0.0", "--cloud-provider=", "--leader-elect=true", - "--kubeconfig=" + pki.KubeControllerConfigPath, + "--kubeconfig=" + pki.GetConfigPath(pki.KubeControllerCertName), "--enable-hostpath-provisioner=false", "--node-monitor-grace-period=40s", "--pod-eviction-timeout=5m0s", @@ -39,8 +39,8 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService, a "--allocate-node-cidrs=true", "--cluster-cidr=" + kubeControllerService.ClusterCIDR, "--service-cluster-ip-range=" + kubeControllerService.ServiceClusterIPRange, - "--service-account-private-key-file=" + pki.KubeAPIKeyPath, - "--root-ca-file=" + pki.CACertPath, + "--service-account-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName), + "--root-ca-file=" + pki.GetCertPath(pki.CACertName), }, } if authorizationMode == RBACAuthorizationMode { diff --git a/services/kubelet.go b/services/kubelet.go index 0ec3ae58..3d68bbfa 100644 --- a/services/kubelet.go +++ b/services/kubelet.go @@ -42,7 +42,7 @@ func buildKubeletConfig(host *hosts.Host, kubeletService v3.KubeletService, unsc "--resolv-conf=/etc/resolv.conf", "--allow-privileged=true", "--cloud-provider=", - "--kubeconfig=" + pki.KubeNodeConfigPath, + "--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName), "--require-kubeconfig=True", }, } diff --git a/services/kubeproxy.go b/services/kubeproxy.go index 113a50bf..cc823f96 100644 --- a/services/kubeproxy.go +++ b/services/kubeproxy.go @@ -30,7 +30,7 @@ func buildKubeproxyConfig(host *hosts.Host, kubeproxyService v3.KubeproxyService "kube-proxy", "--v=2", "--healthz-bind-address=0.0.0.0", - "--kubeconfig=" + pki.KubeProxyConfigPath, + "--kubeconfig=" + pki.GetConfigPath(pki.KubeProxyCertName), }, } hostCfg := &container.HostConfig{ diff --git a/services/scheduler.go b/services/scheduler.go index 0b2a2931..4d1ca669 100644 --- a/services/scheduler.go +++ b/services/scheduler.go @@ -31,7 +31,7 @@ func buildSchedulerConfig(host *hosts.Host, schedulerService v3.SchedulerService "--leader-elect=true", "--v=2", "--address=0.0.0.0", - "--kubeconfig=" + pki.KubeSchedulerConfigPath, + "--kubeconfig=" + pki.GetConfigPath(pki.KubeSchedulerCertName), }, } hostCfg := &container.HostConfig{ diff --git a/templates/calico.go b/templates/calico.go index e9d3c5ea..4ed6d771 100644 --- a/templates/calico.go +++ b/templates/calico.go @@ -104,9 +104,9 @@ data: { "type": "calico", "etcd_endpoints": "{{.EtcdEndpoints}}", - "etcd_key_file": "", - "etcd_cert_file": "", - "etcd_ca_cert_file": "", + "etcd_key_file": "{{.ClientKeyPath}}", + "etcd_cert_file": "{{.ClientCertPath}}", + "etcd_ca_cert_file": "{{.ClientCAPath}}", "log_level": "info", "mtu": 1500, "ipam": { @@ -115,9 +115,9 @@ data: "policy": { "type": "k8s", "k8s_api_root": "{{.APIRoot}}", - "k8s_client_certificate": "{{.ClientCert}}", - "k8s_client_key": "{{.ClientKey}}", - "k8s_certificate_authority": "{{.ClientCA}}" + "k8s_client_certificate": "{{.ClientCertPath}}", + "k8s_client_key": "{{.ClientKeyPath}}", + "k8s_certificate_authority": "{{.ClientCAPath}}" }, "kubernetes": { "kubeconfig": "{{.KubeCfg}}" @@ -131,16 +131,12 @@ data: ] } - # If you're using TLS enabled etcd uncomment the following. - # You must also populate the Secret below with these files. - etcd_ca: "" # "/calico-secrets/etcd-ca" - etcd_cert: "" # "/calico-secrets/etcd-cert" - etcd_key: "" # "/calico-secrets/etcd-key" + etcd_ca: "/calico-secrets/etcd-ca" + etcd_cert: "/calico-secrets/etcd-cert" + etcd_key: "/calico-secrets/etcd-key" --- -# The following contains k8s Secrets for use with a TLS enabled etcd cluster. -# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/ apiVersion: v1 kind: Secret type: Opaque @@ -148,13 +144,9 @@ metadata: name: calico-etcd-secrets namespace: kube-system data: - # Populate the following files with etcd TLS configuration if desired, but leave blank if - # not using TLS for etcd. - # This self-hosted install expects three files with the following names. The values - # should be base64 encoded strings of the entire contents of each file. - # etcd-key: null - # etcd-cert: null - # etcd-ca: null + etcd-key: {{.ClientKey}} + etcd-cert: {{.ClientCert}} + etcd-ca: {{.ClientCA}} --- diff --git a/templates/canal.go b/templates/canal.go index edc3b522..c5a16594 100644 --- a/templates/canal.go +++ b/templates/canal.go @@ -156,9 +156,9 @@ data: "policy": { "type": "k8s", "k8s_api_root": "{{.APIRoot}}", - "k8s_client_certificate": "{{.ClientCert}}", - "k8s_client_key": "{{.ClientKey}}", - "k8s_certificate_authority": "{{.ClientCA}}" + "k8s_client_certificate": "{{.ClientCertPath}}", + "k8s_client_key": "{{.ClientKeyPath}}", + "k8s_certificate_authority": "{{.ClientCAPath}}" }, "kubernetes": { "kubeconfig": "{{.KubeCfg}}"