rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-09-17 23:49:06 +00:00
Code Issues Releases Wiki Activity

Merge pull request #174 from galal-hussein/etcd_tls

Browse Source 
Add TLS to etcd
This commit is contained in:
Alena Prokharchyk
2018-01-25 10:54:32 -08:00
committed by GitHub
parent 1bcaaef8ca 0ef0a17010
commit 24a20b1977
37 changed files with 1743 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
cluster/certificates.go
View File

@@ -6,6 +6,7 @@ import (
"fmt"
"time"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/k8s"
"github.com/rancher/rke/log"
"github.com/rancher/rke/pki"
@@ -21,7 +22,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
kubeCluster.Certificates = currentCluster.Certificates
} else {
log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.EtcdHosts[0].Address)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath)
if err != nil {
return err
}
@@ -33,7 +34,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
kubeCluster.Certificates, err = pki.StartCertificatesGeneration(ctx,
kubeCluster.ControlPlaneHosts,
kubeCluster.WorkerHosts,
kubeCluster.EtcdHosts,
kubeCluster.ClusterDomain,
kubeCluster.LocalKubeConfigPath,
kubeCluster.KubernetesServiceIP)
@@ -41,7 +42,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err)
}
log.Infof(ctx, "[certificates] Temporarily saving certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage]); err != nil {
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage], pki.TempCertPath); err != nil {
return err
}
log.Infof(ctx, "[certificates] Saved certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
@@ -56,32 +57,31 @@ func regenerateAPICertificate(c *Cluster, certificates map[string]pki.Certificat
caCrt := certificates[pki.CACertName].Certificate
caKey := certificates[pki.CACertName].Key
kubeAPIKey := certificates[pki.KubeAPICertName].Key
kubeAPICert, err := pki.GenerateCertWithKey(pki.KubeAPICertName, kubeAPIKey, caCrt, caKey, kubeAPIAltNames)
kubeAPICert, _, err := pki.GenerateSignedCertAndKey(caCrt, caKey, true, pki.KubeAPICertName, kubeAPIAltNames, kubeAPIKey, nil)
if err != nil {
return nil, err
}
certificates[pki.KubeAPICertName] = pki.CertificatePKI{
Certificate: kubeAPICert,
Key: kubeAPIKey,
Config: certificates[pki.KubeAPICertName].Config,
EnvName: certificates[pki.KubeAPICertName].EnvName,
ConfigEnvName: certificates[pki.KubeAPICertName].ConfigEnvName,
KeyEnvName: certificates[pki.KubeAPICertName].KeyEnvName,
}
certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey)
return certificates, nil
}
func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset) (map[string]pki.CertificatePKI, error) {
func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcdHosts []*hosts.Host) (map[string]pki.CertificatePKI, error) {
log.Infof(ctx, "[certificates] Getting Cluster certificates from Kubernetes")
certificatesNames := []string{
pki.CACertName,
pki.KubeAPICertName,
pki.KubeNodeName,
pki.KubeProxyName,
pki.KubeControllerName,
pki.KubeSchedulerName,
pki.KubeAdminCommonName,
pki.KubeNodeCertName,
pki.KubeProxyCertName,
pki.KubeControllerCertName,
pki.KubeSchedulerCertName,
pki.KubeAdminCertName,
}
for _, etcdHost := range etcdHosts {
etcdName := pki.GetEtcdCrtName(etcdHost.InternalAddress)
certificatesNames = append(certificatesNames, etcdName)
}
certMap := make(map[string]pki.CertificatePKI)
for _, certName := range certificatesNames {
secret, err := k8s.GetSecret(kubeClient, certName)

8
cluster/cluster.go
View File

@@ -221,7 +221,7 @@ func GetLocalKubeConfig(configPath, configDir string) string {
func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config")
var workingConfig, newConfig string
currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCommonName]
currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName]
caCrt := kubeCluster.Certificates[pki.CACertName].Certificate
for _, cpHost := range kubeCluster.ControlPlaneHosts {
if (currentKubeConfig == pki.CertificatePKI{}) {
@@ -232,7 +232,7 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
caData := string(cert.EncodeCertPEM(caCrt))
crtData := string(cert.EncodeCertPEM(currentKubeConfig.Certificate))
keyData := string(cert.EncodePrivateKeyPEM(currentKubeConfig.Key))
newConfig = pki.GetKubeConfigX509WithData(kubeURL, pki.KubeAdminCommonName, caData, crtData, keyData)
newConfig = pki.GetKubeConfigX509WithData(kubeURL, pki.KubeAdminCertName, caData, crtData, keyData)
}
if err := pki.DeployAdminConfig(ctx, newConfig, kubeCluster.LocalKubeConfigPath); err != nil {
return fmt.Errorf("Failed to redeploy local admin config with new host")
@@ -244,7 +244,7 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
}
}
currentKubeConfig.Config = workingConfig
kubeCluster.Certificates[pki.KubeAdminCommonName] = currentKubeConfig
kubeCluster.Certificates[pki.KubeAdminCertName] = currentKubeConfig
return nil
}
@@ -271,7 +271,7 @@ func getLocalAdminConfigWithNewAddress(localConfigPath, cpAddress string) string
config.Host = fmt.Sprintf("https://%s:6443", cpAddress)
return pki.GetKubeConfigX509WithData(
"https://"+cpAddress+":6443",
pki.KubeAdminCommonName,
pki.KubeAdminCertName,
string(config.CAData),
string(config.CertData),
string(config.KeyData))

14
cluster/hosts.go
View File

@@ -72,22 +72,18 @@ func (c *Cluster) InvertIndexHosts() error {
func (c *Cluster) SetUpHosts(ctx context.Context) error {
if c.Authentication.Strategy == X509AuthenticationProvider {
log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes")
err := pki.DeployCertificatesOnMasters(ctx, c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
if err != nil {
if err := pki.DeployCertificatesOnMasters(ctx, c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil {
return err
}
err = pki.DeployCertificatesOnWorkers(ctx, c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
if err != nil {
if err := pki.DeployCertificatesOnWorkers(ctx, c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil {
return err
}
// Deploying worker certs on etcd hosts as well
err = pki.DeployCertificatesOnWorkers(ctx, c.EtcdHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
if err != nil {
// Deploying etcd certificates
if err := pki.DeployCertificatesOnEtcd(ctx, c.EtcdHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil {
return err
}
err = pki.DeployAdminConfig(ctx, c.Certificates[pki.KubeAdminCommonName].Config, c.LocalKubeConfigPath)
if err != nil {
if err := pki.DeployAdminConfig(ctx, c.Certificates[pki.KubeAdminCertName].Config, c.LocalKubeConfigPath); err != nil {
return err
}
log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes")

39
cluster/network.go
View File

@@ -8,6 +8,8 @@ import (
"net"
"strings"
b64 "encoding/base64"
"github.com/docker/docker/api/types/container"
"github.com/docker/go-connections/nat"
"github.com/rancher/rke/docker"
@@ -18,6 +20,7 @@ import (
"github.com/rancher/rke/templates"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"k8s.io/client-go/util/cert"
)
const (
@@ -66,10 +69,14 @@ const (
APIRoot = "APIRoot"
// kubernetes client certificates and kubeconfig paths
ClientCert = "ClientCert"
ClientKey = "ClientKey"
ClientCA = "ClientCA"
KubeCfg = "KubeCfg"
ClientCert = "ClientCert"
ClientCertPath = "ClientCertPath"
ClientKey = "ClientKey"
ClientKeyPath = "ClientKeyPath"
ClientCA = "ClientCA"
ClientCAPath = "ClientCAPath"
KubeCfg = "KubeCfg"
ClusterCIDR = "ClusterCIDR"
// Images key names
@@ -120,13 +127,20 @@ func (c *Cluster) doFlannelDeploy(ctx context.Context) error {
}
func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
clientCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.KubeNodeCertName].Certificate))
clientkey := b64.StdEncoding.EncodeToString(cert.EncodePrivateKeyPEM(c.Certificates[pki.KubeNodeCertName].Key))
clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
caCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.CACertName].Certificate))
calicoConfig := map[string]string{
EtcdEndpoints: services.GetEtcdConnString(c.EtcdHosts),
APIRoot: "https://127.0.0.1:6443",
ClientCert: pki.KubeNodeCertPath,
ClientKey: pki.KubeNodeKeyPath,
ClientCA: pki.CACertPath,
KubeCfg: pki.KubeNodeConfigPath,
ClientCert: clientCert,
ClientCertPath: pki.GetCertPath(pki.KubeNodeCertName),
ClientKey: clientkey,
ClientKeyPath: pki.GetKeyPath(pki.KubeNodeCertName),
ClientCA: caCert,
ClientCAPath: pki.GetCertPath(pki.CACertName),
KubeCfg: clientConfig,
ClusterCIDR: c.ClusterCIDR,
CNIImage: c.Network.Options[CalicoCNIImage],
NodeImage: c.Network.Options[CalicoNodeImage],
@@ -143,12 +157,13 @@ func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
}
func (c *Cluster) doCanalDeploy(ctx context.Context) error {
clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
canalConfig := map[string]string{
ClientCert: pki.KubeNodeCertPath,
ClientCertPath: pki.GetCertPath(pki.KubeNodeCertName),
APIRoot: "https://127.0.0.1:6443",
ClientKey: pki.KubeNodeKeyPath,
ClientCA: pki.CACertPath,
KubeCfg: pki.KubeNodeConfigPath,
ClientKeyPath: pki.GetKeyPath(pki.KubeNodeCertName),
ClientCAPath: pki.GetCertPath(pki.CACertName),
KubeCfg: clientConfig,
ClusterCIDR: c.ClusterCIDR,
NodeImage: c.Network.Options[CanalNodeImage],
CNIImage: c.Network.Options[CanalCNIImage],

33
cluster/reconcile.go
View File

@@ -7,9 +7,11 @@ import (
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/k8s"
"github.com/rancher/rke/log"
"github.com/rancher/rke/pki"
"github.com/rancher/rke/services"
"github.com/sirupsen/logrus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/cert"
)
const (
@@ -155,10 +157,14 @@ func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd b
}
func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, kubeClient *kubernetes.Clientset) error {
log.Infof(ctx, "[reconcile] Check etcd hosts to be deleted")
logrus.Infof("[reconcile] Check etcd hosts to be deleted")
// get tls for the first current etcd host
clientCert := cert.EncodeCertPEM(currentCluster.Certificates[pki.KubeNodeCertName].Certificate)
clientkey := cert.EncodePrivateKeyPEM(currentCluster.Certificates[pki.KubeNodeCertName].Key)
etcdToDelete := hosts.GetToDeleteHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts)
for _, etcdHost := range etcdToDelete {
if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil {
if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil {
log.Warnf(ctx, "[reconcile] %v", err)
continue
}
@@ -174,15 +180,34 @@ func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, ku
}
log.Infof(ctx, "[reconcile] Check etcd hosts to be added")
etcdToAdd := hosts.GetToAddHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts)
crtMap := currentCluster.Certificates
var err error
for _, etcdHost := range etcdToAdd {
etcdHost.ToAddEtcdMember = true
// Generate new certificate for the new etcd member
crtMap, err = pki.RegenerateEtcdCertificate(
ctx,
crtMap,
etcdHost,
kubeCluster.EtcdHosts,
kubeCluster.ClusterDomain,
kubeCluster.KubernetesServiceIP)
if err != nil {
return err
}
}
currentCluster.Certificates = crtMap
for _, etcdHost := range etcdToAdd {
if err := services.AddEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil {
// deploy certificates on new etcd host
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts, etcdHost, currentCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage], pki.CertPathPrefix); err != nil {
return err
}
if err := services.AddEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil {
return err
}
etcdHost.ToAddEtcdMember = false
if err := services.ReloadEtcdCluster(ctx, kubeCluster.EtcdHosts, kubeCluster.Services.Etcd, currentCluster.LocalConnDialerFactory); err != nil {
if err := services.ReloadEtcdCluster(ctx, kubeCluster.EtcdHosts, kubeCluster.Services.Etcd, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil {
return err
}
}

8
cluster/state.go
View File

@@ -58,7 +58,10 @@ func (c *Cluster) GetClusterState(ctx context.Context) (*Cluster, error) {
currentCluster = getStateFromKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath)
// Get previous kubernetes certificates
if currentCluster != nil {
currentCluster.Certificates, err = getClusterCerts(ctx, c.KubeClient)
if err := currentCluster.InvertIndexHosts(); err != nil {
return nil, fmt.Errorf("Failed to classify hosts from fetched cluster: %v", err)
}
currentCluster.Certificates, err = getClusterCerts(ctx, c.KubeClient, currentCluster.EtcdHosts)
currentCluster.DockerDialerFactory = c.DockerDialerFactory
if err != nil {
return nil, fmt.Errorf("Failed to Get Kubernetes certificates: %v", err)
@@ -66,9 +69,6 @@ func (c *Cluster) GetClusterState(ctx context.Context) (*Cluster, error) {
// setting cluster defaults for the fetched cluster as well
currentCluster.setClusterDefaults(ctx)
if err := currentCluster.InvertIndexHosts(); err != nil {
return nil, fmt.Errorf("Failed to classify hosts from fetched cluster: %v", err)
}
currentCluster.Certificates, err = regenerateAPICertificate(c, currentCluster.Certificates)
if err != nil {
return nil, fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err)

4
cmd/up.go
View File

@@ -110,8 +110,8 @@ func ClusterUp(
APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate))
clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCommonName].Certificate))
clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCommonName].Key))
clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate))
clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
log.Infof(ctx, "Finished building Kubernetes cluster successfully")
return APIURL, caCrt, clientCert, clientKey, nil

58
pki/constants.go
View File

@@ -1,65 +1,25 @@
package pki
const (
CertPathPrefix = "/etc/kubernetes/ssl/"
CertificatesServiceName = "certificates"
CrtDownloaderContainer = "cert-deployer"
CertFetcherContainer = "cert-fetcher"
CertificatesSecretName = "k8s-certs"
TempCertPath = "/etc/kubernetes/.tmp/"
CACertName = "kube-ca"
CACertENVName = "KUBE_CA"
CAKeyENVName = "KUBE_CA_KEY"
CACertPath = "/etc/kubernetes/ssl/kube-ca.pem"
CAKeyPath = "/etc/kubernetes/ssl/kube-ca-key.pem"
CACertName = "kube-ca"
KubeAPICertName = "kube-apiserver"
KubeControllerCertName = "kube-controller-manager"
KubeSchedulerCertName = "kube-scheduler"
KubeProxyCertName = "kube-proxy"
KubeNodeCertName = "kube-node"
EtcdCertName = "kube-etcd"
KubeAPICertName = "kube-apiserver"
KubeAPICertENVName = "KUBE_API"
KubeAPIKeyENVName = "KUBE_API_KEY"
KubeAPICertPath = "/etc/kubernetes/ssl/kube-api.pem"
KubeAPIKeyPath = "/etc/kubernetes/ssl/kube-api-key.pem"
KubeControllerName = "kube-controller-manager"
KubeControllerCommonName = "system:kube-controller-manager"
KubeControllerCertENVName = "KUBE_CONTROLLER_MANAGER"
KubeControllerKeyENVName = "KUBE_CONTROLLER_MANAGER_KEY"
KubeControllerConfigENVName = "KUBECFG_CONTROLLER_MANAGER"
KubeControllerCertPath = "/etc/kubernetes/ssl/kube-controller-manager.pem"
KubeControllerKeyPath = "/etc/kubernetes/ssl/kube-controller-manager-key.pem"
KubeControllerConfigPath = "/etc/kubernetes/ssl/kubecfg-controller-manager.yaml"
KubeSchedulerName = "kube-scheduler"
KubeSchedulerCommonName = "system:kube-scheduler"
KubeSchedulerCertENVName = "KUBE_SCHEDULER"
KubeSchedulerKeyENVName = "KUBE_SCHEDULER_KEY"
KubeSchedulerConfigENVName = "KUBECFG_SCHEDULER"
KubeSchedulerCertPath = "/etc/kubernetes/ssl/kube-scheduler.pem"
KubeSchedulerKeyPath = "/etc/kubernetes/ssl/kube-scheduler-key.pem"
KubeSchedulerConfigPath = "/etc/kubernetes/ssl/kubecfg-scheduler.yaml"
KubeProxyName = "kube-proxy"
KubeProxyCommonName = "system:kube-proxy"
KubeProxyCertENVName = "KUBE_PROXY"
KubeProxyKeyENVName = "KUBE_PROXY_KEY"
KubeProxyConfigENVName = "KUBECFG_KUBE_PROXY"
KubeProxyCertPath = "/etc/kubernetes/ssl/kube-proxy.pem"
KubeProxyKeyPath = "/etc/kubernetes/ssl/kube-proxy-key.pem"
KubeProxyConfigPath = "/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml"
KubeNodeName = "kube-node"
KubeNodeCommonName = "system:node"
KubeNodeOrganizationName = "system:nodes"
KubeNodeCertENVName = "KUBE_NODE"
KubeNodeKeyENVName = "KUBE_NODE_KEY"
KubeNodeConfigENVName = "KUBECFG_KUBE_NODE"
KubeNodeCertPath = "/etc/kubernetes/ssl/kube-node.pem"
KubeNodeKeyPath = "/etc/kubernetes/ssl/kube-node-key.pem"
KubeNodeConfigPath = "/etc/kubernetes/ssl/kubecfg-kube-node.yaml"
KubeAdminCommonName = "kube-admin"
KubeAdminCertName = "kube-admin"
KubeAdminOrganizationName = "system:masters"
KubeAdminConfigPrefix = ".kube_config_"
KubeAdminConfigENVName = "KUBECFG_ADMIN"
KubeAdminCertEnvName = "KUBE_ADMIN"
KubeAdminKeyEnvName = "KUBE_ADMIN_KEY"
)

202
pki/deploy.go
View File

@@ -24,10 +24,10 @@ func DeployCertificatesOnMasters(ctx context.Context, cpHosts []*hosts.Host, crt
crtList := []string{
CACertName,
KubeAPICertName,
KubeControllerName,
KubeSchedulerName,
KubeProxyName,
KubeNodeName,
KubeControllerCertName,
KubeSchedulerCertName,
KubeProxyCertName,
KubeNodeCertName,
}
env := []string{}
for _, crtName := range crtList {
@@ -48,8 +48,8 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host,
// list of certificates that should be deployed on the workers
crtList := []string{
CACertName,
KubeProxyName,
KubeNodeName,
KubeProxyCertName,
KubeNodeCertName,
}
env := []string{}
for _, crtName := range crtList {
@@ -66,6 +66,31 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host,
return nil
}
func DeployCertificatesOnEtcd(ctx context.Context, etcdHosts []*hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error {
// list of certificates that should be deployed on the etcd
crtList := []string{
CACertName,
KubeProxyCertName,
KubeNodeCertName,
}
for _, host := range etcdHosts {
crtList = append(crtList, GetEtcdCrtName(host.InternalAddress))
}
env := []string{}
for _, crtName := range crtList {
c := crtMap[crtName]
env = append(env, c.ToEnv()...)
}
for i := range etcdHosts {
err := doRunDeployer(ctx, etcdHosts[i], env, certDownloaderImage)
if err != nil {
return err
}
}
return nil
}
func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string, certDownloaderImage string) error {
// remove existing container. Only way it's still here is if previous deployment failed
isRunning := false
@@ -135,19 +160,22 @@ func RemoveAdminConfig(ctx context.Context, localConfigPath string) {
log.Infof(ctx, "Local admin Kubeconfig removed successfully")
}
func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error {
func DeployCertificatesOnHost(ctx context.Context, extraHosts []*hosts.Host, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage, certPath string) error {
crtList := []string{
CACertName,
KubeAPICertName,
KubeControllerName,
KubeSchedulerName,
KubeProxyName,
KubeNodeName,
KubeAdminCommonName,
KubeControllerCertName,
KubeSchedulerCertName,
KubeProxyCertName,
KubeNodeCertName,
KubeAdminCertName,
}
for _, host := range extraHosts {
// Deploy etcd certificates
crtList = append(crtList, GetEtcdCrtName(host.InternalAddress))
}
env := []string{
"CRTS_DEPLOY_PATH=" + TempCertPath,
"CRTS_DEPLOY_PATH=" + certPath,
}
for _, crtName := range crtList {
c := crtMap[crtName]
@@ -157,24 +185,27 @@ func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[
return doRunDeployer(ctx, host, env, certDownloaderImage)
}
func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) {
func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) {
// rebuilding the certificates. This should look better after refactoring pki
tmpCerts := make(map[string]CertificatePKI)
certEnvMap := map[string][]string{
CACertName: []string{CACertPath, CAKeyPath},
KubeAPICertName: []string{KubeAPICertPath, KubeAPIKeyPath},
KubeControllerName: []string{KubeControllerCertPath, KubeControllerKeyPath, KubeControllerConfigPath},
KubeSchedulerName: []string{KubeSchedulerCertPath, KubeSchedulerKeyPath, KubeSchedulerConfigPath},
KubeProxyName: []string{KubeProxyCertPath, KubeProxyKeyPath, KubeProxyConfigPath},
KubeNodeName: []string{KubeNodeCertPath, KubeNodeKeyPath, KubeNodeConfigPath},
KubeAdminCommonName: []string{"kube-admin.pem", "kube-admin-key.pem", "kubecfg-admin.yaml"},
crtList := map[string]bool{
CACertName: false,
KubeAPICertName: false,
KubeControllerCertName: true,
KubeSchedulerCertName: true,
KubeProxyCertName: true,
KubeNodeCertName: true,
KubeAdminCertName: true,
}
for _, etcdHost := range extraHosts {
// Fetch etcd certificates
crtList[GetEtcdCrtName(etcdHost.InternalAddress)] = false
}
// get files from hosts
for crtName, certEnv := range certEnvMap {
for certName, config := range crtList {
certificate := CertificatePKI{}
crt, err := fetchFileFromHost(ctx, getTempPath(certEnv[0]), image, host)
crt, err := fetchFileFromHost(ctx, GetCertTempPath(certName), image, host)
if err != nil {
if strings.Contains(err.Error(), "no such file or directory") ||
strings.Contains(err.Error(), "Could not find the file") {
@@ -182,10 +213,10 @@ func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, loc
}
return nil, err
}
key, err := fetchFileFromHost(ctx, getTempPath(certEnv[1]), image, host)
key, err := fetchFileFromHost(ctx, GetKeyTempPath(certName), image, host)
if len(certEnv) > 2 {
config, err := fetchFileFromHost(ctx, getTempPath(certEnv[2]), image, host)
if config {
config, err := fetchFileFromHost(ctx, GetConfigTempPath(certName), image, host)
if err != nil {
return nil, err
}
@@ -201,14 +232,14 @@ func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, loc
}
certificate.Certificate = parsedCert[0]
certificate.Key = parsedKey.(*rsa.PrivateKey)
tmpCerts[crtName] = certificate
logrus.Debugf("[certificates] Recovered certificate: %s", crtName)
tmpCerts[certName] = certificate
logrus.Debugf("[certificates] Recovered certificate: %s", certName)
}
if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CertFetcherContainer); err != nil {
return nil, err
}
return populateCertMap(tmpCerts, localConfigPath), nil
return populateCertMap(tmpCerts, localConfigPath, extraHosts), nil
}
@@ -244,96 +275,31 @@ func getTempPath(s string) string {
return TempCertPath + path.Base(s)
}
func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string) map[string]CertificatePKI {
func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI {
certs := make(map[string]CertificatePKI)
//CACert
certs[CACertName] = CertificatePKI{
Certificate: tmpCerts[CACertName].Certificate,
Key: tmpCerts[CACertName].Key,
Name: CACertName,
EnvName: CACertENVName,
KeyEnvName: CAKeyENVName,
Path: CACertPath,
KeyPath: CAKeyPath,
}
//KubeAPI
certs[KubeAPICertName] = CertificatePKI{
Certificate: tmpCerts[KubeAPICertName].Certificate,
Key: tmpCerts[KubeAPICertName].Key,
Name: KubeAPICertName,
EnvName: KubeAPICertENVName,
KeyEnvName: KubeAPIKeyENVName,
Path: KubeAPICertPath,
KeyPath: KubeAPIKeyPath,
}
//kubeController
certs[KubeControllerName] = CertificatePKI{
Certificate: tmpCerts[KubeControllerName].Certificate,
Key: tmpCerts[KubeControllerName].Key,
Config: tmpCerts[KubeControllerName].Config,
Name: KubeControllerName,
CommonName: KubeControllerCommonName,
EnvName: KubeControllerCertENVName,
KeyEnvName: KubeControllerKeyENVName,
Path: KubeControllerCertPath,
KeyPath: KubeControllerKeyPath,
ConfigEnvName: KubeControllerConfigENVName,
ConfigPath: KubeControllerConfigPath,
}
//KubeScheduler
certs[KubeSchedulerName] = CertificatePKI{
Certificate: tmpCerts[KubeSchedulerName].Certificate,
Key: tmpCerts[KubeSchedulerName].Key,
Config: tmpCerts[KubeSchedulerName].Config,
Name: KubeSchedulerName,
CommonName: KubeSchedulerCommonName,
EnvName: KubeSchedulerCertENVName,
KeyEnvName: KubeSchedulerKeyENVName,
Path: KubeSchedulerCertPath,
KeyPath: KubeSchedulerKeyPath,
ConfigEnvName: KubeSchedulerConfigENVName,
ConfigPath: KubeSchedulerConfigPath,
}
// CACert
certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key)
// KubeAPI
certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key)
// kubeController
certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key)
// KubeScheduler
certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key)
// KubeProxy
certs[KubeProxyName] = CertificatePKI{
Certificate: tmpCerts[KubeProxyName].Certificate,
Key: tmpCerts[KubeProxyName].Key,
Config: tmpCerts[KubeProxyName].Config,
Name: KubeProxyName,
CommonName: KubeProxyCommonName,
EnvName: KubeProxyCertENVName,
Path: KubeProxyCertPath,
KeyEnvName: KubeProxyKeyENVName,
KeyPath: KubeProxyKeyPath,
ConfigEnvName: KubeProxyConfigENVName,
ConfigPath: KubeProxyConfigPath,
}
certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key)
// KubeNode
certs[KubeNodeName] = CertificatePKI{
Certificate: tmpCerts[KubeNodeName].Certificate,
Key: tmpCerts[KubeNodeName].Key,
Config: tmpCerts[KubeNodeName].Config,
Name: KubeNodeName,
CommonName: KubeNodeCommonName,
OUName: KubeNodeOrganizationName,
EnvName: KubeNodeCertENVName,
KeyEnvName: KubeNodeKeyENVName,
Path: KubeNodeCertPath,
KeyPath: KubeNodeKeyPath,
ConfigEnvName: KubeNodeConfigENVName,
ConfigPath: KubeNodeCommonName,
certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key)
// KubeAdmin
kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key)
kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config
kubeAdminCertObj.ConfigPath = localConfigPath
certs[KubeAdminCertName] = kubeAdminCertObj
// etcd
for _, host := range extraHosts {
etcdName := GetEtcdCrtName(host.InternalAddress)
etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key
certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
}
certs[KubeAdminCommonName] = CertificatePKI{
Certificate: tmpCerts[KubeAdminCommonName].Certificate,
Key: tmpCerts[KubeAdminCommonName].Key,
Config: tmpCerts[KubeAdminCommonName].Config,
CommonName: KubeAdminCommonName,
OUName: KubeAdminOrganizationName,
ConfigEnvName: KubeAdminConfigENVName,
ConfigPath: localConfigPath,
EnvName: KubeAdminCertEnvName,
KeyEnvName: KubeAdminKeyEnvName,
}
return certs
}

291
pki/pki.go
View File

@@ -4,7 +4,6 @@ import (
"context"
"crypto/rsa"
"crypto/x509"
"fmt"
"net"
"github.com/rancher/rke/hosts"
@@ -29,16 +28,16 @@ type CertificatePKI struct {
}
// StartCertificatesGeneration ...
func StartCertificatesGeneration(ctx context.Context, cpHosts []*hosts.Host, workerHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
log.Infof(ctx, "[certificates] Generating kubernetes certificates")
certs, err := generateCerts(ctx, cpHosts, clusterDomain, localConfigPath, KubernetesServiceIP)
func StartCertificatesGeneration(ctx context.Context, cpHosts, etcdHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
logrus.Infof("[certificates] Generating kubernetes certificates")
certs, err := generateCerts(ctx, cpHosts, etcdHosts, clusterDomain, localConfigPath, KubernetesServiceIP)
if err != nil {
return nil, err
}
return certs, nil
}
func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
func generateCerts(ctx context.Context, cpHosts, etcdHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
certs := make(map[string]CertificatePKI)
// generate CA certificate and key
log.Infof(ctx, "[certificates] Generating CA kubernetes certificates")
@@ -46,270 +45,100 @@ func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, lo
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] CA Certificate: %s", string(cert.EncodeCertPEM(caCrt)))
certs[CACertName] = CertificatePKI{
Certificate: caCrt,
Key: caKey,
Name: CACertName,
EnvName: CACertENVName,
KeyEnvName: CAKeyENVName,
Path: CACertPath,
KeyPath: CAKeyPath,
}
certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey)
// generate API certificate and key
log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates")
kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, KubernetesServiceIP)
kubeAPICrt, kubeAPIKey, err := GenerateKubeAPICertAndKey(caCrt, caKey, kubeAPIAltNames)
kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, nil, nil)
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] Kube API Certificate: %s", string(cert.EncodeCertPEM(kubeAPICrt)))
certs[KubeAPICertName] = CertificatePKI{
Certificate: kubeAPICrt,
Key: kubeAPIKey,
Name: KubeAPICertName,
EnvName: KubeAPICertENVName,
KeyEnvName: KubeAPIKeyENVName,
Path: KubeAPICertPath,
KeyPath: KubeAPIKeyPath,
}
certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey)
// generate Kube controller-manager certificate and key
log.Infof(ctx, "[certificates] Generating Kube Controller certificates")
kubeControllerCrt, kubeControllerKey, err := generateClientCertAndKey(caCrt, caKey, KubeControllerCommonName, []string{})
kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil)
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] Kube Controller Certificate: %s", string(cert.EncodeCertPEM(kubeControllerCrt)))
certs[KubeControllerName] = CertificatePKI{
Certificate: kubeControllerCrt,
Key: kubeControllerKey,
Config: getKubeConfigX509("https://127.0.0.1:6443", KubeControllerName, CACertPath, KubeControllerCertPath, KubeControllerKeyPath),
Name: KubeControllerName,
CommonName: KubeControllerCommonName,
EnvName: KubeControllerCertENVName,
KeyEnvName: KubeControllerKeyENVName,
Path: KubeControllerCertPath,
KeyPath: KubeControllerKeyPath,
ConfigEnvName: KubeControllerConfigENVName,
ConfigPath: KubeControllerConfigPath,
}
certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey)
// generate Kube scheduler certificate and key
log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates")
kubeSchedulerCrt, kubeSchedulerKey, err := generateClientCertAndKey(caCrt, caKey, KubeSchedulerCommonName, []string{})
kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil)
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] Kube Scheduler Certificate: %s", string(cert.EncodeCertPEM(kubeSchedulerCrt)))
certs[KubeSchedulerName] = CertificatePKI{
Certificate: kubeSchedulerCrt,
Key: kubeSchedulerKey,
Config: getKubeConfigX509("https://127.0.0.1:6443", KubeSchedulerName, CACertPath, KubeSchedulerCertPath, KubeSchedulerKeyPath),
Name: KubeSchedulerName,
CommonName: KubeSchedulerCommonName,
EnvName: KubeSchedulerCertENVName,
KeyEnvName: KubeSchedulerKeyENVName,
Path: KubeSchedulerCertPath,
KeyPath: KubeSchedulerKeyPath,
ConfigEnvName: KubeSchedulerConfigENVName,
ConfigPath: KubeSchedulerConfigPath,
}
certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey)
// generate Kube Proxy certificate and key
log.Infof(ctx, "[certificates] Generating Kube Proxy certificates")
kubeProxyCrt, kubeProxyKey, err := generateClientCertAndKey(caCrt, caKey, KubeProxyCommonName, []string{})
kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil)
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] Kube Proxy Certificate: %s", string(cert.EncodeCertPEM(kubeProxyCrt)))
certs[KubeProxyName] = CertificatePKI{
Certificate: kubeProxyCrt,
Key: kubeProxyKey,
Config: getKubeConfigX509("https://127.0.0.1:6443", KubeProxyName, CACertPath, KubeProxyCertPath, KubeProxyKeyPath),
Name: KubeProxyName,
CommonName: KubeProxyCommonName,
EnvName: KubeProxyCertENVName,
Path: KubeProxyCertPath,
KeyEnvName: KubeProxyKeyENVName,
KeyPath: KubeProxyKeyPath,
ConfigEnvName: KubeProxyConfigENVName,
ConfigPath: KubeProxyConfigPath,
}
certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey)
// generate Kubelet certificate and key
log.Infof(ctx, "[certificates] Generating Node certificate")
nodeCrt, nodeKey, err := generateClientCertAndKey(caCrt, caKey, KubeNodeCommonName, []string{KubeNodeOrganizationName})
nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName})
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] Node Certificate: %s", string(cert.EncodeCertPEM(kubeProxyCrt)))
certs[KubeNodeName] = CertificatePKI{
Certificate: nodeCrt,
Key: nodeKey,
Config: getKubeConfigX509("https://127.0.0.1:6443", KubeNodeName, CACertPath, KubeNodeCertPath, KubeNodeKeyPath),
Name: KubeNodeName,
CommonName: KubeNodeCommonName,
OUName: KubeNodeOrganizationName,
EnvName: KubeNodeCertENVName,
KeyEnvName: KubeNodeKeyENVName,
Path: KubeNodeCertPath,
KeyPath: KubeNodeKeyPath,
ConfigEnvName: KubeNodeConfigENVName,
ConfigPath: KubeNodeCommonName,
}
log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig")
kubeAdminCrt, kubeAdminKey, err := generateClientCertAndKey(caCrt, caKey, KubeAdminCommonName, []string{KubeAdminOrganizationName})
certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey)
// generate Admin certificate and key
logrus.Infof("[certificates] Generating admin certificates and kubeconfig")
kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName})
if err != nil {
return nil, err
}
logrus.Debugf("[certificates] Admin Certificate: %s", string(cert.EncodeCertPEM(kubeAdminCrt)))
certs[KubeAdminCommonName] = CertificatePKI{
Certificate: kubeAdminCrt,
Key: kubeAdminKey,
Config: GetKubeConfigX509WithData(
"https://"+cpHosts[0].Address+":6443",
KubeAdminCommonName,
string(cert.EncodeCertPEM(caCrt)),
string(cert.EncodeCertPEM(kubeAdminCrt)),
string(cert.EncodePrivateKeyPEM(kubeAdminKey))),
CommonName: KubeAdminCommonName,
OUName: KubeAdminOrganizationName,
ConfigEnvName: KubeAdminConfigENVName,
ConfigPath: localConfigPath,
EnvName: KubeAdminCertEnvName,
KeyEnvName: KubeAdminKeyEnvName,
kubeAdminConfig := GetKubeConfigX509WithData(
"https://"+cpHosts[0].Address+":6443",
KubeAdminCertName,
string(cert.EncodeCertPEM(caCrt)),
string(cert.EncodeCertPEM(kubeAdminCrt)),
string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
kubeAdminCertObj.Config = kubeAdminConfig
kubeAdminCertObj.ConfigPath = localConfigPath
certs[KubeAdminCertName] = kubeAdminCertObj
etcdAltNames := GetAltNames(etcdHosts, clusterDomain, KubernetesServiceIP)
for _, host := range etcdHosts {
logrus.Infof("[certificates] Generating etcd-%s certificate and key", host.InternalAddress)
etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
if err != nil {
return nil, err
}
etcdName := GetEtcdCrtName(host.InternalAddress)
certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
}
return certs, nil
}
func generateClientCertAndKey(caCrt *x509.Certificate, caKey *rsa.PrivateKey, commonName string, orgs []string) (*x509.Certificate, *rsa.PrivateKey, error) {
rootKey, err := cert.NewPrivateKey()
func RegenerateEtcdCertificate(
ctx context.Context,
crtMap map[string]CertificatePKI,
etcdHost *hosts.Host,
etcdHosts []*hosts.Host,
clusterDomain string,
KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
log.Infof(ctx, "[certificates] Regenerating new etcd-%s certificate and key", etcdHost.InternalAddress)
caCrt := crtMap[CACertName].Certificate
caKey := crtMap[CACertName].Key
etcdAltNames := GetAltNames(etcdHosts, clusterDomain, KubernetesServiceIP)
etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err)
return nil, err
}
caConfig := cert.Config{
CommonName: commonName,
Organization: orgs,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err)
}
return clientCert, rootKey, nil
}
func GenerateKubeAPICertAndKey(caCrt *x509.Certificate, caKey *rsa.PrivateKey, altNames *cert.AltNames) (*x509.Certificate, *rsa.PrivateKey, error) {
rootKey, err := cert.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate private key for kube-apiserver certificate: %v", err)
}
caConfig := cert.Config{
CommonName: KubeAPICertName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
AltNames: *altNames,
}
kubeCACert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate kube-apiserver certificate: %v", err)
}
return kubeCACert, rootKey, nil
}
func GenerateCertWithKey(commonName string, key *rsa.PrivateKey, caCrt *x509.Certificate, caKey *rsa.PrivateKey, altNames *cert.AltNames) (*x509.Certificate, error) {
caConfig := cert.Config{
CommonName: commonName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth,
x509.ExtKeyUsageServerAuth},
AltNames: *altNames,
}
cert, err := cert.NewSignedCert(caConfig, key, caCrt, caKey)
if err != nil {
return nil, fmt.Errorf("Failed to generate certificate with existing key: %v", err)
}
return cert, nil
}
func generateCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
rootKey, err := cert.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)
}
caConfig := cert.Config{
CommonName: CACertName,
}
kubeCACert, err := cert.NewSelfSignedCACert(caConfig, rootKey)
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate CA certificate: %v", err)
}
return kubeCACert, rootKey, nil
}
func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceIP net.IP) *cert.AltNames {
ips := []net.IP{}
dnsNames := []string{}
for _, host := range cpHosts {
// Check if node address is a valid IP
if nodeIP := net.ParseIP(host.Address); nodeIP != nil {
ips = append(ips, nodeIP)
} else {
dnsNames = append(dnsNames, host.Address)
}
// Check if node internal address is a valid IP
if len(host.InternalAddress) != 0 && host.InternalAddress != host.Address {
if internalIP := net.ParseIP(host.InternalAddress); internalIP != nil {
ips = append(ips, internalIP)
} else {
dnsNames = append(dnsNames, host.InternalAddress)
}
}
// Add hostname to the ALT dns names
if len(host.HostnameOverride) != 0 && host.HostnameOverride != host.Address {
dnsNames = append(dnsNames, host.HostnameOverride)
}
}
ips = append(ips, net.ParseIP("127.0.0.1"))
ips = append(ips, KubernetesServiceIP)
dnsNames = append(dnsNames, []string{
"localhost",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc." + clusterDomain,
}...)
return &cert.AltNames{
IPs: ips,
DNSNames: dnsNames,
}
}
func (c *CertificatePKI) ToEnv() []string {
env := []string{
c.CertToEnv(),
c.KeyToEnv(),
}
if c.Config != "" {
env = append(env, c.ConfigToEnv())
}
return env
}
func (c *CertificatePKI) CertToEnv() string {
encodedCrt := cert.EncodeCertPEM(c.Certificate)
return fmt.Sprintf("%s=%s", c.EnvName, string(encodedCrt))
}
func (c *CertificatePKI) KeyToEnv() string {
encodedKey := cert.EncodePrivateKeyPEM(c.Key)
return fmt.Sprintf("%s=%s", c.KeyEnvName, string(encodedKey))
}
func (c *CertificatePKI) ConfigToEnv() string {
return fmt.Sprintf("%s=%s", c.ConfigEnvName, c.Config)
etcdName := GetEtcdCrtName(etcdHost.InternalAddress)
crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress)
return crtMap, nil
}

10
pki/pki_test.go
View File

@@ -38,11 +38,11 @@ func TestPKI(t *testing.T) {
certificatesToVerify := []string{
KubeAPICertName,
KubeNodeName,
KubeProxyName,
KubeControllerName,
KubeSchedulerName,
KubeAdminCommonName,
KubeNodeCertName,
KubeProxyCertName,
KubeControllerCertName,
KubeSchedulerCertName,
KubeAdminCertName,
}
opts := x509.VerifyOptions{
Roots: roots,

209
pki/util.go Normal file
View File

@@ -0,0 +1,209 @@
package pki
import (
"crypto/rsa"
"crypto/x509"
"fmt"
"net"
"strings"
"github.com/rancher/rke/hosts"
"k8s.io/client-go/util/cert"
)
func GenerateSignedCertAndKey(
caCrt *x509.Certificate,
caKey *rsa.PrivateKey,
serverCrt bool,
commonName string,
altNames *cert.AltNames,
reusedKey *rsa.PrivateKey,
orgs []string) (*x509.Certificate, *rsa.PrivateKey, error) {
// Generate a generic signed certificate
var rootKey *rsa.PrivateKey
var err error
rootKey = reusedKey
if reusedKey == nil {
rootKey, err = cert.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err)
}
}
usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
if serverCrt {
usages = append(usages, x509.ExtKeyUsageServerAuth)
}
if altNames == nil {
altNames = &cert.AltNames{}
}
caConfig := cert.Config{
CommonName: commonName,
Organization: orgs,
Usages: usages,
AltNames: *altNames,
}
clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err)
}
return clientCert, rootKey, nil
}
func generateCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
rootKey, err := cert.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)
}
caConfig := cert.Config{
CommonName: CACertName,
}
kubeCACert, err := cert.NewSelfSignedCACert(caConfig, rootKey)
if err != nil {
return nil, nil, fmt.Errorf("Failed to generate CA certificate: %v", err)
}
return kubeCACert, rootKey, nil
}
func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceIP net.IP) *cert.AltNames {
ips := []net.IP{}
dnsNames := []string{}
for _, host := range cpHosts {
// Check if node address is a valid IP
if nodeIP := net.ParseIP(host.Address); nodeIP != nil {
ips = append(ips, nodeIP)
} else {
dnsNames = append(dnsNames, host.Address)
}
// Check if node internal address is a valid IP
if len(host.InternalAddress) != 0 && host.InternalAddress != host.Address {
if internalIP := net.ParseIP(host.InternalAddress); internalIP != nil {
ips = append(ips, internalIP)
} else {
dnsNames = append(dnsNames, host.InternalAddress)
}
}
// Add hostname to the ALT dns names
if len(host.HostnameOverride) != 0 && host.HostnameOverride != host.Address {
dnsNames = append(dnsNames, host.HostnameOverride)
}
}
ips = append(ips, net.ParseIP("127.0.0.1"))
ips = append(ips, KubernetesServiceIP)
dnsNames = append(dnsNames, []string{
"localhost",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc." + clusterDomain,
}...)
return &cert.AltNames{
IPs: ips,
DNSNames: dnsNames,
}
}
func (c *CertificatePKI) ToEnv() []string {
env := []string{
c.CertToEnv(),
c.KeyToEnv(),
}
if c.Config != "" {
env = append(env, c.ConfigToEnv())
}
return env
}
func (c *CertificatePKI) CertToEnv() string {
encodedCrt := cert.EncodeCertPEM(c.Certificate)
return fmt.Sprintf("%s=%s", c.EnvName, string(encodedCrt))
}
func (c *CertificatePKI) KeyToEnv() string {
encodedKey := cert.EncodePrivateKeyPEM(c.Key)
return fmt.Sprintf("%s=%s", c.KeyEnvName, string(encodedKey))
}
func (c *CertificatePKI) ConfigToEnv() string {
return fmt.Sprintf("%s=%s", c.ConfigEnvName, c.Config)
}
func getEnvFromName(name string) string {
return strings.Replace(strings.ToUpper(name), "-", "_", -1)
}
func getKeyEnvFromEnv(env string) string {
return fmt.Sprintf("%s_KEY", env)
}
func getConfigEnvFromEnv(env string) string {
return fmt.Sprintf("KUBECFG_%s", env)
}
func GetEtcdCrtName(address string) string {
newAddress := strings.Replace(address, ".", "-", -1)
return fmt.Sprintf("%s-%s", EtcdCertName, newAddress)
}
func GetCertPath(name string) string {
return fmt.Sprintf("%s%s.pem", CertPathPrefix, name)
}
func GetKeyPath(name string) string {
return fmt.Sprintf("%s%s-key.pem", CertPathPrefix, name)
}
func GetConfigPath(name string) string {
return fmt.Sprintf("%skubecfg-%s.yaml", CertPathPrefix, name)
}
func GetCertTempPath(name string) string {
return fmt.Sprintf("%s%s.pem", TempCertPath, name)
}
func GetKeyTempPath(name string) string {
return fmt.Sprintf("%s%s-key.pem", TempCertPath, name)
}
func GetConfigTempPath(name string) string {
return fmt.Sprintf("%skubecfg-%s.yaml", TempCertPath, name)
}
func ToCertObject(componentName, commonName, ouName string, cert *x509.Certificate, key *rsa.PrivateKey) CertificatePKI {
var config, configPath, configEnvName string
if len(commonName) == 0 {
commonName = getDefaultCN(componentName)
}
envName := getEnvFromName(componentName)
keyEnvName := getKeyEnvFromEnv(envName)
caCertPath := GetCertPath(CACertName)
path := GetCertPath(componentName)
keyPath := GetKeyPath(componentName)
if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) {
config = getKubeConfigX509("https://127.0.0.1:6443", componentName, caCertPath, path, keyPath)
configPath = GetConfigPath(componentName)
configEnvName = getConfigEnvFromEnv(envName)
}
return CertificatePKI{
Certificate: cert,
Key: key,
Config: config,
Name: componentName,
CommonName: commonName,
OUName: ouName,
EnvName: envName,
KeyEnvName: keyEnvName,
ConfigEnvName: configEnvName,
Path: path,
KeyPath: keyPath,
ConfigPath: configPath,
}
}
func getDefaultCN(name string) string {
return fmt.Sprintf("system:%s", name)
}

49
services/etcd.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/rancher/rke/docker"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/log"
"github.com/rancher/rke/pki"
"github.com/rancher/types/apis/management.cattle.io/v3"
"github.com/sirupsen/logrus"
)
@@ -19,7 +20,8 @@ func RunEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.E
log.Infof(ctx, "[%s] Building up Etcd Plane..", ETCDRole)
initCluster := getEtcdInitialCluster(etcdHosts)
for _, host := range etcdHosts {
imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster)
nodeName := pki.GetEtcdCrtName(host.InternalAddress)
imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster, nodeName)
err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole)
if err != nil {
return err
@@ -57,7 +59,7 @@ func RemoveEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, force bool) e
return nil
}
func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster string) (*container.Config, *container.HostConfig) {
func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster, nodeName string) (*container.Config, *container.HostConfig) {
clusterState := "new"
if host.ExistingEtcdCluster {
clusterState = "existing"
@@ -67,18 +69,29 @@ func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster s
Cmd: []string{"/usr/local/bin/etcd",
"--name=etcd-" + host.HostnameOverride,
"--data-dir=/etcd-data",
"--advertise-client-urls=http://" + host.InternalAddress + ":2379,http://" + host.InternalAddress + ":4001",
"--listen-client-urls=http://0.0.0.0:2379",
"--initial-advertise-peer-urls=http://" + host.InternalAddress + ":2380",
"--listen-peer-urls=http://0.0.0.0:2380",
"--advertise-client-urls=https://" + host.InternalAddress + ":2379,https://" + host.InternalAddress + ":4001",
"--listen-client-urls=https://0.0.0.0:2379",
"--initial-advertise-peer-urls=https://" + host.InternalAddress + ":2380",
"--listen-peer-urls=https://0.0.0.0:2380",
"--initial-cluster-token=etcd-cluster-1",
"--initial-cluster=" + initCluster,
"--initial-cluster-state=" + clusterState},
"--initial-cluster-state=" + clusterState,
"--peer-client-cert-auth",
"--client-cert-auth",
"--trusted-ca-file=" + pki.GetCertPath(pki.CACertName),
"--peer-trusted-ca-file=" + pki.GetCertPath(pki.CACertName),
"--cert-file=" + pki.GetCertPath(nodeName),
"--key-file=" + pki.GetKeyPath(nodeName),
"--peer-cert-file=" + pki.GetCertPath(nodeName),
"--peer-key-file=" + pki.GetKeyPath(nodeName),
},
}
hostCfg := &container.HostConfig{
RestartPolicy: container.RestartPolicy{Name: "always"},
Binds: []string{
"/var/lib/etcd:/etcd-data"},
"/var/lib/etcd:/etcd-data",
"/etc/kubernetes:/etc/kubernetes",
},
NetworkMode: "host",
}
for arg, value := range etcdService.ExtraArgs {
@@ -89,12 +102,12 @@ func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster s
return imageCfg, hostCfg
}
func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory) error {
func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
log.Infof(ctx, "[add/%s] Adding member [etcd-%s] to etcd cluster", ETCDRole, etcdHost.HostnameOverride)
peerURL := fmt.Sprintf("http://%s:2380", etcdHost.InternalAddress)
peerURL := fmt.Sprintf("https://%s:2380", etcdHost.InternalAddress)
added := false
for _, host := range etcdHosts {
etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory)
etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory, cert, key)
if err != nil {
logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err)
continue
@@ -114,12 +127,12 @@ func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts
return nil
}
func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory) error {
func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
log.Infof(ctx, "[remove/%s] Removing member [etcd-%s] from etcd cluster", ETCDRole, etcdHost.HostnameOverride)
var mID string
removed := false
for _, host := range etcdHosts {
etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory)
etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory, cert, key)
if err != nil {
logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err)
continue
@@ -150,7 +163,7 @@ func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*ho
return nil
}
func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.ETCDService, localConnDialerFactory hosts.DialerFactory) error {
func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.ETCDService, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
readyEtcdHosts := []*hosts.Host{}
for _, host := range etcdHosts {
if !host.ToAddEtcdMember {
@@ -160,16 +173,20 @@ func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService
}
initCluster := getEtcdInitialCluster(readyEtcdHosts)
for _, host := range readyEtcdHosts {
imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster)
imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster, pki.GetEtcdCrtName(host.InternalAddress))
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole); err != nil {
return err
}
}
time.Sleep(10 * time.Second)
var healthy bool
for _, host := range readyEtcdHosts {
if healthy := isEtcdHealthy(ctx, localConnDialerFactory, host); healthy {
if healthy = isEtcdHealthy(ctx, localConnDialerFactory, host, cert, key); healthy {
break
}
}
if !healthy {
return fmt.Errorf("[etcd] Etcd Cluster is not healthy")
}
return nil
}

6
services/etcd_test.go
View File

@@ -5,11 +5,12 @@ import (
"testing"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/pki"
"github.com/rancher/types/apis/management.cattle.io/v3"
)
const (
TestInitEtcdClusterString = "etcd-etcd1=http://1.1.1.1:2380,etcd-etcd2=http://2.2.2.2:2380"
TestInitEtcdClusterString = "etcd-etcd1=https://1.1.1.1:2380,etcd-etcd2=https://2.2.2.2:2380"
TestEtcdImage = "etcd/etcdImage:latest"
TestEtcdNamePrefix = "--name=etcd-"
TestEtcdVolumeBind = "/var/lib/etcd:/etcd-data"
@@ -46,7 +47,8 @@ func TestEtcdConfig(t *testing.T) {
assertEqual(t, initCluster, TestInitEtcdClusterString, "")
for _, host := range etcdHosts {
imageCfg, hostCfg := buildEtcdConfig(host, etcdService, TestInitEtcdClusterString)
nodeName := pki.GetEtcdCrtName(host.InternalAddress)
imageCfg, hostCfg := buildEtcdConfig(host, etcdService, TestInitEtcdClusterString, nodeName)
assertEqual(t, isStringInSlice(TestEtcdNamePrefix+host.HostnameOverride, imageCfg.Cmd), true,
fmt.Sprintf("Failed to find [%s] in Etcd command", TestEtcdNamePrefix+host.HostnameOverride))
assertEqual(t, TestEtcdImage, imageCfg.Image,

52
services/etcd_util.go
View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
@@ -14,36 +15,48 @@ import (
"github.com/sirupsen/logrus"
)
func getEtcdClient(ctx context.Context, etcdHost *hosts.Host, localConnDialerFactory hosts.DialerFactory) (etcdclient.Client, error) {
func getEtcdClient(ctx context.Context, etcdHost *hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) (etcdclient.Client, error) {
dialer, err := getEtcdDialer(localConnDialerFactory, etcdHost)
if err != nil {
return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", etcdHost.Address, err)
}
tlsConfig, err := getEtcdTLSConfig(cert, key)
if err != nil {
return nil, err
}
var DefaultEtcdTransport etcdclient.CancelableTransport = &http.Transport{
Dial: dialer,
Dial: dialer,
TLSClientConfig: tlsConfig,
TLSHandshakeTimeout: 10 * time.Second,
}
cfg := etcdclient.Config{
Endpoints: []string{"http://127.0.0.1:2379"},
Endpoints: []string{"https://127.0.0.1:2379"},
Transport: DefaultEtcdTransport,
}
return etcdclient.New(cfg)
}
func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFactory, host *hosts.Host) bool {
func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFactory, host *hosts.Host, cert, key []byte) bool {
logrus.Debugf("[etcd] Check etcd cluster health")
for i := 0; i < 3; i++ {
dialer, err := getEtcdDialer(localConnDialerFactory, host)
if err != nil {
logrus.Debugf("Failed to create a dialer for host [%s]: %v", host.Address, err)
time.Sleep(5 * time.Second)
continue
return false
}
tlsConfig, err := getEtcdTLSConfig(cert, key)
if err != nil {
logrus.Debugf("[etcd] Failed to create etcd tls config for host [%s]: %v", host.Address, err)
return false
}
hc := http.Client{
Transport: &http.Transport{
Dial: dialer,
Dial: dialer,
TLSClientConfig: tlsConfig,
TLSHandshakeTimeout: 10 * time.Second,
},
}
healthy, err := getHealthEtcd(hc, host)
@@ -62,7 +75,7 @@ func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFacto
func getHealthEtcd(hc http.Client, host *hosts.Host) (string, error) {
healthy := struct{ Health string }{}
resp, err := hc.Get("http://127.0.0.1:2379/health")
resp, err := hc.Get("https://127.0.0.1:2379/health")
if err != nil {
return healthy.Health, fmt.Errorf("Failed to get /health for host [%s]: %v", host.Address, err)
}
@@ -80,7 +93,7 @@ func getHealthEtcd(hc http.Client, host *hosts.Host) (string, error) {
func getEtcdInitialCluster(hosts []*hosts.Host) string {
initialCluster := ""
for i, host := range hosts {
initialCluster += fmt.Sprintf("etcd-%s=http://%s:2380", host.HostnameOverride, host.InternalAddress)
initialCluster += fmt.Sprintf("etcd-%s=https://%s:2380", host.HostnameOverride, host.InternalAddress)
if i < (len(hosts) - 1) {
initialCluster += ","
}
@@ -102,10 +115,27 @@ func getEtcdDialer(localConnDialerFactory hosts.DialerFactory, etcdHost *hosts.H
func GetEtcdConnString(hosts []*hosts.Host) string {
connString := ""
for i, host := range hosts {
connString += "http://" + host.InternalAddress + ":2379"
connString += "https://" + host.InternalAddress + ":2379"
if i < (len(hosts) - 1) {
connString += ","
}
}
return connString
}
func getEtcdTLSConfig(certificate, key []byte) (*tls.Config, error) {
// get tls config
x509Pair, err := tls.X509KeyPair([]byte(certificate), []byte(key))
if err != nil {
return nil, err
}
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
Certificates: []tls.Certificate{x509Pair},
}
if err != nil {
return nil, err
}
return tlsConfig, nil
}

11
services/kubeapi.go
View File

@@ -41,10 +41,13 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd
"--runtime-config=batch/v2alpha1",
"--runtime-config=authentication.k8s.io/v1beta1=true",
"--storage-backend=etcd3",
"--client-ca-file=" + pki.CACertPath,
"--tls-cert-file=" + pki.KubeAPICertPath,
"--tls-private-key-file=" + pki.KubeAPIKeyPath,
"--service-account-key-file=" + pki.KubeAPIKeyPath},
"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
"--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName),
"--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
"--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
"--etcd-cafile=" + pki.GetCertPath(pki.CACertName),
"--etcd-certfile=" + pki.GetCertPath(pki.KubeAPICertName),
"--etcd-keyfile=" + pki.GetKeyPath(pki.KubeAPICertName)},
}
imageCfg.Cmd = append(imageCfg.Cmd, "--etcd-servers="+etcdConnString)

2
services/kubeapi_test.go
View File

@@ -9,7 +9,7 @@ import (
)
const (
TestEtcdConnString = "http://1.1.1.1:2379,http://2.2.2.2:2379"
TestEtcdConnString = "https://1.1.1.1:2379,https://2.2.2.2:2379"
TestKubeAPIImage = "rancher/k8s:latest"
TestInsecureBindAddress = "--insecure-bind-address=127.0.0.1"
TestKubeAPIVolumeBind = "/etc/kubernetes:/etc/kubernetes"

6
services/kubecontroller.go
View File

@@ -31,7 +31,7 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService, a
"--address=0.0.0.0",
"--cloud-provider=",
"--leader-elect=true",
"--kubeconfig=" + pki.KubeControllerConfigPath,
"--kubeconfig=" + pki.GetConfigPath(pki.KubeControllerCertName),
"--enable-hostpath-provisioner=false",
"--node-monitor-grace-period=40s",
"--pod-eviction-timeout=5m0s",
@@ -39,8 +39,8 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService, a
"--allocate-node-cidrs=true",
"--cluster-cidr=" + kubeControllerService.ClusterCIDR,
"--service-cluster-ip-range=" + kubeControllerService.ServiceClusterIPRange,
"--service-account-private-key-file=" + pki.KubeAPIKeyPath,
"--root-ca-file=" + pki.CACertPath,
"--service-account-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
"--root-ca-file=" + pki.GetCertPath(pki.CACertName),
},
}
if authorizationMode == RBACAuthorizationMode {

2
services/kubelet.go
View File

@@ -42,7 +42,7 @@ func buildKubeletConfig(host *hosts.Host, kubeletService v3.KubeletService, unsc
"--resolv-conf=/etc/resolv.conf",
"--allow-privileged=true",
"--cloud-provider=",
"--kubeconfig=" + pki.KubeNodeConfigPath,
"--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName),
"--require-kubeconfig=True",
},
}

2
services/kubeproxy.go
View File

@@ -30,7 +30,7 @@ func buildKubeproxyConfig(host *hosts.Host, kubeproxyService v3.KubeproxyService
"kube-proxy",
"--v=2",
"--healthz-bind-address=0.0.0.0",
"--kubeconfig=" + pki.KubeProxyConfigPath,
"--kubeconfig=" + pki.GetConfigPath(pki.KubeProxyCertName),
},
}
hostCfg := &container.HostConfig{

2
services/scheduler.go
View File

@@ -31,7 +31,7 @@ func buildSchedulerConfig(host *hosts.Host, schedulerService v3.SchedulerService
"--leader-elect=true",
"--v=2",
"--address=0.0.0.0",
"--kubeconfig=" + pki.KubeSchedulerConfigPath,
"--kubeconfig=" + pki.GetConfigPath(pki.KubeSchedulerCertName),
},
}
hostCfg := &container.HostConfig{

32
templates/calico.go
View File

@@ -104,9 +104,9 @@ data:
{
"type": "calico",
"etcd_endpoints": "{{.EtcdEndpoints}}",
"etcd_key_file": "",
"etcd_cert_file": "",
"etcd_ca_cert_file": "",
"etcd_key_file": "{{.ClientKeyPath}}",
"etcd_cert_file": "{{.ClientCertPath}}",
"etcd_ca_cert_file": "{{.ClientCAPath}}",
"log_level": "info",
"mtu": 1500,
"ipam": {
@@ -115,9 +115,9 @@ data:
"policy": {
"type": "k8s",
"k8s_api_root": "{{.APIRoot}}",
"k8s_client_certificate": "{{.ClientCert}}",
"k8s_client_key": "{{.ClientKey}}",
"k8s_certificate_authority": "{{.ClientCA}}"
"k8s_client_certificate": "{{.ClientCertPath}}",
"k8s_client_key": "{{.ClientKeyPath}}",
"k8s_certificate_authority": "{{.ClientCAPath}}"
},
"kubernetes": {
"kubeconfig": "{{.KubeCfg}}"
@@ -131,16 +131,12 @@ data:
]
}
# If you're using TLS enabled etcd uncomment the following.
# You must also populate the Secret below with these files.
etcd_ca: "" # "/calico-secrets/etcd-ca"
etcd_cert: "" # "/calico-secrets/etcd-cert"
etcd_key: "" # "/calico-secrets/etcd-key"
etcd_ca: "/calico-secrets/etcd-ca"
etcd_cert: "/calico-secrets/etcd-cert"
etcd_key: "/calico-secrets/etcd-key"
---
# The following contains k8s Secrets for use with a TLS enabled etcd cluster.
# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
apiVersion: v1
kind: Secret
type: Opaque
@@ -148,13 +144,9 @@ metadata:
name: calico-etcd-secrets
namespace: kube-system
data:
# Populate the following files with etcd TLS configuration if desired, but leave blank if
# not using TLS for etcd.
# This self-hosted install expects three files with the following names. The values
# should be base64 encoded strings of the entire contents of each file.
# etcd-key: null
# etcd-cert: null
# etcd-ca: null
etcd-key: {{.ClientKey}}
etcd-cert: {{.ClientCert}}
etcd-ca: {{.ClientCA}}
---

6
templates/canal.go
View File

@@ -156,9 +156,9 @@ data:
"policy": {
"type": "k8s",
"k8s_api_root": "{{.APIRoot}}",
"k8s_client_certificate": "{{.ClientCert}}",
"k8s_client_key": "{{.ClientKey}}",
"k8s_certificate_authority": "{{.ClientCA}}"
"k8s_client_certificate": "{{.ClientCertPath}}",
"k8s_client_key": "{{.ClientKeyPath}}",
"k8s_certificate_authority": "{{.ClientCAPath}}"
},
"kubernetes": {
"kubeconfig": "{{.KubeCfg}}"

16
vendor/github.com/coreos/etcd/pkg/tlsutil/doc.go generated vendored Normal file
View File

@@ -0,0 +1,16 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package tlsutil provides utility functions for handling TLS.
package tlsutil

72
vendor/github.com/coreos/etcd/pkg/tlsutil/tlsutil.go generated vendored Normal file
View File

@@ -0,0 +1,72 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tlsutil
import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
"io/ioutil"
)
// NewCertPool creates x509 certPool with provided CA files.
func NewCertPool(CAFiles []string) (*x509.CertPool, error) {
certPool := x509.NewCertPool()
for _, CAFile := range CAFiles {
pemByte, err := ioutil.ReadFile(CAFile)
if err != nil {
return nil, err
}
for {
var block *pem.Block
block, pemByte = pem.Decode(pemByte)
if block == nil {
break
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, err
}
certPool.AddCert(cert)
}
}
return certPool, nil
}
// NewCert generates TLS cert by using the given cert,key and parse function.
func NewCert(certfile, keyfile string, parseFunc func([]byte, []byte) (tls.Certificate, error)) (*tls.Certificate, error) {
cert, err := ioutil.ReadFile(certfile)
if err != nil {
return nil, err
}
key, err := ioutil.ReadFile(keyfile)
if err != nil {
return nil, err
}
if parseFunc == nil {
parseFunc = tls.X509KeyPair
}
tlsCert, err := parseFunc(cert, key)
if err != nil {
return nil, err
}
return &tlsCert, nil
}

17
vendor/github.com/coreos/etcd/pkg/transport/doc.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package transport implements various HTTP transport utilities based on Go
// net package.
package transport

94
vendor/github.com/coreos/etcd/pkg/transport/keepalive_listener.go generated vendored Normal file
View File

@@ -0,0 +1,94 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"crypto/tls"
"fmt"
"net"
"time"
)
type keepAliveConn interface {
SetKeepAlive(bool) error
SetKeepAlivePeriod(d time.Duration) error
}
// NewKeepAliveListener returns a listener that listens on the given address.
// Be careful when wrap around KeepAliveListener with another Listener if TLSInfo is not nil.
// Some pkgs (like go/http) might expect Listener to return TLSConn type to start TLS handshake.
// http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html
func NewKeepAliveListener(l net.Listener, scheme string, tlscfg *tls.Config) (net.Listener, error) {
if scheme == "https" {
if tlscfg == nil {
return nil, fmt.Errorf("cannot listen on TLS for given listener: KeyFile and CertFile are not presented")
}
return newTLSKeepaliveListener(l, tlscfg), nil
}
return &keepaliveListener{
Listener: l,
}, nil
}
type keepaliveListener struct{ net.Listener }
func (kln *keepaliveListener) Accept() (net.Conn, error) {
c, err := kln.Listener.Accept()
if err != nil {
return nil, err
}
kac := c.(keepAliveConn)
// detection time: tcp_keepalive_time + tcp_keepalive_probes + tcp_keepalive_intvl
// default on linux: 30 + 8 * 30
// default on osx: 30 + 8 * 75
kac.SetKeepAlive(true)
kac.SetKeepAlivePeriod(30 * time.Second)
return c, nil
}
// A tlsKeepaliveListener implements a network listener (net.Listener) for TLS connections.
type tlsKeepaliveListener struct {
net.Listener
config *tls.Config
}
// Accept waits for and returns the next incoming TLS connection.
// The returned connection c is a *tls.Conn.
func (l *tlsKeepaliveListener) Accept() (c net.Conn, err error) {
c, err = l.Listener.Accept()
if err != nil {
return
}
kac := c.(keepAliveConn)
// detection time: tcp_keepalive_time + tcp_keepalive_probes + tcp_keepalive_intvl
// default on linux: 30 + 8 * 30
// default on osx: 30 + 8 * 75
kac.SetKeepAlive(true)
kac.SetKeepAlivePeriod(30 * time.Second)
c = tls.Server(c, l.config)
return c, nil
}
// NewListener creates a Listener which accepts connections from an inner
// Listener and wraps each connection with Server.
// The configuration config must be non-nil and must have
// at least one certificate.
func newTLSKeepaliveListener(inner net.Listener, config *tls.Config) net.Listener {
l := &tlsKeepaliveListener{}
l.Listener = inner
l.config = config
return l
}

80
vendor/github.com/coreos/etcd/pkg/transport/limit_listen.go generated vendored Normal file
View File

@@ -0,0 +1,80 @@
// Copyright 2013 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package transport provides network utility functions, complementing the more
// common ones in the net package.
package transport
import (
"errors"
"net"
"sync"
"time"
)
var (
ErrNotTCP = errors.New("only tcp connections have keepalive")
)
// LimitListener returns a Listener that accepts at most n simultaneous
// connections from the provided Listener.
func LimitListener(l net.Listener, n int) net.Listener {
return &limitListener{l, make(chan struct{}, n)}
}
type limitListener struct {
net.Listener
sem chan struct{}
}
func (l *limitListener) acquire() { l.sem <- struct{}{} }
func (l *limitListener) release() { <-l.sem }
func (l *limitListener) Accept() (net.Conn, error) {
l.acquire()
c, err := l.Listener.Accept()
if err != nil {
l.release()
return nil, err
}
return &limitListenerConn{Conn: c, release: l.release}, nil
}
type limitListenerConn struct {
net.Conn
releaseOnce sync.Once
release func()
}
func (l *limitListenerConn) Close() error {
err := l.Conn.Close()
l.releaseOnce.Do(l.release)
return err
}
func (l *limitListenerConn) SetKeepAlive(doKeepAlive bool) error {
tcpc, ok := l.Conn.(*net.TCPConn)
if !ok {
return ErrNotTCP
}
return tcpc.SetKeepAlive(doKeepAlive)
}
func (l *limitListenerConn) SetKeepAlivePeriod(d time.Duration) error {
tcpc, ok := l.Conn.(*net.TCPConn)
if !ok {
return ErrNotTCP
}
return tcpc.SetKeepAlivePeriod(d)
}

281
vendor/github.com/coreos/etcd/pkg/transport/listener.go generated vendored Normal file
View File

@@ -0,0 +1,281 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"time"
"github.com/coreos/etcd/pkg/tlsutil"
)
func NewListener(addr, scheme string, tlsinfo *TLSInfo) (l net.Listener, err error) {
if l, err = newListener(addr, scheme); err != nil {
return nil, err
}
return wrapTLS(addr, scheme, tlsinfo, l)
}
func newListener(addr string, scheme string) (net.Listener, error) {
if scheme == "unix" || scheme == "unixs" {
// unix sockets via unix://laddr
return NewUnixListener(addr)
}
return net.Listen("tcp", addr)
}
func wrapTLS(addr, scheme string, tlsinfo *TLSInfo, l net.Listener) (net.Listener, error) {
if scheme != "https" && scheme != "unixs" {
return l, nil
}
return newTLSListener(l, tlsinfo, checkSAN)
}
type TLSInfo struct {
CertFile string
KeyFile string
CAFile string // TODO: deprecate this in v4
TrustedCAFile string
ClientCertAuth bool
CRLFile string
InsecureSkipVerify bool
// ServerName ensures the cert matches the given host in case of discovery / virtual hosting
ServerName string
// HandshakeFailure is optionally called when a connection fails to handshake. The
// connection will be closed immediately afterwards.
HandshakeFailure func(*tls.Conn, error)
selfCert bool
// parseFunc exists to simplify testing. Typically, parseFunc
// should be left nil. In that case, tls.X509KeyPair will be used.
parseFunc func([]byte, []byte) (tls.Certificate, error)
// AllowedCN is a CN which must be provided by a client.
AllowedCN string
}
func (info TLSInfo) String() string {
return fmt.Sprintf("cert = %s, key = %s, ca = %s, trusted-ca = %s, client-cert-auth = %v, crl-file = %s", info.CertFile, info.KeyFile, info.CAFile, info.TrustedCAFile, info.ClientCertAuth, info.CRLFile)
}
func (info TLSInfo) Empty() bool {
return info.CertFile == "" && info.KeyFile == ""
}
func SelfCert(dirpath string, hosts []string) (info TLSInfo, err error) {
if err = os.MkdirAll(dirpath, 0700); err != nil {
return
}
certPath := filepath.Join(dirpath, "cert.pem")
keyPath := filepath.Join(dirpath, "key.pem")
_, errcert := os.Stat(certPath)
_, errkey := os.Stat(keyPath)
if errcert == nil && errkey == nil {
info.CertFile = certPath
info.KeyFile = keyPath
info.selfCert = true
return
}
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return
}
tmpl := x509.Certificate{
SerialNumber: serialNumber,
Subject: pkix.Name{Organization: []string{"etcd"}},
NotBefore: time.Now(),
NotAfter: time.Now().Add(365 * (24 * time.Hour)),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
for _, host := range hosts {
h, _, _ := net.SplitHostPort(host)
if ip := net.ParseIP(h); ip != nil {
tmpl.IPAddresses = append(tmpl.IPAddresses, ip)
} else {
tmpl.DNSNames = append(tmpl.DNSNames, h)
}
}
priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
if err != nil {
return
}
derBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
if err != nil {
return
}
certOut, err := os.Create(certPath)
if err != nil {
return
}
pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
certOut.Close()
b, err := x509.MarshalECPrivateKey(priv)
if err != nil {
return
}
keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return
}
pem.Encode(keyOut, &pem.Block{Type: "EC PRIVATE KEY", Bytes: b})
keyOut.Close()
return SelfCert(dirpath, hosts)
}
func (info TLSInfo) baseConfig() (*tls.Config, error) {
if info.KeyFile == "" || info.CertFile == "" {
return nil, fmt.Errorf("KeyFile and CertFile must both be present[key: %v, cert: %v]", info.KeyFile, info.CertFile)
}
tlsCert, err := tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc)
if err != nil {
return nil, err
}
cfg := &tls.Config{
Certificates: []tls.Certificate{*tlsCert},
MinVersion: tls.VersionTLS12,
ServerName: info.ServerName,
}
if info.AllowedCN != "" {
cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
for _, chains := range verifiedChains {
if len(chains) != 0 {
if info.AllowedCN == chains[0].Subject.CommonName {
return nil
}
}
}
return errors.New("CommonName authentication failed")
}
}
// this only reloads certs when there's a client request
// TODO: support server-side refresh (e.g. inotify, SIGHUP), caching
cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
return tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc)
}
cfg.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
return tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc)
}
return cfg, nil
}
// cafiles returns a list of CA file paths.
func (info TLSInfo) cafiles() []string {
cs := make([]string, 0)
if info.CAFile != "" {
cs = append(cs, info.CAFile)
}
if info.TrustedCAFile != "" {
cs = append(cs, info.TrustedCAFile)
}
return cs
}
// ServerConfig generates a tls.Config object for use by an HTTP server.
func (info TLSInfo) ServerConfig() (*tls.Config, error) {
cfg, err := info.baseConfig()
if err != nil {
return nil, err
}
cfg.ClientAuth = tls.NoClientCert
if info.CAFile != "" || info.ClientCertAuth {
cfg.ClientAuth = tls.RequireAndVerifyClientCert
}
CAFiles := info.cafiles()
if len(CAFiles) > 0 {
cp, err := tlsutil.NewCertPool(CAFiles)
if err != nil {
return nil, err
}
cfg.ClientCAs = cp
}
// "h2" NextProtos is necessary for enabling HTTP2 for go's HTTP server
cfg.NextProtos = []string{"h2"}
return cfg, nil
}
// ClientConfig generates a tls.Config object for use by an HTTP client.
func (info TLSInfo) ClientConfig() (*tls.Config, error) {
var cfg *tls.Config
var err error
if !info.Empty() {
cfg, err = info.baseConfig()
if err != nil {
return nil, err
}
} else {
cfg = &tls.Config{ServerName: info.ServerName}
}
cfg.InsecureSkipVerify = info.InsecureSkipVerify
CAFiles := info.cafiles()
if len(CAFiles) > 0 {
cfg.RootCAs, err = tlsutil.NewCertPool(CAFiles)
if err != nil {
return nil, err
}
}
if info.selfCert {
cfg.InsecureSkipVerify = true
}
return cfg, nil
}
// IsClosedConnError returns true if the error is from closing listener, cmux.
// copied from golang.org/x/net/http2/http2.go
func IsClosedConnError(err error) bool {
// 'use of closed network connection' (Go <=1.8)
// 'use of closed file or network connection' (Go >1.8, internal/poll.ErrClosing)
// 'mux: listener closed' (cmux.ErrListenerClosed)
return err != nil && strings.Contains(err.Error(), "closed")
}

272
vendor/github.com/coreos/etcd/pkg/transport/listener_tls.go generated vendored Normal file
View File

@@ -0,0 +1,272 @@
// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"strings"
"sync"
)
// tlsListener overrides a TLS listener so it will reject client
// certificates with insufficient SAN credentials or CRL revoked
// certificates.
type tlsListener struct {
net.Listener
connc chan net.Conn
donec chan struct{}
err error
handshakeFailure func(*tls.Conn, error)
check tlsCheckFunc
}
type tlsCheckFunc func(context.Context, *tls.Conn) error
// NewTLSListener handshakes TLS connections and performs optional CRL checking.
func NewTLSListener(l net.Listener, tlsinfo *TLSInfo) (net.Listener, error) {
check := func(context.Context, *tls.Conn) error { return nil }
return newTLSListener(l, tlsinfo, check)
}
func newTLSListener(l net.Listener, tlsinfo *TLSInfo, check tlsCheckFunc) (net.Listener, error) {
if tlsinfo == nil || tlsinfo.Empty() {
l.Close()
return nil, fmt.Errorf("cannot listen on TLS for %s: KeyFile and CertFile are not presented", l.Addr().String())
}
tlscfg, err := tlsinfo.ServerConfig()
if err != nil {
return nil, err
}
hf := tlsinfo.HandshakeFailure
if hf == nil {
hf = func(*tls.Conn, error) {}
}
if len(tlsinfo.CRLFile) > 0 {
prevCheck := check
check = func(ctx context.Context, tlsConn *tls.Conn) error {
if err := prevCheck(ctx, tlsConn); err != nil {
return err
}
st := tlsConn.ConnectionState()
if certs := st.PeerCertificates; len(certs) > 0 {
return checkCRL(tlsinfo.CRLFile, certs)
}
return nil
}
}
tlsl := &tlsListener{
Listener: tls.NewListener(l, tlscfg),
connc: make(chan net.Conn),
donec: make(chan struct{}),
handshakeFailure: hf,
check: check,
}
go tlsl.acceptLoop()
return tlsl, nil
}
func (l *tlsListener) Accept() (net.Conn, error) {
select {
case conn := <-l.connc:
return conn, nil
case <-l.donec:
return nil, l.err
}
}
func checkSAN(ctx context.Context, tlsConn *tls.Conn) error {
st := tlsConn.ConnectionState()
if certs := st.PeerCertificates; len(certs) > 0 {
addr := tlsConn.RemoteAddr().String()
return checkCertSAN(ctx, certs[0], addr)
}
return nil
}
// acceptLoop launches each TLS handshake in a separate goroutine
// to prevent a hanging TLS connection from blocking other connections.
func (l *tlsListener) acceptLoop() {
var wg sync.WaitGroup
var pendingMu sync.Mutex
pending := make(map[net.Conn]struct{})
ctx, cancel := context.WithCancel(context.Background())
defer func() {
cancel()
pendingMu.Lock()
for c := range pending {
c.Close()
}
pendingMu.Unlock()
wg.Wait()
close(l.donec)
}()
for {
conn, err := l.Listener.Accept()
if err != nil {
l.err = err
return
}
pendingMu.Lock()
pending[conn] = struct{}{}
pendingMu.Unlock()
wg.Add(1)
go func() {
defer func() {
if conn != nil {
conn.Close()
}
wg.Done()
}()
tlsConn := conn.(*tls.Conn)
herr := tlsConn.Handshake()
pendingMu.Lock()
delete(pending, conn)
pendingMu.Unlock()
if herr != nil {
l.handshakeFailure(tlsConn, herr)
return
}
if err := l.check(ctx, tlsConn); err != nil {
l.handshakeFailure(tlsConn, err)
return
}
select {
case l.connc <- tlsConn:
conn = nil
case <-ctx.Done():
}
}()
}
}
func checkCRL(crlPath string, cert []*x509.Certificate) error {
// TODO: cache
crlBytes, err := ioutil.ReadFile(crlPath)
if err != nil {
return err
}
certList, err := x509.ParseCRL(crlBytes)
if err != nil {
return err
}
revokedSerials := make(map[string]struct{})
for _, rc := range certList.TBSCertList.RevokedCertificates {
revokedSerials[string(rc.SerialNumber.Bytes())] = struct{}{}
}
for _, c := range cert {
serial := string(c.SerialNumber.Bytes())
if _, ok := revokedSerials[serial]; ok {
return fmt.Errorf("transport: certificate serial %x revoked", serial)
}
}
return nil
}
func checkCertSAN(ctx context.Context, cert *x509.Certificate, remoteAddr string) error {
if len(cert.IPAddresses) == 0 && len(cert.DNSNames) == 0 {
return nil
}
h, _, herr := net.SplitHostPort(remoteAddr)
if herr != nil {
return herr
}
if len(cert.IPAddresses) > 0 {
cerr := cert.VerifyHostname(h)
if cerr == nil {
return nil
}
if len(cert.DNSNames) == 0 {
return cerr
}
}
if len(cert.DNSNames) > 0 {
ok, err := isHostInDNS(ctx, h, cert.DNSNames)
if ok {
return nil
}
errStr := ""
if err != nil {
errStr = " (" + err.Error() + ")"
}
return fmt.Errorf("tls: %q does not match any of DNSNames %q"+errStr, h, cert.DNSNames)
}
return nil
}
func isHostInDNS(ctx context.Context, host string, dnsNames []string) (ok bool, err error) {
// reverse lookup
wildcards, names := []string{}, []string{}
for _, dns := range dnsNames {
if strings.HasPrefix(dns, "*.") {
wildcards = append(wildcards, dns[1:])
} else {
names = append(names, dns)
}
}
lnames, lerr := net.DefaultResolver.LookupAddr(ctx, host)
for _, name := range lnames {
// strip trailing '.' from PTR record
if name[len(name)-1] == '.' {
name = name[:len(name)-1]
}
for _, wc := range wildcards {
if strings.HasSuffix(name, wc) {
return true, nil
}
}
for _, n := range names {
if n == name {
return true, nil
}
}
}
err = lerr
// forward lookup
for _, dns := range names {
addrs, lerr := net.DefaultResolver.LookupHost(ctx, dns)
if lerr != nil {
err = lerr
continue
}
for _, addr := range addrs {
if addr == host {
return true, nil
}
}
}
return false, err
}
func (l *tlsListener) Close() error {
err := l.Listener.Close()
<-l.donec
return err
}

44
vendor/github.com/coreos/etcd/pkg/transport/timeout_conn.go generated vendored Normal file
View File

@@ -0,0 +1,44 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net"
"time"
)
type timeoutConn struct {
net.Conn
wtimeoutd time.Duration
rdtimeoutd time.Duration
}
func (c timeoutConn) Write(b []byte) (n int, err error) {
if c.wtimeoutd > 0 {
if err := c.SetWriteDeadline(time.Now().Add(c.wtimeoutd)); err != nil {
return 0, err
}
}
return c.Conn.Write(b)
}
func (c timeoutConn) Read(b []byte) (n int, err error) {
if c.rdtimeoutd > 0 {
if err := c.SetReadDeadline(time.Now().Add(c.rdtimeoutd)); err != nil {
return 0, err
}
}
return c.Conn.Read(b)
}

36
vendor/github.com/coreos/etcd/pkg/transport/timeout_dialer.go generated vendored Normal file
View File

@@ -0,0 +1,36 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net"
"time"
)
type rwTimeoutDialer struct {
wtimeoutd time.Duration
rdtimeoutd time.Duration
net.Dialer
}
func (d *rwTimeoutDialer) Dial(network, address string) (net.Conn, error) {
conn, err := d.Dialer.Dial(network, address)
tconn := &timeoutConn{
rdtimeoutd: d.rdtimeoutd,
wtimeoutd: d.wtimeoutd,
Conn: conn,
}
return tconn, err
}

57
vendor/github.com/coreos/etcd/pkg/transport/timeout_listener.go generated vendored Normal file
View File

@@ -0,0 +1,57 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net"
"time"
)
// NewTimeoutListener returns a listener that listens on the given address.
// If read/write on the accepted connection blocks longer than its time limit,
// it will return timeout error.
func NewTimeoutListener(addr string, scheme string, tlsinfo *TLSInfo, rdtimeoutd, wtimeoutd time.Duration) (net.Listener, error) {
ln, err := newListener(addr, scheme)
if err != nil {
return nil, err
}
ln = &rwTimeoutListener{
Listener: ln,
rdtimeoutd: rdtimeoutd,
wtimeoutd: wtimeoutd,
}
if ln, err = wrapTLS(addr, scheme, tlsinfo, ln); err != nil {
return nil, err
}
return ln, nil
}
type rwTimeoutListener struct {
net.Listener
wtimeoutd time.Duration
rdtimeoutd time.Duration
}
func (rwln *rwTimeoutListener) Accept() (net.Conn, error) {
c, err := rwln.Listener.Accept()
if err != nil {
return nil, err
}
return timeoutConn{
Conn: c,
wtimeoutd: rwln.wtimeoutd,
rdtimeoutd: rwln.rdtimeoutd,
}, nil
}

51
vendor/github.com/coreos/etcd/pkg/transport/timeout_transport.go generated vendored Normal file
View File

@@ -0,0 +1,51 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net"
"net/http"
"time"
)
// NewTimeoutTransport returns a transport created using the given TLS info.
// If read/write on the created connection blocks longer than its time limit,
// it will return timeout error.
// If read/write timeout is set, transport will not be able to reuse connection.
func NewTimeoutTransport(info TLSInfo, dialtimeoutd, rdtimeoutd, wtimeoutd time.Duration) (*http.Transport, error) {
tr, err := NewTransport(info, dialtimeoutd)
if err != nil {
return nil, err
}
if rdtimeoutd != 0 || wtimeoutd != 0 {
// the timed out connection will timeout soon after it is idle.
// it should not be put back to http transport as an idle connection for future usage.
tr.MaxIdleConnsPerHost = -1
} else {
// allow more idle connections between peers to avoid unnecessary port allocation.
tr.MaxIdleConnsPerHost = 1024
}
tr.Dial = (&rwTimeoutDialer{
Dialer: net.Dialer{
Timeout: dialtimeoutd,
KeepAlive: 30 * time.Second,
},
rdtimeoutd: rdtimeoutd,
wtimeoutd: wtimeoutd,
}).Dial
return tr, nil
}

49
vendor/github.com/coreos/etcd/pkg/transport/tls.go generated vendored Normal file
View File

@@ -0,0 +1,49 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"fmt"
"strings"
"time"
)
// ValidateSecureEndpoints scans the given endpoints against tls info, returning only those
// endpoints that could be validated as secure.
func ValidateSecureEndpoints(tlsInfo TLSInfo, eps []string) ([]string, error) {
t, err := NewTransport(tlsInfo, 5*time.Second)
if err != nil {
return nil, err
}
var errs []string
var endpoints []string
for _, ep := range eps {
if !strings.HasPrefix(ep, "https://") {
errs = append(errs, fmt.Sprintf("%q is insecure", ep))
continue
}
conn, cerr := t.Dial("tcp", ep[len("https://"):])
if cerr != nil {
errs = append(errs, fmt.Sprintf("%q failed to dial (%v)", ep, cerr))
continue
}
conn.Close()
endpoints = append(endpoints, ep)
}
if len(errs) != 0 {
err = fmt.Errorf("%s", strings.Join(errs, ","))
}
return endpoints, err
}

71
vendor/github.com/coreos/etcd/pkg/transport/transport.go generated vendored Normal file
View File

@@ -0,0 +1,71 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net"
"net/http"
"strings"
"time"
)
type unixTransport struct{ *http.Transport }
func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, error) {
cfg, err := info.ClientConfig()
if err != nil {
return nil, err
}
t := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: dialtimeoutd,
// value taken from http.DefaultTransport
KeepAlive: 30 * time.Second,
}).Dial,
// value taken from http.DefaultTransport
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: cfg,
}
dialer := (&net.Dialer{
Timeout: dialtimeoutd,
KeepAlive: 30 * time.Second,
})
dial := func(net, addr string) (net.Conn, error) {
return dialer.Dial("unix", addr)
}
tu := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: cfg,
}
ut := &unixTransport{tu}
t.RegisterProtocol("unix", ut)
t.RegisterProtocol("unixs", ut)
return t, nil
}
func (urt *unixTransport) RoundTrip(req *http.Request) (*http.Response, error) {
url := *req.URL
req.URL = &url
req.URL.Scheme = strings.Replace(req.URL.Scheme, "unix", "http", 1)
return urt.Transport.RoundTrip(req)
}

40
vendor/github.com/coreos/etcd/pkg/transport/unix_listener.go generated vendored Normal file
View File

@@ -0,0 +1,40 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net"
"os"
)
type unixListener struct{ net.Listener }
func NewUnixListener(addr string) (net.Listener, error) {
if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
return nil, err
}
l, err := net.Listen("unix", addr)
if err != nil {
return nil, err
}
return &unixListener{l}, nil
}
func (ul *unixListener) Close() error {
if err := os.Remove(ul.Addr().String()); err != nil && !os.IsNotExist(err) {
return err
}
return ul.Listener.Close()
}