From 9ee750ec01209ac456a23663ad7fd893b805d1cf Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Thu, 3 Jan 2019 01:02:34 +0200 Subject: [PATCH] Adding csr generation and custom certs --- cluster/certificates.go | 10 +- cluster/cluster.go | 10 +- cluster/defaults.go | 5 +- cluster/state.go | 40 +++-- cmd/cert.go | 63 ++++++++ cmd/common.go | 3 + cmd/up.go | 12 +- pki/pki.go | 33 ++-- pki/services.go | 246 ++++++++++++++++++++++++---- pki/util.go | 345 ++++++++++++++++++++++++++++++++++++++-- 10 files changed, 688 insertions(+), 79 deletions(-) diff --git a/cluster/certificates.go b/cluster/certificates.go index 3fc2b2a3..843b50ea 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -33,7 +33,7 @@ func regenerateAPICertificate(c *Cluster, certificates map[string]pki.Certificat if err != nil { return nil, err } - certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey) + certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey, nil) return certificates, nil } @@ -114,7 +114,7 @@ func GetClusterCertsFromKubernetes(ctx context.Context, kubeCluster *Cluster) (m kubeAPICert := certMap[pki.KubeAPICertName] if certMap[pki.ServiceAccountTokenKeyName].Key == nil { log.Infof(ctx, "[certificates] Creating service account token key") - certMap[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key) + certMap[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key, nil) } log.Infof(ctx, "[certificates] Successfully fetched Cluster certificates from Kubernetes") return certMap, nil @@ -137,7 +137,7 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi if err != nil { return nil, err } - certificates[pki.RequestHeaderCACertName] = pki.ToCertObject(pki.RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey) + certificates[pki.RequestHeaderCACertName] = pki.ToCertObject(pki.RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey, nil) //generate API server proxy client key and certs logrus.Debugf("[certificates] Regenerating Kubernetes API server proxy client certificates") @@ -145,7 +145,7 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi if err != nil { return nil, err } - certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey) + certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey, nil) return certificates, nil } @@ -196,7 +196,7 @@ func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags, pki.ServiceAccountTokenKeyName, "", c.Certificates[pki.ServiceAccountTokenKeyName].Certificate, - privateKey.(*rsa.PrivateKey)) + privateKey.(*rsa.PrivateKey), nil) } } clusterState.DesiredState.CertificatesBundle = c.Certificates diff --git a/cluster/cluster.go b/cluster/cluster.go index 33314773..ca3ce40b 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -34,6 +34,7 @@ type Cluster struct { CloudConfigFile string ControlPlaneHosts []*hosts.Host Certificates map[string]pki.CertificatePKI + CertificateDir string ClusterDomain string ClusterCIDR string ClusterDNSServer string @@ -49,8 +50,8 @@ type Cluster struct { LocalConnDialerFactory hosts.DialerFactory PrivateRegistriesMap map[string]v3.PrivateRegistry StateFilePath string - UseKubectlDeploy bool UpdateWorkersOnly bool + UseKubectlDeploy bool v3.RancherKubernetesEngineConfig `yaml:",inline"` WorkerHosts []*hosts.Host } @@ -152,16 +153,19 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin ConfigPath: flags.ClusterFilePath, ConfigDir: flags.ConfigDir, DinD: flags.DinD, + CertificateDir: flags.CertificateDir, StateFilePath: GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir), PrivateRegistriesMap: make(map[string]v3.PrivateRegistry), } if len(c.ConfigPath) == 0 { c.ConfigPath = pki.ClusterConfig } - - // set kube_config and state file + // set kube_config, state file, and certificate dir c.LocalKubeConfigPath = pki.GetLocalKubeConfig(c.ConfigPath, c.ConfigDir) c.StateFilePath = GetStateFilePath(c.ConfigPath, c.ConfigDir) + if len(c.CertificateDir) == 0 { + c.CertificateDir = GetCertificateDirPath(c.ConfigPath, c.ConfigDir) + } // Setting cluster Defaults c.setClusterDefaults(ctx) diff --git a/cluster/defaults.go b/cluster/defaults.go index 9381d644..92ab9fff 100644 --- a/cluster/defaults.go +++ b/cluster/defaults.go @@ -52,10 +52,13 @@ const ( ) type ExternalFlags struct { - ConfigDir string + CertificateDir string ClusterFilePath string DinD bool + ConfigDir string + CustomCerts bool DisablePortCheck bool + GenerateCSR bool Local bool UpdateOnly bool } diff --git a/cluster/state.go b/cluster/state.go index 67f49699..eccab5c8 100644 --- a/cluster/state.go +++ b/cluster/state.go @@ -22,6 +22,7 @@ import ( const ( stateFileExt = ".rkestate" + certDirExt = "_certs" ) type FullState struct { @@ -150,6 +151,20 @@ func RebuildState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf }, } + if flags.CustomCerts { + certBundle, err := pki.ReadCertsAndKeysFromDir(flags.CertificateDir) + if err != nil { + return nil, fmt.Errorf("Failed to read certificates from dir [%s]: %v", flags.CertificateDir, err) + } + // make sure all custom certs are included + if err := pki.ValidateBundleContent(rkeConfig, certBundle, flags.ClusterFilePath, flags.ConfigDir); err != nil { + return nil, fmt.Errorf("Failed to validates certificates from dir [%s]: %v", flags.CertificateDir, err) + } + newState.DesiredState.CertificatesBundle = certBundle + newState.CurrentState = oldState.CurrentState + return newState, nil + } + // Rebuilding the certificates of the desired state if oldState.DesiredState.CertificatesBundle == nil { // Get the certificate Bundle @@ -161,15 +176,7 @@ func RebuildState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf } else { // Regenerating etcd certificates for any new etcd nodes pkiCertBundle := oldState.DesiredState.CertificatesBundle - if err := pki.GenerateEtcdCertificates(ctx, pkiCertBundle, *rkeConfig, "", "", false); err != nil { - return nil, err - } - // Regenerating kubeapi certificates for any new kubeapi nodes - if err := pki.GenerateKubeAPICertificate(ctx, pkiCertBundle, *rkeConfig, "", "", false); err != nil { - return nil, err - } - // Regenerating kubeadmin certificates/config - if err := pki.GenerateKubeAdminCertificate(ctx, pkiCertBundle, *rkeConfig, flags.ClusterFilePath, flags.ConfigDir, false); err != nil { + if err := pki.GenerateRKEServicesCerts(ctx, pkiCertBundle, *rkeConfig, flags.ClusterFilePath, flags.ConfigDir, false); err != nil { return nil, err } newState.DesiredState.CertificatesBundle = pkiCertBundle @@ -206,6 +213,21 @@ func GetStateFilePath(configPath, configDir string) string { return trimmedName + stateFileExt } +func GetCertificateDirPath(configPath, configDir string) string { + if configPath == "" { + configPath = pki.ClusterConfig + } + baseDir := filepath.Dir(configPath) + if len(configDir) > 0 { + baseDir = filepath.Dir(configDir) + } + fileName := filepath.Base(configPath) + baseDir += "/" + fullPath := fmt.Sprintf("%s%s", baseDir, fileName) + trimmedName := strings.TrimSuffix(fullPath, filepath.Ext(fullPath)) + return trimmedName + certDirExt +} + func ReadStateFile(ctx context.Context, statePath string) (*FullState, error) { rkeFullState := &FullState{} fp, err := filepath.Abs(statePath) diff --git a/cmd/cert.go b/cmd/cert.go index b9837616..dd651183 100644 --- a/cmd/cert.go +++ b/cmd/cert.go @@ -47,6 +47,23 @@ func CertificateCommand() cli.Command { }, }, }, + cli.Command{ + Name: "generate-csr", + Usage: "Generate certificate sign requests for k8s components", + Action: generateCSRFromCli, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config", + Usage: "Specify an alternate cluster YAML file", + Value: pki.ClusterConfig, + EnvVar: "RKE_CONFIG", + }, + cli.StringFlag{ + Name: "cert-dir", + Usage: "Specify a certificate dir path", + }, + }, + }, }, } } @@ -81,6 +98,28 @@ func rotateRKECertificatesFromCli(ctx *cli.Context) error { return err } +func generateCSRFromCli(ctx *cli.Context) error { + clusterFile, filePath, err := resolveClusterFile(ctx) + if err != nil { + return fmt.Errorf("Failed to resolve cluster file: %v", err) + } + + rkeConfig, err := cluster.ParseConfig(clusterFile) + if err != nil { + return fmt.Errorf("Failed to parse cluster file: %v", err) + } + rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig) + if err != nil { + return err + } + // setting up the flags + externalFlags := cluster.GetExternalFlags(false, false, false, "", filePath) + externalFlags.CertificateDir = ctx.String("cert-dir") + externalFlags.CustomCerts = ctx.Bool("custom-certs") + + return GenerateRKECSRs(context.Background(), rkeConfig, externalFlags) +} + func showRKECertificatesFromCli(ctx *cli.Context) error { return nil } @@ -167,3 +206,27 @@ func rotateRKECertificates(ctx context.Context, kubeCluster *cluster.Cluster, fl } return rkeFullState, nil } + +func GenerateRKECSRs(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, flags cluster.ExternalFlags) error { + log.Infof(ctx, "Generating Kubernetes cluster CSR certificates") + if len(flags.CertificateDir) == 0 { + flags.CertificateDir = cluster.GetCertificateDirPath(flags.ClusterFilePath, flags.ConfigDir) + } + + certBundle, err := pki.ReadCSRsAndKeysFromDir(flags.CertificateDir) + if err != nil { + return err + } + + // initialze the cluster object from the config file + kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags) + if err != nil { + return err + } + + // Generating csrs for kubernetes components + if err := pki.GenerateRKEServicesCSRs(ctx, certBundle, kubeCluster.RancherKubernetesEngineConfig); err != nil { + return err + } + return pki.WriteCertificates(kubeCluster.CertificateDir, certBundle) +} diff --git a/cmd/common.go b/cmd/common.go index 80063d80..05f68346 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -67,6 +67,9 @@ func ClusterInit(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfi log.Infof(ctx, "Initiating Kubernetes cluster") var fullState *cluster.FullState stateFilePath := cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir) + if len(flags.CertificateDir) == 0 { + flags.CertificateDir = cluster.GetCertificateDirPath(flags.ClusterFilePath, flags.ConfigDir) + } rkeFullState, _ := cluster.ReadStateFile(ctx, stateFilePath) kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags) diff --git a/cmd/up.go b/cmd/up.go index 8e9a0f1a..54034118 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -52,6 +52,14 @@ func UpCommand() cli.Command { Name: "init", Usage: "Initiate RKE cluster", }, + cli.StringFlag{ + Name: "cert-dir", + Usage: "Specify a certificate dir path", + }, + cli.BoolFlag{ + Name: "custom-certs", + Usage: "Use custom certificates from a cert dir", + }, } upFlags = append(upFlags, commonFlags...) @@ -257,7 +265,9 @@ func clusterUpFromCli(ctx *cli.Context) error { disablePortCheck := ctx.Bool("disable-port-check") // setting up the flags flags := cluster.GetExternalFlags(false, updateOnly, disablePortCheck, "", filePath) - + // Custom certificates and certificate dir flags + flags.CertificateDir = ctx.String("cert-dir") + flags.CustomCerts = ctx.Bool("custom-certs") if ctx.Bool("init") { return ClusterInit(context.Background(), rkeConfig, hosts.DialersOptions{}, flags) } diff --git a/pki/pki.go b/pki/pki.go index 55ade785..eb994d78 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -18,23 +18,26 @@ import ( ) type CertificatePKI struct { - Certificate *x509.Certificate `json:"-"` - Key *rsa.PrivateKey `json:"-"` - CertificatePEM string `json:"certificatePEM"` - KeyPEM string `json:"keyPEM"` - Config string `json:"config"` - Name string `json:"name"` - CommonName string `json:"commonName"` - OUName string `json:"ouName"` - EnvName string `json:"envName"` - Path string `json:"path"` - KeyEnvName string `json:"keyEnvName"` - KeyPath string `json:"keyPath"` - ConfigEnvName string `json:"configEnvName"` - ConfigPath string `json:"configPath"` + Certificate *x509.Certificate `json:"-"` + Key *rsa.PrivateKey `json:"-"` + CSR *x509.CertificateRequest `json:"-"` + CertificatePEM string `json:"certificatePEM"` + KeyPEM string `json:"keyPEM"` + CSRPEM string `json:"-"` + Config string `json:"config"` + Name string `json:"name"` + CommonName string `json:"commonName"` + OUName string `json:"ouName"` + EnvName string `json:"envName"` + Path string `json:"path"` + KeyEnvName string `json:"keyEnvName"` + KeyPath string `json:"keyPath"` + ConfigEnvName string `json:"configEnvName"` + ConfigPath string `json:"configPath"` } type GenFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig, string, string, bool) error +type CSRFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig) error const ( etcdRole = "etcd" @@ -110,7 +113,7 @@ func RegenerateEtcdCertificate( return nil, err } etcdName := GetEtcdCrtName(etcdHost.InternalAddress) - crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) + crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil) log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress) return crtMap, nil } diff --git a/pki/services.go b/pki/services.go index 35556666..0bbfca44 100644 --- a/pki/services.go +++ b/pki/services.go @@ -34,65 +34,166 @@ func GenerateKubeAPICertificate(ctx context.Context, certs map[string]Certificat if err != nil { return err } - certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey) + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey, nil) // handle service account tokens in old clusters apiCert := certs[KubeAPICertName] if certs[ServiceAccountTokenKeyName].Key == nil { log.Infof(ctx, "[certificates] Generating Service account token key") - certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", apiCert.Certificate, apiCert.Key) + certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", apiCert.Certificate, apiCert.Key, nil) } return nil } +func GenerateKubeAPICSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + // generate API csr and key + kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange) + if err != nil { + return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err) + } + clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain + cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole) + kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, kubernetesServiceIP, rkeConfig.Authentication.SANs) + kubeAPICert := certs[KubeAPICertName].Certificate + oldKubeAPICSR := certs[KubeAPICertName].CSR + if oldKubeAPICSR != nil && + reflect.DeepEqual(kubeAPIAltNames.DNSNames, oldKubeAPICSR.DNSNames) && + deepEqualIPsAltNames(kubeAPIAltNames.IPs, oldKubeAPICSR.IPAddresses) { + return nil + } + log.Infof(ctx, "[certificates] Generating Kubernetes API server csr") + kubeAPICSR, kubeAPIKey, err := GenerateCertSigningRequestAndKey(true, KubeAPICertName, kubeAPIAltNames, certs[KubeAPICertName].Key, nil) + if err != nil { + return err + } + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICert, kubeAPIKey, kubeAPICSR) + return nil +} + func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Kube controller-manager certificate and key - log.Infof(ctx, "[certificates] Generating Kube Controller certificates") caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key - kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil) + if certs[KubeControllerCertName].Certificate != nil { + return nil + } + log.Infof(ctx, "[certificates] Generating Kube Controller certificates") + kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, certs[KubeControllerCertName].Key, nil) if err != nil { return err } - certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey) + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey, nil) + return nil +} + +func GenerateKubeControllerCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + // generate Kube controller-manager csr and key + kubeControllerCrt := certs[KubeControllerCertName].Certificate + kubeControllerCSRPEM := certs[KubeControllerCertName].CSRPEM + if kubeControllerCSRPEM != "" { + return nil + } + log.Infof(ctx, "[certificates] Generating Kube Controller csr") + kubeControllerCSR, kubeControllerKey, err := GenerateCertSigningRequestAndKey(false, getDefaultCN(KubeControllerCertName), nil, certs[KubeControllerCertName].Key, nil) + if err != nil { + return err + } + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey, kubeControllerCSR) return nil } func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Kube scheduler certificate and key - log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates") caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key - kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil) + if certs[KubeSchedulerCertName].Certificate != nil { + return nil + } + log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates") + kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, certs[KubeSchedulerCertName].Key, nil) if err != nil { return err } - certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey) + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey, nil) + return nil +} + +func GenerateKubeSchedulerCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + // generate Kube scheduler csr and key + kubeSchedulerCrt := certs[KubeSchedulerCertName].Certificate + kubeSchedulerCSRPEM := certs[KubeSchedulerCertName].CSRPEM + if kubeSchedulerCSRPEM != "" { + return nil + } + log.Infof(ctx, "[certificates] Generating Kube Scheduler csr") + kubeSchedulerCSR, kubeSchedulerKey, err := GenerateCertSigningRequestAndKey(false, getDefaultCN(KubeSchedulerCertName), nil, certs[KubeSchedulerCertName].Key, nil) + if err != nil { + return err + } + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey, kubeSchedulerCSR) return nil } func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Kube Proxy certificate and key - log.Infof(ctx, "[certificates] Generating Kube Proxy certificates") caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key - kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil) + if certs[KubeProxyCertName].Certificate != nil { + return nil + } + log.Infof(ctx, "[certificates] Generating Kube Proxy certificates") + kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, certs[KubeProxyCertName].Key, nil) if err != nil { return err } - certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey) + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey, nil) + return nil +} + +func GenerateKubeProxyCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + // generate Kube Proxy csr and key + kubeProxyCrt := certs[KubeProxyCertName].Certificate + kubeProxyCSRPEM := certs[KubeProxyCertName].CSRPEM + if kubeProxyCSRPEM != "" { + return nil + } + log.Infof(ctx, "[certificates] Generating Kube Proxy csr") + kubeProxyCSR, kubeProxyKey, err := GenerateCertSigningRequestAndKey(false, getDefaultCN(KubeProxyCertName), nil, certs[KubeProxyCertName].Key, nil) + if err != nil { + return err + } + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey, kubeProxyCSR) return nil } func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate kubelet certificate - log.Infof(ctx, "[certificates] Generating Node certificate") caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key - nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName}) + if certs[KubeNodeCertName].Certificate != nil { + return nil + } + log.Infof(ctx, "[certificates] Generating Node certificate") + nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, certs[KubeNodeCertName].Key, []string{KubeNodeOrganizationName}) if err != nil { return err } - certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey) + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey, nil) + return nil +} + +func GenerateKubeNodeCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + // generate kubelet csr and key + nodeCrt := certs[KubeNodeCertName].Certificate + nodeCSRPEM := certs[KubeNodeCertName].CSRPEM + if nodeCSRPEM != "" { + return nil + } + log.Infof(ctx, "[certificates] Generating Node csr and key") + nodeCSR, nodeKey, err := GenerateCertSigningRequestAndKey(false, KubeNodeCommonName, nil, certs[KubeNodeCertName].Key, []string{KubeNodeOrganizationName}) + if err != nil { + return err + } + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey, nodeCSR) return nil } @@ -106,11 +207,11 @@ func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]Certific configPath = ClusterConfig } localKubeConfigPath := GetLocalKubeConfig(configPath, configDir) - kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName}) + kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, certs[KubeAdminCertName].Key, []string{KubeAdminOrganizationName}) if err != nil { return err } - kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey) + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey, nil) if len(cpHosts) > 0 { kubeAdminConfig := GetKubeConfigX509WithData( "https://"+cpHosts[0].Address+":6443", @@ -128,16 +229,52 @@ func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]Certific return nil } -func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { - //generate API server proxy client key and certs - log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates") - caCrt := certs[RequestHeaderCACertName].Certificate - caKey := certs[RequestHeaderCACertName].Key - apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, APIProxyClientCertName, nil, nil, nil) +func GenerateKubeAdminCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + // generate Admin certificate and key + kubeAdminCrt := certs[KubeAdminCertName].Certificate + kubeAdminCSRPEM := certs[KubeAdminCertName].CSRPEM + if kubeAdminCSRPEM != "" { + return nil + } + kubeAdminCSR, kubeAdminKey, err := GenerateCertSigningRequestAndKey(false, KubeAdminCertName, nil, certs[KubeAdminCertName].Key, []string{KubeAdminOrganizationName}) if err != nil { return err } - certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey) + log.Infof(ctx, "[certificates] Generating admin csr and kubeconfig") + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey, kubeAdminCSR) + certs[KubeAdminCertName] = kubeAdminCertObj + return nil +} + +func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { + //generate API server proxy client key and certs + caCrt := certs[RequestHeaderCACertName].Certificate + caKey := certs[RequestHeaderCACertName].Key + if certs[APIProxyClientCertName].Certificate != nil { + return nil + } + log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates") + apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, APIProxyClientCertName, nil, certs[APIProxyClientCertName].Key, nil) + if err != nil { + return err + } + certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey, nil) + return nil +} + +func GenerateAPIProxyClientCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + //generate API server proxy client key and certs + apiserverProxyClientCrt := certs[APIProxyClientCertName].Certificate + apiserverProxyClientCSRPEM := certs[APIProxyClientCertName].CSRPEM + if apiserverProxyClientCSRPEM != "" { + return nil + } + log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client csr") + apiserverProxyClientCSR, apiserverProxyClientKey, err := GenerateCertSigningRequestAndKey(true, APIProxyClientCertName, nil, certs[APIProxyClientCertName].Key, nil) + if err != nil { + return err + } + certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey, apiserverProxyClientCSR) return nil } @@ -150,13 +287,13 @@ func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]Cert if err != nil { return err } - certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey)) + certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey), nil) caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert)) if err != nil { return err } - certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil) + certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil, nil) return nil } @@ -176,20 +313,48 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP continue } log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress) - etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil) + etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, certs[etcdName].Key, nil) if err != nil { return err } - certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil) + } + return nil +} + +func GenerateEtcdCSRs(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange) + if err != nil { + return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err) + } + clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain + etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole) + etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{}) + for _, host := range etcdHosts { + etcdName := GetEtcdCrtName(host.InternalAddress) + etcdCrt := certs[etcdName].Certificate + etcdCSRPEM := certs[etcdName].CSRPEM + if etcdCSRPEM != "" { + return nil + } + log.Infof(ctx, "[certificates] Generating etcd-%s csr and key", host.InternalAddress) + etcdCSR, etcdKey, err := GenerateCertSigningRequestAndKey(true, EtcdCertName, etcdAltNames, certs[etcdName].Key, nil) + if err != nil { + return err + } + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, etcdCSR) } return nil } func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate service account token key - var privateAPIKey *rsa.PrivateKey + privateAPIKey := certs[ServiceAccountTokenKeyName].Key caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key + if certs[ServiceAccountTokenKeyName].Certificate != nil { + return nil + } // handle rotation on old clusters if certs[ServiceAccountTokenKeyName].Key == nil { privateAPIKey = certs[KubeAPICertName].Key @@ -198,7 +363,7 @@ func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePK if err != nil { return fmt.Errorf("Failed to generate private key for service account token: %v", err) } - certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", tokenCrt, tokenKey) + certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", tokenCrt, tokenKey, nil) return nil } @@ -209,7 +374,7 @@ func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, co if err != nil { return err } - certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey) + certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey, nil) // generate request header client CA certificate and key log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates") @@ -217,7 +382,7 @@ func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, co if err != nil { return err } - certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey) + certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey, nil) return nil } @@ -243,3 +408,22 @@ func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificateP } return nil } + +func GenerateRKEServicesCSRs(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error { + RKECerts := []CSRFunc{ + GenerateKubeAPICSR, + GenerateKubeControllerCSR, + GenerateKubeSchedulerCSR, + GenerateKubeProxyCSR, + GenerateKubeNodeCSR, + GenerateKubeAdminCSR, + GenerateAPIProxyClientCSR, + GenerateEtcdCSRs, + } + for _, csr := range RKECerts { + if err := csr(ctx, certs, rkeConfig); err != nil { + return err + } + } + return nil +} diff --git a/pki/util.go b/pki/util.go index c5f10289..74f420bb 100644 --- a/pki/util.go +++ b/pki/util.go @@ -5,11 +5,14 @@ import ( "crypto/rsa" "crypto/x509" "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" + "io/ioutil" "math" "math/big" "net" + "os" "path" "path/filepath" "reflect" @@ -18,6 +21,7 @@ import ( "github.com/rancher/rke/hosts" "github.com/rancher/types/apis/management.cattle.io/v3" + "github.com/sirupsen/logrus" "k8s.io/client-go/util/cert" ) @@ -59,6 +63,43 @@ func GenerateSignedCertAndKey( return clientCert, rootKey, nil } +func GenerateCertSigningRequestAndKey( + serverCrt bool, + commonName string, + altNames *cert.AltNames, + reusedKey *rsa.PrivateKey, + orgs []string) ([]byte, *rsa.PrivateKey, error) { + // Generate a generic signed certificate + var rootKey *rsa.PrivateKey + var err error + rootKey = reusedKey + if reusedKey == nil { + rootKey, err = cert.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err) + } + } + usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + if serverCrt { + usages = append(usages, x509.ExtKeyUsageServerAuth) + } + if altNames == nil { + altNames = &cert.AltNames{} + } + caConfig := cert.Config{ + CommonName: commonName, + Organization: orgs, + Usages: usages, + AltNames: *altNames, + } + clientCSR, err := newCertSigningRequest(caConfig, rootKey) + + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err) + } + return clientCSR, rootKey, nil +} + func GenerateCACertAndKey(commonName string, privateKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { var err error rootKey := privateKey @@ -197,8 +238,10 @@ func GetConfigTempPath(name string) string { return fmt.Sprintf("%skubecfg-%s.yaml", TempCertPath, name) } -func ToCertObject(componentName, commonName, ouName string, certificate *x509.Certificate, key *rsa.PrivateKey) CertificatePKI { - var config, configPath, configEnvName string +func ToCertObject(componentName, commonName, ouName string, certificate *x509.Certificate, key *rsa.PrivateKey, csrASN1 []byte) CertificatePKI { + var config, configPath, configEnvName, certificatePEM, keyPEM string + var csr *x509.CertificateRequest + var csrPEM []byte if len(commonName) == 0 { commonName = getDefaultCN(componentName) } @@ -208,8 +251,18 @@ func ToCertObject(componentName, commonName, ouName string, certificate *x509.Ce caCertPath := GetCertPath(CACertName) path := GetCertPath(componentName) keyPath := GetKeyPath(componentName) - certificatePEM := string(cert.EncodeCertPEM(certificate)) - keyPEM := string(cert.EncodePrivateKeyPEM(key)) + if certificate != nil { + certificatePEM = string(cert.EncodeCertPEM(certificate)) + } + if key != nil { + keyPEM = string(cert.EncodePrivateKeyPEM(key)) + } + if csrASN1 != nil { + csr, _ = x509.ParseCertificateRequest(csrASN1) + csrPEM = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE REQUEST", Bytes: csrASN1, + }) + } if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) && componentName != ServiceAccountTokenKeyName { config = getKubeConfigX509("https://127.0.0.1:6443", "local", componentName, caCertPath, path, keyPath) @@ -220,8 +273,10 @@ func ToCertObject(componentName, commonName, ouName string, certificate *x509.Ce return CertificatePKI{ Certificate: certificate, Key: key, + CSR: csr, CertificatePEM: certificatePEM, KeyPEM: keyPEM, + CSRPEM: string(csrPEM), Config: config, Name: componentName, CommonName: commonName, @@ -318,19 +373,19 @@ func getTempPath(s string) string { func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI { certs := make(map[string]CertificatePKI) // CACert - certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key) + certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key, nil) // KubeAPI - certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key) + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key, nil) // kubeController - certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key) + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key, nil) // KubeScheduler - certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key) + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key, nil) // KubeProxy - certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key) + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key, nil) // KubeNode - certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key) + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key, nil) // KubeAdmin - kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key) + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key, nil) kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config kubeAdminCertObj.ConfigPath = localConfigPath certs[KubeAdminCertName] = kubeAdminCertObj @@ -338,7 +393,7 @@ func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, for _, host := range extraHosts { etcdName := GetEtcdCrtName(host.InternalAddress) etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key - certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil) } return certs @@ -377,6 +432,25 @@ func newSignedCert(cfg cert.Config, key *rsa.PrivateKey, caCert *x509.Certificat return x509.ParseCertificate(certDERBytes) } +func newCertSigningRequest(cfg cert.Config, key *rsa.PrivateKey) ([]byte, error) { + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + } + return x509.CreateCertificateRequest(cryptorand.Reader, &certTmpl, key) +} + func isFileNotFoundErr(e error) bool { if strings.Contains(e.Error(), "no such file or directory") || strings.Contains(e.Error(), "Could not find the file") || @@ -400,10 +474,17 @@ func deepEqualIPsAltNames(oldIPs, newIPs []net.IP) bool { } func TransformPEMToObject(in map[string]CertificatePKI) map[string]CertificatePKI { + var certificate *x509.Certificate out := map[string]CertificatePKI{} for k, v := range in { certs, _ := cert.ParseCertsPEM([]byte(v.CertificatePEM)) key, _ := cert.ParsePrivateKeyPEM([]byte(v.KeyPEM)) + if len(certs) > 0 { + certificate = certs[0] + } + if key != nil { + key = key.(*rsa.PrivateKey) + } o := CertificatePKI{ ConfigEnvName: v.ConfigEnvName, Name: v.Name, @@ -415,12 +496,248 @@ func TransformPEMToObject(in map[string]CertificatePKI) map[string]CertificatePK KeyEnvName: v.KeyEnvName, KeyPath: v.KeyPath, ConfigPath: v.ConfigPath, - Certificate: certs[0], - Key: key.(*rsa.PrivateKey), + Certificate: certificate, CertificatePEM: v.CertificatePEM, KeyPEM: v.KeyPEM, } + if key != nil { + o.Key = key.(*rsa.PrivateKey) + } + out[k] = o } return out } + +func ReadCSRsAndKeysFromDir(certDir string) (map[string]CertificatePKI, error) { + certMap := make(map[string]CertificatePKI) + if _, err := os.Stat(certDir); os.IsNotExist(err) { + return certMap, nil + } + + files, err := ioutil.ReadDir(certDir) + if err != nil { + return nil, err + } + + for _, file := range files { + if strings.Contains(file.Name(), "-csr.pem") { + certName := strings.TrimSuffix(file.Name(), "-csr.pem") + logrus.Debugf("[certificates] Loading %s csr from directory [%s]", certName, certDir) + // fetching csr + csrASN1, err := getCSRFromFile(certDir, certName+"-csr.pem") + if err != nil { + return nil, err + } + // fetching key + key, err := getKeyFromFile(certDir, certName+"-key.pem") + if err != nil { + return nil, err + } + certMap[certName] = ToCertObject(certName, getCommonName(certName), getOUName(certName), nil, key, csrASN1) + } + } + + return certMap, nil +} + +func ReadCertsAndKeysFromDir(certDir string) (map[string]CertificatePKI, error) { + certMap := make(map[string]CertificatePKI) + if _, err := os.Stat(certDir); os.IsNotExist(err) { + return certMap, nil + } + + files, err := ioutil.ReadDir(certDir) + if err != nil { + return nil, err + } + + for _, file := range files { + logrus.Debugf("[certificates] reading file %s from directory [%s]", file.Name(), certDir) + // fetching cert + cert, err := getCertFromFile(certDir, file.Name()) + if err != nil { + continue + } + // fetching the cert's key + certName := strings.TrimSuffix(file.Name(), ".pem") + key, err := getKeyFromFile(certDir, certName+"-key.pem") + if err != nil { + continue + } + certMap[certName] = ToCertObject(certName, getCommonName(certName), getOUName(certName), cert, key, nil) + } + + return certMap, nil +} + +func getCommonName(certName string) string { + switch certName { + case KubeNodeCertName: + return KubeNodeCommonName + default: + return certName + } +} + +func getOUName(certName string) string { + switch certName { + case KubeNodeCertName: + return KubeNodeOrganizationName + case KubeAdminCertName: + return KubeAdminOrganizationName + default: + return "" + } +} + +func getCertFromFile(certDir string, fileName string) (*x509.Certificate, error) { + var certificate *x509.Certificate + certPEM, _ := ioutil.ReadFile(filepath.Join(certDir, fileName)) + if len(certPEM) > 0 { + certificates, err := cert.ParseCertsPEM(certPEM) + if err != nil { + return nil, fmt.Errorf("failed to read certificate [%s]: %v", fileName, err) + } + certificate = certificates[0] + } + return certificate, nil +} + +func getKeyFromFile(certDir string, fileName string) (*rsa.PrivateKey, error) { + var key *rsa.PrivateKey + keyPEM, _ := ioutil.ReadFile(filepath.Join(certDir, fileName)) + if len(keyPEM) > 0 { + keyInterface, err := cert.ParsePrivateKeyPEM(keyPEM) + if err != nil { + return nil, fmt.Errorf("failed to read key [%s]: %v", fileName, err) + } + key = keyInterface.(*rsa.PrivateKey) + } + return key, nil +} + +func getCSRFromFile(certDir string, fileName string) ([]byte, error) { + csrPEM, err := ioutil.ReadFile(filepath.Join(certDir, fileName)) + if err != nil { + return nil, fmt.Errorf("failed to read csr [%s]: %v", fileName, err) + } + csrASN1, _ := pem.Decode(csrPEM) + return csrASN1.Bytes, nil +} + +func WriteCertificates(certDirPath string, certBundle map[string]CertificatePKI) error { + if _, err := os.Stat(certDirPath); os.IsNotExist(err) { + err = os.MkdirAll(certDirPath, 0755) + if err != nil { + return err + } + } + + for certName, cert := range certBundle { + if cert.CertificatePEM != "" { + certificatePath := filepath.Join(certDirPath, certName+".pem") + if err := ioutil.WriteFile(certificatePath, []byte(cert.CertificatePEM), 0640); err != nil { + return fmt.Errorf("Failed to write certificate to path %v: %v", certificatePath, err) + } + logrus.Debugf("Successfully Deployed certificate file at [%s]", certificatePath) + } + + if cert.KeyPEM != "" { + keyPath := filepath.Join(certDirPath, certName+"-key.pem") + if err := ioutil.WriteFile(keyPath, []byte(cert.KeyPEM), 0640); err != nil { + return fmt.Errorf("Failed to write key to path %v: %v", keyPath, err) + } + logrus.Debugf("Successfully Deployed key file at [%s]", keyPath) + } + + if cert.CSRPEM != "" { + csrPath := filepath.Join(certDirPath, certName+"-csr.pem") + if err := ioutil.WriteFile(csrPath, []byte(cert.CSRPEM), 0640); err != nil { + return fmt.Errorf("Failed to write csr to path %v: %v", csrPath, err) + } + logrus.Debugf("Successfully Deployed csr file at [%s]", csrPath) + } + } + logrus.Infof("Successfully Deployed certificates at [%s]", certDirPath) + return nil +} + +func ValidateBundleContent(rkeConfig *v3.RancherKubernetesEngineConfig, certBundle map[string]CertificatePKI, configPath, configDir string) error { + // ensure all needed certs exists + // make sure all CA Certs exist + if certBundle[CACertName].Certificate == nil { + return fmt.Errorf("Failed to find master CA certificate") + } + if certBundle[RequestHeaderCACertName].Certificate == nil { + logrus.Warnf("Failed to find RequestHeader CA certificate, using master CA certificate") + certBundle[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, RequestHeaderCACertName, "", certBundle[CACertName].Certificate, nil, nil) + } + // make sure all components exists + ComponentsCerts := []string{ + KubeAPICertName, + KubeControllerCertName, + KubeSchedulerCertName, + KubeProxyCertName, + KubeNodeCertName, + KubeAdminCertName, + APIProxyClientCertName, + } + for _, certName := range ComponentsCerts { + if certBundle[certName].Certificate == nil || certBundle[certName].Key == nil { + return fmt.Errorf("Failed to find [%s] Certificate or Key", certName) + } + } + etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole) + for _, host := range etcdHosts { + etcdName := GetEtcdCrtName(host.InternalAddress) + if certBundle[etcdName].Certificate == nil || certBundle[etcdName].Key == nil { + return fmt.Errorf("Failed to find etcd [%s] Certificate or Key", etcdName) + } + } + // Configure kubeconfig + cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole) + localKubeConfigPath := GetLocalKubeConfig(configPath, configDir) + if len(cpHosts) > 0 { + kubeAdminCertObj := certBundle[KubeAdminCertName] + kubeAdminConfig := GetKubeConfigX509WithData( + "https://"+cpHosts[0].Address+":6443", + rkeConfig.ClusterName, + KubeAdminCertName, + string(cert.EncodeCertPEM(certBundle[CACertName].Certificate)), + string(cert.EncodeCertPEM(certBundle[KubeAdminCertName].Certificate)), + string(cert.EncodePrivateKeyPEM(certBundle[KubeAdminCertName].Key))) + kubeAdminCertObj.Config = kubeAdminConfig + kubeAdminCertObj.ConfigPath = localKubeConfigPath + certBundle[KubeAdminCertName] = kubeAdminCertObj + } + return validateCAIssuer(rkeConfig, certBundle) +} + +func validateCAIssuer(rkeConfig *v3.RancherKubernetesEngineConfig, certBundle map[string]CertificatePKI) error { + // make sure all certs are signed by CA cert + caCert := certBundle[CACertName].Certificate + ComponentsCerts := []string{ + KubeAPICertName, + KubeControllerCertName, + KubeSchedulerCertName, + KubeProxyCertName, + KubeNodeCertName, + KubeAdminCertName, + } + etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole) + for _, host := range etcdHosts { + etcdName := GetEtcdCrtName(host.InternalAddress) + ComponentsCerts = append(ComponentsCerts, etcdName) + } + for _, componentCert := range ComponentsCerts { + if certBundle[componentCert].Certificate.Issuer.CommonName != caCert.Subject.CommonName { + return fmt.Errorf("Component [%s] is not signed by the custom CA certificate", componentCert) + } + } + requestHeaderCACert := certBundle[RequestHeaderCACertName].Certificate + if certBundle[APIProxyClientCertName].Certificate.Issuer.CommonName != requestHeaderCACert.Subject.CommonName { + return fmt.Errorf("Component [%s] is not signed by the custom Request Header CA certificate", APIProxyClientCertName) + } + return nil +}