rke/cluster/certificates.go

package cluster

import (
	"context"
	"crypto/rsa"
	"fmt"
	"strings"

	"github.com/rancher/rke/hosts"
	"github.com/rancher/rke/k8s"
	"github.com/rancher/rke/log"
	"github.com/rancher/rke/pki"
	"github.com/rancher/rke/pki/cert"
	"github.com/rancher/rke/services"
)

func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Cluster, fullState *FullState) error {
	if kubeCluster.AuthnStrategies[AuthnX509Provider] {
		kubeCluster.Certificates = fullState.DesiredState.CertificatesBundle
		compareCerts(ctx, kubeCluster, currentCluster)
		return nil
	}
	return nil
}

func GetClusterCertsFromKubernetes(ctx context.Context, kubeCluster *Cluster) (map[string]pki.CertificatePKI, error) {
	log.Infof(ctx, "[certificates] Getting Cluster certificates from Kubernetes")

	k8sClient, err := k8s.NewClient(kubeCluster.LocalKubeConfigPath, kubeCluster.K8sWrapTransport)
	if err != nil {
		return nil, fmt.Errorf("Failed to create Kubernetes Client: %v", err)
	}
	certificatesNames := []string{
		pki.CACertName,
		pki.KubeAPICertName,
		pki.KubeNodeCertName,
		pki.KubeProxyCertName,
		pki.KubeControllerCertName,
		pki.KubeSchedulerCertName,
		pki.KubeAdminCertName,
		pki.APIProxyClientCertName,
		pki.RequestHeaderCACertName,
		pki.ServiceAccountTokenKeyName,
	}

	for _, etcdHost := range kubeCluster.EtcdHosts {
		etcdName := pki.GetCrtNameForHost(etcdHost, pki.EtcdCertName)
		certificatesNames = append(certificatesNames, etcdName)
	}

	certMap := make(map[string]pki.CertificatePKI)
	for _, certName := range certificatesNames {
		secret, err := k8s.GetSystemSecret(k8sClient, certName)
		if err != nil && !strings.HasPrefix(certName, "kube-etcd") &&
			!strings.Contains(certName, pki.RequestHeaderCACertName) &&
			!strings.Contains(certName, pki.APIProxyClientCertName) &&
			!strings.Contains(certName, pki.ServiceAccountTokenKeyName) {
			return nil, err
		}
		// If I can't find an etcd, requestheader, or proxy client cert, I will not fail and will create it later.
		if (secret == nil || secret.Data == nil) &&
			(strings.HasPrefix(certName, "kube-etcd") ||
				strings.Contains(certName, pki.RequestHeaderCACertName) ||
				strings.Contains(certName, pki.APIProxyClientCertName) ||
				strings.Contains(certName, pki.ServiceAccountTokenKeyName)) {
			certMap[certName] = pki.CertificatePKI{}
			continue
		}

		secretCert, err := cert.ParseCertsPEM(secret.Data["Certificate"])
		if err != nil {
			return nil, fmt.Errorf("Failed to parse certificate of %s: %v", certName, err)
		}
		secretKey, err := cert.ParsePrivateKeyPEM(secret.Data["Key"])
		if err != nil {
			return nil, fmt.Errorf("Failed to parse private key of %s: %v", certName, err)
		}
		secretConfig := string(secret.Data["Config"])
		if len(secretCert) == 0 || secretKey == nil {
			return nil, fmt.Errorf("certificate or key of %s is not found", certName)
		}
		certificatePEM := string(cert.EncodeCertPEM(secretCert[0]))
		keyPEM := string(cert.EncodePrivateKeyPEM(secretKey.(*rsa.PrivateKey)))

		certMap[certName] = pki.CertificatePKI{
			Certificate:    secretCert[0],
			Key:            secretKey.(*rsa.PrivateKey),
			CertificatePEM: certificatePEM,
			KeyPEM:         keyPEM,
			Config:         secretConfig,
			EnvName:        string(secret.Data["EnvName"]),
			ConfigEnvName:  string(secret.Data["ConfigEnvName"]),
			KeyEnvName:     string(secret.Data["KeyEnvName"]),
			Path:           string(secret.Data["Path"]),
			KeyPath:        string(secret.Data["KeyPath"]),
			ConfigPath:     string(secret.Data["ConfigPath"]),
		}
	}
	// Handle service account token key issue
	kubeAPICert := certMap[pki.KubeAPICertName]
	if certMap[pki.ServiceAccountTokenKeyName].Key == nil {
		log.Infof(ctx, "[certificates] Creating service account token key")
		certMap[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key, nil)
	}
	log.Infof(ctx, "[certificates] Successfully fetched Cluster certificates from Kubernetes")
	return certMap, nil
}

func (c *Cluster) getBackupHosts() []*hosts.Host {
	var backupHosts []*hosts.Host
	if len(c.Services.Etcd.ExternalURLs) > 0 {
		backupHosts = c.ControlPlaneHosts
	} else {
		// Save certificates on etcd and controlplane hosts
		backupHosts = hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, nil)
	}
	return backupHosts
}

func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags, clusterState *FullState) error {
	var (
		serviceAccountTokenKey string
	)
	componentsCertsFuncMap := map[string][]pki.GenFunc{
		services.KubeAPIContainerName:        []pki.GenFunc{pki.GenerateKubeAPICertificate},
		services.KubeControllerContainerName: []pki.GenFunc{pki.GenerateKubeControllerCertificate},
		services.SchedulerContainerName:      []pki.GenFunc{pki.GenerateKubeSchedulerCertificate},
		services.KubeproxyContainerName:      []pki.GenFunc{pki.GenerateKubeProxyCertificate},
		services.KubeletContainerName:        []pki.GenFunc{pki.GenerateKubeNodeCertificate},
		services.EtcdContainerName:           []pki.GenFunc{pki.GenerateEtcdCertificates},
	}
	if c.IsKubeletGenerateServingCertificateEnabled() {
		componentsCertsFuncMap[services.KubeletContainerName] = append(componentsCertsFuncMap[services.KubeletContainerName], pki.GenerateKubeletCertificate)
	}
	rotateFlags := c.RancherKubernetesEngineConfig.RotateCertificates
	if rotateFlags.CACertificates {
		// rotate CA cert and RequestHeader CA cert
		if err := pki.GenerateRKECACerts(ctx, c.Certificates, flags.ClusterFilePath, flags.ConfigDir); err != nil {
			return err
		}
		rotateFlags.Services = nil
	}
	for _, k8sComponent := range rotateFlags.Services {
		genFunctions := componentsCertsFuncMap[k8sComponent]
		if genFunctions != nil {
			for _, genFunc := range genFunctions {
				if err := genFunc(ctx, c.Certificates, c.RancherKubernetesEngineConfig, flags.ClusterFilePath, flags.ConfigDir, true); err != nil {
					return err
				}
			}
		}
	}
	// to handle kontainer engine sending empty string for services
	if len(rotateFlags.Services) == 0 || (len(rotateFlags.Services) == 1 && rotateFlags.Services[0] == "") {
		// do not rotate service account token
		if c.Certificates[pki.ServiceAccountTokenKeyName].Key != nil {
			serviceAccountTokenKey = string(cert.EncodePrivateKeyPEM(c.Certificates[pki.ServiceAccountTokenKeyName].Key))
		}
		// check for legacy clusters prior to requestheaderca
		if c.Certificates[pki.RequestHeaderCACertName].Certificate == nil {
			if err := pki.GenerateRKERequestHeaderCACert(ctx, c.Certificates, flags.ClusterFilePath, flags.ConfigDir); err != nil {
				return err
			}
		}
		if err := pki.GenerateRKEServicesCerts(ctx, c.Certificates, c.RancherKubernetesEngineConfig, flags.ClusterFilePath, flags.ConfigDir, true); err != nil {
			return err
		}
		if serviceAccountTokenKey != "" {
			privateKey, err := cert.ParsePrivateKeyPEM([]byte(serviceAccountTokenKey))
			if err != nil {
				return err
			}
			c.Certificates[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(
				pki.ServiceAccountTokenKeyName,
				pki.ServiceAccountTokenKeyName,
				"",
				c.Certificates[pki.ServiceAccountTokenKeyName].Certificate,
				privateKey.(*rsa.PrivateKey), nil)
		}
	}
	clusterState.DesiredState.CertificatesBundle = c.Certificates
	return nil
}

func GetClusterCertsFromNodes(ctx context.Context, kubeCluster *Cluster) (map[string]pki.CertificatePKI, error) {
	log.Infof(ctx, "[certificates] Fetching kubernetes certificates from nodes")
	var err error
	backupHosts := hosts.GetUniqueHostList(kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts, nil)
	var certificates map[string]pki.CertificatePKI
	for _, host := range backupHosts {
		certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, host, kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
		if certificates != nil {
			// Handle service account token key issue
			kubeAPICert := certificates[pki.KubeAPICertName]
			if certificates[pki.ServiceAccountTokenKeyName].Key == nil {
				log.Infof(ctx, "[certificates] Creating service account token key")
				certificates[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key, nil)
			}
			return certificates, nil
		}
	}
	// reporting the last error only.
	return nil, err
}

func compareCerts(ctx context.Context, kubeCluster, currentCluster *Cluster) {
	// check if relevant certs were changed and if so set force deploy to true
	// to deploy certs with new SANs
	if currentCluster != nil {
		for _, certName := range []string{
			pki.KubeAPICertName,
			pki.EtcdCertName,
		} {
			currentCert := currentCluster.Certificates[certName]
			desiredCert := kubeCluster.Certificates[certName]
			if desiredCert.CertificatePEM != currentCert.CertificatePEM {
				log.Infof(ctx, "[certificates] %s certificate changed, force deploying certs", certName)
				kubeCluster.ForceDeployCerts = true
				return
			}
		}
	}
}

func (c *Cluster) IsKubeletGenerateServingCertificateEnabled() bool {
	if c == nil {
		return false
	}
	if c.Services.Kubelet.GenerateServingCertificate {
		return true
	}
	return false
}