diff --git a/cluster.yml b/cluster.yml index 18911265..118c6a51 100644 --- a/cluster.yml +++ b/cluster.yml @@ -71,7 +71,7 @@ services: system_images: alpine: alpine:latest nginx_proxy: rancher/rke-nginx-proxy:0.1.0 - cert_downloader: rancher/rke-cert-deployer:0.1.0 + cert_downloader: rancher/rke-cert-deployer:0.1.1 kubedns_image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5 dnsmasq_image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5 kubedns_sidecar_image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5 diff --git a/cluster/certificates.go b/cluster/certificates.go index 768d560a..31a95959 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -20,6 +20,17 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust if currentCluster != nil { kubeCluster.Certificates = currentCluster.Certificates } else { + log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.EtcdHosts[0].Address) + kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath) + if err != nil { + return err + } + if kubeCluster.Certificates != nil { + log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.EtcdHosts[0].Address) + return nil + } + log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.EtcdHosts[0].Address) + kubeCluster.Certificates, err = pki.StartCertificatesGeneration(ctx, kubeCluster.ControlPlaneHosts, kubeCluster.WorkerHosts, @@ -29,6 +40,11 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust if err != nil { return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err) } + log.Infof(ctx, "[certificates] Temporarily saving certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address) + if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage]); err != nil { + return err + } + log.Infof(ctx, "[certificates] Saved certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address) } } return nil diff --git a/cluster/defaults.go b/cluster/defaults.go index 9dc5a46d..8cec86a0 100644 --- a/cluster/defaults.go +++ b/cluster/defaults.go @@ -20,7 +20,7 @@ const ( DefaultInfraContainerImage = "gcr.io/google_containers/pause-amd64:3.0" DefaultAplineImage = "alpine:latest" DefaultNginxProxyImage = "rancher/rke-nginx-proxy:0.1.0" - DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.0" + DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.1" DefaultServiceSidekickImage = "rancher/rke-service-sidekick:0.1.0" DefaultEtcdImage = "quay.io/coreos/etcd:latest" diff --git a/docker/docker.go b/docker/docker.go index 086966ca..ab1788d9 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -1,6 +1,7 @@ package docker import ( + "archive/tar" "context" "fmt" "io" @@ -271,3 +272,20 @@ func IsSupportedDockerVersion(info types.Info, K8sVersion string) (bool, error) } return false, nil } + +func ReadFileFromContainer(ctx context.Context, dClient *client.Client, hostname, container, filePath string) (string, error) { + reader, _, err := dClient.CopyFromContainer(ctx, container, filePath) + if err != nil { + return "", fmt.Errorf("Failed to copy file [%s] from container [%s] on host [%s]: %v", filePath, container, hostname, err) + } + defer reader.Close() + tarReader := tar.NewReader(reader) + if _, err := tarReader.Next(); err != nil { + return "", err + } + file, err := ioutil.ReadAll(tarReader) + if err != nil { + return "", err + } + return string(file), nil +} diff --git a/hosts/hosts.go b/hosts/hosts.go index 18c96257..cc69724f 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -33,6 +33,7 @@ const ( ToCleanCNIConf = "/etc/cni" ToCleanCNIBin = "/opt/cni" ToCleanCalicoRun = "/var/run/calico" + ToCleanTempCertPath = "/etc/kubernetes/.tmp/" CleanerContainerName = "kube-cleaner" ) @@ -44,6 +45,7 @@ func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string) error { ToCleanCNIConf, ToCleanCNIBin, ToCleanCalicoRun, + ToCleanTempCertPath, } return h.CleanUp(ctx, toCleanPaths, cleanerImage) } diff --git a/package/certs-deployer/build-and-push.sh b/package/certs-deployer/build-and-push.sh index a3960004..9cbb68ce 100755 --- a/package/certs-deployer/build-and-push.sh +++ b/package/certs-deployer/build-and-push.sh @@ -2,5 +2,5 @@ ACCT=${ACCT:-rancher} -docker build -t $ACCT/rke-cert-deployer:0.1.0 . -docker push $ACCT/rke-cert-deployer:0.1.0 +docker build -t $ACCT/rke-cert-deployer:0.1.1 . +docker push $ACCT/rke-cert-deployer:0.1.1 diff --git a/package/certs-deployer/entrypoint.sh b/package/certs-deployer/entrypoint.sh index e6bc7c05..b02b8c61 100755 --- a/package/certs-deployer/entrypoint.sh +++ b/package/certs-deployer/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -x -SSL_CRTS_DIR=/etc/kubernetes/ssl +SSL_CRTS_DIR=${CRTS_DEPLOY_PATH:-/etc/kubernetes/ssl} mkdir -p $SSL_CRTS_DIR for i in $(env | grep -o KUBE_.*=); do diff --git a/pki/constants.go b/pki/constants.go index af2bed7f..92f022a2 100644 --- a/pki/constants.go +++ b/pki/constants.go @@ -3,7 +3,9 @@ package pki const ( CertificatesServiceName = "certificates" CrtDownloaderContainer = "cert-deployer" + CertFetcherContainer = "cert-fetcher" CertificatesSecretName = "k8s-certs" + TempCertPath = "/etc/kubernetes/.tmp/" CACertName = "kube-ca" CACertENVName = "KUBE_CA" @@ -58,4 +60,6 @@ const ( KubeAdminOrganizationName = "system:masters" KubeAdminConfigPrefix = ".kube_config_" KubeAdminConfigENVName = "KUBECFG_ADMIN" + KubeAdminCertEnvName = "KUBE_ADMIN" + KubeAdminKeyEnvName = "KUBE_ADMIN_KEY" ) diff --git a/pki/deploy.go b/pki/deploy.go index 089d395d..26c5d7d7 100644 --- a/pki/deploy.go +++ b/pki/deploy.go @@ -2,9 +2,12 @@ package pki import ( "context" + "crypto/rsa" "fmt" "io/ioutil" "os" + "path" + "strings" "time" "github.com/docker/docker/api/types" @@ -13,6 +16,7 @@ import ( "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" "github.com/sirupsen/logrus" + "k8s.io/client-go/util/cert" ) func DeployCertificatesOnMasters(ctx context.Context, cpHosts []*hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error { @@ -63,6 +67,17 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host, } func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string, certDownloaderImage string) error { + // remove existing container. Only way it's still here is if previous deployment failed + isRunning := false + isRunning, err := docker.IsContainerRunning(ctx, host.DClient, host.Address, CrtDownloaderContainer, true) + if err != nil { + return err + } + if isRunning { + if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CrtDownloaderContainer); err != nil { + return err + } + } if err := docker.UseLocalOrPull(ctx, host.DClient, host.Address, certDownloaderImage, CertificatesServiceName); err != nil { return err } @@ -119,3 +134,205 @@ func RemoveAdminConfig(ctx context.Context, localConfigPath string) { } log.Infof(ctx, "Local admin Kubeconfig removed successfully") } + +func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error { + crtList := []string{ + CACertName, + KubeAPICertName, + KubeControllerName, + KubeSchedulerName, + KubeProxyName, + KubeNodeName, + KubeAdminCommonName, + } + + env := []string{ + "CRTS_DEPLOY_PATH=" + TempCertPath, + } + for _, crtName := range crtList { + c := crtMap[crtName] + // We don't need to edit the cert paths, they are not used in deployment + env = append(env, c.ToEnv()...) + } + return doRunDeployer(ctx, host, env, certDownloaderImage) +} + +func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) { + // rebuilding the certificates. This should look better after refactoring pki + tmpCerts := make(map[string]CertificatePKI) + + certEnvMap := map[string][]string{ + CACertName: []string{CACertPath, CAKeyPath}, + KubeAPICertName: []string{KubeAPICertPath, KubeAPIKeyPath}, + KubeControllerName: []string{KubeControllerCertPath, KubeControllerKeyPath, KubeControllerConfigPath}, + KubeSchedulerName: []string{KubeSchedulerCertPath, KubeSchedulerKeyPath, KubeSchedulerConfigPath}, + KubeProxyName: []string{KubeProxyCertPath, KubeProxyKeyPath, KubeProxyConfigPath}, + KubeNodeName: []string{KubeNodeCertPath, KubeNodeKeyPath, KubeNodeConfigPath}, + KubeAdminCommonName: []string{"kube-admin.pem", "kube-admin-key.pem", "kubecfg-admin.yaml"}, + } + // get files from hosts + + for crtName, certEnv := range certEnvMap { + certificate := CertificatePKI{} + crt, err := fetchFileFromHost(ctx, getTempPath(certEnv[0]), image, host) + if err != nil { + if strings.Contains(err.Error(), "no such file or directory") { + return nil, nil + } + return nil, err + } + key, err := fetchFileFromHost(ctx, getTempPath(certEnv[1]), image, host) + + if len(certEnv) > 2 { + config, err := fetchFileFromHost(ctx, getTempPath(certEnv[2]), image, host) + if err != nil { + return nil, err + } + certificate.Config = config + } + parsedCert, err := cert.ParseCertsPEM([]byte(crt)) + if err != nil { + return nil, err + } + parsedKey, err := cert.ParsePrivateKeyPEM([]byte(key)) + if err != nil { + return nil, err + } + certificate.Certificate = parsedCert[0] + certificate.Key = parsedKey.(*rsa.PrivateKey) + tmpCerts[crtName] = certificate + logrus.Debugf("[certificates] Recovered certificate: %s", crtName) + } + + if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CertFetcherContainer); err != nil { + return nil, err + } + return populateCertMap(tmpCerts, localConfigPath), nil + +} + +func fetchFileFromHost(ctx context.Context, filePath, image string, host *hosts.Host) (string, error) { + + imageCfg := &container.Config{ + Image: image, + } + hostCfg := &container.HostConfig{ + Binds: []string{ + "/etc/kubernetes:/etc/kubernetes", + }, + Privileged: true, + } + isRunning, err := docker.IsContainerRunning(ctx, host.DClient, host.Address, CertFetcherContainer, true) + if err != nil { + return "", err + } + if !isRunning { + if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, CertFetcherContainer, host.Address, "certificates"); err != nil { + return "", err + } + } + file, err := docker.ReadFileFromContainer(ctx, host.DClient, host.Address, CertFetcherContainer, filePath) + if err != nil { + return "", err + } + + return file, nil +} + +func getTempPath(s string) string { + return TempCertPath + path.Base(s) +} + +func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string) map[string]CertificatePKI { + certs := make(map[string]CertificatePKI) + //CACert + certs[CACertName] = CertificatePKI{ + Certificate: tmpCerts[CACertName].Certificate, + Key: tmpCerts[CACertName].Key, + Name: CACertName, + EnvName: CACertENVName, + KeyEnvName: CAKeyENVName, + Path: CACertPath, + KeyPath: CAKeyPath, + } + //KubeAPI + certs[KubeAPICertName] = CertificatePKI{ + Certificate: tmpCerts[KubeAPICertName].Certificate, + Key: tmpCerts[KubeAPICertName].Key, + Name: KubeAPICertName, + EnvName: KubeAPICertENVName, + KeyEnvName: KubeAPIKeyENVName, + Path: KubeAPICertPath, + KeyPath: KubeAPIKeyPath, + } + //kubeController + certs[KubeControllerName] = CertificatePKI{ + Certificate: tmpCerts[KubeControllerName].Certificate, + Key: tmpCerts[KubeControllerName].Key, + Config: tmpCerts[KubeControllerName].Config, + Name: KubeControllerName, + CommonName: KubeControllerCommonName, + EnvName: KubeControllerCertENVName, + KeyEnvName: KubeControllerKeyENVName, + Path: KubeControllerCertPath, + KeyPath: KubeControllerKeyPath, + ConfigEnvName: KubeControllerConfigENVName, + ConfigPath: KubeControllerConfigPath, + } + //KubeScheduler + certs[KubeSchedulerName] = CertificatePKI{ + Certificate: tmpCerts[KubeSchedulerName].Certificate, + Key: tmpCerts[KubeSchedulerName].Key, + Config: tmpCerts[KubeSchedulerName].Config, + Name: KubeSchedulerName, + CommonName: KubeSchedulerCommonName, + EnvName: KubeSchedulerCertENVName, + KeyEnvName: KubeSchedulerKeyENVName, + Path: KubeSchedulerCertPath, + KeyPath: KubeSchedulerKeyPath, + ConfigEnvName: KubeSchedulerConfigENVName, + ConfigPath: KubeSchedulerConfigPath, + } + // KubeProxy + certs[KubeProxyName] = CertificatePKI{ + Certificate: tmpCerts[KubeProxyName].Certificate, + Key: tmpCerts[KubeProxyName].Key, + Config: tmpCerts[KubeProxyName].Config, + Name: KubeProxyName, + CommonName: KubeProxyCommonName, + EnvName: KubeProxyCertENVName, + Path: KubeProxyCertPath, + KeyEnvName: KubeProxyKeyENVName, + KeyPath: KubeProxyKeyPath, + ConfigEnvName: KubeProxyConfigENVName, + ConfigPath: KubeProxyConfigPath, + } + // KubeNode + certs[KubeNodeName] = CertificatePKI{ + Certificate: tmpCerts[KubeNodeName].Certificate, + Key: tmpCerts[KubeNodeName].Key, + Config: tmpCerts[KubeNodeName].Config, + Name: KubeNodeName, + CommonName: KubeNodeCommonName, + OUName: KubeNodeOrganizationName, + EnvName: KubeNodeCertENVName, + KeyEnvName: KubeNodeKeyENVName, + Path: KubeNodeCertPath, + KeyPath: KubeNodeKeyPath, + ConfigEnvName: KubeNodeConfigENVName, + ConfigPath: KubeNodeCommonName, + } + + certs[KubeAdminCommonName] = CertificatePKI{ + Certificate: tmpCerts[KubeAdminCommonName].Certificate, + Key: tmpCerts[KubeAdminCommonName].Key, + Config: tmpCerts[KubeAdminCommonName].Config, + CommonName: KubeAdminCommonName, + OUName: KubeAdminOrganizationName, + ConfigEnvName: KubeAdminConfigENVName, + ConfigPath: localConfigPath, + EnvName: KubeAdminCertEnvName, + KeyEnvName: KubeAdminKeyEnvName, + } + return certs +} diff --git a/pki/pki.go b/pki/pki.go index f24cffd7..39dd3c8f 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -178,6 +178,8 @@ func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, lo OUName: KubeAdminOrganizationName, ConfigEnvName: KubeAdminConfigENVName, ConfigPath: localConfigPath, + EnvName: KubeAdminCertEnvName, + KeyEnvName: KubeAdminKeyEnvName, } return certs, nil }