rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-05-10 09:24:32 +00:00
Code Issues Releases Wiki Activity

recoverable provisioning

Browse Source
This commit is contained in:
moelsayed 2018-01-11 01:03:08 +02:00
parent 34199e7f4a
commit 4c08db1d53
10 changed files with 264 additions and 5 deletions

2
cluster.yml
View File

@ -71,7 +71,7 @@ services:
system_images:
alpine: alpine:latest
nginx_proxy: rancher/rke-nginx-proxy:0.1.0
cert_downloader: rancher/rke-cert-deployer:0.1.0
cert_downloader: rancher/rke-cert-deployer:0.1.1
kubedns_image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5
dnsmasq_image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5
kubedns_sidecar_image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5

16
cluster/certificates.go
View File

@ -20,6 +20,17 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
if currentCluster != nil {
kubeCluster.Certificates = currentCluster.Certificates
} else {
log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.EtcdHosts[0].Address)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath)
if err != nil {
return err
}
if kubeCluster.Certificates != nil {
log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.EtcdHosts[0].Address)
return nil
}
log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.EtcdHosts[0].Address)
kubeCluster.Certificates, err = pki.StartCertificatesGeneration(ctx,
kubeCluster.ControlPlaneHosts,
kubeCluster.WorkerHosts,
@ -29,6 +40,11 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
if err != nil {
return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err)
}
log.Infof(ctx, "[certificates] Temporarily saving certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage]); err != nil {
return err
}
log.Infof(ctx, "[certificates] Saved certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
}
}
return nil

2
cluster/defaults.go
View File

@ -20,7 +20,7 @@ const (
DefaultInfraContainerImage = "gcr.io/google_containers/pause-amd64:3.0"
DefaultAplineImage = "alpine:latest"
DefaultNginxProxyImage = "rancher/rke-nginx-proxy:0.1.0"
DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.0"
DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.1"
DefaultServiceSidekickImage = "rancher/rke-service-sidekick:0.1.0"
DefaultEtcdImage = "quay.io/coreos/etcd:latest"

18
docker/docker.go
View File

@ -1,6 +1,7 @@
package docker
import (
"archive/tar"
"context"
"fmt"
"io"
@ -271,3 +272,20 @@ func IsSupportedDockerVersion(info types.Info, K8sVersion string) (bool, error)
}
return false, nil
}
func ReadFileFromContainer(ctx context.Context, dClient *client.Client, hostname, container, filePath string) (string, error) {
reader, _, err := dClient.CopyFromContainer(ctx, container, filePath)
if err != nil {
return "", fmt.Errorf("Failed to copy file [%s] from container [%s] on host [%s]: %v", filePath, container, hostname, err)
}
defer reader.Close()
tarReader := tar.NewReader(reader)
if _, err := tarReader.Next(); err != nil {
return "", err
}
file, err := ioutil.ReadAll(tarReader)
if err != nil {
return "", err
}
return string(file), nil
}

2
hosts/hosts.go
View File

@ -33,6 +33,7 @@ const (
ToCleanCNIConf = "/etc/cni"
ToCleanCNIBin = "/opt/cni"
ToCleanCalicoRun = "/var/run/calico"
ToCleanTempCertPath = "/etc/kubernetes/.tmp/"
CleanerContainerName = "kube-cleaner"
)
@ -44,6 +45,7 @@ func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string) error {
ToCleanCNIConf,
ToCleanCNIBin,
ToCleanCalicoRun,
ToCleanTempCertPath,
}
return h.CleanUp(ctx, toCleanPaths, cleanerImage)
}

4
package/certs-deployer/build-and-push.sh
View File

@ -2,5 +2,5 @@
ACCT=${ACCT:-rancher}
docker build -t $ACCT/rke-cert-deployer:0.1.0 .
docker push $ACCT/rke-cert-deployer:0.1.0
docker build -t $ACCT/rke-cert-deployer:0.1.1 .
docker push $ACCT/rke-cert-deployer:0.1.1

2
package/certs-deployer/entrypoint.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash -x
SSL_CRTS_DIR=/etc/kubernetes/ssl
SSL_CRTS_DIR=${CRTS_DEPLOY_PATH:-/etc/kubernetes/ssl}
mkdir -p $SSL_CRTS_DIR
for i in $(env | grep -o KUBE_.*=); do

4
pki/constants.go
View File

@ -3,7 +3,9 @@ package pki
const (
CertificatesServiceName = "certificates"
CrtDownloaderContainer = "cert-deployer"
CertFetcherContainer = "cert-fetcher"
CertificatesSecretName = "k8s-certs"
TempCertPath = "/etc/kubernetes/.tmp/"
CACertName = "kube-ca"
CACertENVName = "KUBE_CA"
@ -58,4 +60,6 @@ const (
KubeAdminOrganizationName = "system:masters"
KubeAdminConfigPrefix = ".kube_config_"
KubeAdminConfigENVName = "KUBECFG_ADMIN"
KubeAdminCertEnvName = "KUBE_ADMIN"
KubeAdminKeyEnvName = "KUBE_ADMIN_KEY"
)

217
pki/deploy.go
View File

@ -2,9 +2,12 @@ package pki
import (
"context"
"crypto/rsa"
"fmt"
"io/ioutil"
"os"
"path"
"strings"
"time"
"github.com/docker/docker/api/types"
@ -13,6 +16,7 @@ import (
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/log"
"github.com/sirupsen/logrus"
"k8s.io/client-go/util/cert"
)
func DeployCertificatesOnMasters(ctx context.Context, cpHosts []*hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error {
@ -63,6 +67,17 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host,
}
func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string, certDownloaderImage string) error {
// remove existing container. Only way it's still here is if previous deployment failed
isRunning := false
isRunning, err := docker.IsContainerRunning(ctx, host.DClient, host.Address, CrtDownloaderContainer, true)
if err != nil {
return err
}
if isRunning {
if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CrtDownloaderContainer); err != nil {
return err
}
}
if err := docker.UseLocalOrPull(ctx, host.DClient, host.Address, certDownloaderImage, CertificatesServiceName); err != nil {
return err
}
@ -119,3 +134,205 @@ func RemoveAdminConfig(ctx context.Context, localConfigPath string) {
}
log.Infof(ctx, "Local admin Kubeconfig removed successfully")
}
func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error {
crtList := []string{
CACertName,
KubeAPICertName,
KubeControllerName,
KubeSchedulerName,
KubeProxyName,
KubeNodeName,
KubeAdminCommonName,
}
env := []string{
"CRTS_DEPLOY_PATH=" + TempCertPath,
}
for _, crtName := range crtList {
c := crtMap[crtName]
// We don't need to edit the cert paths, they are not used in deployment
env = append(env, c.ToEnv()...)
}
return doRunDeployer(ctx, host, env, certDownloaderImage)
}
func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) {
// rebuilding the certificates. This should look better after refactoring pki
tmpCerts := make(map[string]CertificatePKI)
certEnvMap := map[string][]string{
CACertName: []string{CACertPath, CAKeyPath},
KubeAPICertName: []string{KubeAPICertPath, KubeAPIKeyPath},
KubeControllerName: []string{KubeControllerCertPath, KubeControllerKeyPath, KubeControllerConfigPath},
KubeSchedulerName: []string{KubeSchedulerCertPath, KubeSchedulerKeyPath, KubeSchedulerConfigPath},
KubeProxyName: []string{KubeProxyCertPath, KubeProxyKeyPath, KubeProxyConfigPath},
KubeNodeName: []string{KubeNodeCertPath, KubeNodeKeyPath, KubeNodeConfigPath},
KubeAdminCommonName: []string{"kube-admin.pem", "kube-admin-key.pem", "kubecfg-admin.yaml"},
}
// get files from hosts
for crtName, certEnv := range certEnvMap {
certificate := CertificatePKI{}
crt, err := fetchFileFromHost(ctx, getTempPath(certEnv[0]), image, host)
if err != nil {
if strings.Contains(err.Error(), "no such file or directory") {
return nil, nil
}
return nil, err
}
key, err := fetchFileFromHost(ctx, getTempPath(certEnv[1]), image, host)
if len(certEnv) > 2 {
config, err := fetchFileFromHost(ctx, getTempPath(certEnv[2]), image, host)
if err != nil {
return nil, err
}
certificate.Config = config
}
parsedCert, err := cert.ParseCertsPEM([]byte(crt))
if err != nil {
return nil, err
}
parsedKey, err := cert.ParsePrivateKeyPEM([]byte(key))
if err != nil {
return nil, err
}
certificate.Certificate = parsedCert[0]
certificate.Key = parsedKey.(*rsa.PrivateKey)
tmpCerts[crtName] = certificate
logrus.Debugf("[certificates] Recovered certificate: %s", crtName)
}
if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CertFetcherContainer); err != nil {
return nil, err
}
return populateCertMap(tmpCerts, localConfigPath), nil
}
func fetchFileFromHost(ctx context.Context, filePath, image string, host *hosts.Host) (string, error) {
imageCfg := &container.Config{
Image: image,
}
hostCfg := &container.HostConfig{
Binds: []string{
"/etc/kubernetes:/etc/kubernetes",
},
Privileged: true,
}
isRunning, err := docker.IsContainerRunning(ctx, host.DClient, host.Address, CertFetcherContainer, true)
if err != nil {
return "", err
}
if !isRunning {
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, CertFetcherContainer, host.Address, "certificates"); err != nil {
return "", err
}
}
file, err := docker.ReadFileFromContainer(ctx, host.DClient, host.Address, CertFetcherContainer, filePath)
if err != nil {
return "", err
}
return file, nil
}
func getTempPath(s string) string {
return TempCertPath + path.Base(s)
}
func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string) map[string]CertificatePKI {
certs := make(map[string]CertificatePKI)
//CACert
certs[CACertName] = CertificatePKI{
Certificate: tmpCerts[CACertName].Certificate,
Key: tmpCerts[CACertName].Key,
Name: CACertName,
EnvName: CACertENVName,
KeyEnvName: CAKeyENVName,
Path: CACertPath,
KeyPath: CAKeyPath,
}
//KubeAPI
certs[KubeAPICertName] = CertificatePKI{
Certificate: tmpCerts[KubeAPICertName].Certificate,
Key: tmpCerts[KubeAPICertName].Key,
Name: KubeAPICertName,
EnvName: KubeAPICertENVName,
KeyEnvName: KubeAPIKeyENVName,
Path: KubeAPICertPath,
KeyPath: KubeAPIKeyPath,
}
//kubeController
certs[KubeControllerName] = CertificatePKI{
Certificate: tmpCerts[KubeControllerName].Certificate,
Key: tmpCerts[KubeControllerName].Key,
Config: tmpCerts[KubeControllerName].Config,
Name: KubeControllerName,
CommonName: KubeControllerCommonName,
EnvName: KubeControllerCertENVName,
KeyEnvName: KubeControllerKeyENVName,
Path: KubeControllerCertPath,
KeyPath: KubeControllerKeyPath,
ConfigEnvName: KubeControllerConfigENVName,
ConfigPath: KubeControllerConfigPath,
}
//KubeScheduler
certs[KubeSchedulerName] = CertificatePKI{
Certificate: tmpCerts[KubeSchedulerName].Certificate,
Key: tmpCerts[KubeSchedulerName].Key,
Config: tmpCerts[KubeSchedulerName].Config,
Name: KubeSchedulerName,
CommonName: KubeSchedulerCommonName,
EnvName: KubeSchedulerCertENVName,
KeyEnvName: KubeSchedulerKeyENVName,
Path: KubeSchedulerCertPath,
KeyPath: KubeSchedulerKeyPath,
ConfigEnvName: KubeSchedulerConfigENVName,
ConfigPath: KubeSchedulerConfigPath,
}
// KubeProxy
certs[KubeProxyName] = CertificatePKI{
Certificate: tmpCerts[KubeProxyName].Certificate,
Key: tmpCerts[KubeProxyName].Key,
Config: tmpCerts[KubeProxyName].Config,
Name: KubeProxyName,
CommonName: KubeProxyCommonName,
EnvName: KubeProxyCertENVName,
Path: KubeProxyCertPath,
KeyEnvName: KubeProxyKeyENVName,
KeyPath: KubeProxyKeyPath,
ConfigEnvName: KubeProxyConfigENVName,
ConfigPath: KubeProxyConfigPath,
}
// KubeNode
certs[KubeNodeName] = CertificatePKI{
Certificate: tmpCerts[KubeNodeName].Certificate,
Key: tmpCerts[KubeNodeName].Key,
Config: tmpCerts[KubeNodeName].Config,
Name: KubeNodeName,
CommonName: KubeNodeCommonName,
OUName: KubeNodeOrganizationName,
EnvName: KubeNodeCertENVName,
KeyEnvName: KubeNodeKeyENVName,
Path: KubeNodeCertPath,
KeyPath: KubeNodeKeyPath,
ConfigEnvName: KubeNodeConfigENVName,
ConfigPath: KubeNodeCommonName,
}
certs[KubeAdminCommonName] = CertificatePKI{
Certificate: tmpCerts[KubeAdminCommonName].Certificate,
Key: tmpCerts[KubeAdminCommonName].Key,
Config: tmpCerts[KubeAdminCommonName].Config,
CommonName: KubeAdminCommonName,
OUName: KubeAdminOrganizationName,
ConfigEnvName: KubeAdminConfigENVName,
ConfigPath: localConfigPath,
EnvName: KubeAdminCertEnvName,
KeyEnvName: KubeAdminKeyEnvName,
}
return certs
}

2
pki/pki.go
View File

@ -178,6 +178,8 @@ func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, lo
OUName: KubeAdminOrganizationName,
ConfigEnvName: KubeAdminConfigENVName,
ConfigPath: localConfigPath,
EnvName: KubeAdminCertEnvName,
KeyEnvName: KubeAdminKeyEnvName,
}
return certs, nil
}