diff --git a/cluster/addons.go b/cluster/addons.go index 3f74ab19..1fac0667 100644 --- a/cluster/addons.go +++ b/cluster/addons.go @@ -28,6 +28,9 @@ const ( IngressAddonJobName = "rke-ingress-controller-deploy-job" IngressAddonDeleteJobName = "rke-ingress-controller-delete-job" MetricsServerAddonResourceName = "rke-metrics-addon" + NginxIngressAddonAppName = "ingress-nginx" + KubeDNSAddonAppName = "kube-dns" + KubeDNSAutoscalerAppName = "kube-dns-autoscaler" ) type ingressOptions struct { diff --git a/cluster/certificates.go b/cluster/certificates.go index e75ad84e..1f1234d9 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -71,7 +71,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust if err := rebuildLocalAdminConfig(ctx, kubeCluster); err != nil { return err } - kubeCluster.Certificates, err = regenerateAPICertificate(kubeCluster, kubeCluster.Certificates) + err = pki.GenerateKubeAPICertificate(ctx, kubeCluster.Certificates, kubeCluster.RancherKubernetesEngineConfig, "", "") if err != nil { return fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err) } @@ -129,6 +129,7 @@ func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcd pki.KubeAdminCertName, pki.APIProxyClientCertName, pki.RequestHeaderCACertName, + pki.ServiceAccountTokenKeyName, } for _, etcdHost := range etcdHosts { @@ -141,14 +142,16 @@ func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcd secret, err := k8s.GetSecret(kubeClient, certName) if err != nil && !strings.HasPrefix(certName, "kube-etcd") && !strings.Contains(certName, pki.RequestHeaderCACertName) && - !strings.Contains(certName, pki.APIProxyClientCertName) { + !strings.Contains(certName, pki.APIProxyClientCertName) && + !strings.Contains(certName, pki.ServiceAccountTokenKeyName) { return nil, err } // If I can't find an etcd, requestheader, or proxy client cert, I will not fail and will create it later. if (secret == nil || secret.Data == nil) && (strings.HasPrefix(certName, "kube-etcd") || strings.Contains(certName, pki.RequestHeaderCACertName) || - strings.Contains(certName, pki.APIProxyClientCertName)) { + strings.Contains(certName, pki.APIProxyClientCertName) || + strings.Contains(certName, pki.ServiceAccountTokenKeyName)) { certMap[certName] = pki.CertificatePKI{} continue } @@ -346,7 +349,7 @@ func (c *Cluster) getBackupHosts() []*hosts.Host { func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.CertificatePKI) (map[string]pki.CertificatePKI, error) { logrus.Debugf("[certificates] Regenerating Kubernetes API server aggregation layer requestheader client CA certificates") - requestHeaderCACrt, requestHeaderCAKey, err := pki.GenerateCACertAndKey(pki.RequestHeaderCACertName) + requestHeaderCACrt, requestHeaderCAKey, err := pki.GenerateCACertAndKey(pki.RequestHeaderCACertName, nil) if err != nil { return nil, err } @@ -361,3 +364,54 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey) return certificates, nil } + +func RotateRKECertificates(ctx context.Context, c *Cluster, configPath, configDir string, components []string, rotateCACerts bool) error { + var ( + serviceAccountTokenKey string + ) + componentsCertsFuncMap := map[string]pki.GenFunc{ + services.KubeAPIContainerName: pki.GenerateKubeAPICertificate, + services.KubeControllerContainerName: pki.GenerateKubeControllerCertificate, + services.SchedulerContainerName: pki.GenerateKubeSchedulerCertificate, + services.KubeproxyContainerName: pki.GenerateKubeProxyCertificate, + services.KubeletContainerName: pki.GenerateKubeNodeCertificate, + services.EtcdContainerName: pki.GenerateEtcdCertificates, + } + if rotateCACerts { + // rotate CA cert and RequestHeader CA cert + if err := pki.GenerateRKECACerts(ctx, c.Certificates, configPath, configDir); err != nil { + return err + } + components = nil + } + for _, k8sComponent := range components { + genFunc := componentsCertsFuncMap[k8sComponent] + if genFunc != nil { + if err := genFunc(ctx, c.Certificates, c.RancherKubernetesEngineConfig, configPath, configDir); err != nil { + return err + } + } + } + if len(components) == 0 { + // do not rotate service account token + if c.Certificates[pki.ServiceAccountTokenKeyName].Key != nil { + serviceAccountTokenKey = string(cert.EncodePrivateKeyPEM(c.Certificates[pki.ServiceAccountTokenKeyName].Key)) + } + if err := pki.GenerateRKEServicesCerts(ctx, c.Certificates, c.RancherKubernetesEngineConfig, configPath, configDir); err != nil { + return err + } + if serviceAccountTokenKey != "" { + privateKey, err := cert.ParsePrivateKeyPEM([]byte(serviceAccountTokenKey)) + if err != nil { + return err + } + c.Certificates[pki.ServiceAccountTokenKeyName] = pki.ToCertObject( + pki.ServiceAccountTokenKeyName, + pki.ServiceAccountTokenKeyName, + "", + c.Certificates[pki.ServiceAccountTokenKeyName].Certificate, + privateKey.(*rsa.PrivateKey)) + } + } + return nil +} diff --git a/cluster/cluster.go b/cluster/cluster.go index 019cb64e..bcf899d6 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -67,6 +67,10 @@ const ( WorkerPlane = "workerPlan" EtcdPlane = "etcd" + KubeAppLabel = "k8s-app" + AppLabel = "app" + NameLabel = "name" + WorkerThreads = util.WorkerThreads ) @@ -458,3 +462,47 @@ func ConfigureCluster( } return nil } + +func RestartClusterPods(ctx context.Context, kubeCluster *Cluster) error { + log.Infof(ctx, "Restarting network, ingress, and metrics pods") + // this will remove the pods created by RKE and let the controller creates them again + kubeClient, err := k8s.NewClient(kubeCluster.LocalKubeConfigPath, kubeCluster.K8sWrapTransport) + if err != nil { + return fmt.Errorf("Failed to initialize new kubernetes client: %v", err) + } + labelsList := []string{ + fmt.Sprintf("%s=%s", KubeAppLabel, CalicoNetworkPlugin), + fmt.Sprintf("%s=%s", KubeAppLabel, FlannelNetworkPlugin), + fmt.Sprintf("%s=%s", KubeAppLabel, CanalNetworkPlugin), + fmt.Sprintf("%s=%s", NameLabel, WeaveNetworkPlugin), + fmt.Sprintf("%s=%s", AppLabel, NginxIngressAddonAppName), + fmt.Sprintf("%s=%s", KubeAppLabel, DefaultMonitoringProvider), + fmt.Sprintf("%s=%s", KubeAppLabel, KubeDNSAddonAppName), + fmt.Sprintf("%s=%s", KubeAppLabel, KubeDNSAutoscalerAppName), + } + var errgrp errgroup.Group + labelQueue := util.GetObjectQueue(labelsList) + for w := 0; w < services.WorkerThreads; w++ { + errgrp.Go(func() error { + var errList []error + for label := range labelQueue { + runLabel := label.(string) + // list pods to be deleted + pods, err := k8s.ListPodsByLabel(kubeClient, runLabel) + if err != nil { + errList = append(errList, err) + } + // delete pods + err = k8s.DeletePods(kubeClient, pods) + if err != nil { + errList = append(errList, err) + } + } + return util.ErrList(errList) + }) + } + if err := errgrp.Wait(); err != nil { + return err + } + return nil +} diff --git a/cluster/hosts.go b/cluster/hosts.go index 7050fccf..c3b2d14a 100644 --- a/cluster/hosts.go +++ b/cluster/hosts.go @@ -116,7 +116,7 @@ func (c *Cluster) InvertIndexHosts() error { return nil } -func (c *Cluster) SetUpHosts(ctx context.Context) error { +func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error { if c.Authentication.Strategy == X509AuthenticationProvider { log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes") hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) @@ -127,7 +127,7 @@ func (c *Cluster) SetUpHosts(ctx context.Context) error { errgrp.Go(func() error { var errList []error for host := range hostsQueue { - err := pki.DeployCertificatesOnPlaneHost(ctx, host.(*hosts.Host), c.RancherKubernetesEngineConfig, c.Certificates, c.SystemImages.CertDownloader, c.PrivateRegistriesMap) + err := pki.DeployCertificatesOnPlaneHost(ctx, host.(*hosts.Host), c.RancherKubernetesEngineConfig, c.Certificates, c.SystemImages.CertDownloader, c.PrivateRegistriesMap, rotateCerts) if err != nil { errList = append(errList, err) } diff --git a/cluster/plan.go b/cluster/plan.go index e98af2e6..aaa6dcca 100644 --- a/cluster/plan.go +++ b/cluster/plan.go @@ -134,7 +134,7 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process { "tls-private-key-file": pki.GetKeyPath(pki.KubeAPICertName), "kubelet-client-certificate": pki.GetCertPath(pki.KubeAPICertName), "kubelet-client-key": pki.GetKeyPath(pki.KubeAPICertName), - "service-account-key-file": pki.GetKeyPath(pki.KubeAPICertName), + "service-account-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName), "etcd-cafile": etcdCAClientCert, "etcd-certfile": etcdClientCert, "etcd-keyfile": etcdClientKey, @@ -249,7 +249,7 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process { "allocate-node-cidrs": "true", "cluster-cidr": c.ClusterCIDR, "service-cluster-ip-range": c.Services.KubeController.ServiceClusterIPRange, - "service-account-private-key-file": pki.GetKeyPath(pki.KubeAPICertName), + "service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName), "root-ca-file": pki.GetCertPath(pki.CACertName), } if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName { diff --git a/cluster/reconcile.go b/cluster/reconcile.go index 946ca995..60a598a4 100644 --- a/cluster/reconcile.go +++ b/cluster/reconcile.go @@ -47,6 +47,12 @@ func ReconcileCluster(ctx context.Context, kubeCluster, currentCluster *Cluster, if err := reconcileControl(ctx, currentCluster, kubeCluster, kubeClient); err != nil { return err } + // Handle service account token key issue + kubeAPICert := currentCluster.Certificates[pki.KubeAPICertName] + if currentCluster.Certificates[pki.ServiceAccountTokenKeyName].Key == nil { + log.Infof(ctx, "[certificates] Creating service account token key") + currentCluster.Certificates[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key) + } log.Infof(ctx, "[reconcile] Reconciled cluster state successfully") return nil } diff --git a/cmd/cert.go b/cmd/cert.go new file mode 100644 index 00000000..6783ad8a --- /dev/null +++ b/cmd/cert.go @@ -0,0 +1,141 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/rancher/rke/cluster" + "github.com/rancher/rke/hosts" + "github.com/rancher/rke/k8s" + "github.com/rancher/rke/log" + "github.com/rancher/rke/pki" + "github.com/rancher/rke/services" + "github.com/rancher/types/apis/management.cattle.io/v3" + "github.com/urfave/cli" +) + +func CertificateCommand() cli.Command { + return cli.Command{ + Name: "cert", + Usage: "Certificates management for RKE cluster", + Subcommands: cli.Commands{ + cli.Command{ + Name: "rotate", + Usage: "Rotate RKE cluster certificates", + Action: rotateRKECertificatesFromCli, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config", + Usage: "Specify an alternate cluster YAML file", + Value: pki.ClusterConfig, + EnvVar: "RKE_CONFIG", + }, + cli.StringSliceFlag{ + Name: "service", + Usage: fmt.Sprintf("Specify a k8s service to rotate certs, (allowed values: %s, %s, %s, %s, %s, %s)", + services.KubeAPIContainerName, + services.KubeControllerContainerName, + services.SchedulerContainerName, + services.KubeletContainerName, + services.KubeproxyContainerName, + services.EtcdContainerName, + ), + }, + cli.BoolFlag{ + Name: "rotate-ca", + Usage: "Rotate all certificates including CA certs", + }, + }, + }, + }, + } +} + +func rotateRKECertificatesFromCli(ctx *cli.Context) error { + k8sComponent := ctx.StringSlice("service") + rotateCACert := ctx.Bool("rotate-ca") + clusterFile, filePath, err := resolveClusterFile(ctx) + if err != nil { + return fmt.Errorf("Failed to resolve cluster file: %v", err) + } + clusterFilePath = filePath + + rkeConfig, err := cluster.ParseConfig(clusterFile) + if err != nil { + return fmt.Errorf("Failed to parse cluster file: %v", err) + } + rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig) + if err != nil { + return err + } + + return RotateRKECertificates(context.Background(), rkeConfig, nil, nil, nil, false, "", k8sComponent, rotateCACert) +} + +func showRKECertificatesFromCli(ctx *cli.Context) error { + return nil +} + +func RotateRKECertificates( + ctx context.Context, + rkeConfig *v3.RancherKubernetesEngineConfig, + dockerDialerFactory, localConnDialerFactory hosts.DialerFactory, + k8sWrapTransport k8s.WrapTransport, + local bool, configDir string, components []string, rotateCACerts bool) error { + + log.Infof(ctx, "Rotating Kubernetes cluster certificates") + kubeCluster, err := cluster.ParseCluster(ctx, rkeConfig, clusterFilePath, configDir, dockerDialerFactory, localConnDialerFactory, k8sWrapTransport) + if err != nil { + return err + } + + if err := kubeCluster.TunnelHosts(ctx, local); err != nil { + return err + } + + currentCluster, err := kubeCluster.GetClusterState(ctx) + if err != nil { + return err + } + + if err := cluster.SetUpAuthentication(ctx, kubeCluster, currentCluster); err != nil { + return err + } + + if err := cluster.RotateRKECertificates(ctx, kubeCluster, clusterFilePath, configDir, components, rotateCACerts); err != nil { + return err + } + + if err := kubeCluster.SetUpHosts(ctx, true); err != nil { + return err + } + // Restarting Kubernetes components + servicesMap := make(map[string]bool) + for _, component := range components { + servicesMap[component] = true + } + + if len(components) == 0 || rotateCACerts || servicesMap[services.EtcdContainerName] { + if err := services.RestartEtcdPlane(ctx, kubeCluster.EtcdHosts); err != nil { + return err + } + } + + if err := services.RestartControlPlane(ctx, kubeCluster.ControlPlaneHosts); err != nil { + return err + } + + allHosts := hosts.GetUniqueHostList(kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts, kubeCluster.WorkerHosts) + if err := services.RestartWorkerPlane(ctx, allHosts); err != nil { + return err + } + + if err := kubeCluster.SaveClusterState(ctx, &kubeCluster.RancherKubernetesEngineConfig); err != nil { + return err + } + + if rotateCACerts { + return cluster.RestartClusterPods(ctx, kubeCluster) + } + return nil +} diff --git a/cmd/up.go b/cmd/up.go index 824b545c..ffe5890d 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -109,7 +109,7 @@ func ClusterUp( if len(kubeCluster.ControlPlaneHosts) > 0 { APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443") } - err = kubeCluster.SetUpHosts(ctx) + err = kubeCluster.SetUpHosts(ctx, false) if err != nil { return APIURL, caCrt, clientCert, clientKey, nil, err } diff --git a/docker/docker.go b/docker/docker.go index e4ebf341..7d39cc26 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -28,7 +28,7 @@ import ( const ( DockerRegistryURL = "docker.io" // RestartTimeout in seconds - RestartTimeout = 30 + RestartTimeout = 5 // StopTimeout in seconds StopTimeout = 5 ) @@ -230,6 +230,14 @@ func RemoveContainer(ctx context.Context, dClient *client.Client, hostname strin return nil } +func RestartContainer(ctx context.Context, dClient *client.Client, hostname, containerName string) error { + restartTimeout := RestartTimeout * time.Second + err := dClient.ContainerRestart(ctx, containerName, &restartTimeout) + if err != nil { + return fmt.Errorf("Can't restart Docker container [%s] for host [%s]: %v", containerName, hostname, err) + } + return nil +} func StopContainer(ctx context.Context, dClient *client.Client, hostname string, containerName string) error { // define the stop timeout stopTimeoutDuration := StopTimeout * time.Second @@ -455,3 +463,23 @@ func GetKubeletDockerConfig(prsMap map[string]v3.PrivateRegistry) (string, error } return string(cfg), nil } + +func DoRestartContainer(ctx context.Context, dClient *client.Client, containerName, hostname string) error { + logrus.Debugf("[restart/%s] Checking if container is running on host [%s]", containerName, hostname) + // not using the wrapper to check if the error is a NotFound error + _, err := dClient.ContainerInspect(ctx, containerName) + if err != nil { + if client.IsErrNotFound(err) { + logrus.Debugf("[restart/%s] Container doesn't exist on host [%s]", containerName, hostname) + return nil + } + return err + } + logrus.Debugf("[restart/%s] Restarting container on host [%s]", containerName, hostname) + err = RestartContainer(ctx, dClient, hostname, containerName) + if err != nil { + return err + } + log.Infof(ctx, "[restart/%s] Successfully restarted container on host [%s]", containerName, hostname) + return nil +} diff --git a/k8s/pod.go b/k8s/pod.go new file mode 100644 index 00000000..14bb079f --- /dev/null +++ b/k8s/pod.go @@ -0,0 +1,24 @@ +package k8s + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +func DeletePods(k8sClient *kubernetes.Clientset, podList *v1.PodList) error { + for _, pod := range podList.Items { + if err := k8sClient.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{}); err != nil { + return err + } + } + return nil +} + +func ListPodsByLabel(k8sClient *kubernetes.Clientset, label string) (*v1.PodList, error) { + pods, err := k8sClient.CoreV1().Pods("").List(metav1.ListOptions{LabelSelector: label}) + if err != nil { + return nil, err + } + return pods, nil +} diff --git a/main.go b/main.go index eac2ad4e..eeace539 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ func mainErr() error { cmd.VersionCommand(), cmd.ConfigCommand(), cmd.EtcdCommand(), + cmd.CertificateCommand(), } app.Flags = []cli.Flag{ cli.BoolFlag{ diff --git a/pki/constants.go b/pki/constants.go index 20026517..4adf9189 100644 --- a/pki/constants.go +++ b/pki/constants.go @@ -1,5 +1,7 @@ package pki +import "time" + const ( CertPathPrefix = "/etc/kubernetes/ssl/" CertificatesServiceName = "certificates" @@ -12,17 +14,18 @@ const ( ClusterStateEnv = "CLUSTER_STATE" BundleCertPath = "/backup/pki.bundle.tar.gz" - CACertName = "kube-ca" - RequestHeaderCACertName = "kube-apiserver-requestheader-ca" - KubeAPICertName = "kube-apiserver" - KubeControllerCertName = "kube-controller-manager" - KubeSchedulerCertName = "kube-scheduler" - KubeProxyCertName = "kube-proxy" - KubeNodeCertName = "kube-node" - EtcdCertName = "kube-etcd" - EtcdClientCACertName = "kube-etcd-client-ca" - EtcdClientCertName = "kube-etcd-client" - APIProxyClientCertName = "kube-apiserver-proxy-client" + CACertName = "kube-ca" + RequestHeaderCACertName = "kube-apiserver-requestheader-ca" + KubeAPICertName = "kube-apiserver" + KubeControllerCertName = "kube-controller-manager" + KubeSchedulerCertName = "kube-scheduler" + KubeProxyCertName = "kube-proxy" + KubeNodeCertName = "kube-node" + EtcdCertName = "kube-etcd" + EtcdClientCACertName = "kube-etcd-client-ca" + EtcdClientCertName = "kube-etcd-client" + APIProxyClientCertName = "kube-apiserver-proxy-client" + ServiceAccountTokenKeyName = "kube-service-account-token" KubeNodeCommonName = "system:node" KubeNodeOrganizationName = "system:nodes" @@ -30,4 +33,5 @@ const ( KubeAdminCertName = "kube-admin" KubeAdminOrganizationName = "system:masters" KubeAdminConfigPrefix = "kube_config_" + duration365d = time.Hour * 24 * 365 ) diff --git a/pki/deploy.go b/pki/deploy.go index e4af0d3a..272fb6a4 100644 --- a/pki/deploy.go +++ b/pki/deploy.go @@ -24,12 +24,15 @@ const ( StateDeployerContainerName = "cluster-state-deployer" ) -func DeployCertificatesOnPlaneHost(ctx context.Context, host *hosts.Host, rkeConfig v3.RancherKubernetesEngineConfig, crtMap map[string]CertificatePKI, certDownloaderImage string, prsMap map[string]v3.PrivateRegistry) error { +func DeployCertificatesOnPlaneHost(ctx context.Context, host *hosts.Host, rkeConfig v3.RancherKubernetesEngineConfig, crtMap map[string]CertificatePKI, certDownloaderImage string, prsMap map[string]v3.PrivateRegistry, rotateCerts bool) error { crtBundle := GenerateRKENodeCerts(ctx, rkeConfig, host.Address, crtMap) env := []string{} for _, crt := range crtBundle { env = append(env, crt.ToEnv()...) } + if rotateCerts { + env = append(env, "FORCE_DEPLOY=true") + } return doRunDeployer(ctx, host, env, certDownloaderImage, prsMap) } @@ -153,15 +156,16 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho tmpCerts := make(map[string]CertificatePKI) crtList := map[string]bool{ - CACertName: false, - KubeAPICertName: false, - KubeControllerCertName: true, - KubeSchedulerCertName: true, - KubeProxyCertName: true, - KubeNodeCertName: true, - KubeAdminCertName: false, - RequestHeaderCACertName: false, - APIProxyClientCertName: false, + CACertName: false, + KubeAPICertName: false, + KubeControllerCertName: true, + KubeSchedulerCertName: true, + KubeProxyCertName: true, + KubeNodeCertName: true, + KubeAdminCertName: false, + RequestHeaderCACertName: false, + APIProxyClientCertName: false, + ServiceAccountTokenKeyName: false, } for _, etcdHost := range extraHosts { @@ -175,7 +179,8 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho // I will only exit with an error if it's not a not-found-error and this is not an etcd certificate if err != nil && (!strings.HasPrefix(certName, "kube-etcd") && !strings.Contains(certName, APIProxyClientCertName) && - !strings.Contains(certName, RequestHeaderCACertName)) { + !strings.Contains(certName, RequestHeaderCACertName) && + !strings.Contains(certName, ServiceAccountTokenKeyName)) { // IsErrNotFound doesn't catch this because it's a custom error if isFileNotFoundErr(err) { return nil, nil @@ -186,7 +191,8 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho // If I can't find an etcd or api aggregator cert, I will not fail and will create it later. if crt == "" && (strings.HasPrefix(certName, "kube-etcd") || strings.Contains(certName, APIProxyClientCertName) || - strings.Contains(certName, RequestHeaderCACertName)) { + strings.Contains(certName, RequestHeaderCACertName) || + strings.Contains(certName, ServiceAccountTokenKeyName)) { tmpCerts[certName] = CertificatePKI{} continue } @@ -247,45 +253,3 @@ func FetchFileFromHost(ctx context.Context, filePath, image string, host *hosts. return file, nil } - -func getTempPath(s string) string { - return TempCertPath + path.Base(s) -} - -func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI { - certs := make(map[string]CertificatePKI) - // CACert - certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key) - // KubeAPI - certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key) - // kubeController - certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key) - // KubeScheduler - certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key) - // KubeProxy - certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key) - // KubeNode - certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key) - // KubeAdmin - kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key) - kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config - kubeAdminCertObj.ConfigPath = localConfigPath - certs[KubeAdminCertName] = kubeAdminCertObj - // etcd - for _, host := range extraHosts { - etcdName := GetEtcdCrtName(host.InternalAddress) - etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key - certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) - } - - return certs -} - -func isFileNotFoundErr(e error) bool { - if strings.Contains(e.Error(), "no such file or directory") || - strings.Contains(e.Error(), "Could not find the file") || - strings.Contains(e.Error(), "No such container:path:") { - return true - } - return false -} diff --git a/pki/pki.go b/pki/pki.go index 09f08c0f..f33e8dc5 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -15,7 +15,6 @@ import ( "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" "github.com/rancher/types/apis/management.cattle.io/v3" - "k8s.io/client-go/util/cert" ) type CertificatePKI struct { @@ -33,6 +32,8 @@ type CertificatePKI struct { ConfigPath string } +type GenFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig, string, string) error + const ( etcdRole = "etcd" controlRole = "controlplane" @@ -42,131 +43,14 @@ const ( func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) (map[string]CertificatePKI, error) { certs := make(map[string]CertificatePKI) - // generate CA certificate and key - log.Infof(ctx, "[certificates] Generating CA kubernetes certificates") - caCrt, caKey, err := GenerateCACertAndKey(CACertName) - if err != nil { - return nil, err + // generate RKE CA certificates + if err := GenerateRKECACerts(ctx, certs, configPath, configDir); err != nil { + return certs, err } - certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey) - - // generate API certificate and key - log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates") - kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange) - clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain - cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole) - etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole) - if err != nil { - return nil, fmt.Errorf("Failed to get Kubernetes Service IP: %v", err) + // Generating certificates for kubernetes components + if err := GenerateRKEServicesCerts(ctx, certs, rkeConfig, configPath, configDir); err != nil { + return certs, err } - kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, kubernetesServiceIP, rkeConfig.Authentication.SANs) - kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, nil, nil) - if err != nil { - return nil, err - } - certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey) - - // generate Kube controller-manager certificate and key - log.Infof(ctx, "[certificates] Generating Kube Controller certificates") - kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil) - if err != nil { - return nil, err - } - certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey) - - // generate Kube scheduler certificate and key - log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates") - kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil) - if err != nil { - return nil, err - } - certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey) - - // generate Kube Proxy certificate and key - log.Infof(ctx, "[certificates] Generating Kube Proxy certificates") - kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil) - if err != nil { - return nil, err - } - certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey) - - log.Infof(ctx, "[certificates] Generating Node certificate") - nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName}) - if err != nil { - return nil, err - } - certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey) - - // generate Admin certificate and key - log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig") - if len(configPath) == 0 { - configPath = ClusterConfig - } - localKubeConfigPath := GetLocalKubeConfig(configPath, configDir) - kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName}) - if err != nil { - return nil, err - } - kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey) - if len(cpHosts) > 0 { - kubeAdminConfig := GetKubeConfigX509WithData( - "https://"+cpHosts[0].Address+":6443", - rkeConfig.ClusterName, - KubeAdminCertName, - string(cert.EncodeCertPEM(caCrt)), - string(cert.EncodeCertPEM(kubeAdminCrt)), - string(cert.EncodePrivateKeyPEM(kubeAdminKey))) - kubeAdminCertObj.Config = kubeAdminConfig - kubeAdminCertObj.ConfigPath = localKubeConfigPath - } else { - kubeAdminCertObj.Config = "" - } - certs[KubeAdminCertName] = kubeAdminCertObj - // generate etcd certificate and key - if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 { - clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert)) - if err != nil { - return nil, err - } - clientKey, err := cert.ParsePrivateKeyPEM([]byte(rkeConfig.Services.Etcd.Key)) - if err != nil { - return nil, err - } - certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey)) - - caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert)) - if err != nil { - return nil, err - } - certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil) - } - etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{}) - for _, host := range etcdHosts { - log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress) - etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil) - if err != nil { - return nil, err - } - etcdName := GetEtcdCrtName(host.InternalAddress) - certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) - } - - // generate request header client CA certificate and key - log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates") - requestHeaderCACrt, requestHeaderCAKey, err := GenerateCACertAndKey(RequestHeaderCACertName) - if err != nil { - return nil, err - } - certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey) - - //generate API server proxy client key and certs - log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates") - apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(requestHeaderCACrt, requestHeaderCAKey, true, APIProxyClientCertName, nil, nil, nil) - if err != nil { - return nil, err - } - certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey) - return certs, nil } diff --git a/pki/services.go b/pki/services.go new file mode 100644 index 00000000..7d4c2f2b --- /dev/null +++ b/pki/services.go @@ -0,0 +1,234 @@ +package pki + +import ( + "context" + "crypto/rsa" + "fmt" + + "github.com/rancher/rke/hosts" + "github.com/rancher/rke/log" + "github.com/rancher/types/apis/management.cattle.io/v3" + "k8s.io/client-go/util/cert" +) + +func GenerateKubeAPICertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate API certificate and key + log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates") + var privateAPIKey *rsa.PrivateKey + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange) + if err != nil { + return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err) + } + clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain + cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole) + kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, kubernetesServiceIP, rkeConfig.Authentication.SANs) + // handle rotation on old clusters + if certs[ServiceAccountTokenKeyName].Key == nil { + privateAPIKey = certs[KubeAPICertName].Key + } + kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, privateAPIKey, nil) + if err != nil { + return err + } + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey) + return nil +} + +func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate Kube controller-manager certificate and key + log.Infof(ctx, "[certificates] Generating Kube Controller certificates") + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil) + if err != nil { + return err + } + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey) + return nil +} + +func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate Kube scheduler certificate and key + log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates") + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil) + if err != nil { + return err + } + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey) + return nil +} + +func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate Kube Proxy certificate and key + log.Infof(ctx, "[certificates] Generating Kube Proxy certificates") + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil) + if err != nil { + return err + } + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey) + return nil +} + +func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate kubelet certificate + log.Infof(ctx, "[certificates] Generating Node certificate") + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName}) + if err != nil { + return err + } + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey) + return nil +} + +func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate Admin certificate and key + log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig") + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole) + if len(configPath) == 0 { + configPath = ClusterConfig + } + localKubeConfigPath := GetLocalKubeConfig(configPath, configDir) + kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName}) + if err != nil { + return err + } + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey) + if len(cpHosts) > 0 { + kubeAdminConfig := GetKubeConfigX509WithData( + "https://"+cpHosts[0].Address+":6443", + rkeConfig.ClusterName, + KubeAdminCertName, + string(cert.EncodeCertPEM(caCrt)), + string(cert.EncodeCertPEM(kubeAdminCrt)), + string(cert.EncodePrivateKeyPEM(kubeAdminKey))) + kubeAdminCertObj.Config = kubeAdminConfig + kubeAdminCertObj.ConfigPath = localKubeConfigPath + } else { + kubeAdminCertObj.Config = "" + } + certs[KubeAdminCertName] = kubeAdminCertObj + return nil +} + +func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + //generate API server proxy client key and certs + log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates") + caCrt := certs[RequestHeaderCACertName].Certificate + caKey := certs[RequestHeaderCACertName].Key + apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, APIProxyClientCertName, nil, nil, nil) + if err != nil { + return err + } + certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey) + return nil +} + +func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert)) + if err != nil { + return err + } + clientKey, err := cert.ParsePrivateKeyPEM([]byte(rkeConfig.Services.Etcd.Key)) + if err != nil { + return err + } + certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey)) + + caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert)) + if err != nil { + return err + } + certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil) + return nil +} + +func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange) + if err != nil { + return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err) + } + clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain + etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole) + etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{}) + for _, host := range etcdHosts { + log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress) + etcdName := GetEtcdCrtName(host.InternalAddress) + etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil) + if err != nil { + return err + } + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) + } + return nil +} + +func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + // generate service account token key + var privateAPIKey *rsa.PrivateKey + caCrt := certs[CACertName].Certificate + caKey := certs[CACertName].Key + // handle rotation on old clusters + if certs[ServiceAccountTokenKeyName].Key == nil { + privateAPIKey = certs[KubeAPICertName].Key + } + tokenCrt, tokenKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, ServiceAccountTokenKeyName, nil, privateAPIKey, nil) + if err != nil { + return fmt.Errorf("Failed to generate private key for service account token: %v", err) + } + certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", tokenCrt, tokenKey) + return nil +} + +func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, configPath, configDir string) error { + // generate kubernetes CA certificate and key + log.Infof(ctx, "[certificates] Generating CA kubernetes certificates") + caCrt, caKey, err := GenerateCACertAndKey(CACertName, certs[CACertName].Key) + if err != nil { + return err + } + certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey) + + // generate request header client CA certificate and key + log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates") + requestHeaderCACrt, requestHeaderCAKey, err := GenerateCACertAndKey(RequestHeaderCACertName, nil) + if err != nil { + return err + } + certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey) + return nil +} + +func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { + RKECerts := []GenFunc{ + GenerateKubeAPICertificate, + GenerateServiceTokenKey, + GenerateKubeControllerCertificate, + GenerateKubeSchedulerCertificate, + GenerateKubeProxyCertificate, + GenerateKubeNodeCertificate, + GenerateKubeAdminCertificate, + GenerateAPIProxyClientCertificate, + GenerateEtcdCertificates, + } + for _, gen := range RKECerts { + if err := gen(ctx, certs, rkeConfig, configPath, configDir); err != nil { + return err + } + } + if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 { + return GenerateExternalEtcdCertificates(ctx, certs, rkeConfig, configPath, configDir) + } + return nil +} diff --git a/pki/util.go b/pki/util.go index 4bfddf00..48cff673 100644 --- a/pki/util.go +++ b/pki/util.go @@ -1,12 +1,19 @@ package pki import ( + cryptorand "crypto/rand" "crypto/rsa" "crypto/x509" + "crypto/x509/pkix" + "errors" "fmt" + "math" + "math/big" "net" + "path" "path/filepath" "strings" + "time" "github.com/rancher/rke/hosts" "github.com/rancher/types/apis/management.cattle.io/v3" @@ -44,17 +51,21 @@ func GenerateSignedCertAndKey( Usages: usages, AltNames: *altNames, } - clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey) + clientCert, err := newSignedCert(caConfig, rootKey, caCrt, caKey) if err != nil { return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err) } return clientCert, rootKey, nil } -func GenerateCACertAndKey(commonName string) (*x509.Certificate, *rsa.PrivateKey, error) { - rootKey, err := cert.NewPrivateKey() - if err != nil { - return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err) +func GenerateCACertAndKey(commonName string, privateKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + var err error + rootKey := privateKey + if rootKey == nil { + rootKey, err = cert.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err) + } } caConfig := cert.Config{ CommonName: commonName, @@ -197,7 +208,7 @@ func ToCertObject(componentName, commonName, ouName string, cert *x509.Certifica path := GetCertPath(componentName) keyPath := GetKeyPath(componentName) - if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) { + if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) && componentName != ServiceAccountTokenKeyName { config = getKubeConfigX509("https://127.0.0.1:6443", "local", componentName, caCertPath, path, keyPath) configPath = GetConfigPath(componentName) configEnvName = getConfigEnvFromEnv(envName) @@ -227,6 +238,7 @@ func getControlCertKeys() []string { return []string{ CACertName, KubeAPICertName, + ServiceAccountTokenKeyName, KubeControllerCertName, KubeSchedulerCertName, KubeProxyCertName, @@ -293,3 +305,78 @@ func strKeyToEnv(crtName, key string) string { envName := getEnvFromName(crtName) return fmt.Sprintf("%s=%s", getKeyEnvFromEnv(envName), key) } + +func getTempPath(s string) string { + return TempCertPath + path.Base(s) +} + +func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI { + certs := make(map[string]CertificatePKI) + // CACert + certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key) + // KubeAPI + certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key) + // kubeController + certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key) + // KubeScheduler + certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key) + // KubeProxy + certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key) + // KubeNode + certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key) + // KubeAdmin + kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key) + kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config + kubeAdminCertObj.ConfigPath = localConfigPath + certs[KubeAdminCertName] = kubeAdminCertObj + // etcd + for _, host := range extraHosts { + etcdName := GetEtcdCrtName(host.InternalAddress) + etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key + certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey) + } + + return certs +} + +// Overriding k8s.io/client-go/util/cert.NewSignedCert function to extend the expiration date to 10 years instead of 1 year +func newSignedCert(cfg cert.Config, key *rsa.PrivateKey, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, error) { + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(duration365d * 10).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} + +func isFileNotFoundErr(e error) bool { + if strings.Contains(e.Error(), "no such file or directory") || + strings.Contains(e.Error(), "Could not find the file") || + strings.Contains(e.Error(), "No such container:path:") { + return true + } + return false +} diff --git a/services/controlplane.go b/services/controlplane.go index 0e6a7fc9..ebcf3286 100644 --- a/services/controlplane.go +++ b/services/controlplane.go @@ -82,6 +82,42 @@ func RemoveControlPlane(ctx context.Context, controlHosts []*hosts.Host, force b return nil } +func RestartControlPlane(ctx context.Context, controlHosts []*hosts.Host) error { + log.Infof(ctx, "[%s] Restarting the Controller Plane..", ControlRole) + var errgrp errgroup.Group + + hostsQueue := util.GetObjectQueue(controlHosts) + for w := 0; w < WorkerThreads; w++ { + errgrp.Go(func() error { + var errList []error + for host := range hostsQueue { + runHost := host.(*hosts.Host) + // restart KubeAPI + if err := restartKubeAPI(ctx, runHost); err != nil { + errList = append(errList, err) + } + + // restart KubeController + if err := restartKubeController(ctx, runHost); err != nil { + errList = append(errList, err) + } + + // restart scheduler + err := restartScheduler(ctx, runHost) + if err != nil { + errList = append(errList, err) + } + } + return util.ErrList(errList) + }) + } + if err := errgrp.Wait(); err != nil { + return err + } + log.Infof(ctx, "[%s] Successfully restarted Controller Plane..", ControlRole) + return nil +} + func doDeployControlHost(ctx context.Context, host *hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, alpineImage string, certMap map[string]pki.CertificatePKI) error { if host.IsWorker { if err := removeNginxProxy(ctx, host); err != nil { diff --git a/services/etcd.go b/services/etcd.go index e8892ed3..ff342e70 100644 --- a/services/etcd.go +++ b/services/etcd.go @@ -72,6 +72,30 @@ func RunEtcdPlane( return nil } +func RestartEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host) error { + log.Infof(ctx, "[%s] Restarting up etcd plane..", ETCDRole) + var errgrp errgroup.Group + + hostsQueue := util.GetObjectQueue(etcdHosts) + for w := 0; w < WorkerThreads; w++ { + errgrp.Go(func() error { + var errList []error + for host := range hostsQueue { + runHost := host.(*hosts.Host) + if err := docker.DoRestartContainer(ctx, runHost.DClient, EtcdContainerName, runHost.Address); err != nil { + errList = append(errList, err) + } + } + return util.ErrList(errList) + }) + } + if err := errgrp.Wait(); err != nil { + return err + } + log.Infof(ctx, "[%s] Successfully restarted etcd plane..", ETCDRole) + return nil +} + func RemoveEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, force bool) error { log.Infof(ctx, "[%s] Tearing down etcd plane..", ETCDRole) diff --git a/services/kubeapi.go b/services/kubeapi.go index c8ae06d9..d8ec3050 100644 --- a/services/kubeapi.go +++ b/services/kubeapi.go @@ -24,3 +24,7 @@ func runKubeAPI(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p func removeKubeAPI(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, KubeAPIContainerName, host.Address) } + +func restartKubeAPI(ctx context.Context, host *hosts.Host) error { + return docker.DoRestartContainer(ctx, host.DClient, KubeAPIContainerName, host.Address) +} diff --git a/services/kubecontroller.go b/services/kubecontroller.go index 7043aa0c..a90bf28f 100644 --- a/services/kubecontroller.go +++ b/services/kubecontroller.go @@ -22,3 +22,7 @@ func runKubeController(ctx context.Context, host *hosts.Host, df hosts.DialerFac func removeKubeController(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, KubeControllerContainerName, host.Address) } + +func restartKubeController(ctx context.Context, host *hosts.Host) error { + return docker.DoRestartContainer(ctx, host.DClient, KubeControllerContainerName, host.Address) +} diff --git a/services/kubelet.go b/services/kubelet.go index 2256b677..52627e3c 100644 --- a/services/kubelet.go +++ b/services/kubelet.go @@ -23,3 +23,7 @@ func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p func removeKubelet(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, KubeletContainerName, host.Address) } + +func restartKubelet(ctx context.Context, host *hosts.Host) error { + return docker.DoRestartContainer(ctx, host.DClient, KubeletContainerName, host.Address) +} diff --git a/services/kubeproxy.go b/services/kubeproxy.go index 70a37043..5d6140c5 100644 --- a/services/kubeproxy.go +++ b/services/kubeproxy.go @@ -22,3 +22,7 @@ func runKubeproxy(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, func removeKubeproxy(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, KubeproxyContainerName, host.Address) } + +func restartKubeproxy(ctx context.Context, host *hosts.Host) error { + return docker.DoRestartContainer(ctx, host.DClient, KubeproxyContainerName, host.Address) +} diff --git a/services/proxy.go b/services/proxy.go index 9a664fd9..b3aa13aa 100644 --- a/services/proxy.go +++ b/services/proxy.go @@ -24,3 +24,7 @@ func runNginxProxy(ctx context.Context, host *hosts.Host, prsMap map[string]v3.P func removeNginxProxy(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, NginxProxyContainerName, host.Address) } + +func restartNginxProxy(ctx context.Context, host *hosts.Host) error { + return docker.DoRestartContainer(ctx, host.DClient, NginxProxyContainerName, host.Address) +} diff --git a/services/scheduler.go b/services/scheduler.go index 84e2b3ac..a98a440e 100644 --- a/services/scheduler.go +++ b/services/scheduler.go @@ -22,3 +22,7 @@ func runScheduler(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, func removeScheduler(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, SchedulerContainerName, host.Address) } + +func restartScheduler(ctx context.Context, host *hosts.Host) error { + return docker.DoRestartContainer(ctx, host.DClient, SchedulerContainerName, host.Address) +} diff --git a/services/workerplane.go b/services/workerplane.go index 800fe57f..c4716cba 100644 --- a/services/workerplane.go +++ b/services/workerplane.go @@ -99,6 +99,37 @@ func RemoveWorkerPlane(ctx context.Context, workerHosts []*hosts.Host, force boo return nil } +func RestartWorkerPlane(ctx context.Context, workerHosts []*hosts.Host) error { + log.Infof(ctx, "[%s] Restarting Worker Plane..", WorkerRole) + var errgrp errgroup.Group + + hostsQueue := util.GetObjectQueue(workerHosts) + for w := 0; w < WorkerThreads; w++ { + errgrp.Go(func() error { + var errList []error + for host := range hostsQueue { + runHost := host.(*hosts.Host) + if err := restartKubelet(ctx, runHost); err != nil { + errList = append(errList, err) + } + if err := restartKubeproxy(ctx, runHost); err != nil { + errList = append(errList, err) + } + if err := restartNginxProxy(ctx, runHost); err != nil { + errList = append(errList, err) + } + } + return util.ErrList(errList) + }) + } + if err := errgrp.Wait(); err != nil { + return err + } + log.Infof(ctx, "[%s] Successfully restarted Worker Plane..", WorkerRole) + + return nil +} + func doDeployWorkerPlane(ctx context.Context, host *hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, certMap map[string]pki.CertificatePKI, alpineImage string) error {