diff --git a/cluster/certificates.go b/cluster/certificates.go index 5e4872f3..e7495e04 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -19,6 +19,11 @@ import ( "k8s.io/client-go/util/cert" ) +type RotateCertificatesFlags struct { + RotateCACerts bool + RotateComponents []string +} + func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Cluster, fullState *FullState) error { if kubeCluster.Authentication.Strategy == X509AuthenticationProvider { kubeCluster.Certificates = fullState.DesiredState.CertificatesBundle @@ -274,7 +279,7 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi return certificates, nil } -func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags) error { +func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags, rotateflags RotateCertificatesFlags, clusterState *FullState) error { var ( serviceAccountTokenKey string ) @@ -286,27 +291,27 @@ func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags) services.KubeletContainerName: pki.GenerateKubeNodeCertificate, services.EtcdContainerName: pki.GenerateEtcdCertificates, } - if flags.RotateCACerts { + if rotateflags.RotateCACerts { // rotate CA cert and RequestHeader CA cert if err := pki.GenerateRKECACerts(ctx, c.Certificates, flags.ClusterFilePath, flags.ConfigDir); err != nil { return err } - flags.RotateComponents = nil + rotateflags.RotateComponents = nil } - for _, k8sComponent := range flags.RotateComponents { + for _, k8sComponent := range rotateflags.RotateComponents { genFunc := componentsCertsFuncMap[k8sComponent] if genFunc != nil { - if err := genFunc(ctx, c.Certificates, c.RancherKubernetesEngineConfig, flags.ClusterFilePath, flags.ConfigDir); err != nil { + if err := genFunc(ctx, c.Certificates, c.RancherKubernetesEngineConfig, flags.ClusterFilePath, flags.ConfigDir, true); err != nil { return err } } } - if len(flags.RotateComponents) == 0 { + if len(rotateflags.RotateComponents) == 0 { // do not rotate service account token if c.Certificates[pki.ServiceAccountTokenKeyName].Key != nil { serviceAccountTokenKey = string(cert.EncodePrivateKeyPEM(c.Certificates[pki.ServiceAccountTokenKeyName].Key)) } - if err := pki.GenerateRKEServicesCerts(ctx, c.Certificates, c.RancherKubernetesEngineConfig, flags.ClusterFilePath, flags.ConfigDir); err != nil { + if err := pki.GenerateRKEServicesCerts(ctx, c.Certificates, c.RancherKubernetesEngineConfig, flags.ClusterFilePath, flags.ConfigDir, true); err != nil { return err } if serviceAccountTokenKey != "" { @@ -322,5 +327,14 @@ func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags) privateKey.(*rsa.PrivateKey)) } } + clusterState.DesiredState.CertificatesBundle = c.Certificates + clusterState.DesiredState.RancherKubernetesEngineConfig = &c.RancherKubernetesEngineConfig return nil } + +func GetRotateCertsFlags(rotateCACerts bool, components []string) RotateCertificatesFlags { + return RotateCertificatesFlags{ + RotateCACerts: rotateCACerts, + RotateComponents: components, + } +} diff --git a/cluster/defaults.go b/cluster/defaults.go index debc4ecb..0f675d6d 100644 --- a/cluster/defaults.go +++ b/cluster/defaults.go @@ -50,8 +50,6 @@ type ExternalFlags struct { ClusterFilePath string DisablePortCheck bool Local bool - RotateCACerts bool - RotateComponents []string UpdateOnly bool } @@ -292,14 +290,12 @@ func (c *Cluster) setCloudProvider() error { return nil } -func GetExternalFlags(local, rotateca, updateOnly, disablePortCheck bool, RotateComponents []string, configDir, clusterFilePath string) ExternalFlags { +func GetExternalFlags(local, updateOnly, disablePortCheck bool, configDir, clusterFilePath string) ExternalFlags { return ExternalFlags{ Local: local, UpdateOnly: updateOnly, DisablePortCheck: disablePortCheck, ConfigDir: configDir, ClusterFilePath: clusterFilePath, - RotateCACerts: rotateca, - RotateComponents: RotateComponents, } } diff --git a/cluster/state.go b/cluster/state.go index ac6f4414..67f49699 100644 --- a/cluster/state.go +++ b/cluster/state.go @@ -47,7 +47,7 @@ func (c *Cluster) GetClusterState(ctx context.Context, fullState *FullState) (*C } // resetup external flags - flags := GetExternalFlags(false, false, false, false, nil, c.ConfigDir, c.ConfigPath) + flags := GetExternalFlags(false, false, false, c.ConfigDir, c.ConfigPath) currentCluster, err := InitClusterObject(ctx, fullState.CurrentState.RancherKubernetesEngineConfig, flags) if err != nil { return nil, err @@ -161,15 +161,15 @@ func RebuildState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf } else { // Regenerating etcd certificates for any new etcd nodes pkiCertBundle := oldState.DesiredState.CertificatesBundle - if err := pki.GenerateEtcdCertificates(ctx, pkiCertBundle, *rkeConfig, "", ""); err != nil { + if err := pki.GenerateEtcdCertificates(ctx, pkiCertBundle, *rkeConfig, "", "", false); err != nil { return nil, err } // Regenerating kubeapi certificates for any new kubeapi nodes - if err := pki.GenerateKubeAPICertificate(ctx, pkiCertBundle, *rkeConfig, "", ""); err != nil { + if err := pki.GenerateKubeAPICertificate(ctx, pkiCertBundle, *rkeConfig, "", "", false); err != nil { return nil, err } // Regenerating kubeadmin certificates/config - if err := pki.GenerateKubeAdminCertificate(ctx, pkiCertBundle, *rkeConfig, flags.ClusterFilePath, flags.ConfigDir); err != nil { + if err := pki.GenerateKubeAdminCertificate(ctx, pkiCertBundle, *rkeConfig, flags.ClusterFilePath, flags.ConfigDir, false); err != nil { return nil, err } newState.DesiredState.CertificatesBundle = pkiCertBundle diff --git a/cmd/cert.go b/cmd/cert.go index f93ac38b..d4af1bd4 100644 --- a/cmd/cert.go +++ b/cmd/cert.go @@ -67,18 +67,26 @@ func rotateRKECertificatesFromCli(ctx *cli.Context) error { return err } // setting up the flags - flags := cluster.GetExternalFlags(false, rotateCACert, false, false, k8sComponent, "", filePath) + externalFlags := cluster.GetExternalFlags(false, false, false, "", filePath) + rotateFlags := cluster.GetRotateCertsFlags(rotateCACert, k8sComponent) - return RotateRKECertificates(context.Background(), rkeConfig, hosts.DialersOptions{}, flags) + if err := RotateRKECertificates(context.Background(), rkeConfig, hosts.DialersOptions{}, externalFlags, rotateFlags); err != nil { + return err + } + return RebuildClusterWithRotatedCertificates(context.Background(), rkeConfig, hosts.DialersOptions{}, externalFlags, rotateFlags) } func showRKECertificatesFromCli(ctx *cli.Context) error { return nil } -func RotateRKECertificates(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, dialersOptions hosts.DialersOptions, flags cluster.ExternalFlags) error { +func RebuildClusterWithRotatedCertificates(ctx context.Context, + rkeConfig *v3.RancherKubernetesEngineConfig, + dialersOptions hosts.DialersOptions, + flags cluster.ExternalFlags, + rotateFlags cluster.RotateCertificatesFlags) error { - log.Infof(ctx, "Rotating Kubernetes cluster certificates") + log.Infof(ctx, "Rebuilding Kubernetes cluster with rotated certificates") clusterState, err := cluster.ReadStateFile(ctx, cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir)) if err != nil { return err @@ -96,29 +104,25 @@ func RotateRKECertificates(ctx context.Context, rkeConfig *v3.RancherKubernetesE return err } - currentCluster, err := kubeCluster.GetClusterState(ctx, clusterState) - if err != nil { - return err - } - - if err := cluster.SetUpAuthentication(ctx, kubeCluster, currentCluster, clusterState); err != nil { - return err - } - - if err := cluster.RotateRKECertificates(ctx, kubeCluster, flags); err != nil { + if err := cluster.SetUpAuthentication(ctx, kubeCluster, nil, clusterState); err != nil { return err } if err := kubeCluster.SetUpHosts(ctx, true); err != nil { return err } + // Save new State + if err := kubeCluster.UpdateClusterCurrentState(ctx, clusterState); err != nil { + return err + } + // Restarting Kubernetes components servicesMap := make(map[string]bool) - for _, component := range flags.RotateComponents { + for _, component := range rotateFlags.RotateComponents { servicesMap[component] = true } - if len(flags.RotateComponents) == 0 || flags.RotateCACerts || servicesMap[services.EtcdContainerName] { + if len(rotateFlags.RotateComponents) == 0 || rotateFlags.RotateCACerts || servicesMap[services.EtcdContainerName] { if err := services.RestartEtcdPlane(ctx, kubeCluster.EtcdHosts); err != nil { return err } @@ -133,8 +137,48 @@ func RotateRKECertificates(ctx context.Context, rkeConfig *v3.RancherKubernetesE return err } - if flags.RotateCACerts { + if rotateFlags.RotateCACerts { return cluster.RestartClusterPods(ctx, kubeCluster) } return nil } + +func RotateRKECertificates(ctx context.Context, + rkeConfig *v3.RancherKubernetesEngineConfig, + dialersOptions hosts.DialersOptions, + flags cluster.ExternalFlags, + rotateFlags cluster.RotateCertificatesFlags) error { + log.Infof(ctx, "Rotating Kubernetes cluster certificates") + stateFilePath := cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir) + clusterState, _ := cluster.ReadStateFile(ctx, stateFilePath) + + kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags) + if err != nil { + return err + } + + if err := kubeCluster.SetupDialers(ctx, dialersOptions); err != nil { + return err + } + + err = doUpgradeLegacyCluster(ctx, kubeCluster, clusterState) + if err != nil { + log.Warnf(ctx, "[state] can't fetch legacy cluster state from Kubernetes") + } + + currentCluster, err := kubeCluster.GetClusterState(ctx, clusterState) + if err != nil { + return err + } + if currentCluster == nil { + return fmt.Errorf("Failed to rotate certificates: can't find old certificates") + } + if err := cluster.RotateRKECertificates(ctx, currentCluster, flags, rotateFlags, clusterState); err != nil { + return err + } + rkeState := cluster.FullState{ + DesiredState: clusterState.DesiredState, + CurrentState: clusterState.CurrentState, + } + return rkeState.WriteStateFile(ctx, stateFilePath) +} diff --git a/cmd/etcd.go b/cmd/etcd.go index 3cfec002..8070daa4 100644 --- a/cmd/etcd.go +++ b/cmd/etcd.go @@ -130,7 +130,7 @@ func SnapshotSaveEtcdHostsFromCli(ctx *cli.Context) error { logrus.Warnf("Name of the snapshot is not specified using [%s]", etcdSnapshotName) } // setting up the flags - flags := cluster.GetExternalFlags(false, false, false, false, nil, "", filePath) + flags := cluster.GetExternalFlags(false, false, false, "", filePath) return SnapshotSaveEtcdHosts(context.Background(), rkeConfig, hosts.DialersOptions{}, flags, etcdSnapshotName) } @@ -155,7 +155,7 @@ func RestoreEtcdSnapshotFromCli(ctx *cli.Context) error { return fmt.Errorf("You must specify the snapshot name to restore") } // setting up the flags - flags := cluster.GetExternalFlags(false, false, false, false, nil, "", filePath) + flags := cluster.GetExternalFlags(false, false, false, "", filePath) return RestoreEtcdSnapshot(context.Background(), rkeConfig, hosts.DialersOptions{}, flags, etcdSnapshotName) diff --git a/cmd/remove.go b/cmd/remove.go index c1a35c31..361215e2 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -114,7 +114,7 @@ func clusterRemoveFromCli(ctx *cli.Context) error { } // setting up the flags - flags := cluster.GetExternalFlags(false, false, false, false, nil, "", filePath) + flags := cluster.GetExternalFlags(false, false, false, "", filePath) return ClusterRemove(context.Background(), rkeConfig, hosts.DialersOptions{}, flags) } @@ -138,7 +138,7 @@ func clusterRemoveLocal(ctx *cli.Context) error { return err } // setting up the flags - flags := cluster.GetExternalFlags(true, false, false, false, nil, "", filePath) + flags := cluster.GetExternalFlags(true, false, false, "", filePath) return ClusterRemove(context.Background(), rkeConfig, hosts.DialersOptions{}, flags) } diff --git a/cmd/up.go b/cmd/up.go index c1471f0a..6a4546ad 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -281,7 +281,7 @@ func clusterUpFromCli(ctx *cli.Context) error { updateOnly := ctx.Bool("update-only") disablePortCheck := ctx.Bool("disable-port-check") // setting up the flags - flags := cluster.GetExternalFlags(false, false, updateOnly, disablePortCheck, nil, "", filePath) + flags := cluster.GetExternalFlags(false, updateOnly, disablePortCheck, "", filePath) if ctx.Bool("init") { return ClusterInit(context.Background(), rkeConfig, hosts.DialersOptions{}, flags) @@ -313,7 +313,7 @@ func clusterUpLocal(ctx *cli.Context) error { // setting up the dialers dialers := hosts.GetDialerOptions(nil, hosts.LocalHealthcheckFactory, nil) // setting up the flags - flags := cluster.GetExternalFlags(true, false, false, false, nil, "", filePath) + flags := cluster.GetExternalFlags(true, false, false, "", filePath) if ctx.Bool("init") { return ClusterInit(context.Background(), rkeConfig, dialers, flags) @@ -339,7 +339,7 @@ func clusterUpDind(ctx *cli.Context) error { // setting up the dialers dialers := hosts.GetDialerOptions(hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil) // setting up flags - flags := cluster.GetExternalFlags(false, false, false, disablePortCheck, nil, "", filePath) + flags := cluster.GetExternalFlags(false, false, disablePortCheck, "", filePath) if ctx.Bool("init") { return ClusterInit(context.Background(), rkeConfig, dialers, flags) diff --git a/pki/pki.go b/pki/pki.go index c916f336..12c8ccd5 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -34,7 +34,7 @@ type CertificatePKI struct { ConfigPath string `json:"configPath"` } -type GenFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig, string, string) error +type GenFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig, string, string, bool) error const ( etcdRole = "etcd" @@ -50,7 +50,7 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC return certs, err } // Generating certificates for kubernetes components - if err := GenerateRKEServicesCerts(ctx, certs, rkeConfig, configPath, configDir); err != nil { + if err := GenerateRKEServicesCerts(ctx, certs, rkeConfig, configPath, configDir, false); err != nil { return certs, err } return certs, nil diff --git a/pki/services.go b/pki/services.go index 47b12b70..ef003264 100644 --- a/pki/services.go +++ b/pki/services.go @@ -12,7 +12,7 @@ import ( "k8s.io/client-go/util/cert" ) -func GenerateKubeAPICertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateKubeAPICertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate API certificate and key caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key @@ -44,7 +44,7 @@ func GenerateKubeAPICertificate(ctx context.Context, certs map[string]Certificat return nil } -func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Kube controller-manager certificate and key log.Infof(ctx, "[certificates] Generating Kube Controller certificates") caCrt := certs[CACertName].Certificate @@ -57,7 +57,7 @@ func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]Cer return nil } -func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Kube scheduler certificate and key log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates") caCrt := certs[CACertName].Certificate @@ -70,7 +70,7 @@ func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]Cert return nil } -func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Kube Proxy certificate and key log.Infof(ctx, "[certificates] Generating Kube Proxy certificates") caCrt := certs[CACertName].Certificate @@ -83,7 +83,7 @@ func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]Certific return nil } -func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate kubelet certificate log.Infof(ctx, "[certificates] Generating Node certificate") caCrt := certs[CACertName].Certificate @@ -96,7 +96,7 @@ func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]Certifica return nil } -func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate Admin certificate and key log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig") caCrt := certs[CACertName].Certificate @@ -128,7 +128,7 @@ func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]Certific return nil } -func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { //generate API server proxy client key and certs log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates") caCrt := certs[RequestHeaderCACertName].Certificate @@ -141,7 +141,7 @@ func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]Cer return nil } -func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert)) if err != nil { return err @@ -160,7 +160,7 @@ func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]Cert return nil } -func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { caCrt := certs[CACertName].Certificate caKey := certs[CACertName].Key kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange) @@ -172,7 +172,7 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{}) for _, host := range etcdHosts { etcdName := GetEtcdCrtName(host.InternalAddress) - if _, ok := certs[etcdName]; ok { + if _, ok := certs[etcdName]; ok && !rotate { continue } log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress) @@ -185,7 +185,7 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP return nil } -func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { // generate service account token key var privateAPIKey *rsa.PrivateKey caCrt := certs[CACertName].Certificate @@ -221,7 +221,7 @@ func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, co return nil } -func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error { +func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error { RKECerts := []GenFunc{ GenerateKubeAPICertificate, GenerateServiceTokenKey, @@ -234,12 +234,12 @@ func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificateP GenerateEtcdCertificates, } for _, gen := range RKECerts { - if err := gen(ctx, certs, rkeConfig, configPath, configDir); err != nil { + if err := gen(ctx, certs, rkeConfig, configPath, configDir, rotate); err != nil { return err } } if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 { - return GenerateExternalEtcdCertificates(ctx, certs, rkeConfig, configPath, configDir) + return GenerateExternalEtcdCertificates(ctx, certs, rkeConfig, configPath, configDir, false) } return nil }