renew-embedded-certs

This commit is contained in:
fabriziopandini 2019-05-06 16:48:24 +02:00
parent 2dc3509b6d
commit cf7f8acae2
6 changed files with 640 additions and 173 deletions

View File

@ -24,6 +24,7 @@ import (
kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
@ -34,11 +35,17 @@ import (
) )
var ( var (
genericLongDesc = normalizer.LongDesc(` genericCertRenewLongDesc = normalizer.LongDesc(`
Renew the %[1]s, and save them into %[2]s.cert and %[2]s.key files. Renew the %[1]s, and save them into %[2]s.cert and %[2]s.key files.
Extra attributes such as SANs will be based on the existing certificates, there is no need to resupply them. Extra attributes such as SANs will be based on the existing certificates, there is no need to resupply them.
`) `)
genericCertRenewEmbeddedLongDesc = normalizer.LongDesc(`
Renew the certificate embedded in the kubeconfig file %s.
Kubeconfig attributes and certificate extra attributes such as SANs will be based on the existing kubeconfig/certificates, there is no need to resupply them.
`)
allLongDesc = normalizer.LongDesc(` allLongDesc = normalizer.LongDesc(`
Renew all known certificates necessary to run the control plane. Renewals are run unconditionally, regardless Renew all known certificates necessary to run the control plane. Renewals are run unconditionally, regardless
of expiration date. Renewals can also be run individually for more control. of expiration date. Renewals can also be run individually for more control.
@ -66,7 +73,7 @@ func newCmdCertsRenewal() *cobra.Command {
RunE: cmdutil.SubCmdRunE("renew"), RunE: cmdutil.SubCmdRunE("renew"),
} }
cmd.AddCommand(getRenewSubCommands()...) cmd.AddCommand(getRenewSubCommands(kubeadmconstants.KubernetesDir)...)
return cmd return cmd
} }
@ -80,8 +87,15 @@ type renewConfig struct {
csrPath string csrPath string
} }
func getRenewSubCommands() []*cobra.Command { func getRenewSubCommands(kdir string) []*cobra.Command {
cfg := &renewConfig{} cfg := &renewConfig{
cfg: kubeadmapiv1beta2.InitConfiguration{
ClusterConfiguration: kubeadmapiv1beta2.ClusterConfiguration{
// Setting kubernetes version to a default value in order to allow a not necessary internet lookup
KubernetesVersion: constants.CurrentKubernetesVersion.String(),
},
},
}
// Default values for the cobra help text // Default values for the cobra help text
kubeadmscheme.Scheme.Default(&cfg.cfg) kubeadmscheme.Scheme.Default(&cfg.cfg)
@ -95,9 +109,11 @@ func getRenewSubCommands() []*cobra.Command {
// Don't offer to renew CAs; would cause serious consequences // Don't offer to renew CAs; would cause serious consequences
for _, cert := range certs { for _, cert := range certs {
// get the cobra.Command skeleton for this command // get the cobra.Command skeleton for this command
cmd := generateRenewalCommand(cert, cfg) cmd := generateCertRenewalCommand(cert, cfg)
// get the implementation of renewing this certificate // get the implementation of renewing this certificate
renewalFunc := generateRenewalFunction(cert, caCert, cfg) renewalFunc := func(cert *certsphase.KubeadmCert, caCert *certsphase.KubeadmCert) func() {
return func() { renewCert(cert, caCert, cfg) }
}(cert, caCert)
// install the implementation into the command // install the implementation into the command
cmd.Run = func(*cobra.Command, []string) { renewalFunc() } cmd.Run = func(*cobra.Command, []string) { renewalFunc() }
cmdList = append(cmdList, cmd) cmdList = append(cmdList, cmd)
@ -106,6 +122,27 @@ func getRenewSubCommands() []*cobra.Command {
} }
} }
kubeconfigs := []string{
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
//NB. we are escluding KubeletKubeConfig from renewal because management of this certificate is delegated to kubelet
}
for _, k := range kubeconfigs {
// get the cobra.Command skeleton for this command
cmd := generateEmbeddedCertRenewalCommand(k, cfg)
// get the implementation of renewing this certificate
renewalFunc := func(kdir, k string) func() {
return func() { renewEmbeddedCert(kdir, k, cfg) }
}(kdir, k)
// install the implementation into the command
cmd.Run = func(*cobra.Command, []string) { renewalFunc() }
cmdList = append(cmdList, cmd)
// Collect renewal functions for `renew all`
funcList = append(funcList, renewalFunc)
}
allCmd := &cobra.Command{ allCmd := &cobra.Command{
Use: "all", Use: "all",
Short: "Renew all available certificates", Short: "Renew all available certificates",
@ -131,53 +168,100 @@ func addFlags(cmd *cobra.Command, cfg *renewConfig) {
cmd.Flags().BoolVar(&cfg.useAPI, "use-api", cfg.useAPI, "Use the Kubernetes certificate API to renew certificates") cmd.Flags().BoolVar(&cfg.useAPI, "use-api", cfg.useAPI, "Use the Kubernetes certificate API to renew certificates")
} }
func generateRenewalFunction(cert *certsphase.KubeadmCert, caCert *certsphase.KubeadmCert, cfg *renewConfig) func() { func renewCert(cert *certsphase.KubeadmCert, caCert *certsphase.KubeadmCert, cfg *renewConfig) {
return func() { internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfg.cfgPath, &cfg.cfg)
internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfg.cfgPath, &cfg.cfg) kubeadmutil.CheckErr(err)
// if the renewal operation is set to generate only CSR request
if cfg.useCSR {
// trigger CSR generation in the csrPath, or if this one is missing, in the CertificateDir
path := cfg.csrPath
if path == "" {
path = cfg.cfg.CertificatesDir
}
err := certsphase.CreateCSR(cert, internalcfg, path)
kubeadmutil.CheckErr(err)
return
}
// otherwise, the renewal operation has to actually renew a certificate
var externalCA bool
switch caCert.BaseName {
case kubeadmconstants.CACertAndKeyBaseName:
// Check if an external CA is provided by the user (when the CA Cert is present but the CA Key is not)
externalCA, _ = certsphase.UsingExternalCA(&internalcfg.ClusterConfiguration)
case kubeadmconstants.FrontProxyCACertAndKeyBaseName:
// Check if an external Front-Proxy CA is provided by the user (when the Front-Proxy CA Cert is present but the Front-Proxy CA Key is not)
externalCA, _ = certsphase.UsingExternalFrontProxyCA(&internalcfg.ClusterConfiguration)
default:
externalCA = false
}
if !externalCA {
renewer, err := getRenewer(cfg, caCert.BaseName)
kubeadmutil.CheckErr(err) kubeadmutil.CheckErr(err)
if cfg.useCSR { err = renewal.RenewExistingCert(internalcfg.CertificatesDir, cert.BaseName, renewer)
path := cfg.csrPath kubeadmutil.CheckErr(err)
if path == "" {
path = cfg.cfg.CertificatesDir
}
err := certsphase.CreateCSR(cert, internalcfg, path)
kubeadmutil.CheckErr(err)
return
}
var externalCA bool fmt.Printf("Certificate %s renewed\n", cert.Name)
switch caCert.BaseName { return
case kubeadmconstants.CACertAndKeyBaseName:
// Check if an external CA is provided by the user (when the CA Cert is present but the CA Key is not)
externalCA, _ = certsphase.UsingExternalCA(&internalcfg.ClusterConfiguration)
case kubeadmconstants.FrontProxyCACertAndKeyBaseName:
// Check if an external Front-Proxy CA is provided by the user (when the Front-Proxy CA Cert is present but the Front-Proxy CA Key is not)
externalCA, _ = certsphase.UsingExternalFrontProxyCA(&internalcfg.ClusterConfiguration)
default:
externalCA = false
}
if !externalCA {
renewer, err := getRenewer(cfg, caCert.BaseName)
kubeadmutil.CheckErr(err)
err = renewal.RenewExistingCert(internalcfg.CertificatesDir, cert.BaseName, renewer)
kubeadmutil.CheckErr(err)
fmt.Printf("Certificate %s renewed\n", cert.Name)
return
}
fmt.Printf("Detected external %s, certificate %s can't be renewed\n", cert.CAName, cert.Name)
} }
fmt.Printf("Detected external %s, certificate %s can't be renewed\n", cert.CAName, cert.Name)
} }
func generateRenewalCommand(cert *certsphase.KubeadmCert, cfg *renewConfig) *cobra.Command { func renewEmbeddedCert(kdir, k string, cfg *renewConfig) {
internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfg.cfgPath, &cfg.cfg)
kubeadmutil.CheckErr(err)
// if the renewal operation is set to generate only CSR request
if cfg.useCSR {
// trigger CSR generation in the csrPath, or if this one is missing, in the CertificateDir
path := cfg.csrPath
if path == "" {
path = cfg.cfg.CertificatesDir
}
err := certsphase.CreateCSR(nil, internalcfg, path)
kubeadmutil.CheckErr(err)
return
}
// otherwise, the renewal operation has to actually renew a certificate
// Check if an external CA is provided by the user (when the CA Cert is present but the CA Key is not)
externalCA, _ := certsphase.UsingExternalCA(&internalcfg.ClusterConfiguration)
if !externalCA {
renewer, err := getRenewer(cfg, certsphase.KubeadmCertRootCA.BaseName)
kubeadmutil.CheckErr(err)
err = renewal.RenewEmbeddedClientCert(kdir, k, renewer)
kubeadmutil.CheckErr(err)
fmt.Printf("Certificate embedded in %s renewed\n", k)
return
}
fmt.Printf("Detected external CA, certificate embedded in %s can't be renewed\n", k)
}
func generateCertRenewalCommand(cert *certsphase.KubeadmCert, cfg *renewConfig) *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: cert.Name, Use: cert.Name,
Short: fmt.Sprintf("Generate the %s", cert.LongName), Short: fmt.Sprintf("Renew the %s", cert.LongName),
Long: fmt.Sprintf(genericLongDesc, cert.LongName, cert.BaseName), Long: fmt.Sprintf(genericCertRenewLongDesc, cert.LongName, cert.BaseName),
}
addFlags(cmd, cfg)
return cmd
}
func generateEmbeddedCertRenewalCommand(k string, cfg *renewConfig) *cobra.Command {
cmd := &cobra.Command{
Use: k,
Short: fmt.Sprintf("Renew the certificate embedded in %s", k),
Long: fmt.Sprintf(genericCertRenewEmbeddedLongDesc, k),
} }
addFlags(cmd, cfg) addFlags(cmd, cfg)
return cmd return cmd

View File

@ -17,13 +17,11 @@ limitations under the License.
package alpha package alpha
import ( import (
"crypto/rand" "crypto"
"crypto/rsa"
"crypto/x509" "crypto/x509"
"crypto/x509/pkix"
"fmt" "fmt"
"math/big"
"os" "os"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -31,7 +29,8 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test" testutil "k8s.io/kubernetes/cmd/kubeadm/test"
cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd" cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd"
@ -81,144 +80,198 @@ func TestCommandsGenerated(t *testing.T) {
func TestRunRenewCommands(t *testing.T) { func TestRunRenewCommands(t *testing.T) {
tests := []struct { tests := []struct {
command string command string
baseNames []string CAs []*certsphase.KubeadmCert
caBaseNames []string Certs []*certsphase.KubeadmCert
KubeconfigFiles []string
}{ }{
{ {
command: "all", command: "all",
baseNames: []string{ CAs: []*certsphase.KubeadmCert{
kubeadmconstants.APIServerCertAndKeyBaseName, &certsphase.KubeadmCertRootCA,
kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, &certsphase.KubeadmCertFrontProxyCA,
kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, &certsphase.KubeadmCertEtcdCA,
kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
kubeadmconstants.EtcdServerCertAndKeyBaseName,
kubeadmconstants.EtcdPeerCertAndKeyBaseName,
kubeadmconstants.EtcdHealthcheckClientCertAndKeyBaseName,
}, },
caBaseNames: []string{ Certs: []*certsphase.KubeadmCert{
kubeadmconstants.CACertAndKeyBaseName, &certsphase.KubeadmCertAPIServer,
kubeadmconstants.FrontProxyCACertAndKeyBaseName, &certsphase.KubeadmCertKubeletClient,
kubeadmconstants.EtcdCACertAndKeyBaseName, &certsphase.KubeadmCertFrontProxyClient,
&certsphase.KubeadmCertEtcdAPIClient,
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
},
KubeconfigFiles: []string{
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
}, },
}, },
{ {
command: "apiserver", command: "apiserver",
baseNames: []string{kubeadmconstants.APIServerCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.CACertAndKeyBaseName}, &certsphase.KubeadmCertRootCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertAPIServer,
},
}, },
{ {
command: "apiserver-kubelet-client", command: "apiserver-kubelet-client",
baseNames: []string{kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.CACertAndKeyBaseName}, &certsphase.KubeadmCertRootCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertKubeletClient,
},
}, },
{ {
command: "apiserver-etcd-client", command: "apiserver-etcd-client",
baseNames: []string{kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.EtcdCACertAndKeyBaseName}, &certsphase.KubeadmCertEtcdCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdAPIClient,
},
}, },
{ {
command: "front-proxy-client", command: "front-proxy-client",
baseNames: []string{kubeadmconstants.FrontProxyClientCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.FrontProxyCACertAndKeyBaseName}, &certsphase.KubeadmCertFrontProxyCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertFrontProxyClient,
},
}, },
{ {
command: "etcd-server", command: "etcd-server",
baseNames: []string{kubeadmconstants.EtcdServerCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.EtcdCACertAndKeyBaseName}, &certsphase.KubeadmCertEtcdCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
},
}, },
{ {
command: "etcd-peer", command: "etcd-peer",
baseNames: []string{kubeadmconstants.EtcdPeerCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.EtcdCACertAndKeyBaseName}, &certsphase.KubeadmCertEtcdCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdPeer,
},
}, },
{ {
command: "etcd-healthcheck-client", command: "etcd-healthcheck-client",
baseNames: []string{kubeadmconstants.EtcdHealthcheckClientCertAndKeyBaseName}, CAs: []*certsphase.KubeadmCert{
caBaseNames: []string{kubeadmconstants.EtcdCACertAndKeyBaseName}, &certsphase.KubeadmCertEtcdCA,
},
Certs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdHealthcheck,
},
},
{
command: "admin.conf",
CAs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertRootCA,
},
KubeconfigFiles: []string{
kubeadmconstants.AdminKubeConfigFileName,
},
},
{
command: "scheduler.conf",
CAs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertRootCA,
},
KubeconfigFiles: []string{
kubeadmconstants.SchedulerKubeConfigFileName,
},
},
{
command: "controller-manager.conf",
CAs: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertRootCA,
},
KubeconfigFiles: []string{
kubeadmconstants.ControllerManagerKubeConfigFileName,
},
}, },
} }
renewCmds := getRenewSubCommands()
for _, test := range tests { for _, test := range tests {
t.Run(test.command, func(t *testing.T) { t.Run(test.command, func(t *testing.T) {
tmpDir := testutil.SetupTempDir(t) tmpDir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t) cfg := testutil.GetDefaultInternalConfig(t)
cfg.CertificatesDir = tmpDir
for _, caBaseName := range test.caBaseNames { // Generate all the CA
if err := pkiutil.WriteCertAndKey(tmpDir, caBaseName, caCert, caKey); err != nil { CACerts := map[string]*x509.Certificate{}
t.Fatalf("couldn't write out CA: %v", err) CAKeys := map[string]crypto.Signer{}
for _, ca := range test.CAs {
caCert, caKey, err := ca.CreateAsCA(cfg)
if err != nil {
t.Fatalf("couldn't write out CA %s: %v", ca.Name, err)
} }
CACerts[ca.Name] = caCert
CAKeys[ca.Name] = caKey
} }
certTmpl := x509.Certificate{ // Generate all the signed certificates (and store creation time)
Subject: pkix.Name{ createTime := map[string]time.Time{}
CommonName: "test-cert", for _, cert := range test.Certs {
Organization: []string{"sig-cluster-lifecycle"}, caCert := CACerts[cert.CAName]
}, caKey := CAKeys[cert.CAName]
DNSNames: []string{"test-domain.space"}, if err := cert.CreateFromCA(cfg, caCert, caKey); err != nil {
SerialNumber: new(big.Int).SetInt64(0), t.Fatalf("couldn't write certificate %s: %v", cert.Name, err)
NotBefore: time.Now().Add(-time.Hour * 24 * 365),
NotAfter: time.Now().Add(-time.Hour),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("couldn't generate private key: %v", err)
}
certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, key.Public(), caKey)
if err != nil {
t.Fatalf("couldn't generate private key: %v", err)
}
cert, err := x509.ParseCertificate(certDERBytes)
if err != nil {
t.Fatalf("couldn't generate private key: %v", err)
}
for _, baseName := range test.baseNames {
if err := pkiutil.WriteCertAndKey(tmpDir, baseName, cert, key); err != nil {
t.Fatalf("couldn't write out initial certificate")
} }
file, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("%s.crt", cert.BaseName)))
if err != nil {
t.Fatalf("couldn't get certificate %s: %v", cert.Name, err)
}
createTime[cert.Name] = file.ModTime()
} }
// Generate all the kubeconfig files with embedded certs(and store creation time)
for _, kubeConfig := range test.KubeconfigFiles {
if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpDir, cfg); err != nil {
t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err)
}
file, err := os.Stat(filepath.Join(tmpDir, kubeConfig))
if err != nil {
t.Fatalf("couldn't get kubeconfig %s: %v", kubeConfig, err)
}
createTime[kubeConfig] = file.ModTime()
}
// exec renew
renewCmds := getRenewSubCommands(tmpDir)
cmdtestutil.RunSubCommand(t, renewCmds, test.command, fmt.Sprintf("--cert-dir=%s", tmpDir)) cmdtestutil.RunSubCommand(t, renewCmds, test.command, fmt.Sprintf("--cert-dir=%s", tmpDir))
for _, baseName := range test.baseNames { // read renewed certificates and check the file is modified
newCert, newKey, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpDir, baseName) for _, cert := range test.Certs {
file, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("%s.crt", cert.BaseName)))
if err != nil { if err != nil {
t.Fatalf("couldn't load renewed certificate: %v", err) t.Fatalf("couldn't get certificate %s: %v", cert.Name, err)
} }
if createTime[cert.Name] == file.ModTime() {
certstestutil.AssertCertificateIsSignedByCa(t, newCert, caCert) t.Errorf("certificate %s was not renewed as expected", cert.Name)
pool := x509.NewCertPool()
pool.AddCert(caCert)
_, err = newCert.Verify(x509.VerifyOptions{
DNSName: "test-domain.space",
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
if err != nil {
t.Errorf("couldn't verify renewed cert: %v", err)
}
switch pubKey := newCert.PublicKey.(type) {
case *rsa.PublicKey:
if pubKey.N.Cmp(newKey.(*rsa.PrivateKey).N) != 0 {
t.Error("private key does not match public key")
}
default:
t.Errorf("unknown public key type %T", newCert.PublicKey)
} }
} }
// ead renewed kubeconfig files and check the file is modified
for _, kubeConfig := range test.KubeconfigFiles {
file, err := os.Stat(filepath.Join(tmpDir, kubeConfig))
if err != nil {
t.Fatalf("couldn't get kubeconfig %s: %v", kubeConfig, err)
}
if createTime[kubeConfig] == file.ModTime() {
t.Errorf("kubeconfig %s was not renewed as expected", kubeConfig)
}
}
}) })
} }
} }
@ -228,7 +281,7 @@ func TestRenewUsingCSR(t *testing.T) {
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
cert := &certs.KubeadmCertEtcdServer cert := &certs.KubeadmCertEtcdServer
renewCmds := getRenewSubCommands() renewCmds := getRenewSubCommands(tmpDir)
cmdtestutil.RunSubCommand(t, renewCmds, cert.Name, "--csr-only", "--csr-dir="+tmpDir) cmdtestutil.RunSubCommand(t, renewCmds, cert.Name, "--csr-only", "--csr-dir="+tmpDir)
if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(tmpDir, cert.BaseName); err != nil { if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(tmpDir, cert.BaseName); err != nil {

View File

@ -18,9 +18,12 @@ package renewal
import ( import (
"crypto/x509" "crypto/x509"
"path/filepath"
"github.com/pkg/errors" "github.com/pkg/errors"
"k8s.io/client-go/tools/clientcmd"
certutil "k8s.io/client-go/util/cert" certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
) )
@ -49,6 +52,72 @@ func RenewExistingCert(certsDir, baseName string, impl Interface) error {
return nil return nil
} }
// RenewEmbeddedClientCert loads a kubeconfig file, uses the renew interface to renew the client certificate
// embedded in it, and then saves the resulting kubeconfig and key over the old one.
func RenewEmbeddedClientCert(kubeConfigFileDir, kubeConfigFileName string, impl Interface) error {
kubeConfigFilePath := filepath.Join(kubeConfigFileDir, kubeConfigFileName)
// try to load the kubeconfig file
kubeconfig, err := clientcmd.LoadFromFile(kubeConfigFilePath)
if err != nil {
return errors.Wrapf(err, "failed to load kubeconfig file %s", kubeConfigFilePath)
}
// get current context
if _, ok := kubeconfig.Contexts[kubeconfig.CurrentContext]; !ok {
return errors.Errorf("invalid kubeconfig file %s: missing context %s", kubeConfigFilePath, kubeconfig.CurrentContext)
}
// get cluster info for current context and ensure a server certificate is embedded in it
clusterName := kubeconfig.Contexts[kubeconfig.CurrentContext].Cluster
if _, ok := kubeconfig.Clusters[clusterName]; !ok {
return errors.Errorf("invalid kubeconfig file %s: missing cluster %s", kubeConfigFilePath, clusterName)
}
cluster := kubeconfig.Clusters[clusterName]
if len(cluster.CertificateAuthorityData) == 0 {
return errors.Errorf("kubeconfig file %s does not have and embedded server certificate", kubeConfigFilePath)
}
// get auth info for current context and ensure a client certificate is embedded in it
authInfoName := kubeconfig.Contexts[kubeconfig.CurrentContext].AuthInfo
if _, ok := kubeconfig.AuthInfos[authInfoName]; !ok {
return errors.Errorf("invalid kubeconfig file %s: missing authInfo %s", kubeConfigFilePath, authInfoName)
}
authInfo := kubeconfig.AuthInfos[authInfoName]
if len(authInfo.ClientCertificateData) == 0 {
return errors.Errorf("kubeconfig file %s does not have and embedded client certificate", kubeConfigFilePath)
}
// parse the client certificate, retrive the cert config and then renew it
certs, err := certutil.ParseCertsPEM(authInfo.ClientCertificateData)
if err != nil {
return errors.Wrapf(err, "kubeconfig file %s does not contain a valid client certificate", kubeConfigFilePath)
}
cfg := certToConfig(certs[0])
newCert, newKey, err := impl.Renew(cfg)
if err != nil {
return errors.Wrapf(err, "failed to renew certificate embedded in %s", kubeConfigFilePath)
}
// encodes the new key
encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(newKey)
if err != nil {
return errors.Wrapf(err, "failed to marshal private key to PEM")
}
// create a kubeconfig copy with the new client certs
newConfig := kubeconfig.DeepCopy()
newConfig.AuthInfos[authInfoName].ClientKeyData = encodedClientKey
newConfig.AuthInfos[authInfoName].ClientCertificateData = pkiutil.EncodeCertPEM(newCert)
// writes the kubeconfig to disk
return clientcmd.WriteToFile(*newConfig, kubeConfigFilePath)
}
func certToConfig(cert *x509.Certificate) *certutil.Config { func certToConfig(cert *x509.Certificate) *certutil.Config {
return &certutil.Config{ return &certutil.Config{
CommonName: cert.Subject.CommonName, CommonName: cert.Subject.CommonName,

View File

@ -17,11 +17,13 @@ limitations under the License.
package renewal package renewal
import ( import (
"bytes"
"crypto" "crypto"
"crypto/x509" "crypto/x509"
"crypto/x509/pkix" "crypto/x509/pkix"
"net" "net"
"os" "os"
"path/filepath"
"testing" "testing"
"time" "time"
@ -31,8 +33,11 @@ import (
"k8s.io/apimachinery/pkg/watch" "k8s.io/apimachinery/pkg/watch"
fakecerts "k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake" fakecerts "k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake"
k8stesting "k8s.io/client-go/testing" k8stesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/clientcmd"
certutil "k8s.io/client-go/util/cert" certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
certtestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" certtestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test" testutil "k8s.io/kubernetes/cmd/kubeadm/test"
) )
@ -186,6 +191,7 @@ func TestCertToConfig(t *testing.T) {
} }
func TestRenewExistingCert(t *testing.T) { func TestRenewExistingCert(t *testing.T) {
// creates a CA, a certificate, and save it to a file
cfg := &certutil.Config{ cfg := &certutil.Config{
CommonName: "test-common-name", CommonName: "test-common-name",
Organization: []string{"sig-cluster-lifecycle"}, Organization: []string{"sig-cluster-lifecycle"},
@ -214,21 +220,136 @@ func TestRenewExistingCert(t *testing.T) {
t.Fatalf("couldn't write out certificate") t.Fatalf("couldn't write out certificate")
} }
// makes some time pass
time.Sleep(1 * time.Second)
// renew the certificate
renewer := NewFileRenewal(caCert, caKey) renewer := NewFileRenewal(caCert, caKey)
if err := RenewExistingCert(dir, "server", renewer); err != nil { if err := RenewExistingCert(dir, "server", renewer); err != nil {
t.Fatalf("couldn't renew certificate: %v", err) t.Fatalf("couldn't renew certificate: %v", err)
} }
// reads the renewed certificate
newCert, err := pkiutil.TryLoadCertFromDisk(dir, "server") newCert, err := pkiutil.TryLoadCertFromDisk(dir, "server")
if err != nil { if err != nil {
t.Fatalf("couldn't load created certificate: %v", err) t.Fatalf("couldn't load created certificate: %v", err)
} }
// check the new certificate is changed, has an newer expiration date, but preserve all the
// other attributes
if newCert.SerialNumber.Cmp(cert.SerialNumber) == 0 { if newCert.SerialNumber.Cmp(cert.SerialNumber) == 0 {
t.Fatal("expected new certificate, but renewed certificate has same serial number") t.Fatal("expected new certificate, but renewed certificate has same serial number")
} }
if !newCert.NotAfter.After(cert.NotAfter) {
t.Fatalf("expected new certificate with updated expiration, but renewed certificate has the same serial number: saw %s, expected greather than %s", newCert.NotAfter, cert.NotAfter)
}
certtestutil.AssertCertificateIsSignedByCa(t, newCert, caCert)
certtestutil.AssertCertificateHasClientAuthUsage(t, newCert)
certtestutil.AssertCertificateHasOrganizations(t, newCert, cfg.Organization...)
certtestutil.AssertCertificateHasCommonName(t, newCert, cfg.CommonName)
certtestutil.AssertCertificateHasDNSNames(t, newCert, cfg.AltNames.DNSNames...)
certtestutil.AssertCertificateHasIPAddresses(t, newCert, cfg.AltNames.IPs...)
}
func TestRenewEmbeddedClientCert(t *testing.T) {
// creates a CA, a client certificate, and then embeds it into a kubeconfig file
caCertCfg := &certutil.Config{CommonName: "kubernetes"}
caCert, caKey, err := pkiutil.NewCertificateAuthority(caCertCfg)
if err != nil {
t.Fatalf("couldn't create CA: %v", err)
}
cfg := &certutil.Config{
CommonName: "test-common-name",
Organization: []string{"sig-cluster-lifecycle"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
AltNames: certutil.AltNames{
IPs: []net.IP{net.ParseIP("10.100.0.1")},
DNSNames: []string{"test-domain.space"},
},
}
cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg)
if err != nil {
t.Fatalf("couldn't generate certificate: %v", err)
}
encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(key)
if err != nil {
t.Fatalf("failed to marshal private key to PEM: %v", err)
}
certificateAuthorityData := pkiutil.EncodeCertPEM(caCert)
config := kubeconfigutil.CreateWithCerts(
"https://localhost:1234",
"kubernetes-test",
"user-test",
certificateAuthorityData,
encodedClientKey,
pkiutil.EncodeCertPEM(cert),
)
dir := testutil.SetupTempDir(t)
defer os.RemoveAll(dir)
kubeconfigPath := filepath.Join(dir, "k.conf")
if err := clientcmd.WriteToFile(*config, kubeconfigPath); err != nil {
t.Fatalf("couldn't write out certificate")
}
// makes some time pass
time.Sleep(1 * time.Second)
// renew the embedded certificate
renewer := NewFileRenewal(caCert, caKey)
if err := RenewEmbeddedClientCert(dir, "k.conf", renewer); err != nil {
t.Fatalf("couldn't renew embedded certificate: %v", err)
}
// reads the kubeconfig file and gets the renewed certificate
newConfig, err := clientcmd.LoadFromFile(kubeconfigPath)
if err != nil {
t.Fatalf("failed to load kubeconfig file %s: %v", kubeconfigPath, err)
}
if newConfig.Contexts[config.CurrentContext].Cluster != "kubernetes-test" {
t.Fatalf("invalid cluster. expected kubernetes-test, saw %s", newConfig.Contexts[config.CurrentContext].Cluster)
}
cluster := newConfig.Clusters["kubernetes-test"]
if !bytes.Equal(cluster.CertificateAuthorityData, certificateAuthorityData) {
t.Fatalf("invalid cluster. CertificateAuthorityData does not contain expected value")
}
if newConfig.Contexts[config.CurrentContext].AuthInfo != "user-test" {
t.Fatalf("invalid AuthInfo. expected user-test, saw %s", newConfig.Contexts[config.CurrentContext].AuthInfo)
}
authInfo := newConfig.AuthInfos["user-test"]
newCerts, err := certutil.ParseCertsPEM(authInfo.ClientCertificateData)
if err != nil {
t.Fatalf("couldn't load created certificate: %v", err)
}
// check the new certificate is changed, has an newer expiration date, but preserve all the
// other attributes
newCert := newCerts[0]
if newCert.SerialNumber.Cmp(cert.SerialNumber) == 0 {
t.Fatal("expected new certificate, but renewed certificate has same serial number")
}
if !newCert.NotAfter.After(cert.NotAfter) {
t.Fatalf("expected new certificate with updated expiration, but renewed certificate has same serial number: saw %s, expected greather than %s", newCert.NotAfter, cert.NotAfter)
}
certtestutil.AssertCertificateIsSignedByCa(t, newCert, caCert) certtestutil.AssertCertificateIsSignedByCa(t, newCert, caCert)
certtestutil.AssertCertificateHasClientAuthUsage(t, newCert) certtestutil.AssertCertificateHasClientAuthUsage(t, newCert)
certtestutil.AssertCertificateHasOrganizations(t, newCert, cfg.Organization...) certtestutil.AssertCertificateHasOrganizations(t, newCert, cfg.Organization...)

View File

@ -202,8 +202,8 @@ func upgradeComponent(component string, renewCerts bool, waiter apiclient.Waiter
// if certificate renewal should be performed // if certificate renewal should be performed
if renewCerts { if renewCerts {
// renew all the certificates used by the current component // renew all the certificates used by the current component
if err := renewCertsByComponent(cfg, component); err != nil { if err := renewCertsByComponent(cfg, constants.KubernetesDir, component); err != nil {
return errors.Wrapf(err, "failed to renew certificates for component %q", component) return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to renew certificates for component %q", component), pathMgr, recoverEtcd)
} }
} }
@ -450,6 +450,14 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
} }
} }
if renewCerts {
// renew the certificate embedded in the admin.conf file
err := renewEmbeddedCertsByName(cfg, constants.KubernetesDir, constants.AdminKubeConfigFileName)
if err != nil {
return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.AdminKubeConfigFileName), pathMgr, false)
}
}
// Remove the temporary directories used on a best-effort (don't fail if the calls error out) // Remove the temporary directories used on a best-effort (don't fail if the calls error out)
// The calls are set here by design; we should _not_ use "defer" above as that would remove the directories // The calls are set here by design; we should _not_ use "defer" above as that would remove the directories
// even in the "fail and rollback" case, where we want the directories preserved for the user. // even in the "fail and rollback" case, where we want the directories preserved for the user.
@ -495,7 +503,7 @@ func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathMa
// renewCertsByComponent takes charge of renewing certificates used by a specific component before // renewCertsByComponent takes charge of renewing certificates used by a specific component before
// the static pod of the component is upgraded // the static pod of the component is upgraded
func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string) error { func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, kubernetesDir, component string) error {
// if the cluster is using a local etcd // if the cluster is using a local etcd
if cfg.Etcd.Local != nil { if cfg.Etcd.Local != nil {
if component == constants.Etcd || component == constants.KubeAPIServer { if component == constants.Etcd || component == constants.KubeAPIServer {
@ -576,5 +584,42 @@ func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string)
} }
} }
} }
if component == constants.KubeControllerManager {
// renew the certificate embedded in the controller-manager.conf file
err := renewEmbeddedCertsByName(cfg, kubernetesDir, constants.ControllerManagerKubeConfigFileName)
if err != nil {
return errors.Wrapf(err, "failed to upgrade the %s certificates", constants.ControllerManagerKubeConfigFileName)
}
}
if component == constants.KubeScheduler {
// renew the certificate embedded in the scheduler.conf file
err := renewEmbeddedCertsByName(cfg, kubernetesDir, constants.SchedulerKubeConfigFileName)
if err != nil {
return errors.Wrapf(err, "failed to upgrade the %s certificates", constants.SchedulerKubeConfigFileName)
}
}
return nil
}
func renewEmbeddedCertsByName(cfg *kubeadmapi.InitConfiguration, kubernetesDir, kubeConfigFile string) error {
// Checks if an external CA is provided by the user (when the CA Cert is present but the CA Key is not)
// if not, then CA is managed by kubeadm, so it is possible to renew all the certificates signed by ca
// and used by the apis server (the apiserver certificate and the apiserver-kubelet-client certificate)
externalCA, _ := certsphase.UsingExternalCA(&cfg.ClusterConfiguration)
if !externalCA {
// try to load ca
caCert, caKey, err := certsphase.LoadCertificateAuthority(cfg.CertificatesDir, certsphase.KubeadmCertRootCA.BaseName)
if err != nil {
return errors.Wrapf(err, "failed to upgrade the %s certificates", kubeConfigFile)
}
// create a renewer for certificates signed by CA
renewer := renewal.NewFileRenewal(caCert, caKey)
// renew the certificate embedded in the controller-manager.conf file
fmt.Printf("[upgrade/staticpods] Renewing certificate embedded in %q \n", kubeConfigFile)
if err := renewal.RenewEmbeddedClientCert(kubernetesDir, kubeConfigFile, renewer); err != nil {
return errors.Wrapf(err, "failed to renew certificate embedded in %s", kubeConfigFile)
}
}
return nil return nil
} }

View File

@ -18,6 +18,7 @@ package upgrade
import ( import (
"crypto/sha256" "crypto/sha256"
"crypto/x509"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"math/big" "math/big"
@ -31,11 +32,15 @@ import (
"github.com/coreos/etcd/pkg/transport" "github.com/coreos/etcd/pkg/transport"
"github.com/pkg/errors" "github.com/pkg/errors"
"k8s.io/client-go/tools/clientcmd"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
@ -318,6 +323,7 @@ func TestStaticPodControlPlane(t *testing.T) {
description string description string
waitErrsToReturn map[string]error waitErrsToReturn map[string]error
moveFileFunc func(string, string) error moveFileFunc func(string, string) error
skipKubeConfig string
expectedErr bool expectedErr bool
manifestShouldChange bool manifestShouldChange bool
}{ }{
@ -424,6 +430,34 @@ func TestStaticPodControlPlane(t *testing.T) {
expectedErr: true, expectedErr: true,
manifestShouldChange: false, manifestShouldChange: false,
}, },
{
description: "any cert renew error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy)",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
},
skipKubeConfig: kubeadmconstants.SchedulerKubeConfigFileName,
expectedErr: true,
manifestShouldChange: false,
},
{
description: "any cert renew error should result in a rollback and an abort; even though this is admin.conf (kube-apiserver and kube-controller-manager and kube-scheduler healthy)",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
},
skipKubeConfig: kubeadmconstants.AdminKubeConfigFileName,
expectedErr: true,
manifestShouldChange: false,
},
} }
for _, rt := range tests { for _, rt := range tests {
@ -463,6 +497,19 @@ func TestStaticPodControlPlane(t *testing.T) {
t.Logf("Wrote certs to %s\n", oldcfg.CertificatesDir) t.Logf("Wrote certs to %s\n", oldcfg.CertificatesDir)
for _, kubeConfig := range []string{
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
} {
if rt.skipKubeConfig == kubeConfig {
continue
}
if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, constants.KubernetesDir, oldcfg); err != nil {
t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err)
}
}
// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg)
if err != nil { if err != nil {
@ -642,16 +689,17 @@ func TestRenewCertsByComponent(t *testing.T) {
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t) caCert, caKey := certstestutil.SetupCertificateAuthorithy(t)
tests := []struct { tests := []struct {
name string name string
component string component string
externalCA bool externalCA bool
externalFrontProxyCA bool externalFrontProxyCA bool
skipCreateEtcdCA bool skipCreateEtcdCA bool
shouldErrorOnRenew bool shouldErrorOnRenew bool
certsShouldExist []*certsphase.KubeadmCert certsShouldExist []*certsphase.KubeadmCert
kubeConfigShouldExist []string
}{ }{
{ {
name: "all certs exist, should be rotated for etcd", name: "all CA exist, all certs should be rotated for etcd",
component: constants.Etcd, component: constants.Etcd,
certsShouldExist: []*certsphase.KubeadmCert{ certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer, &certsphase.KubeadmCertEtcdServer,
@ -660,7 +708,7 @@ func TestRenewCertsByComponent(t *testing.T) {
}, },
}, },
{ {
name: "all certs exist, should be rotated for apiserver", name: "all CA exist, all certs should be rotated for apiserver",
component: constants.KubeAPIServer, component: constants.KubeAPIServer,
certsShouldExist: []*certsphase.KubeadmCert{ certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdAPIClient, &certsphase.KubeadmCertEtcdAPIClient,
@ -670,7 +718,7 @@ func TestRenewCertsByComponent(t *testing.T) {
}, },
}, },
{ {
name: "external CA, renew only certificates not signed by CA", name: "external CA, renew only certificates not signed by CA for apiserver",
component: constants.KubeAPIServer, component: constants.KubeAPIServer,
certsShouldExist: []*certsphase.KubeadmCert{ certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdAPIClient, &certsphase.KubeadmCertEtcdAPIClient,
@ -679,7 +727,7 @@ func TestRenewCertsByComponent(t *testing.T) {
externalCA: true, externalCA: true,
}, },
{ {
name: "external front-proxy-CA, renew only certificates not signed by front-proxy-CA", name: "external front-proxy-CA, renew only certificates not signed by front-proxy-CA for apiserver",
component: constants.KubeAPIServer, component: constants.KubeAPIServer,
certsShouldExist: []*certsphase.KubeadmCert{ certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdAPIClient, &certsphase.KubeadmCertEtcdAPIClient,
@ -689,8 +737,18 @@ func TestRenewCertsByComponent(t *testing.T) {
externalFrontProxyCA: true, externalFrontProxyCA: true,
}, },
{ {
name: "ignores other compnonents", name: "all CA exist, should be rotated for scheduler",
component: constants.KubeScheduler, component: constants.KubeScheduler,
kubeConfigShouldExist: []string{
kubeadmconstants.SchedulerKubeConfigFileName,
},
},
{
name: "all CA exist, should be rotated for controller manager",
component: constants.KubeControllerManager,
kubeConfigShouldExist: []string{
kubeadmconstants.ControllerManagerKubeConfigFileName,
},
}, },
{ {
name: "missing a cert to renew", name: "missing a cert to renew",
@ -736,25 +794,36 @@ func TestRenewCertsByComponent(t *testing.T) {
} }
} }
// Create expected certs certMaps := make(map[string]big.Int)
// Create expected certs and load to recorde the serial numbers
for _, kubeCert := range test.certsShouldExist { for _, kubeCert := range test.certsShouldExist {
if err := kubeCert.CreateFromCA(cfg, caCert, caKey); err != nil { if err := kubeCert.CreateFromCA(cfg, caCert, caKey); err != nil {
t.Fatalf("couldn't renew certificate %q: %v", kubeCert.Name, err) t.Fatalf("couldn't create certificate %q: %v", kubeCert.Name, err)
} }
}
// Load expected certs to check if serial numbers changes
certMaps := make(map[*certsphase.KubeadmCert]big.Int)
for _, kubeCert := range test.certsShouldExist {
cert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName) cert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
if err != nil { if err != nil {
t.Fatalf("couldn't load certificate %q: %v", kubeCert.Name, err) t.Fatalf("couldn't load certificate %q: %v", kubeCert.Name, err)
} }
certMaps[kubeCert] = *cert.SerialNumber certMaps[kubeCert.Name] = *cert.SerialNumber
}
// Create expected kubeconfigs
for _, kubeConfig := range test.kubeConfigShouldExist {
if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpDir, cfg); err != nil {
t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err)
}
newCerts, err := getEmbeddedCerts(tmpDir, kubeConfig)
if err != nil {
t.Fatalf("error reading embedded certs from %s: %v", kubeConfig, err)
}
certMaps[kubeConfig] = *newCerts[0].SerialNumber
} }
// Renew everything // Renew everything
err := renewCertsByComponent(cfg, test.component) err := renewCertsByComponent(cfg, tmpDir, test.component)
if test.shouldErrorOnRenew { if test.shouldErrorOnRenew {
if err == nil { if err == nil {
t.Fatal("expected renewal error, got nothing") t.Fatal("expected renewal error, got nothing")
@ -767,17 +836,43 @@ func TestRenewCertsByComponent(t *testing.T) {
} }
// See if the certificate serial numbers change // See if the certificate serial numbers change
for kubeCert, cert := range certMaps { for _, kubeCert := range test.certsShouldExist {
newCert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName) newCert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
if err != nil { if err != nil {
t.Errorf("couldn't load new certificate %q: %v", kubeCert.Name, err) t.Errorf("couldn't load new certificate %q: %v", kubeCert.Name, err)
continue continue
} }
if cert.Cmp(newCert.SerialNumber) == 0 { oldSerial, _ := certMaps[kubeCert.Name]
if oldSerial.Cmp(newCert.SerialNumber) == 0 {
t.Errorf("certifitate %v was not reissued", kubeCert.Name) t.Errorf("certifitate %v was not reissued", kubeCert.Name)
} }
} }
// See if the embedded certificate serial numbers change
for _, kubeConfig := range test.kubeConfigShouldExist {
newCerts, err := getEmbeddedCerts(tmpDir, kubeConfig)
if err != nil {
t.Fatalf("error reading embedded certs from %s: %v", kubeConfig, err)
}
oldSerial, _ := certMaps[kubeConfig]
if oldSerial.Cmp(newCerts[0].SerialNumber) == 0 {
t.Errorf("certifitate %v was not reissued", kubeConfig)
}
}
}) })
} }
} }
func getEmbeddedCerts(tmpDir, kubeConfig string) ([]*x509.Certificate, error) {
kubeconfigPath := filepath.Join(tmpDir, kubeConfig)
newConfig, err := clientcmd.LoadFromFile(kubeconfigPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to load kubeconfig file %s", kubeconfigPath)
}
authInfoName := newConfig.Contexts[newConfig.CurrentContext].AuthInfo
authInfo := newConfig.AuthInfos[authInfoName]
return certutil.ParseCertsPEM(authInfo.ClientCertificateData)
}