mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #76862 from fabriziopandini/fix-upgrade-certs-renew
kubeadm: fix certs renewal during upgrade
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
c88b7cdd58
@ -52,6 +52,7 @@ type applyFlags struct {
|
||||
force bool
|
||||
dryRun bool
|
||||
etcdUpgrade bool
|
||||
renewCerts bool
|
||||
criSocket string
|
||||
imagePullTimeout time.Duration
|
||||
}
|
||||
@ -67,6 +68,7 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command {
|
||||
applyPlanFlags: apf,
|
||||
imagePullTimeout: defaultImagePullTimeout,
|
||||
etcdUpgrade: true,
|
||||
renewCerts: true,
|
||||
// Don't set criSocket to a default value here, as this will override the setting in the stored config in RunApply below.
|
||||
}
|
||||
|
||||
@ -90,6 +92,7 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command {
|
||||
cmd.Flags().BoolVarP(&flags.force, "force", "f", flags.force, "Force upgrading although some requirements might not be met. This also implies non-interactive mode.")
|
||||
cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output what actions would be performed.")
|
||||
cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.")
|
||||
cmd.Flags().BoolVar(&flags.renewCerts, "certificate-renewal", flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
|
||||
cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.")
|
||||
|
||||
// The CRI socket flag is deprecated here, since it should be taken from the NodeRegistrationOptions for the current
|
||||
@ -231,7 +234,7 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w
|
||||
}
|
||||
|
||||
// Don't save etcd backup directory if etcd is HA, as this could cause corruption
|
||||
return PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade)
|
||||
return PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts)
|
||||
}
|
||||
|
||||
// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration.
|
||||
@ -241,14 +244,14 @@ func GetPathManagerForUpgrade(internalcfg *kubeadmapi.InitConfiguration, etcdUpg
|
||||
}
|
||||
|
||||
// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
|
||||
func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) error {
|
||||
func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error {
|
||||
pathManager, err := GetPathManagerForUpgrade(internalcfg, etcdUpgrade)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing
|
||||
return upgrade.StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, nil, nil)
|
||||
return upgrade.StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil)
|
||||
}
|
||||
|
||||
// DryRunStaticPodUpgrade fakes an upgrade of the control plane
|
||||
|
||||
@ -66,6 +66,7 @@ type controlplaneUpgradeFlags struct {
|
||||
advertiseAddress string
|
||||
nodeName string
|
||||
etcdUpgrade bool
|
||||
renewCerts bool
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
@ -114,6 +115,7 @@ func NewCmdUpgradeControlPlane() *cobra.Command {
|
||||
kubeConfigPath: constants.GetKubeletKubeConfigPath(),
|
||||
advertiseAddress: "",
|
||||
etcdUpgrade: true,
|
||||
renewCerts: true,
|
||||
dryRun: false,
|
||||
}
|
||||
|
||||
@ -151,6 +153,7 @@ func NewCmdUpgradeControlPlane() *cobra.Command {
|
||||
options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath)
|
||||
cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output the actions that would be performed.")
|
||||
cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.")
|
||||
cmd.Flags().BoolVar(&flags.renewCerts, "certificate-renewal", flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
|
||||
return cmd
|
||||
}
|
||||
|
||||
@ -221,18 +224,13 @@ func RunUpgradeControlPlane(flags *controlplaneUpgradeFlags) error {
|
||||
return errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
|
||||
}
|
||||
|
||||
// Rotate API server certificate if needed
|
||||
if err := upgrade.BackupAPIServerCertIfNeeded(cfg, flags.dryRun); err != nil {
|
||||
return errors.Wrap(err, "unable to rotate API server certificate")
|
||||
}
|
||||
|
||||
// Upgrade the control plane and etcd if installed on this node
|
||||
fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion)
|
||||
if flags.dryRun {
|
||||
return DryRunStaticPodUpgrade(cfg)
|
||||
}
|
||||
|
||||
if err := PerformStaticPodUpgrade(client, waiter, cfg, flags.etcdUpgrade); err != nil {
|
||||
if err := PerformStaticPodUpgrade(client, waiter, cfg, flags.etcdUpgrade, flags.renewCerts); err != nil {
|
||||
return errors.Wrap(err, "couldn't complete the static pod upgrade")
|
||||
}
|
||||
|
||||
|
||||
@ -15,7 +15,6 @@ go_library(
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||
"//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library",
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/images:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/addons/dns:go_default_library",
|
||||
@ -45,7 +44,6 @@ go_library(
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
|
||||
"//vendor/github.com/pkg/errors:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@ -17,7 +17,6 @@ limitations under the License.
|
||||
package upgrade
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
@ -29,15 +28,12 @@ import (
|
||||
errorsutil "k8s.io/apimachinery/pkg/util/errors"
|
||||
"k8s.io/apimachinery/pkg/util/version"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
certutil "k8s.io/client-go/util/cert"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
|
||||
nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
|
||||
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
|
||||
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
|
||||
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
|
||||
@ -101,11 +97,6 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
// Rotate the kube-apiserver cert and key if needed
|
||||
if err := BackupAPIServerCertIfNeeded(cfg, dryRun); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
// Upgrade kube-dns/CoreDNS and kube-proxy
|
||||
if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil {
|
||||
errs = append(errs, err)
|
||||
@ -152,37 +143,6 @@ func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.ClusterConfigurati
|
||||
}, 10)
|
||||
}
|
||||
|
||||
// BackupAPIServerCertIfNeeded rotates the kube-apiserver certificate if older than 180 days
|
||||
func BackupAPIServerCertIfNeeded(cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
|
||||
certAndKeyDir := kubeadmapiv1beta2.DefaultCertificatesDir
|
||||
shouldBackup, err := shouldBackupAPIServerCertAndKey(certAndKeyDir)
|
||||
if err != nil {
|
||||
// Don't fail the upgrade phase if failing to determine to backup kube-apiserver cert and key.
|
||||
return errors.Wrap(err, "[postupgrade] WARNING: failed to determine to backup kube-apiserver cert and key")
|
||||
}
|
||||
|
||||
if !shouldBackup {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If dry-running, just say that this would happen to the user and exit
|
||||
if dryRun {
|
||||
fmt.Println("[postupgrade] Would rotate the API server certificate and key.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Don't fail the upgrade phase if failing to backup kube-apiserver cert and key, just continue rotating the cert
|
||||
// TODO: We might want to reconsider this choice.
|
||||
if err := backupAPIServerCertAndKey(certAndKeyDir); err != nil {
|
||||
fmt.Printf("[postupgrade] WARNING: failed to backup kube-apiserver cert and key: %v\n", err)
|
||||
}
|
||||
return certsphase.CreateCertAndKeyFilesWithCA(
|
||||
&certsphase.KubeadmCertAPIServer,
|
||||
&certsphase.KubeadmCertRootCA,
|
||||
cfg,
|
||||
)
|
||||
}
|
||||
|
||||
func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
|
||||
kubeletDir, err := GetKubeletDir(dryRun)
|
||||
if err != nil {
|
||||
@ -228,20 +188,6 @@ func GetKubeletDir(dryRun bool) (string, error) {
|
||||
return kubeadmconstants.KubeletRunDirectory, nil
|
||||
}
|
||||
|
||||
// backupAPIServerCertAndKey backups the old cert and key of kube-apiserver to a specified directory.
|
||||
func backupAPIServerCertAndKey(certAndKeyDir string) error {
|
||||
subDir := filepath.Join(certAndKeyDir, "expired")
|
||||
if err := os.Mkdir(subDir, 0700); err != nil {
|
||||
return errors.Wrapf(err, "failed to created backup directory %s", subDir)
|
||||
}
|
||||
|
||||
filesToMove := map[string]string{
|
||||
filepath.Join(certAndKeyDir, kubeadmconstants.APIServerCertName): filepath.Join(subDir, kubeadmconstants.APIServerCertName),
|
||||
filepath.Join(certAndKeyDir, kubeadmconstants.APIServerKeyName): filepath.Join(subDir, kubeadmconstants.APIServerKeyName),
|
||||
}
|
||||
return moveFiles(filesToMove)
|
||||
}
|
||||
|
||||
// moveFiles moves files from one directory to another.
|
||||
func moveFiles(files map[string]string) error {
|
||||
filesToRecover := map[string]string{}
|
||||
@ -264,21 +210,3 @@ func rollbackFiles(files map[string]string, originalErr error) error {
|
||||
}
|
||||
return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
|
||||
}
|
||||
|
||||
// shouldBackupAPIServerCertAndKey checks if the cert of kube-apiserver will be expired in 180 days.
|
||||
func shouldBackupAPIServerCertAndKey(certAndKeyDir string) (bool, error) {
|
||||
apiServerCert := filepath.Join(certAndKeyDir, kubeadmconstants.APIServerCertName)
|
||||
certs, err := certutil.CertsFromFile(apiServerCert)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "couldn't load the certificate file %s", apiServerCert)
|
||||
}
|
||||
if len(certs) == 0 {
|
||||
return false, errors.New("no certificate data found")
|
||||
}
|
||||
|
||||
if time.Since(certs[0].NotBefore) > expiry {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@ -21,40 +21,13 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
|
||||
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
||||
)
|
||||
|
||||
func TestBackupAPIServerCertAndKey(t *testing.T) {
|
||||
tmpdir := testutil.SetupTempDir(t)
|
||||
defer os.RemoveAll(tmpdir)
|
||||
os.Chmod(tmpdir, 0766)
|
||||
|
||||
certPath := filepath.Join(tmpdir, constants.APIServerCertName)
|
||||
certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create cert file %s: %v", certPath, err)
|
||||
}
|
||||
defer certFile.Close()
|
||||
|
||||
keyPath := filepath.Join(tmpdir, constants.APIServerKeyName)
|
||||
keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create key file %s: %v", keyPath, err)
|
||||
}
|
||||
defer keyFile.Close()
|
||||
|
||||
if err := backupAPIServerCertAndKey(tmpdir); err != nil {
|
||||
t.Fatalf("Failed to backup cert and key in dir %s: %v", tmpdir, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFiles(t *testing.T) {
|
||||
tmpdir := testutil.SetupTempDir(t)
|
||||
defer os.RemoveAll(tmpdir)
|
||||
@ -128,59 +101,3 @@ func TestRollbackFiles(t *testing.T) {
|
||||
t.Fatalf("Expected error contains %q, got %v", errString, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
|
||||
cfg := &kubeadmapi.InitConfiguration{
|
||||
LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
|
||||
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
|
||||
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
|
||||
},
|
||||
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "test-node"},
|
||||
}
|
||||
|
||||
for desc, test := range map[string]struct {
|
||||
adjustedExpiry time.Duration
|
||||
expected bool
|
||||
}{
|
||||
"default: cert not older than 180 days doesn't needs to backup": {
|
||||
expected: false,
|
||||
},
|
||||
"cert older than 180 days need to backup": {
|
||||
adjustedExpiry: expiry + 100*time.Hour,
|
||||
expected: true,
|
||||
},
|
||||
} {
|
||||
t.Run(desc, func(t *testing.T) {
|
||||
tmpdir := testutil.SetupTempDir(t)
|
||||
defer os.RemoveAll(tmpdir)
|
||||
cfg.CertificatesDir = tmpdir
|
||||
|
||||
caCert, caKey, err := certsphase.KubeadmCertRootCA.CreateAsCA(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed creation of ca cert and key: %v", err)
|
||||
}
|
||||
caCert.NotBefore = caCert.NotBefore.Add(-test.adjustedExpiry).UTC()
|
||||
|
||||
err = certsphase.KubeadmCertAPIServer.CreateFromCA(cfg, caCert, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %s: failed creation of cert and key: %v", desc, err)
|
||||
}
|
||||
|
||||
certAndKey := []string{filepath.Join(tmpdir, constants.APIServerCertName), filepath.Join(tmpdir, constants.APIServerKeyName)}
|
||||
for _, path := range certAndKey {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
t.Fatalf("Test %s: %s not exist: %v", desc, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
shouldBackup, err := shouldBackupAPIServerCertAndKey(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %s: failed to check shouldBackupAPIServerCertAndKey: %v", desc, err)
|
||||
}
|
||||
|
||||
if shouldBackup != test.expected {
|
||||
t.Fatalf("Test %s: shouldBackupAPIServerCertAndKey expected %v, got %v", desc, test.expected, shouldBackup)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -167,7 +167,7 @@ func (spm *KubeStaticPodPathManager) CleanupDirs() error {
|
||||
return utilerrors.NewAggregate(errlist)
|
||||
}
|
||||
|
||||
func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string) error {
|
||||
func upgradeComponent(component string, renewCerts bool, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string) error {
|
||||
// Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd
|
||||
// manifests only for the case when component is Etcd
|
||||
recoverEtcd := false
|
||||
@ -176,9 +176,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
|
||||
recoverEtcd = true
|
||||
}
|
||||
|
||||
if err := renewCerts(cfg, component); err != nil {
|
||||
return errors.Wrapf(err, "failed to renew certificates for component %q", component)
|
||||
}
|
||||
fmt.Printf("[upgrade/staticpods] Preparing for %q upgrade\n", component)
|
||||
|
||||
// The old manifest is here; in the /etc/kubernetes/manifests/
|
||||
currentManifestPath := pathMgr.RealManifestPath(component)
|
||||
@ -201,6 +199,14 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
|
||||
return nil
|
||||
}
|
||||
|
||||
// if certificate renewal should be performed
|
||||
if renewCerts {
|
||||
// renew all the certificates used by the current component
|
||||
if err := renewCertsByComponent(cfg, component); err != nil {
|
||||
return errors.Wrapf(err, "failed to renew certificates for component %q", component)
|
||||
}
|
||||
}
|
||||
|
||||
// Move the old manifest into the old-manifests directory
|
||||
if err := pathMgr.MoveFile(currentManifestPath, backupManifestPath); err != nil {
|
||||
return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
|
||||
@ -239,7 +245,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
|
||||
}
|
||||
|
||||
// performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error.
|
||||
func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
|
||||
func performEtcdStaticPodUpgrade(renewCerts bool, client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
|
||||
// Add etcd static pod spec only if external etcd is not configured
|
||||
if cfg.Etcd.External != nil {
|
||||
return false, errors.New("external etcd detected, won't try to change any etcd state")
|
||||
@ -303,7 +309,7 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa
|
||||
retryInterval := 15 * time.Second
|
||||
|
||||
// Perform etcd upgrade using common to all control plane components function
|
||||
if err := upgradeComponent(constants.Etcd, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil {
|
||||
if err := upgradeComponent(constants.Etcd, renewCerts, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil {
|
||||
fmt.Printf("[upgrade/etcd] Failed to upgrade etcd: %v\n", err)
|
||||
// Since upgrade component failed, the old etcd manifest has either been restored or was never touched
|
||||
// Now we need to check the health of etcd cluster if it is up with old manifest
|
||||
@ -379,7 +385,7 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa
|
||||
}
|
||||
|
||||
// StaticPodControlPlane upgrades a static pod-hosted control plane
|
||||
func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
|
||||
func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
|
||||
recoverManifests := map[string]string{}
|
||||
var isExternalEtcd bool
|
||||
|
||||
@ -422,7 +428,7 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
|
||||
fmt.Printf("[upgrade/etcd] Upgrading to TLS for %s\n", constants.Etcd)
|
||||
|
||||
// Perform etcd upgrade using common to all control plane components function
|
||||
fatal, err := performEtcdStaticPodUpgrade(client, waiter, pathMgr, cfg, recoverManifests, oldEtcdClient, newEtcdClient)
|
||||
fatal, err := performEtcdStaticPodUpgrade(renewCerts, client, waiter, pathMgr, cfg, recoverManifests, oldEtcdClient, newEtcdClient)
|
||||
if err != nil {
|
||||
if fatal {
|
||||
return err
|
||||
@ -439,7 +445,7 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
|
||||
}
|
||||
|
||||
for _, component := range constants.ControlPlaneComponents {
|
||||
if err = upgradeComponent(component, waiter, pathMgr, cfg, beforePodHashMap[component], recoverManifests); err != nil {
|
||||
if err = upgradeComponent(component, renewCerts, waiter, pathMgr, cfg, beforePodHashMap[component], recoverManifests); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -487,34 +493,88 @@ func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathMa
|
||||
return nil
|
||||
}
|
||||
|
||||
func renewCerts(cfg *kubeadmapi.InitConfiguration, component string) error {
|
||||
// renewCertsByComponent takes charge of renewing certificates used by a specific component before
|
||||
// the static pod of the component is upgraded
|
||||
func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string) error {
|
||||
// if the cluster is using a local etcd
|
||||
if cfg.Etcd.Local != nil {
|
||||
// ensure etcd certs are loaded for etcd and kube-apiserver
|
||||
if component == constants.Etcd || component == constants.KubeAPIServer {
|
||||
// try to load the etcd CA
|
||||
caCert, caKey, err := certsphase.LoadCertificateAuthority(cfg.CertificatesDir, certsphase.KubeadmCertEtcdCA.BaseName)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to upgrade the %s CA certificate and key", constants.Etcd)
|
||||
}
|
||||
// create a renewer for certificates signed by etcd CA
|
||||
renewer := renewal.NewFileRenewal(caCert, caKey)
|
||||
|
||||
// then, if upgrading the etcd component, renew all the certificates signed by etcd CA and used
|
||||
// by etcd itself (the etcd-server, the etcd-peer and the etcd-healthcheck-client certificate)
|
||||
if component == constants.Etcd {
|
||||
for _, cert := range []*certsphase.KubeadmCert{
|
||||
&certsphase.KubeadmCertEtcdServer,
|
||||
&certsphase.KubeadmCertEtcdPeer,
|
||||
&certsphase.KubeadmCertEtcdHealthcheck,
|
||||
} {
|
||||
fmt.Printf("[upgrade/staticpods] Renewing %q certificate\n", cert.BaseName)
|
||||
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
|
||||
return errors.Wrapf(err, "failed to renew %s certificate and key", cert.Name)
|
||||
return errors.Wrapf(err, "failed to renew %s certificates", cert.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
// if upgrading the apiserver component, renew the certificate signed by etcd CA and used
|
||||
// by the apiserver (the apiserver-etcd-client certificate)
|
||||
if component == constants.KubeAPIServer {
|
||||
cert := certsphase.KubeadmCertEtcdAPIClient
|
||||
fmt.Printf("[upgrade/staticpods] Renewing %q certificate\n", cert.BaseName)
|
||||
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
|
||||
return errors.Wrapf(err, "failed to renew %s certificate and key", cert.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if component == constants.KubeAPIServer {
|
||||
// Checks if an external CA is provided by the user (when the CA Cert is present but the CA Key is not)
|
||||
// if not, then CA is managed by kubeadm, so it is possible to renew all the certificates signed by ca
|
||||
// and used the apis server (the apiserver certificate and the apiserver-kubelet-client certificate)
|
||||
externalCA, _ := certsphase.UsingExternalCA(&cfg.ClusterConfiguration)
|
||||
if !externalCA {
|
||||
// try to load ca
|
||||
caCert, caKey, err := certsphase.LoadCertificateAuthority(cfg.CertificatesDir, certsphase.KubeadmCertRootCA.BaseName)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to upgrade the %s certificates", constants.KubeAPIServer)
|
||||
}
|
||||
// create a renewer for certificates signed by CA
|
||||
renewer := renewal.NewFileRenewal(caCert, caKey)
|
||||
// renew the certificates
|
||||
for _, cert := range []*certsphase.KubeadmCert{
|
||||
&certsphase.KubeadmCertAPIServer,
|
||||
&certsphase.KubeadmCertKubeletClient,
|
||||
} {
|
||||
fmt.Printf("[upgrade/staticpods] Renewing %q certificate\n", cert.BaseName)
|
||||
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
|
||||
return errors.Wrapf(err, "failed to renew %s certificate and key", cert.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Checks if an external Front-Proxy CA is provided by the user (when the Front-Proxy CA Cert is present but the Front-Proxy CA Key is not)
|
||||
// if not, then Front-Proxy CA is managed by kubeadm, so it is possible to renew all the certificates signed by ca
|
||||
// and used the apis server (the front-proxy-client certificate)
|
||||
externalFrontProxyCA, _ := certsphase.UsingExternalFrontProxyCA(&cfg.ClusterConfiguration)
|
||||
if !externalFrontProxyCA {
|
||||
// try to load front-proxy-ca
|
||||
caCert, caKey, err := certsphase.LoadCertificateAuthority(cfg.CertificatesDir, certsphase.KubeadmCertFrontProxyCA.BaseName)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to upgrade the %s certificates", constants.KubeAPIServer)
|
||||
}
|
||||
// create a renewer for certificates signed by Front-Proxy CA
|
||||
renewer := renewal.NewFileRenewal(caCert, caKey)
|
||||
// renew the certificates
|
||||
cert := certsphase.KubeadmCertFrontProxyClient
|
||||
fmt.Printf("[upgrade/staticpods] Renewing %q certificate\n", cert.BaseName)
|
||||
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
|
||||
return errors.Wrapf(err, "failed to renew %s certificate and key", cert.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -505,6 +505,7 @@ func TestStaticPodControlPlane(t *testing.T) {
|
||||
pathMgr,
|
||||
newcfg,
|
||||
true,
|
||||
true,
|
||||
fakeTLSEtcdClient{
|
||||
TLS: false,
|
||||
},
|
||||
@ -637,19 +638,20 @@ func TestCleanupDirs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenewCerts(t *testing.T) {
|
||||
func TestRenewCertsByComponent(t *testing.T) {
|
||||
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t)
|
||||
t.Run("all certs exist, should be rotated", func(t *testing.T) {
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
component string
|
||||
skipCreateCA bool
|
||||
shouldErrorOnRenew bool
|
||||
certsShouldExist []*certsphase.KubeadmCert
|
||||
name string
|
||||
component string
|
||||
externalCA bool
|
||||
externalFrontProxyCA bool
|
||||
skipCreateEtcdCA bool
|
||||
shouldErrorOnRenew bool
|
||||
certsShouldExist []*certsphase.KubeadmCert
|
||||
}{
|
||||
{
|
||||
name: "all certs exist, should be rotated",
|
||||
name: "all certs exist, should be rotated for etcd",
|
||||
component: constants.Etcd,
|
||||
certsShouldExist: []*certsphase.KubeadmCert{
|
||||
&certsphase.KubeadmCertEtcdServer,
|
||||
@ -658,16 +660,37 @@ func TestRenewCerts(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "just renew API cert",
|
||||
name: "all certs exist, should be rotated for apiserver",
|
||||
component: constants.KubeAPIServer,
|
||||
certsShouldExist: []*certsphase.KubeadmCert{
|
||||
&certsphase.KubeadmCertEtcdAPIClient,
|
||||
&certsphase.KubeadmCertAPIServer,
|
||||
&certsphase.KubeadmCertKubeletClient,
|
||||
&certsphase.KubeadmCertFrontProxyClient,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignores other compnonents",
|
||||
skipCreateCA: true,
|
||||
component: constants.KubeScheduler,
|
||||
name: "external CA, renew only certificates not signed by CA",
|
||||
component: constants.KubeAPIServer,
|
||||
certsShouldExist: []*certsphase.KubeadmCert{
|
||||
&certsphase.KubeadmCertEtcdAPIClient,
|
||||
&certsphase.KubeadmCertFrontProxyClient,
|
||||
},
|
||||
externalCA: true,
|
||||
},
|
||||
{
|
||||
name: "external front-proxy-CA, renew only certificates not signed by front-proxy-CA",
|
||||
component: constants.KubeAPIServer,
|
||||
certsShouldExist: []*certsphase.KubeadmCert{
|
||||
&certsphase.KubeadmCertEtcdAPIClient,
|
||||
&certsphase.KubeadmCertAPIServer,
|
||||
&certsphase.KubeadmCertKubeletClient,
|
||||
},
|
||||
externalFrontProxyCA: true,
|
||||
},
|
||||
{
|
||||
name: "ignores other compnonents",
|
||||
component: constants.KubeScheduler,
|
||||
},
|
||||
{
|
||||
name: "missing a cert to renew",
|
||||
@ -681,7 +704,7 @@ func TestRenewCerts(t *testing.T) {
|
||||
{
|
||||
name: "no CA, cannot continue",
|
||||
component: constants.Etcd,
|
||||
skipCreateCA: true,
|
||||
skipCreateEtcdCA: true,
|
||||
shouldErrorOnRenew: true,
|
||||
},
|
||||
}
|
||||
@ -695,9 +718,21 @@ func TestRenewCerts(t *testing.T) {
|
||||
cfg := testutil.GetDefaultInternalConfig(t)
|
||||
cfg.CertificatesDir = tmpDir
|
||||
|
||||
if !test.skipCreateCA {
|
||||
if err := pkiutil.WriteCertAndKey(tmpDir, constants.CACertAndKeyBaseName, caCert, caKey); err != nil {
|
||||
t.Fatalf("couldn't write out CA: %v", err)
|
||||
}
|
||||
if test.externalCA {
|
||||
os.Remove(filepath.Join(tmpDir, constants.CAKeyName))
|
||||
}
|
||||
if err := pkiutil.WriteCertAndKey(tmpDir, constants.FrontProxyCACertAndKeyBaseName, caCert, caKey); err != nil {
|
||||
t.Fatalf("couldn't write out front-proxy-CA: %v", err)
|
||||
}
|
||||
if test.externalFrontProxyCA {
|
||||
os.Remove(filepath.Join(tmpDir, constants.FrontProxyCAKeyName))
|
||||
}
|
||||
if !test.skipCreateEtcdCA {
|
||||
if err := pkiutil.WriteCertAndKey(tmpDir, constants.EtcdCACertAndKeyBaseName, caCert, caKey); err != nil {
|
||||
t.Fatalf("couldn't write out CA: %v", err)
|
||||
t.Fatalf("couldn't write out etcd-CA: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -719,7 +754,7 @@ func TestRenewCerts(t *testing.T) {
|
||||
}
|
||||
|
||||
// Renew everything
|
||||
err := renewCerts(cfg, test.component)
|
||||
err := renewCertsByComponent(cfg, test.component)
|
||||
if test.shouldErrorOnRenew {
|
||||
if err == nil {
|
||||
t.Fatal("expected renewal error, got nothing")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user