Merge pull request #55998 from xiangpengzhao/regen-apiserver-crt

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Regenerate API server serving certificates when upgrading.

**What this PR does / why we need it**:
TODO: 
- [x] check the age of crt.
- [x] check the new version number.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes https://github.com/kubernetes/kubeadm/issues/548

**Special notes for your reviewer**:
/cc @luxas 

**Release note**:

```release-note
NONE
```

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -167,7 +167,7 @@ func RunApply(flags *applyFlags) error {
 	}
 
 	// Upgrade RBAC rules and addons.
-	if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg); err != nil {
+	if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion); err != nil {
 		return fmt.Errorf("[upgrade/postupgrade] FATAL post-upgrade error: %v", err)
 	}
 

cmd/kubeadm/app/phases/upgrade/BUILD

@@ -8,6 +8,7 @@ go_library(
         "health.go",
         "policy.go",
         "postupgrade.go",
+        "postupgrade_v18_19.go",
         "prepull.go",
         "selfhosted.go",
         "staticpods.go",
@@ -25,6 +26,7 @@ go_library(
         "//cmd/kubeadm/app/phases/addons/proxy:go_default_library",
         "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
         "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/etcd:go_default_library",
         "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
@@ -64,6 +66,7 @@ go_test(
     srcs = [
         "compute_test.go",
         "policy_test.go",
+        "postupgrade_v18_19_test.go",
         "prepull_test.go",
         "staticpods_test.go",
     ],
@@ -73,9 +76,12 @@ go_test(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
         "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/etcd:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/test:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -17,19 +17,24 @@ limitations under the License.
 package upgrade
 
 import (
+	"fmt"
+
 	"k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
 	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
+	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 // PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
 // Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
-func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration) error {
+func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version) error {
 	errs := []error{}
 
 	// Upload currently used configuration to the cluster
@@ -64,6 +69,21 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
 		errs = append(errs, err)
 	}
 
+	certAndKeyDir := kubeadmapiext.DefaultCertificatesDir
+	shouldBackup, err := shouldBackupAPIServerCertAndKey(certAndKeyDir, newK8sVer)
+	// Don't fail the upgrade phase if failing to determine to backup kube-apiserver cert and key.
+	if err != nil {
+		fmt.Printf("[postupgrade] WARNING: failed to determine to backup kube-apiserver cert and key: %v", err)
+	} else if shouldBackup {
+		// Don't fail the upgrade phase if failing to backup kube-apiserver cert and key.
+		if err := backupAPIServerCertAndKey(certAndKeyDir); err != nil {
+			fmt.Printf("[postupgrade] WARNING: failed to backup kube-apiserver cert and key: %v", err)
+		}
+		if err := certsphase.CreateAPIServerCertAndKeyFiles(cfg); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	// Upgrade kube-dns and kube-proxy
 	if err := dns.EnsureDNSAddon(cfg, client); err != nil {
 		errs = append(errs, err)

cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+var v190 = version.MustParseSemantic("v1.9.0")
+var expiry = 180 * 24 * time.Hour
+
+// backupAPIServerCertAndKey backups the old cert and key of kube-apiserver to a specified directory.
+func backupAPIServerCertAndKey(certAndKeyDir string) error {
+	subDir := filepath.Join(certAndKeyDir, "expired")
+	if err := os.Mkdir(subDir, 0766); err != nil {
+		return fmt.Errorf("failed to created backup directory %s: %v", subDir, err)
+	}
+
+	filesToMove := map[string]string{
+		filepath.Join(certAndKeyDir, constants.APIServerCertName): filepath.Join(subDir, constants.APIServerCertName),
+		filepath.Join(certAndKeyDir, constants.APIServerKeyName):  filepath.Join(subDir, constants.APIServerKeyName),
+	}
+	return moveFiles(filesToMove)
+}
+
+// moveFiles moves files from one directory to another.
+func moveFiles(files map[string]string) error {
+	filesToRecover := map[string]string{}
+	for from, to := range files {
+		if err := os.Rename(from, to); err != nil {
+			return rollbackFiles(filesToRecover, err)
+		}
+		filesToRecover[to] = from
+	}
+	return nil
+}
+
+// rollbackFiles moves the files back to the original directory.
+func rollbackFiles(files map[string]string, originalErr error) error {
+	errs := []error{originalErr}
+	for from, to := range files {
+		if err := os.Rename(from, to); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return fmt.Errorf("couldn't move these files: %v. Got errors: %v", files, errors.NewAggregate(errs))
+}
+
+// shouldBackupAPIServerCertAndKey check if the new k8s version is at least 1.9.0
+// and kube-apiserver will be expired in 60 days.
+func shouldBackupAPIServerCertAndKey(certAndKeyDir string, newK8sVer *version.Version) (bool, error) {
+	if newK8sVer.LessThan(v190) {
+		return false, nil
+	}
+
+	apiServerCert := filepath.Join(certAndKeyDir, constants.APIServerCertName)
+	data, err := ioutil.ReadFile(apiServerCert)
+	if err != nil {
+		return false, fmt.Errorf("failed to read kube-apiserver certificate from disk: %v", err)
+	}
+
+	block, _ := pem.Decode(data)
+	if block == nil {
+		return false, fmt.Errorf("expected the kube-apiserver certificate to be PEM encoded")
+	}
+
+	certs, err := x509.ParseCertificates(block.Bytes)
+	if err != nil {
+		return false, fmt.Errorf("unable to parse certificate data: %v", err)
+	}
+	if len(certs) == 0 {
+		return false, fmt.Errorf("no certificate data found")
+	}
+
+	if time.Now().Sub(certs[0].NotBefore) > expiry {
+		return true, nil
+	}
+
+	return false, nil
+}

cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19_test.go

@@ -0,0 +1,192 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
+	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+func TestBackupAPIServerCertAndKey(t *testing.T) {
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+	os.Chmod(tmpdir, 0766)
+
+	certPath := filepath.Join(tmpdir, constants.APIServerCertName)
+	certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create cert file %s: %v", certPath, err)
+	}
+	defer certFile.Close()
+
+	keyPath := filepath.Join(tmpdir, constants.APIServerKeyName)
+	keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create key file %s: %v", keyPath, err)
+	}
+	defer keyFile.Close()
+
+	if err := backupAPIServerCertAndKey(tmpdir); err != nil {
+		t.Fatalf("Failed to backup cert and key in dir %s: %v", tmpdir, err)
+	}
+}
+
+func TestMoveFiles(t *testing.T) {
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+	os.Chmod(tmpdir, 0766)
+
+	certPath := filepath.Join(tmpdir, constants.APIServerCertName)
+	certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create cert file %s: %v", certPath, err)
+	}
+	defer certFile.Close()
+
+	keyPath := filepath.Join(tmpdir, constants.APIServerKeyName)
+	keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create key file %s: %v", keyPath, err)
+	}
+	defer keyFile.Close()
+
+	subDir := filepath.Join(tmpdir, "expired")
+	if err := os.Mkdir(subDir, 0766); err != nil {
+		t.Fatalf("Failed to create backup directory %s: %v", subDir, err)
+	}
+
+	filesToMove := map[string]string{
+		filepath.Join(tmpdir, constants.APIServerCertName): filepath.Join(subDir, constants.APIServerCertName),
+		filepath.Join(tmpdir, constants.APIServerKeyName):  filepath.Join(subDir, constants.APIServerKeyName),
+	}
+
+	if err := moveFiles(filesToMove); err != nil {
+		t.Fatalf("Failed to move files %v: %v", filesToMove, err)
+	}
+}
+
+func TestRollbackFiles(t *testing.T) {
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+	os.Chmod(tmpdir, 0766)
+
+	subDir := filepath.Join(tmpdir, "expired")
+	if err := os.Mkdir(subDir, 0766); err != nil {
+		t.Fatalf("Failed to create backup directory %s: %v", subDir, err)
+	}
+
+	certPath := filepath.Join(subDir, constants.APIServerCertName)
+	certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create cert file %s: %v", certPath, err)
+	}
+	defer certFile.Close()
+
+	keyPath := filepath.Join(subDir, constants.APIServerKeyName)
+	keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create key file %s: %v", keyPath, err)
+	}
+	defer keyFile.Close()
+
+	filesToRollBack := map[string]string{
+		filepath.Join(subDir, constants.APIServerCertName): filepath.Join(tmpdir, constants.APIServerCertName),
+		filepath.Join(subDir, constants.APIServerKeyName):  filepath.Join(tmpdir, constants.APIServerKeyName),
+	}
+
+	errString := "there are files need roll back"
+	originalErr := errors.New(errString)
+	err = rollbackFiles(filesToRollBack, originalErr)
+	if err == nil {
+		t.Fatalf("Expected error contains %q, got nil", errString)
+	}
+	if !strings.Contains(err.Error(), errString) {
+		t.Fatalf("Expected error contains %q, got %v", errString, err)
+	}
+}
+
+func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
+	cfg := &kubeadmapi.MasterConfiguration{
+		API:        kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
+		Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
+		NodeName:   "test-node",
+	}
+	caCert, caKey, err := certsphase.NewCACertAndKey()
+	if err != nil {
+		t.Fatalf("failed creation of ca cert and key: %v", err)
+	}
+
+	for desc, test := range map[string]struct {
+		adjustedExpiry time.Duration
+		k8sVersion     *version.Version
+		expected       bool
+	}{
+		"1.8 version doesn't need to backup": {
+			k8sVersion: version.MustParseSemantic("v1.8.0"),
+			expected:   false,
+		},
+		"1.9 version with cert not older than 180 days doesn't needs to backup": {
+			k8sVersion: version.MustParseSemantic("v1.9.0"),
+			expected:   false,
+		},
+		"1.9 version with cert older than 180 days need to backup": {
+			adjustedExpiry: expiry + 100*time.Hour,
+			k8sVersion:     version.MustParseSemantic("v1.9.0"),
+			expected:       true,
+		},
+	} {
+		caCert.NotBefore = caCert.NotBefore.Add(-test.adjustedExpiry).UTC()
+		apiCert, apiKey, err := certsphase.NewAPIServerCertAndKey(cfg, caCert, caKey)
+		if err != nil {
+			t.Fatalf("Test %s: failed creation of cert and key: %v", desc, err)
+		}
+
+		tmpdir := testutil.SetupTempDir(t)
+		defer os.RemoveAll(tmpdir)
+
+		if err := pkiutil.WriteCertAndKey(tmpdir, constants.APIServerCertAndKeyBaseName, apiCert, apiKey); err != nil {
+			t.Fatalf("Test %s: failure while saving %s certificate and key: %v", desc, constants.APIServerCertAndKeyBaseName, err)
+		}
+
+		certAndKey := []string{filepath.Join(tmpdir, constants.APIServerCertName), filepath.Join(tmpdir, constants.APIServerKeyName)}
+		for _, path := range certAndKey {
+			if _, err := os.Stat(path); os.IsNotExist(err) {
+				t.Fatalf("Test %s: %s not exist: %v", desc, path, err)
+			}
+		}
+
+		shouldBackup, err := shouldBackupAPIServerCertAndKey(tmpdir, test.k8sVersion)
+		if err != nil {
+			t.Fatalf("Test %s: failed to check shouldBackupAPIServerCertAndKey: %v", desc, err)
+		}
+
+		if shouldBackup != test.expected {
+			t.Fatalf("Test %s: shouldBackupAPIServerCertAndKey expected %v, got %v", desc, test.expected, shouldBackup)
+		}
+	}
+}