From bb689eb2bb96505b5833ff73fc53c7bcc2170c28 Mon Sep 17 00:00:00 2001 From: leigh schrandt Date: Fri, 15 Dec 2017 14:50:31 -0700 Subject: [PATCH] Secure etcd API /w TLS on kubeadm init [kubeadm/#594] - Generate Server and Peer cert for etcd - Generate Client cert for apiserver - Add flags / hostMounts for etcd static pod - Add flags / hostMounts for apiserver static pod - Generate certs on upgrade of static-pods for etcd/kube-apiserver - Modify logic for appending etcd flags to staticpod to be safer for external etcd --- cmd/kubeadm/app/cmd/phases/certs.go | 45 +++++ cmd/kubeadm/app/constants/constants.go | 27 +++ cmd/kubeadm/app/phases/certs/certs.go | 133 ++++++++++++++ cmd/kubeadm/app/phases/certs/certs_test.go | 86 +++++++++ cmd/kubeadm/app/phases/certs/doc.go | 8 +- .../app/phases/controlplane/manifests.go | 40 +++-- .../app/phases/controlplane/manifests_test.go | 167 ++++++++++++++++-- cmd/kubeadm/app/phases/etcd/local.go | 24 ++- cmd/kubeadm/app/phases/etcd/local_test.go | 40 ++++- cmd/kubeadm/app/phases/upgrade/staticpods.go | 29 ++- .../app/phases/upgrade/staticpods_test.go | 44 ++++- 11 files changed, 587 insertions(+), 56 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index b63cf1bbdce..5d4e040d7e3 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -74,6 +74,33 @@ var ( If both files already exist, kubeadm skips the generation step and existing files will be used. `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName) + etcdCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the etcd serving certificate and key and saves them into %s and %s files. + + The certificate includes default subject alternative names and additional sans eventually provided by the user; + default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, + kubernetes.default.svc., (that is the .10 address in address space). + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdCertName, kubeadmconstants.EtcdKeyName) + + etcdPeerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the etcd peer certificate and key and saves them into %s and %s files. + + The certificate includes default subject alternative names and additional sans eventually provided by the user; + default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, + kubernetes.default.svc., (that is the .10 address in address space). + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName) + + apiServerEtcdCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the client certificate for the API server to connect to etcd securely and the respective key, + and saves them into %s and %s files. + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName) + saKeyLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the private key for signing service account tokens along with its public key, and saves them into %s and %s files. @@ -157,6 +184,24 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { long: apiServerKubeletCertLongDesc, cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, }, + { + use: "etcd-server", + short: "Generates etcd serving certificate and key", + long: etcdCertLongDesc, + cmdFunc: certsphase.CreateEtcdCertAndKeyFiles, + }, + { + use: "etcd-peer", + short: "Generates etcd peer certificate and key", + long: etcdPeerCertLongDesc, + cmdFunc: certsphase.CreateEtcdPeerCertAndKeyFiles, + }, + { + use: "apiserver-etcd-client", + short: "Generates client certificate for the API server to connect to etcd securely", + long: apiServerEtcdCertLongDesc, + cmdFunc: certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, + }, { use: "sa", short: "Generates a private key for signing service account tokens along with its public key", diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index d6e10000205..a7bd2e52ed5 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -65,6 +65,33 @@ const ( // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" + // EtcdCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdCertAndKeyBaseName = "etcd-server" + // EtcdCertName defines etcd's server certificate name + EtcdCertName = "etcd-server.crt" + // EtcdKeyName defines etcd's server key name + EtcdKeyName = "etcd-server.key" + // EtcdCertCommonName defines etcd's server certificate common name (CN) + EtcdCertCommonName = "kube-etcd" + + // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name + EtcdPeerCertAndKeyBaseName = "etcd-peer" + // EtcdPeerCertName defines etcd's peer certificate name + EtcdPeerCertName = "etcd-peer.crt" + // EtcdPeerKeyName defines etcd's peer key name + EtcdPeerKeyName = "etcd-peer.key" + // EtcdPeerCertCommonName defines etcd's peer certificate common name (CN) + EtcdPeerCertCommonName = "kube-etcd-peer" + + // APIServerEtcdClientCertAndKeyBaseName defines etcd client certificate and key base name + APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" + // APIServerEtcdClientCertName defines etcd client certificate name + APIServerEtcdClientCertName = "apiserver-etcd-client.crt" + // APIServerEtcdClientKeyName defines etcd client key name + APIServerEtcdClientKeyName = "apiserver-etcd-client.key" + // APIServerEtcdClientCertCommonName defines etcd client certificate common name (CN) + APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" + // ServiceAccountKeyBaseName defines SA key base name ServiceAccountKeyBaseName = "sa" // ServiceAccountPublicKeyName defines SA public key base name diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index 258c5b1da26..8d0cff3c8c8 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -40,6 +40,9 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { CreateCACertAndKeyfiles, CreateAPIServerCertAndKeyFiles, CreateAPIServerKubeletClientCertAndKeyFiles, + CreateEtcdCertAndKeyFiles, + CreateEtcdPeerCertAndKeyFiles, + CreateAPIServerEtcdClientCertAndKeyFiles, CreateServiceAccountKeyAndPublicKeyFiles, CreateFrontProxyCACertAndKeyFiles, CreateFrontProxyClientCertAndKeyFiles, @@ -122,6 +125,78 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura ) } +// CreateEtcdCertAndKeyFiles create a new certificate and key file for etcd. +// If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateEtcdCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + etcdCert, etcdKey, err := NewEtcdCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.EtcdCertAndKeyBaseName, + caCert, + etcdCert, + etcdKey, + ) +} + +// CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering. +// If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.EtcdPeerCertAndKeyBaseName, + caCert, + etcdPeerCert, + etcdPeerKey, + ) +} + +// CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd +// If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + apiClientCert, apiClientKey, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, + caCert, + apiClientCert, + apiClientKey, + ) +} + // CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users. // If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { @@ -230,6 +305,64 @@ func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.Pr return apiClientCert, apiClientKey, nil } +// NewEtcdCertAndKey generate CA certificate for etcd, signed by the given CA. +func NewEtcdCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + altNames, err := getAltNames(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failure while composing altnames for etcd: %v", err) + } + + config := certutil.Config{ + CommonName: kubeadmconstants.EtcdCertCommonName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + etcdCert, etcdKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating etcd key and certificate: %v", err) + } + + return etcdCert, etcdKey, nil +} + +// NewEtcdPeerCertAndKey generate CA certificate for etcd peering, signed by the given CA. +func NewEtcdPeerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + altNames, err := getAltNames(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failure while composing altnames for etcd peering: %v", err) + } + + config := certutil.Config{ + CommonName: kubeadmconstants.EtcdPeerCertCommonName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + etcdPeerCert, etcdPeerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating etcd peer key and certificate: %v", err) + } + + return etcdPeerCert, etcdPeerKey, nil +} + +// NewAPIServerEtcdClientCertAndKey generate CA certificate for the apiservers to connect to etcd securely, signed by the given CA. +func NewAPIServerEtcdClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + config := certutil.Config{ + CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName, + Organization: []string{kubeadmconstants.MastersGroup}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating API server etcd client key and certificate: %v", err) + } + + return apiClientCert, apiClientKey, nil +} + // NewServiceAccountSigningKey generate public/private key pairs for signing service account tokens. func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) { diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 1532c5e0403..0a93f1e1869 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -357,6 +357,77 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { certstestutil.AssertCertificateHasOrganizations(t, apiClientCert, kubeadmconstants.MastersGroup) } +func TestNewEtcdCertAndKey(t *testing.T) { + hostname := "valid-hostname" + + advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} + for _, addr := range advertiseAddresses { + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: addr}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdCert, _, err := NewEtcdCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") + certstestutil.AssertCertificateHasIPAddresses(t, etcdCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) + } +} + +func TestNewEtcdPeerCertAndKey(t *testing.T) { + hostname := "valid-hostname" + + advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} + for _, addr := range advertiseAddresses { + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: addr}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdPeerCert, _, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdPeerCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdPeerCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, etcdPeerCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") + certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) + } +} + +func TestNewAPIServerEtcdClientCertAndKey(t *testing.T) { + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + apiEtcdClientCert, _, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, apiEtcdClientCert, caCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, apiEtcdClientCert) + certstestutil.AssertCertificateHasOrganizations(t, apiEtcdClientCert, kubeadmconstants.MastersGroup) +} + func TestNewNewServiceAccountSigningKey(t *testing.T) { key, err := NewServiceAccountSigningKey() @@ -570,6 +641,21 @@ func TestCreateCertificateFilesMethods(t *testing.T) { createFunc: CreateAPIServerKubeletClientCertAndKeyFiles, expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateEtcdCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdCertName, kubeadmconstants.EtcdKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateEtcdPeerCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateAPIServerEtcdClientCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName}, + }, { createFunc: CreateServiceAccountKeyAndPublicKeyFiles, expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName}, diff --git a/cmd/kubeadm/app/phases/certs/doc.go b/cmd/kubeadm/app/phases/certs/doc.go index baf35b138be..ef9ea9689d5 100644 --- a/cmd/kubeadm/app/phases/certs/doc.go +++ b/cmd/kubeadm/app/phases/certs/doc.go @@ -23,7 +23,7 @@ package certs INPUTS: From MasterConfiguration .API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs - .APIServerCertSANs is needed for knowing which DNS names and IPs the API Server serving cert should be valid for + .APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN .Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has .Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to .CertificatesDir is required for knowing where all certificates should be stored @@ -36,6 +36,12 @@ package certs - apiserver.key - apiserver-kubelet-client.crt - apiserver-kubelet-client.key + - etcd-server.crt + - etcd-server.key + - etcd-peer.crt + - etcd-peer.key + - apiserver-etcd-client.crt + - apiserver-etcd-client.key - sa.pub - sa.key - front-proxy-ca.crt diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 8dbe1db716c..169ee18c589 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -190,21 +190,37 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServerExtraArgs)...) command = append(command, getAuthzParameters(cfg.AuthorizationModes)...) - // Check if the user decided to use an external etcd cluster + // If the user set endpoints for an external etcd cluster if len(cfg.Etcd.Endpoints) > 0 { command = append(command, fmt.Sprintf("--etcd-servers=%s", strings.Join(cfg.Etcd.Endpoints, ","))) - } else { - command = append(command, "--etcd-servers=http://127.0.0.1:2379") - } - // Is etcd secured? - if cfg.Etcd.CAFile != "" { - command = append(command, fmt.Sprintf("--etcd-cafile=%s", cfg.Etcd.CAFile)) - } - if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { - etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", cfg.Etcd.CertFile) - etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", cfg.Etcd.KeyFile) - command = append(command, etcdClientFileArg, etcdKeyFileArg) + // Use any user supplied etcd certificates + if cfg.Etcd.CAFile != "" { + command = append(command, fmt.Sprintf("--etcd-cafile=%s", cfg.Etcd.CAFile)) + } + if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", cfg.Etcd.CertFile) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", cfg.Etcd.KeyFile) + command = append(command, etcdClientFileArg, etcdKeyFileArg) + } + } else { + // Default to etcd static pod on localhost + etcdEndpointsArg := "--etcd-servers=https://127.0.0.1:2379" + etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)) + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)) + command = append(command, etcdEndpointsArg, etcdCAFileArg, etcdClientFileArg, etcdKeyFileArg) + + // Warn for unused user supplied variables + if cfg.Etcd.CAFile != "" { + fmt.Printf("[controlplane] Configuration for %s CAFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CAFile, kubeadmconstants.Etcd) + } + if cfg.Etcd.CertFile != "" { + fmt.Printf("[controlplane] Configuration for %s CertFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CertFile, kubeadmconstants.Etcd) + } + if cfg.Etcd.KeyFile != "" { + fmt.Printf("[controlplane] Configuration for %s KeyFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.KeyFile, kubeadmconstants.Etcd) + } } if cfg.CloudProvider != "" { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index e12d700f10e..72110dfdb89 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -224,7 +224,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -258,7 +261,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -292,7 +298,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -327,9 +336,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -369,9 +379,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -406,9 +417,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -443,9 +455,123 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + FeatureGates: map[string]bool{features.HighAvailability: true}, + Etcd: kubeadmapi.Etcd{Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, CAFile: "fuz", CertFile: "fiz", KeyFile: "faz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=https://8.6.4.1:2379,https://8.6.4.2:2379", + "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", + fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"}}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:2380", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{CAFile: "fuz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -483,7 +609,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), "--audit-policy-file=/etc/kubernetes/audit/audit.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", @@ -522,7 +651,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--cloud-provider=gce", }, }, @@ -558,7 +690,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--cloud-provider=aws", }, }, diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 1de1e38adbe..d87005706e2 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -18,6 +18,7 @@ package etcd import ( "fmt" + "path/filepath" "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -28,7 +29,8 @@ import ( ) const ( - etcdVolumeName = "etcd" + etcdVolumeName = "etcd-data" + certsVolumeName = "k8s-certs" ) // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. @@ -50,7 +52,8 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { pathType := v1.HostPathDirectoryOrCreate etcdMounts := map[string]v1.Volume{ - etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir, &pathType), } return staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.Etcd, @@ -58,7 +61,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image), ImagePullPolicy: cfg.ImagePullPolicy, // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner - VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, + VolumeMounts: []v1.VolumeMount{ + staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false), + staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir, false), + }, LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP), }, etcdMounts) } @@ -66,9 +72,17 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { // getEtcdCommand builds the right etcd command from the given config object func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments := map[string]string{ - "listen-client-urls": "http://127.0.0.1:2379", - "advertise-client-urls": "http://127.0.0.1:2379", + "listen-client-urls": "https://127.0.0.1:2379", + "advertise-client-urls": "https://127.0.0.1:2379", "data-dir": cfg.Etcd.DataDir, + "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCertName), + "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdKeyName), + "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "client-cert-auth": "true", + "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), + "peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName), + "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "peer-client-cert-auth": "true", } command := []string{"etcd"} diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 569d2c6c5a1..920680e0599 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -79,9 +79,17 @@ func TestGetEtcdCommand(t *testing.T) { }, expected: []string{ "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", + "--listen-client-urls=https://127.0.0.1:2379", + "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/var/lib/etcd", + "--cert-file=etcd-server.crt", + "--key-file=etcd-server.key", + "--trusted-ca-file=ca.crt", + "--client-cert-auth=true", + "--peer-cert-file=etcd-peer.crt", + "--peer-key-file=etcd-peer.key", + "--peer-trusted-ca-file=ca.crt", + "--peer-client-cert-auth=true", }, }, { @@ -89,16 +97,24 @@ func TestGetEtcdCommand(t *testing.T) { Etcd: kubeadmapi.Etcd{ DataDir: "/var/lib/etcd", ExtraArgs: map[string]string{ - "listen-client-urls": "http://10.0.1.10:2379", - "advertise-client-urls": "http://10.0.1.10:2379", + "listen-client-urls": "https://10.0.1.10:2379", + "advertise-client-urls": "https://10.0.1.10:2379", }, }, }, expected: []string{ "etcd", - "--listen-client-urls=http://10.0.1.10:2379", - "--advertise-client-urls=http://10.0.1.10:2379", + "--listen-client-urls=https://10.0.1.10:2379", + "--advertise-client-urls=https://10.0.1.10:2379", "--data-dir=/var/lib/etcd", + "--cert-file=etcd-server.crt", + "--key-file=etcd-server.key", + "--trusted-ca-file=ca.crt", + "--client-cert-auth=true", + "--peer-cert-file=etcd-peer.crt", + "--peer-key-file=etcd-peer.key", + "--peer-trusted-ca-file=ca.crt", + "--peer-client-cert-auth=true", }, }, { @@ -107,9 +123,17 @@ func TestGetEtcdCommand(t *testing.T) { }, expected: []string{ "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", + "--listen-client-urls=https://127.0.0.1:2379", + "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/etc/foo", + "--cert-file=etcd-server.crt", + "--key-file=etcd-server.key", + "--trusted-ca-file=ca.crt", + "--client-cert-auth=true", + "--peer-cert-file=etcd-peer.crt", + "--peer-key-file=etcd-peer.key", + "--peer-trusted-ca-file=ca.crt", + "--peer-client-cert-auth=true", }, }, } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index dbce0222132..40ab06db67e 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -23,7 +23,8 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -133,6 +134,22 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP if component == constants.Etcd { recoverEtcd = true } + + // ensure etcd certs are generated for etcd and kube-apiserver + if component == constants.Etcd { + if err := certsphase.CreateEtcdCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s certificate: %v", constants.Etcd, err) + } + if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s peer certificate: %v", constants.Etcd, err) + } + } + if component == constants.KubeAPIServer { + if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s %s-client certificate: %v", constants.KubeAPIServer, constants.Etcd, err) + } + } + // The old manifest is here; in the /etc/kubernetes/manifests/ currentManifestPath := pathMgr.RealManifestPath(component) // The new, upgraded manifest will be written here @@ -180,7 +197,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM if len(cfg.Etcd.Endpoints) != 0 { return false, fmt.Errorf("external etcd detected, won't try to change any etcd state") } - // Checking health state of etcd before proceeding with the upgrtade + // Checking health state of etcd before proceeding with the upgrade etcdCluster := util.LocalEtcdCluster{} etcdStatus, err := etcdCluster.GetEtcdClusterStatus() if err != nil { @@ -191,7 +208,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM backupEtcdDir := pathMgr.BackupEtcdDir() runningEtcdDir := cfg.Etcd.DataDir if err := util.CopyDir(runningEtcdDir, backupEtcdDir); err != nil { - return true, fmt.Errorf("fail to back up etcd data: %v", err) + return true, fmt.Errorf("failer to back up etcd data: %v", err) } // Need to check currently used version and version from constants, if differs then upgrade @@ -215,7 +232,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeName, constants.Etcd) if err != nil { - return true, fmt.Errorf("fail to get etcd pod's hash: %v", err) + return true, fmt.Errorf("failed to get etcd pod's hash: %v", err) } // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change @@ -227,7 +244,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM // Perform etcd upgrade using common to all control plane components function if err := upgradeComponent(constants.Etcd, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil { // Since etcd upgrade component failed, the old manifest has been restored - // now we need to check the heatlth of etcd cluster if it came back up with old manifest + // now we need to check the health of etcd cluster if it came back up with old manifest if _, err := etcdCluster.GetEtcdClusterStatus(); err != nil { // At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest if err := rollbackEtcdData(cfg, fmt.Errorf("etcd cluster is not healthy after upgrade: %v rolling back", err), pathMgr); err != nil { @@ -299,7 +316,7 @@ func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager // Write the updated static Pod manifests into the temporary directory fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir()) - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) if err != nil { return fmt.Errorf("error creating init static pod manifest files: %v", err) } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 5b131fbca1b..e9001ce5b72 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -29,7 +29,8 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -49,7 +50,7 @@ apiServerExtraArgs: null authorizationModes: - Node - RBAC -certificatesDir: /etc/kubernetes/pki +certificatesDir: %s cloudProvider: "" controllerManagerExtraArgs: null etcd: @@ -305,12 +306,38 @@ func TestStaticPodControlPlane(t *testing.T) { defer os.RemoveAll(pathMgr.TempManifestDir()) defer os.RemoveAll(pathMgr.BackupManifestDir()) - oldcfg, err := getConfig("v1.7.0") + tempCersDir, err := ioutil.TempDir("", "kubeadm-certs") + if err != nil { + t.Fatalf("couldn't create temporary certificates directory: %v", err) + } + defer os.RemoveAll(tempCersDir) + + oldcfg, err := getConfig("v1.7.0", tempCersDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } + + // Initialize PKI minus any etcd certificates to simulate etcd PKI upgrade + certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{ + certsphase.CreateCACertAndKeyfiles, + certsphase.CreateAPIServerCertAndKeyFiles, + certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, + // certsphase.CreateEtcdCertAndKeyFiles, + // certsphase.CreateEtcdPeerCertAndKeyFiles, + // certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, + certsphase.CreateServiceAccountKeyAndPublicKeyFiles, + certsphase.CreateFrontProxyCACertAndKeyFiles, + certsphase.CreateFrontProxyClientCertAndKeyFiles, + } + for _, action := range certActions { + err := action(oldcfg) + if err != nil { + t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err) + } + } + // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) if err != nil { t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) } @@ -324,7 +351,7 @@ func TestStaticPodControlPlane(t *testing.T) { t.Fatalf("couldn't read temp file: %v", err) } - newcfg, err := getConfig("v1.8.0") + newcfg, err := getConfig("v1.8.0", tempCersDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } @@ -332,9 +359,10 @@ func TestStaticPodControlPlane(t *testing.T) { actualErr := StaticPodControlPlane(waiter, pathMgr, newcfg, false) if (actualErr != nil) != rt.expectedErr { t.Errorf( - "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t", + "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t\n\tactual error: %v", rt.expectedErr, (actualErr != nil), + actualErr, ) } @@ -365,10 +393,10 @@ func getAPIServerHash(dir string) (string, error) { return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil } -func getConfig(version string) (*kubeadmapi.MasterConfiguration, error) { +func getConfig(version string, certsDir string) (*kubeadmapi.MasterConfiguration, error) { externalcfg := &kubeadmapiext.MasterConfiguration{} internalcfg := &kubeadmapi.MasterConfiguration{} - if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, version)), externalcfg); err != nil { + if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, certsDir, version)), externalcfg); err != nil { return nil, fmt.Errorf("unable to decode config: %v", err) } legacyscheme.Scheme.Convert(externalcfg, internalcfg, nil)