From bb689eb2bb96505b5833ff73fc53c7bcc2170c28 Mon Sep 17 00:00:00 2001 From: leigh schrandt Date: Fri, 15 Dec 2017 14:50:31 -0700 Subject: [PATCH 1/4] Secure etcd API /w TLS on kubeadm init [kubeadm/#594] - Generate Server and Peer cert for etcd - Generate Client cert for apiserver - Add flags / hostMounts for etcd static pod - Add flags / hostMounts for apiserver static pod - Generate certs on upgrade of static-pods for etcd/kube-apiserver - Modify logic for appending etcd flags to staticpod to be safer for external etcd --- cmd/kubeadm/app/cmd/phases/certs.go | 45 +++++ cmd/kubeadm/app/constants/constants.go | 27 +++ cmd/kubeadm/app/phases/certs/certs.go | 133 ++++++++++++++ cmd/kubeadm/app/phases/certs/certs_test.go | 86 +++++++++ cmd/kubeadm/app/phases/certs/doc.go | 8 +- .../app/phases/controlplane/manifests.go | 40 +++-- .../app/phases/controlplane/manifests_test.go | 167 ++++++++++++++++-- cmd/kubeadm/app/phases/etcd/local.go | 24 ++- cmd/kubeadm/app/phases/etcd/local_test.go | 40 ++++- cmd/kubeadm/app/phases/upgrade/staticpods.go | 29 ++- .../app/phases/upgrade/staticpods_test.go | 44 ++++- 11 files changed, 587 insertions(+), 56 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index b63cf1bbdce..5d4e040d7e3 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -74,6 +74,33 @@ var ( If both files already exist, kubeadm skips the generation step and existing files will be used. `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName) + etcdCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the etcd serving certificate and key and saves them into %s and %s files. + + The certificate includes default subject alternative names and additional sans eventually provided by the user; + default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, + kubernetes.default.svc., (that is the .10 address in address space). + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdCertName, kubeadmconstants.EtcdKeyName) + + etcdPeerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the etcd peer certificate and key and saves them into %s and %s files. + + The certificate includes default subject alternative names and additional sans eventually provided by the user; + default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, + kubernetes.default.svc., (that is the .10 address in address space). + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName) + + apiServerEtcdCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the client certificate for the API server to connect to etcd securely and the respective key, + and saves them into %s and %s files. + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName) + saKeyLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the private key for signing service account tokens along with its public key, and saves them into %s and %s files. @@ -157,6 +184,24 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { long: apiServerKubeletCertLongDesc, cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, }, + { + use: "etcd-server", + short: "Generates etcd serving certificate and key", + long: etcdCertLongDesc, + cmdFunc: certsphase.CreateEtcdCertAndKeyFiles, + }, + { + use: "etcd-peer", + short: "Generates etcd peer certificate and key", + long: etcdPeerCertLongDesc, + cmdFunc: certsphase.CreateEtcdPeerCertAndKeyFiles, + }, + { + use: "apiserver-etcd-client", + short: "Generates client certificate for the API server to connect to etcd securely", + long: apiServerEtcdCertLongDesc, + cmdFunc: certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, + }, { use: "sa", short: "Generates a private key for signing service account tokens along with its public key", diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index d6e10000205..a7bd2e52ed5 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -65,6 +65,33 @@ const ( // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" + // EtcdCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdCertAndKeyBaseName = "etcd-server" + // EtcdCertName defines etcd's server certificate name + EtcdCertName = "etcd-server.crt" + // EtcdKeyName defines etcd's server key name + EtcdKeyName = "etcd-server.key" + // EtcdCertCommonName defines etcd's server certificate common name (CN) + EtcdCertCommonName = "kube-etcd" + + // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name + EtcdPeerCertAndKeyBaseName = "etcd-peer" + // EtcdPeerCertName defines etcd's peer certificate name + EtcdPeerCertName = "etcd-peer.crt" + // EtcdPeerKeyName defines etcd's peer key name + EtcdPeerKeyName = "etcd-peer.key" + // EtcdPeerCertCommonName defines etcd's peer certificate common name (CN) + EtcdPeerCertCommonName = "kube-etcd-peer" + + // APIServerEtcdClientCertAndKeyBaseName defines etcd client certificate and key base name + APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" + // APIServerEtcdClientCertName defines etcd client certificate name + APIServerEtcdClientCertName = "apiserver-etcd-client.crt" + // APIServerEtcdClientKeyName defines etcd client key name + APIServerEtcdClientKeyName = "apiserver-etcd-client.key" + // APIServerEtcdClientCertCommonName defines etcd client certificate common name (CN) + APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" + // ServiceAccountKeyBaseName defines SA key base name ServiceAccountKeyBaseName = "sa" // ServiceAccountPublicKeyName defines SA public key base name diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index 258c5b1da26..8d0cff3c8c8 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -40,6 +40,9 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { CreateCACertAndKeyfiles, CreateAPIServerCertAndKeyFiles, CreateAPIServerKubeletClientCertAndKeyFiles, + CreateEtcdCertAndKeyFiles, + CreateEtcdPeerCertAndKeyFiles, + CreateAPIServerEtcdClientCertAndKeyFiles, CreateServiceAccountKeyAndPublicKeyFiles, CreateFrontProxyCACertAndKeyFiles, CreateFrontProxyClientCertAndKeyFiles, @@ -122,6 +125,78 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura ) } +// CreateEtcdCertAndKeyFiles create a new certificate and key file for etcd. +// If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateEtcdCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + etcdCert, etcdKey, err := NewEtcdCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.EtcdCertAndKeyBaseName, + caCert, + etcdCert, + etcdKey, + ) +} + +// CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering. +// If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.EtcdPeerCertAndKeyBaseName, + caCert, + etcdPeerCert, + etcdPeerKey, + ) +} + +// CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd +// If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + apiClientCert, apiClientKey, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, + caCert, + apiClientCert, + apiClientKey, + ) +} + // CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users. // If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { @@ -230,6 +305,64 @@ func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.Pr return apiClientCert, apiClientKey, nil } +// NewEtcdCertAndKey generate CA certificate for etcd, signed by the given CA. +func NewEtcdCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + altNames, err := getAltNames(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failure while composing altnames for etcd: %v", err) + } + + config := certutil.Config{ + CommonName: kubeadmconstants.EtcdCertCommonName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + etcdCert, etcdKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating etcd key and certificate: %v", err) + } + + return etcdCert, etcdKey, nil +} + +// NewEtcdPeerCertAndKey generate CA certificate for etcd peering, signed by the given CA. +func NewEtcdPeerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + altNames, err := getAltNames(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failure while composing altnames for etcd peering: %v", err) + } + + config := certutil.Config{ + CommonName: kubeadmconstants.EtcdPeerCertCommonName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + etcdPeerCert, etcdPeerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating etcd peer key and certificate: %v", err) + } + + return etcdPeerCert, etcdPeerKey, nil +} + +// NewAPIServerEtcdClientCertAndKey generate CA certificate for the apiservers to connect to etcd securely, signed by the given CA. +func NewAPIServerEtcdClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + config := certutil.Config{ + CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName, + Organization: []string{kubeadmconstants.MastersGroup}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating API server etcd client key and certificate: %v", err) + } + + return apiClientCert, apiClientKey, nil +} + // NewServiceAccountSigningKey generate public/private key pairs for signing service account tokens. func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) { diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 1532c5e0403..0a93f1e1869 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -357,6 +357,77 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { certstestutil.AssertCertificateHasOrganizations(t, apiClientCert, kubeadmconstants.MastersGroup) } +func TestNewEtcdCertAndKey(t *testing.T) { + hostname := "valid-hostname" + + advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} + for _, addr := range advertiseAddresses { + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: addr}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdCert, _, err := NewEtcdCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") + certstestutil.AssertCertificateHasIPAddresses(t, etcdCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) + } +} + +func TestNewEtcdPeerCertAndKey(t *testing.T) { + hostname := "valid-hostname" + + advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} + for _, addr := range advertiseAddresses { + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: addr}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdPeerCert, _, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdPeerCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdPeerCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, etcdPeerCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") + certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) + } +} + +func TestNewAPIServerEtcdClientCertAndKey(t *testing.T) { + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + apiEtcdClientCert, _, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, apiEtcdClientCert, caCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, apiEtcdClientCert) + certstestutil.AssertCertificateHasOrganizations(t, apiEtcdClientCert, kubeadmconstants.MastersGroup) +} + func TestNewNewServiceAccountSigningKey(t *testing.T) { key, err := NewServiceAccountSigningKey() @@ -570,6 +641,21 @@ func TestCreateCertificateFilesMethods(t *testing.T) { createFunc: CreateAPIServerKubeletClientCertAndKeyFiles, expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateEtcdCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdCertName, kubeadmconstants.EtcdKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateEtcdPeerCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateAPIServerEtcdClientCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName}, + }, { createFunc: CreateServiceAccountKeyAndPublicKeyFiles, expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName}, diff --git a/cmd/kubeadm/app/phases/certs/doc.go b/cmd/kubeadm/app/phases/certs/doc.go index baf35b138be..ef9ea9689d5 100644 --- a/cmd/kubeadm/app/phases/certs/doc.go +++ b/cmd/kubeadm/app/phases/certs/doc.go @@ -23,7 +23,7 @@ package certs INPUTS: From MasterConfiguration .API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs - .APIServerCertSANs is needed for knowing which DNS names and IPs the API Server serving cert should be valid for + .APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN .Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has .Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to .CertificatesDir is required for knowing where all certificates should be stored @@ -36,6 +36,12 @@ package certs - apiserver.key - apiserver-kubelet-client.crt - apiserver-kubelet-client.key + - etcd-server.crt + - etcd-server.key + - etcd-peer.crt + - etcd-peer.key + - apiserver-etcd-client.crt + - apiserver-etcd-client.key - sa.pub - sa.key - front-proxy-ca.crt diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 8dbe1db716c..169ee18c589 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -190,21 +190,37 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServerExtraArgs)...) command = append(command, getAuthzParameters(cfg.AuthorizationModes)...) - // Check if the user decided to use an external etcd cluster + // If the user set endpoints for an external etcd cluster if len(cfg.Etcd.Endpoints) > 0 { command = append(command, fmt.Sprintf("--etcd-servers=%s", strings.Join(cfg.Etcd.Endpoints, ","))) - } else { - command = append(command, "--etcd-servers=http://127.0.0.1:2379") - } - // Is etcd secured? - if cfg.Etcd.CAFile != "" { - command = append(command, fmt.Sprintf("--etcd-cafile=%s", cfg.Etcd.CAFile)) - } - if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { - etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", cfg.Etcd.CertFile) - etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", cfg.Etcd.KeyFile) - command = append(command, etcdClientFileArg, etcdKeyFileArg) + // Use any user supplied etcd certificates + if cfg.Etcd.CAFile != "" { + command = append(command, fmt.Sprintf("--etcd-cafile=%s", cfg.Etcd.CAFile)) + } + if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", cfg.Etcd.CertFile) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", cfg.Etcd.KeyFile) + command = append(command, etcdClientFileArg, etcdKeyFileArg) + } + } else { + // Default to etcd static pod on localhost + etcdEndpointsArg := "--etcd-servers=https://127.0.0.1:2379" + etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)) + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)) + command = append(command, etcdEndpointsArg, etcdCAFileArg, etcdClientFileArg, etcdKeyFileArg) + + // Warn for unused user supplied variables + if cfg.Etcd.CAFile != "" { + fmt.Printf("[controlplane] Configuration for %s CAFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CAFile, kubeadmconstants.Etcd) + } + if cfg.Etcd.CertFile != "" { + fmt.Printf("[controlplane] Configuration for %s CertFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CertFile, kubeadmconstants.Etcd) + } + if cfg.Etcd.KeyFile != "" { + fmt.Printf("[controlplane] Configuration for %s KeyFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.KeyFile, kubeadmconstants.Etcd) + } } if cfg.CloudProvider != "" { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index e12d700f10e..72110dfdb89 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -224,7 +224,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -258,7 +261,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -292,7 +298,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -327,9 +336,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -369,9 +379,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -406,9 +417,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -443,9 +455,123 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + FeatureGates: map[string]bool{features.HighAvailability: true}, + Etcd: kubeadmapi.Etcd{Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, CAFile: "fuz", CertFile: "fiz", KeyFile: "faz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=https://8.6.4.1:2379,https://8.6.4.2:2379", + "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", + fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"}}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:2380", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{CAFile: "fuz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -483,7 +609,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), "--audit-policy-file=/etc/kubernetes/audit/audit.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", @@ -522,7 +651,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--cloud-provider=gce", }, }, @@ -558,7 +690,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--cloud-provider=aws", }, }, diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 1de1e38adbe..d87005706e2 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -18,6 +18,7 @@ package etcd import ( "fmt" + "path/filepath" "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -28,7 +29,8 @@ import ( ) const ( - etcdVolumeName = "etcd" + etcdVolumeName = "etcd-data" + certsVolumeName = "k8s-certs" ) // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. @@ -50,7 +52,8 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { pathType := v1.HostPathDirectoryOrCreate etcdMounts := map[string]v1.Volume{ - etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir, &pathType), } return staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.Etcd, @@ -58,7 +61,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image), ImagePullPolicy: cfg.ImagePullPolicy, // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner - VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, + VolumeMounts: []v1.VolumeMount{ + staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false), + staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir, false), + }, LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP), }, etcdMounts) } @@ -66,9 +72,17 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { // getEtcdCommand builds the right etcd command from the given config object func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments := map[string]string{ - "listen-client-urls": "http://127.0.0.1:2379", - "advertise-client-urls": "http://127.0.0.1:2379", + "listen-client-urls": "https://127.0.0.1:2379", + "advertise-client-urls": "https://127.0.0.1:2379", "data-dir": cfg.Etcd.DataDir, + "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCertName), + "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdKeyName), + "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "client-cert-auth": "true", + "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), + "peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName), + "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "peer-client-cert-auth": "true", } command := []string{"etcd"} diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 569d2c6c5a1..920680e0599 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -79,9 +79,17 @@ func TestGetEtcdCommand(t *testing.T) { }, expected: []string{ "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", + "--listen-client-urls=https://127.0.0.1:2379", + "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/var/lib/etcd", + "--cert-file=etcd-server.crt", + "--key-file=etcd-server.key", + "--trusted-ca-file=ca.crt", + "--client-cert-auth=true", + "--peer-cert-file=etcd-peer.crt", + "--peer-key-file=etcd-peer.key", + "--peer-trusted-ca-file=ca.crt", + "--peer-client-cert-auth=true", }, }, { @@ -89,16 +97,24 @@ func TestGetEtcdCommand(t *testing.T) { Etcd: kubeadmapi.Etcd{ DataDir: "/var/lib/etcd", ExtraArgs: map[string]string{ - "listen-client-urls": "http://10.0.1.10:2379", - "advertise-client-urls": "http://10.0.1.10:2379", + "listen-client-urls": "https://10.0.1.10:2379", + "advertise-client-urls": "https://10.0.1.10:2379", }, }, }, expected: []string{ "etcd", - "--listen-client-urls=http://10.0.1.10:2379", - "--advertise-client-urls=http://10.0.1.10:2379", + "--listen-client-urls=https://10.0.1.10:2379", + "--advertise-client-urls=https://10.0.1.10:2379", "--data-dir=/var/lib/etcd", + "--cert-file=etcd-server.crt", + "--key-file=etcd-server.key", + "--trusted-ca-file=ca.crt", + "--client-cert-auth=true", + "--peer-cert-file=etcd-peer.crt", + "--peer-key-file=etcd-peer.key", + "--peer-trusted-ca-file=ca.crt", + "--peer-client-cert-auth=true", }, }, { @@ -107,9 +123,17 @@ func TestGetEtcdCommand(t *testing.T) { }, expected: []string{ "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", + "--listen-client-urls=https://127.0.0.1:2379", + "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/etc/foo", + "--cert-file=etcd-server.crt", + "--key-file=etcd-server.key", + "--trusted-ca-file=ca.crt", + "--client-cert-auth=true", + "--peer-cert-file=etcd-peer.crt", + "--peer-key-file=etcd-peer.key", + "--peer-trusted-ca-file=ca.crt", + "--peer-client-cert-auth=true", }, }, } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index dbce0222132..40ab06db67e 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -23,7 +23,8 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -133,6 +134,22 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP if component == constants.Etcd { recoverEtcd = true } + + // ensure etcd certs are generated for etcd and kube-apiserver + if component == constants.Etcd { + if err := certsphase.CreateEtcdCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s certificate: %v", constants.Etcd, err) + } + if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s peer certificate: %v", constants.Etcd, err) + } + } + if component == constants.KubeAPIServer { + if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s %s-client certificate: %v", constants.KubeAPIServer, constants.Etcd, err) + } + } + // The old manifest is here; in the /etc/kubernetes/manifests/ currentManifestPath := pathMgr.RealManifestPath(component) // The new, upgraded manifest will be written here @@ -180,7 +197,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM if len(cfg.Etcd.Endpoints) != 0 { return false, fmt.Errorf("external etcd detected, won't try to change any etcd state") } - // Checking health state of etcd before proceeding with the upgrtade + // Checking health state of etcd before proceeding with the upgrade etcdCluster := util.LocalEtcdCluster{} etcdStatus, err := etcdCluster.GetEtcdClusterStatus() if err != nil { @@ -191,7 +208,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM backupEtcdDir := pathMgr.BackupEtcdDir() runningEtcdDir := cfg.Etcd.DataDir if err := util.CopyDir(runningEtcdDir, backupEtcdDir); err != nil { - return true, fmt.Errorf("fail to back up etcd data: %v", err) + return true, fmt.Errorf("failer to back up etcd data: %v", err) } // Need to check currently used version and version from constants, if differs then upgrade @@ -215,7 +232,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeName, constants.Etcd) if err != nil { - return true, fmt.Errorf("fail to get etcd pod's hash: %v", err) + return true, fmt.Errorf("failed to get etcd pod's hash: %v", err) } // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change @@ -227,7 +244,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM // Perform etcd upgrade using common to all control plane components function if err := upgradeComponent(constants.Etcd, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil { // Since etcd upgrade component failed, the old manifest has been restored - // now we need to check the heatlth of etcd cluster if it came back up with old manifest + // now we need to check the health of etcd cluster if it came back up with old manifest if _, err := etcdCluster.GetEtcdClusterStatus(); err != nil { // At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest if err := rollbackEtcdData(cfg, fmt.Errorf("etcd cluster is not healthy after upgrade: %v rolling back", err), pathMgr); err != nil { @@ -299,7 +316,7 @@ func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager // Write the updated static Pod manifests into the temporary directory fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir()) - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) if err != nil { return fmt.Errorf("error creating init static pod manifest files: %v", err) } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 5b131fbca1b..e9001ce5b72 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -29,7 +29,8 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -49,7 +50,7 @@ apiServerExtraArgs: null authorizationModes: - Node - RBAC -certificatesDir: /etc/kubernetes/pki +certificatesDir: %s cloudProvider: "" controllerManagerExtraArgs: null etcd: @@ -305,12 +306,38 @@ func TestStaticPodControlPlane(t *testing.T) { defer os.RemoveAll(pathMgr.TempManifestDir()) defer os.RemoveAll(pathMgr.BackupManifestDir()) - oldcfg, err := getConfig("v1.7.0") + tempCersDir, err := ioutil.TempDir("", "kubeadm-certs") + if err != nil { + t.Fatalf("couldn't create temporary certificates directory: %v", err) + } + defer os.RemoveAll(tempCersDir) + + oldcfg, err := getConfig("v1.7.0", tempCersDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } + + // Initialize PKI minus any etcd certificates to simulate etcd PKI upgrade + certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{ + certsphase.CreateCACertAndKeyfiles, + certsphase.CreateAPIServerCertAndKeyFiles, + certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, + // certsphase.CreateEtcdCertAndKeyFiles, + // certsphase.CreateEtcdPeerCertAndKeyFiles, + // certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, + certsphase.CreateServiceAccountKeyAndPublicKeyFiles, + certsphase.CreateFrontProxyCACertAndKeyFiles, + certsphase.CreateFrontProxyClientCertAndKeyFiles, + } + for _, action := range certActions { + err := action(oldcfg) + if err != nil { + t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err) + } + } + // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) if err != nil { t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) } @@ -324,7 +351,7 @@ func TestStaticPodControlPlane(t *testing.T) { t.Fatalf("couldn't read temp file: %v", err) } - newcfg, err := getConfig("v1.8.0") + newcfg, err := getConfig("v1.8.0", tempCersDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } @@ -332,9 +359,10 @@ func TestStaticPodControlPlane(t *testing.T) { actualErr := StaticPodControlPlane(waiter, pathMgr, newcfg, false) if (actualErr != nil) != rt.expectedErr { t.Errorf( - "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t", + "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t\n\tactual error: %v", rt.expectedErr, (actualErr != nil), + actualErr, ) } @@ -365,10 +393,10 @@ func getAPIServerHash(dir string) (string, error) { return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil } -func getConfig(version string) (*kubeadmapi.MasterConfiguration, error) { +func getConfig(version string, certsDir string) (*kubeadmapi.MasterConfiguration, error) { externalcfg := &kubeadmapiext.MasterConfiguration{} internalcfg := &kubeadmapi.MasterConfiguration{} - if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, version)), externalcfg); err != nil { + if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, certsDir, version)), externalcfg); err != nil { return nil, fmt.Errorf("unable to decode config: %v", err) } legacyscheme.Scheme.Convert(externalcfg, internalcfg, nil) From f5e11a0ce0505def386014cb71a30d0482ffae9d Mon Sep 17 00:00:00 2001 From: leigh schrandt Date: Sun, 17 Dec 2017 23:04:04 -0700 Subject: [PATCH 2/4] Change SANs for etcd serving and peer certs - Place etcd server and peer certs & keys into pki subdir - Move certs.altName functions to pkiutil + add appendSANstoAltNames() Share the append logic for the getAltName functions as suggested by @jamiehannaford. Move functions/tests to certs/pkiutil as suggested by @luxas. Update Bazel BUILD deps - Warn when an APIServerCertSANs or EtcdCertSANs entry is unusable - Add MasterConfiguration.EtcdPeerCertSANs - Move EtcdServerCertSANs and EtcdPeerCertSANs under MasterConfiguration.Etcd --- cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go | 2 + cmd/kubeadm/app/apis/kubeadm/types.go | 6 + .../app/apis/kubeadm/v1alpha1/types.go | 4 + .../v1alpha1/zz_generated.conversion.go | 4 + .../kubeadm/v1alpha1/zz_generated.deepcopy.go | 10 ++ .../app/apis/kubeadm/validation/validation.go | 8 +- .../kubeadm/validation/validation_test.go | 6 +- .../app/apis/kubeadm/zz_generated.deepcopy.go | 10 ++ cmd/kubeadm/app/cmd/phases/certs.go | 26 ++- cmd/kubeadm/app/constants/constants.go | 22 +-- cmd/kubeadm/app/phases/certs/BUILD | 2 - cmd/kubeadm/app/phases/certs/certs.go | 85 ++-------- cmd/kubeadm/app/phases/certs/certs_test.go | 118 +++++--------- cmd/kubeadm/app/phases/certs/doc.go | 10 +- cmd/kubeadm/app/phases/certs/pkiutil/BUILD | 13 +- .../app/phases/certs/pkiutil/pki_helpers.go | 108 +++++++++++++ .../phases/certs/pkiutil/pki_helpers_test.go | 153 ++++++++++++++++++ .../app/phases/controlplane/manifests.go | 6 +- cmd/kubeadm/app/phases/etcd/local.go | 4 +- cmd/kubeadm/app/phases/etcd/local_test.go | 36 ++--- cmd/kubeadm/app/phases/upgrade/staticpods.go | 2 +- .../app/phases/upgrade/staticpods_test.go | 4 +- 22 files changed, 428 insertions(+), 211 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 5b1db1d06a6..10d78f734de 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -43,6 +43,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.AuthorizationModes = []string{"foo"} obj.CertificatesDir = "foo" obj.APIServerCertSANs = []string{"foo"} + obj.Etcd.ServerCertSANs = []string{"foo"} + obj.Etcd.PeerCertSANs = []string{"foo"} obj.Token = "foo" obj.CRISocket = "foo" obj.Etcd.Image = "foo" diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 70edf13f39a..cf6c9db3a1f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -178,6 +178,12 @@ type Etcd struct { Image string // SelfHosted holds configuration for self-hosting etcd. SelfHosted *SelfHostedEtcd + // ServerCertSANs sets extra Subject Alternative Names for the etcd server + // signing cert. This is currently used for the etcd static-pod. + ServerCertSANs []string + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer + // signing cert. This is currently used for the etcd static-pod. + PeerCertSANs []string } // SelfHostedEtcd describes options required to configure self-hosted etcd. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index 066dd7979cf..4d94ad7f208 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -170,6 +170,10 @@ type Etcd struct { Image string `json:"image"` // SelfHosted holds configuration for self-hosting etcd. SelfHosted *SelfHostedEtcd `json:"selfHosted,omitempty"` + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. + ServerCertSANs []string `json:"serverCertSANs,omitempty"` + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. + PeerCertSANs []string `json:"peerCertSANs,omitempty"` } // SelfHostedEtcd describes options required to configure self-hosted etcd. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go index 80c47bf9b90..1f5b4d2d0a0 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -122,6 +122,8 @@ func autoConvert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s co out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.Image = in.Image out.SelfHosted = (*kubeadm.SelfHostedEtcd)(unsafe.Pointer(in.SelfHosted)) + out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) + out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil } @@ -139,6 +141,8 @@ func autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s co out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.Image = in.Image out.SelfHosted = (*SelfHostedEtcd)(unsafe.Pointer(in.SelfHosted)) + out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) + out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go index 4eb4e008641..578c431af99 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go @@ -92,6 +92,16 @@ func (in *Etcd) DeepCopyInto(out *Etcd) { **out = **in } } + if in.ServerCertSANs != nil { + in, out := &in.ServerCertSANs, &out.ServerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PeerCertSANs != nil { + in, out := &in.PeerCertSANs, &out.PeerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index 27c31e0157a..3e8aa557515 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -72,7 +72,9 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateCloudProvider(c.CloudProvider, field.NewPath("cloudprovider"))...) allErrs = append(allErrs, ValidateAuthorizationModes(c.AuthorizationModes, field.NewPath("authorization-modes"))...) allErrs = append(allErrs, ValidateNetworking(&c.Networking, field.NewPath("networking"))...) - allErrs = append(allErrs, ValidateAPIServerCertSANs(c.APIServerCertSANs, field.NewPath("cert-altnames"))...) + allErrs = append(allErrs, ValidateCertSANs(c.APIServerCertSANs, field.NewPath("api-server-cert-altnames"))...) + allErrs = append(allErrs, ValidateCertSANs(c.Etcd.ServerCertSANs, field.NewPath("etcd-server-cert-altnames"))...) + allErrs = append(allErrs, ValidateCertSANs(c.Etcd.PeerCertSANs, field.NewPath("etcd-peer-cert-altnames"))...) allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificates-dir"))...) allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...) allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) @@ -228,8 +230,8 @@ func ValidateToken(t string, fldPath *field.Path) field.ErrorList { return allErrs } -// ValidateAPIServerCertSANs validates alternative names -func ValidateAPIServerCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { +// ValidateCertSANs validates alternative names +func ValidateCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} for _, altname := range altnames { if len(validation.IsDNS1123Subdomain(altname)) != 0 && net.ParseIP(altname) == nil { diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 331ad37609d..29262a998c1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -134,7 +134,7 @@ func TestValidateCloudProvider(t *testing.T) { } } -func TestValidateAPIServerCertSANs(t *testing.T) { +func TestValidateCertSANs(t *testing.T) { var tests = []struct { sans []string expected bool @@ -148,10 +148,10 @@ func TestValidateAPIServerCertSANs(t *testing.T) { {[]string{"my-hostname2", "my.other.subdomain", "2001:db8::10"}, true}, // supported } for _, rt := range tests { - actual := ValidateAPIServerCertSANs(rt.sans, nil) + actual := ValidateCertSANs(rt.sans, nil) if (len(actual) == 0) != rt.expected { t.Errorf( - "failed ValidateAPIServerCertSANs:\n\texpected: %t\n\t actual: %t", + "failed ValidateCertSANs:\n\texpected: %t\n\t actual: %t", rt.expected, (len(actual) == 0), ) diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 93775c57a21..30c6c6c5d92 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -92,6 +92,16 @@ func (in *Etcd) DeepCopyInto(out *Etcd) { **out = **in } } + if in.ServerCertSANs != nil { + in, out := &in.ServerCertSANs, &out.ServerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PeerCertSANs != nil { + in, out := &in.PeerCertSANs, &out.PeerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index 5d4e040d7e3..564b8754749 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -60,8 +60,8 @@ var ( apiServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the API server serving certificate and key and saves them into %s and %s files. - The certificate includes default subject alternative names and additional sans eventually provided by the user; - default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, + The certificate includes default subject alternative names and additional SANs provided by the user; + default SANs are: , , kubernetes, kubernetes.default, kubernetes.default.svc, kubernetes.default.svc., (that is the .10 address in address space). If both files already exist, kubeadm skips the generation step and existing files will be used. @@ -74,27 +74,25 @@ var ( If both files already exist, kubeadm skips the generation step and existing files will be used. `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName) - etcdCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + etcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the etcd serving certificate and key and saves them into %s and %s files. - The certificate includes default subject alternative names and additional sans eventually provided by the user; - default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, - kubernetes.default.svc., (that is the .10 address in address space). + The certificate includes default subject alternative names and additional SANs provided by the user; + default SANs are: localhost, 127.0.0.1. If both files already exist, kubeadm skips the generation step and existing files will be used. - `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdCertName, kubeadmconstants.EtcdKeyName) + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName) etcdPeerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the etcd peer certificate and key and saves them into %s and %s files. - The certificate includes default subject alternative names and additional sans eventually provided by the user; - default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, - kubernetes.default.svc., (that is the .10 address in address space). + The certificate includes default subject alternative names and additional SANs provided by the user; + default SANs are: , . If both files already exist, kubeadm skips the generation step and existing files will be used. `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName) - apiServerEtcdCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + apiServerEtcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the client certificate for the API server to connect to etcd securely and the respective key, and saves them into %s and %s files. @@ -187,8 +185,8 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { { use: "etcd-server", short: "Generates etcd serving certificate and key", - long: etcdCertLongDesc, - cmdFunc: certsphase.CreateEtcdCertAndKeyFiles, + long: etcdServerCertLongDesc, + cmdFunc: certsphase.CreateEtcdServerCertAndKeyFiles, }, { use: "etcd-peer", @@ -199,7 +197,7 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { { use: "apiserver-etcd-client", short: "Generates client certificate for the API server to connect to etcd securely", - long: apiServerEtcdCertLongDesc, + long: apiServerEtcdServerCertLongDesc, cmdFunc: certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, }, { diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index a7bd2e52ed5..c2986240744 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -65,21 +65,21 @@ const ( // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" - // EtcdCertAndKeyBaseName defines etcd's server certificate and key base name - EtcdCertAndKeyBaseName = "etcd-server" - // EtcdCertName defines etcd's server certificate name - EtcdCertName = "etcd-server.crt" - // EtcdKeyName defines etcd's server key name - EtcdKeyName = "etcd-server.key" - // EtcdCertCommonName defines etcd's server certificate common name (CN) - EtcdCertCommonName = "kube-etcd" + // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdServerCertAndKeyBaseName = "etcd/server" + // EtcdServerCertName defines etcd's server certificate name + EtcdServerCertName = "etcd/server.crt" + // EtcdServerKeyName defines etcd's server key name + EtcdServerKeyName = "etcd/server.key" + // EtcdServerCertCommonName defines etcd's server certificate common name (CN) + EtcdServerCertCommonName = "kube-etcd" // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name - EtcdPeerCertAndKeyBaseName = "etcd-peer" + EtcdPeerCertAndKeyBaseName = "etcd/peer" // EtcdPeerCertName defines etcd's peer certificate name - EtcdPeerCertName = "etcd-peer.crt" + EtcdPeerCertName = "etcd/peer.crt" // EtcdPeerKeyName defines etcd's peer key name - EtcdPeerKeyName = "etcd-peer.key" + EtcdPeerKeyName = "etcd/peer.key" // EtcdPeerCertCommonName defines etcd's peer certificate common name (CN) EtcdPeerCertCommonName = "kube-etcd-peer" diff --git a/cmd/kubeadm/app/phases/certs/BUILD b/cmd/kubeadm/app/phases/certs/BUILD index 617c57d09c7..3ac7c0fab75 100644 --- a/cmd/kubeadm/app/phases/certs/BUILD +++ b/cmd/kubeadm/app/phases/certs/BUILD @@ -30,8 +30,6 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", - "//pkg/registry/core/service/ipallocator:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index 8d0cff3c8c8..4c7df216944 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -20,16 +20,13 @@ import ( "crypto/rsa" "crypto/x509" "fmt" - "net" "os" "path/filepath" - "k8s.io/apimachinery/pkg/util/validation" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" - "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) // CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. @@ -40,7 +37,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { CreateCACertAndKeyfiles, CreateAPIServerCertAndKeyFiles, CreateAPIServerKubeletClientCertAndKeyFiles, - CreateEtcdCertAndKeyFiles, + CreateEtcdServerCertAndKeyFiles, CreateEtcdPeerCertAndKeyFiles, CreateAPIServerEtcdClientCertAndKeyFiles, CreateServiceAccountKeyAndPublicKeyFiles, @@ -125,27 +122,27 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura ) } -// CreateEtcdCertAndKeyFiles create a new certificate and key file for etcd. +// CreateEtcdServerCertAndKeyFiles create a new certificate and key file for etcd. // If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. // It assumes the cluster CA certificate and key file exist in the CertificatesDir -func CreateEtcdCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { +func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } - etcdCert, etcdKey, err := NewEtcdCertAndKey(cfg, caCert, caKey) + etcdServerCert, etcdServerKey, err := NewEtcdServerCertAndKey(cfg, caCert, caKey) if err != nil { return err } return writeCertificateFilesIfNotExist( cfg.CertificatesDir, - kubeadmconstants.EtcdCertAndKeyBaseName, + kubeadmconstants.EtcdServerCertAndKeyBaseName, caCert, - etcdCert, - etcdKey, + etcdServerCert, + etcdServerKey, ) } @@ -271,7 +268,7 @@ func NewCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) { // NewAPIServerCertAndKey generate CA certificate for apiserver, signed by the given CA. func NewAPIServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { - altNames, err := getAltNames(cfg) + altNames, err := pkiutil.GetAPIServerAltNames(cfg) if err != nil { return nil, nil, fmt.Errorf("failure while composing altnames for API server: %v", err) } @@ -305,31 +302,31 @@ func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.Pr return apiClientCert, apiClientKey, nil } -// NewEtcdCertAndKey generate CA certificate for etcd, signed by the given CA. -func NewEtcdCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { +// NewEtcdServerCertAndKey generate CA certificate for etcd, signed by the given CA. +func NewEtcdServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { - altNames, err := getAltNames(cfg) + altNames, err := pkiutil.GetEtcdAltNames(cfg) if err != nil { return nil, nil, fmt.Errorf("failure while composing altnames for etcd: %v", err) } config := certutil.Config{ - CommonName: kubeadmconstants.EtcdCertCommonName, + CommonName: kubeadmconstants.EtcdServerCertCommonName, AltNames: *altNames, Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, } - etcdCert, etcdKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + etcdServerCert, etcdServerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) if err != nil { return nil, nil, fmt.Errorf("failure while creating etcd key and certificate: %v", err) } - return etcdCert, etcdKey, nil + return etcdServerCert, etcdServerKey, nil } // NewEtcdPeerCertAndKey generate CA certificate for etcd peering, signed by the given CA. func NewEtcdPeerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { - altNames, err := getAltNames(cfg) + altNames, err := pkiutil.GetEtcdPeerAltNames(cfg) if err != nil { return nil, nil, fmt.Errorf("failure while composing altnames for etcd peering: %v", err) } @@ -636,55 +633,3 @@ func validatePrivatePublicKey(l certKeyLocation) error { } return nil } - -// getAltNames builds an AltNames object for to be used when generating apiserver certificate -func getAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { - - // advertise address - advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress) - if advertiseAddress == nil { - return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress) - } - - // internal IP address for the API server - _, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet) - if err != nil { - return nil, fmt.Errorf("error parsing CIDR %q: %v", cfg.Networking.ServiceSubnet, err) - } - - internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1) - if err != nil { - return nil, fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", svcSubnet.String(), err) - } - - // create AltNames with defaults DNSNames/IPs - altNames := &certutil.AltNames{ - DNSNames: []string{ - cfg.NodeName, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain), - }, - IPs: []net.IP{ - internalAPIServerVirtualIP, - advertiseAddress, - }, - } - - // adds additional SAN - for _, altname := range cfg.APIServerCertSANs { - if ip := net.ParseIP(altname); ip != nil { - altNames.IPs = append(altNames.IPs, ip) - } else if len(validation.IsDNS1123Subdomain(altname)) == 0 { - altNames.DNSNames = append(altNames.DNSNames, altname) - } - } - - // add api server dns advertise address - if len(cfg.API.ControlPlaneEndpoint) > 0 { - altNames.DNSNames = append(altNames.DNSNames, cfg.API.ControlPlaneEndpoint) - } - - return altNames, nil -} diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 0a93f1e1869..58223adf89b 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -258,53 +258,6 @@ func TestWriteKeyFilesIfNotExist(t *testing.T) { } } -func TestGetAltNames(t *testing.T) { - hostname := "valid-hostname" - advertiseIP := "1.2.3.4" - controlPlaneEndpoint := "api.k8s.io" - cfg := &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{AdvertiseAddress: advertiseIP, ControlPlaneEndpoint: controlPlaneEndpoint}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - NodeName: hostname, - APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95"}, - } - - altNames, err := getAltNames(cfg) - if err != nil { - t.Fatalf("failed calling getAltNames: %v", err) - } - - expectedDNSNames := []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", controlPlaneEndpoint} - for _, DNSName := range expectedDNSNames { - found := false - for _, val := range altNames.DNSNames { - if val == DNSName { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain DNSName %s", DNSName) - } - } - - expectedIPAddresses := []string{"10.96.0.1", advertiseIP, "10.1.245.94", "10.1.245.95"} - for _, IPAddress := range expectedIPAddresses { - found := false - for _, val := range altNames.IPs { - if val.Equal(net.ParseIP(IPAddress)) { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain IPAddress %s", IPAddress) - } - } -} - func TestNewCACertAndKey(t *testing.T) { caCert, _, err := NewCACertAndKey() if err != nil { @@ -357,42 +310,50 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { certstestutil.AssertCertificateHasOrganizations(t, apiClientCert, kubeadmconstants.MastersGroup) } -func TestNewEtcdCertAndKey(t *testing.T) { - hostname := "valid-hostname" +func TestNewEtcdServerCertAndKey(t *testing.T) { + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" - advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} - for _, addr := range advertiseAddresses { - cfg := &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{AdvertiseAddress: addr}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - NodeName: "valid-hostname", - } - caCert, caKey, err := NewCACertAndKey() - if err != nil { - t.Fatalf("failed creation of ca cert and key: %v", err) - } - - etcdCert, _, err := NewEtcdCertAndKey(cfg, caCert, caKey) - if err != nil { - t.Fatalf("failed creation of cert and key: %v", err) - } - - certstestutil.AssertCertificateIsSignedByCa(t, etcdCert, caCert) - certstestutil.AssertCertificateHasServerAuthUsage(t, etcdCert) - certstestutil.AssertCertificateHasDNSNames(t, etcdCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") - certstestutil.AssertCertificateHasIPAddresses(t, etcdCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) + cfg := &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ServerCertSANs: []string{ + proxy, + proxyIP, + }, + }, } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdServerCert, _, err := NewEtcdServerCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdServerCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdServerCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdServerCert, "localhost", proxy) + certstestutil.AssertCertificateHasIPAddresses(t, etcdServerCert, net.ParseIP("127.0.0.1"), net.ParseIP(proxyIP)) } func TestNewEtcdPeerCertAndKey(t *testing.T) { hostname := "valid-hostname" + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} for _, addr := range advertiseAddresses { cfg := &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{AdvertiseAddress: addr}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - NodeName: "valid-hostname", + API: kubeadmapi.API{AdvertiseAddress: addr}, + NodeName: hostname, + Etcd: kubeadmapi.Etcd{ + PeerCertSANs: []string{ + proxy, + proxyIP, + }, + }, } caCert, caKey, err := NewCACertAndKey() if err != nil { @@ -407,8 +368,8 @@ func TestNewEtcdPeerCertAndKey(t *testing.T) { certstestutil.AssertCertificateIsSignedByCa(t, etcdPeerCert, caCert) certstestutil.AssertCertificateHasServerAuthUsage(t, etcdPeerCert) certstestutil.AssertCertificateHasClientAuthUsage(t, etcdPeerCert) - certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") - certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) + certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, proxy) + certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP(addr), net.ParseIP(proxyIP)) } } @@ -622,6 +583,9 @@ func TestCreateCertificateFilesMethods(t *testing.T) { kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, + kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName, + kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName, + kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName, kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName, kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName, @@ -643,8 +607,8 @@ func TestCreateCertificateFilesMethods(t *testing.T) { }, { setupFunc: CreateCACertAndKeyfiles, - createFunc: CreateEtcdCertAndKeyFiles, - expectedFiles: []string{kubeadmconstants.EtcdCertName, kubeadmconstants.EtcdKeyName}, + createFunc: CreateEtcdServerCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName}, }, { setupFunc: CreateCACertAndKeyfiles, diff --git a/cmd/kubeadm/app/phases/certs/doc.go b/cmd/kubeadm/app/phases/certs/doc.go index ef9ea9689d5..da538747af7 100644 --- a/cmd/kubeadm/app/phases/certs/doc.go +++ b/cmd/kubeadm/app/phases/certs/doc.go @@ -24,6 +24,8 @@ package certs From MasterConfiguration .API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs .APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN + .Etcd.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN + .Etcd.PeerCertSANs is an optional parameter for adding DNS names and IPs to the etcd peer cert SAN .Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has .Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to .CertificatesDir is required for knowing where all certificates should be stored @@ -36,12 +38,12 @@ package certs - apiserver.key - apiserver-kubelet-client.crt - apiserver-kubelet-client.key - - etcd-server.crt - - etcd-server.key - - etcd-peer.crt - - etcd-peer.key - apiserver-etcd-client.crt - apiserver-etcd-client.key + - etcd/server.crt + - etcd/server.key + - etcd/peer.crt + - etcd/peer.key - sa.pub - sa.key - front-proxy-ca.crt diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/BUILD b/cmd/kubeadm/app/phases/certs/pkiutil/BUILD index cb91095764b..f4c8a743178 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/BUILD +++ b/cmd/kubeadm/app/phases/certs/pkiutil/BUILD @@ -10,14 +10,23 @@ go_test( name = "go_default_test", srcs = ["pki_helpers_test.go"], embed = [":go_default_library"], - deps = ["//vendor/k8s.io/client-go/util/cert:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], ) go_library( name = "go_default_library", srcs = ["pki_helpers.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil", - deps = ["//vendor/k8s.io/client-go/util/cert:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//pkg/registry/core/service/ipallocator:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], ) filegroup( diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go index e6bcb27c955..996688ee685 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go @@ -20,11 +20,16 @@ import ( "crypto/rsa" "crypto/x509" "fmt" + "net" "os" "path/filepath" "time" + "k8s.io/apimachinery/pkg/util/validation" certutil "k8s.io/client-go/util/cert" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) // NewCertificateAuthority creates new certificate and private key for the certificate authority @@ -246,3 +251,106 @@ func pathForKey(pkiPath, name string) string { func pathForPublicKey(pkiPath, name string) string { return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name)) } + +// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate +func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { + // advertise address + advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress) + if advertiseAddress == nil { + return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress) + } + + // internal IP address for the API server + _, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet) + if err != nil { + return nil, fmt.Errorf("error parsing CIDR %q: %v", cfg.Networking.ServiceSubnet, err) + } + + internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1) + if err != nil { + return nil, fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", svcSubnet.String(), err) + } + + // create AltNames with defaults DNSNames/IPs + altNames := &certutil.AltNames{ + DNSNames: []string{ + cfg.NodeName, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain), + }, + IPs: []net.IP{ + internalAPIServerVirtualIP, + advertiseAddress, + }, + } + + // add api server dns advertise address + if len(cfg.API.ControlPlaneEndpoint) > 0 { + altNames.DNSNames = append(altNames.DNSNames, cfg.API.ControlPlaneEndpoint) + } + + appendSANsToAltNames(altNames, cfg.APIServerCertSANs, kubeadmconstants.APIServerCertName) + + return altNames, nil +} + +// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate. +// `localhost` is included in the SAN since this is the interface the etcd static pod listens on. +// Hostname and `API.AdvertiseAddress` are excluded since etcd does not listen on this interface by default. +// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`. +func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { + // create AltNames with defaults DNSNames/IPs + altNames := &certutil.AltNames{ + DNSNames: []string{"localhost"}, + IPs: []net.IP{net.IPv4(127, 0, 0, 1)}, + } + + appendSANsToAltNames(altNames, cfg.Etcd.ServerCertSANs, kubeadmconstants.EtcdServerCertName) + + return altNames, nil +} + +// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate. +// `localhost` is excluded from the SAN since etcd will not refer to itself as a peer. +// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one. +// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`. +func GetEtcdPeerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { + // advertise address + advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress) + if advertiseAddress == nil { + return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress) + } + + // create AltNames with defaults DNSNames/IPs + altNames := &certutil.AltNames{ + DNSNames: []string{cfg.NodeName}, + IPs: []net.IP{advertiseAddress}, + } + + appendSANsToAltNames(altNames, cfg.Etcd.PeerCertSANs, kubeadmconstants.EtcdPeerCertName) + + return altNames, nil +} + +// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert +// altNames is passed in with a pointer, and the struct is modified +// valid IP address strings are parsed and added to altNames.IPs as net.IP's +// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings +// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for +func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) { + for _, altname := range SANs { + if ip := net.ParseIP(altname); ip != nil { + altNames.IPs = append(altNames.IPs, ip) + } else if len(validation.IsDNS1123Subdomain(altname)) == 0 { + altNames.DNSNames = append(altNames.DNSNames, altname) + } else { + fmt.Printf( + "[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n", + altname, + certName, + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go index 34ac26ee14c..ba4057c6c92 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go @@ -21,10 +21,12 @@ import ( "crypto/rsa" "crypto/x509" "io/ioutil" + "net" "os" "testing" certutil "k8s.io/client-go/util/cert" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) func TestNewCertificateAuthority(t *testing.T) { @@ -432,3 +434,154 @@ func TestPathForPublicKey(t *testing.T) { t.Errorf("unexpected certificate path: %s", pubPath) } } + +func TestGetAPIServerAltNames(t *testing.T) { + hostname := "valid-hostname" + advertiseIP := "1.2.3.4" + controlPlaneEndpoint := "api.k8s.io" + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: advertiseIP, ControlPlaneEndpoint: controlPlaneEndpoint}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: hostname, + APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"}, + } + + altNames, err := GetAPIServerAltNames(cfg) + if err != nil { + t.Fatalf("failed calling GetAPIServerAltNames: %v", err) + } + + expectedDNSNames := []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", controlPlaneEndpoint} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{"10.96.0.1", advertiseIP, "10.1.245.94", "10.1.245.95"} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} + +func TestGetEtcdAltNames(t *testing.T) { + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" + cfg := &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ServerCertSANs: []string{ + proxy, + proxyIP, + "1.2.3.L", + "invalid,commas,in,DNS", + }, + }, + } + + altNames, err := GetEtcdAltNames(cfg) + if err != nil { + t.Fatalf("failed calling GetEtcdAltNames: %v", err) + } + + expectedDNSNames := []string{"localhost", proxy} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{"127.0.0.1", proxyIP} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} + +func TestGetEtcdPeerAltNames(t *testing.T) { + hostname := "valid-hostname" + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" + advertiseIP := "1.2.3.4" + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: advertiseIP}, + NodeName: hostname, + Etcd: kubeadmapi.Etcd{ + PeerCertSANs: []string{ + proxy, + proxyIP, + "1.2.3.L", + "invalid,commas,in,DNS", + }, + }, + } + + altNames, err := GetEtcdPeerAltNames(cfg) + if err != nil { + t.Fatalf("failed calling GetEtcdPeerAltNames: %v", err) + } + + expectedDNSNames := []string{hostname, proxy} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{advertiseIP, proxyIP} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 169ee18c589..123763c2dce 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -213,13 +213,13 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio // Warn for unused user supplied variables if cfg.Etcd.CAFile != "" { - fmt.Printf("[controlplane] Configuration for %s CAFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CAFile, kubeadmconstants.Etcd) + fmt.Printf("[controlplane] WARNING: Configuration for %s CAFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CAFile, kubeadmconstants.Etcd) } if cfg.Etcd.CertFile != "" { - fmt.Printf("[controlplane] Configuration for %s CertFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CertFile, kubeadmconstants.Etcd) + fmt.Printf("[controlplane] WARNING: Configuration for %s CertFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CertFile, kubeadmconstants.Etcd) } if cfg.Etcd.KeyFile != "" { - fmt.Printf("[controlplane] Configuration for %s KeyFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.KeyFile, kubeadmconstants.Etcd) + fmt.Printf("[controlplane] WARNING: Configuration for %s KeyFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.KeyFile, kubeadmconstants.Etcd) } } diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index d87005706e2..d25503521f9 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -75,8 +75,8 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { "listen-client-urls": "https://127.0.0.1:2379", "advertise-client-urls": "https://127.0.0.1:2379", "data-dir": cfg.Etcd.DataDir, - "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCertName), - "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdKeyName), + "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), + "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), "client-cert-auth": "true", "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 920680e0599..22008bd38be 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -82,13 +82,13 @@ func TestGetEtcdCommand(t *testing.T) { "--listen-client-urls=https://127.0.0.1:2379", "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/var/lib/etcd", - "--cert-file=etcd-server.crt", - "--key-file=etcd-server.key", - "--trusted-ca-file=ca.crt", + "--cert-file=" + kubeadmconstants.EtcdServerCertName, + "--key-file=" + kubeadmconstants.EtcdServerKeyName, + "--trusted-ca-file=" + kubeadmconstants.CACertName, "--client-cert-auth=true", - "--peer-cert-file=etcd-peer.crt", - "--peer-key-file=etcd-peer.key", - "--peer-trusted-ca-file=ca.crt", + "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, + "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, + "--peer-trusted-ca-file=" + kubeadmconstants.CACertName, "--peer-client-cert-auth=true", }, }, @@ -107,13 +107,13 @@ func TestGetEtcdCommand(t *testing.T) { "--listen-client-urls=https://10.0.1.10:2379", "--advertise-client-urls=https://10.0.1.10:2379", "--data-dir=/var/lib/etcd", - "--cert-file=etcd-server.crt", - "--key-file=etcd-server.key", - "--trusted-ca-file=ca.crt", + "--cert-file=" + kubeadmconstants.EtcdServerCertName, + "--key-file=" + kubeadmconstants.EtcdServerKeyName, + "--trusted-ca-file=" + kubeadmconstants.CACertName, "--client-cert-auth=true", - "--peer-cert-file=etcd-peer.crt", - "--peer-key-file=etcd-peer.key", - "--peer-trusted-ca-file=ca.crt", + "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, + "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, + "--peer-trusted-ca-file=" + kubeadmconstants.CACertName, "--peer-client-cert-auth=true", }, }, @@ -126,13 +126,13 @@ func TestGetEtcdCommand(t *testing.T) { "--listen-client-urls=https://127.0.0.1:2379", "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/etc/foo", - "--cert-file=etcd-server.crt", - "--key-file=etcd-server.key", - "--trusted-ca-file=ca.crt", + "--cert-file=" + kubeadmconstants.EtcdServerCertName, + "--key-file=" + kubeadmconstants.EtcdServerKeyName, + "--trusted-ca-file=" + kubeadmconstants.CACertName, "--client-cert-auth=true", - "--peer-cert-file=etcd-peer.crt", - "--peer-key-file=etcd-peer.key", - "--peer-trusted-ca-file=ca.crt", + "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, + "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, + "--peer-trusted-ca-file=" + kubeadmconstants.CACertName, "--peer-client-cert-auth=true", }, }, diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index 40ab06db67e..ee1016868f9 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -137,7 +137,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP // ensure etcd certs are generated for etcd and kube-apiserver if component == constants.Etcd { - if err := certsphase.CreateEtcdCertAndKeyFiles(cfg); err != nil { + if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil { return fmt.Errorf("failed to upgrade the %s certificate: %v", constants.Etcd, err) } if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil { diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index e9001ce5b72..6bd50ca9d70 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -61,6 +61,8 @@ etcd: extraArgs: null image: "" keyFile: "" + serverCertSANs: null + peerCertSANs: null featureFlags: null imageRepository: k8s.gcr.io kubernetesVersion: %s @@ -322,7 +324,7 @@ func TestStaticPodControlPlane(t *testing.T) { certsphase.CreateCACertAndKeyfiles, certsphase.CreateAPIServerCertAndKeyFiles, certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, - // certsphase.CreateEtcdCertAndKeyFiles, + // certsphase.CreateEtcdServerCertAndKeyFiles, // certsphase.CreateEtcdPeerCertAndKeyFiles, // certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, certsphase.CreateServiceAccountKeyAndPublicKeyFiles, From f61430d7c832efef1d13733738c71f14dd48029b Mon Sep 17 00:00:00 2001 From: leigh schrandt Date: Mon, 18 Dec 2017 00:39:35 -0700 Subject: [PATCH 3/4] Fix typos - Fix typos in tests for upgrade phase - Rename loadCertificateAuthorithy() --> loadCertificateAuthority() - Disambiguate apiKubeletClientCert & apiEtcdClientCert - Parameterize hard-coded certs_test config + log tempCertsDir --- cmd/kubeadm/app/phases/certs/certs.go | 28 +++++++++---------- cmd/kubeadm/app/phases/certs/certs_test.go | 10 +++---- cmd/kubeadm/app/phases/upgrade/staticpods.go | 2 +- .../app/phases/upgrade/staticpods_test.go | 9 +++--- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index 4c7df216944..86583f4cf5d 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -79,7 +79,7 @@ func CreateCACertAndKeyfiles(cfg *kubeadmapi.MasterConfiguration) error { // It assumes the cluster CA certificate and key files should exists into the CertificatesDir func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } @@ -103,12 +103,12 @@ func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { // It assumes the cluster CA certificate and key files should exists into the CertificatesDir func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } - apiClientCert, apiClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) + apiKubeletClientCert, apiKubeletClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) if err != nil { return err } @@ -117,8 +117,8 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, caCert, - apiClientCert, - apiClientKey, + apiKubeletClientCert, + apiKubeletClientKey, ) } @@ -127,7 +127,7 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura // It assumes the cluster CA certificate and key file exist in the CertificatesDir func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } @@ -151,7 +151,7 @@ func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error // It assumes the cluster CA certificate and key file exist in the CertificatesDir func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } @@ -175,12 +175,12 @@ func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { // It assumes the cluster CA certificate and key file exist in the CertificatesDir func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } - apiClientCert, apiClientKey, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + apiEtcdClientCert, apiEtcdClientKey, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) if err != nil { return err } @@ -189,8 +189,8 @@ func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguratio cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, caCert, - apiClientCert, - apiClientKey, + apiEtcdClientCert, + apiEtcdClientKey, ) } @@ -235,7 +235,7 @@ func CreateFrontProxyCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) erro // It assumes the front proxy CAA certificate and key files should exists into the CertificatesDir func CreateFrontProxyClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName) + frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName) if err != nil { return err } @@ -398,8 +398,8 @@ func NewFrontProxyClientCertAndKey(frontProxyCACert *x509.Certificate, frontProx return frontProxyClientCert, frontProxyClientKey, nil } -// loadCertificateAuthorithy loads certificate authority -func loadCertificateAuthorithy(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) { +// loadCertificateAuthority loads certificate authority +func loadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) { // Checks if certificate authority exists in the PKI directory if !pkiutil.CertOrKeyExist(pkiDir, baseName) { return nil, nil, fmt.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir) diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 58223adf89b..55c1ea1f56d 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -275,7 +275,7 @@ func TestNewAPIServerCertAndKey(t *testing.T) { cfg := &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{AdvertiseAddress: addr}, Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - NodeName: "valid-hostname", + NodeName: hostname, } caCert, caKey, err := NewCACertAndKey() if err != nil { @@ -300,14 +300,14 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { t.Fatalf("failed creation of ca cert and key: %v", err) } - apiClientCert, _, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) + apiKubeletClientCert, _, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) if err != nil { t.Fatalf("failed creation of cert and key: %v", err) } - certstestutil.AssertCertificateIsSignedByCa(t, apiClientCert, caCert) - certstestutil.AssertCertificateHasClientAuthUsage(t, apiClientCert) - certstestutil.AssertCertificateHasOrganizations(t, apiClientCert, kubeadmconstants.MastersGroup) + certstestutil.AssertCertificateIsSignedByCa(t, apiKubeletClientCert, caCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, apiKubeletClientCert) + certstestutil.AssertCertificateHasOrganizations(t, apiKubeletClientCert, kubeadmconstants.MastersGroup) } func TestNewEtcdServerCertAndKey(t *testing.T) { diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index ee1016868f9..f79146a91f6 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -208,7 +208,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM backupEtcdDir := pathMgr.BackupEtcdDir() runningEtcdDir := cfg.Etcd.DataDir if err := util.CopyDir(runningEtcdDir, backupEtcdDir); err != nil { - return true, fmt.Errorf("failer to back up etcd data: %v", err) + return true, fmt.Errorf("failed to back up etcd data: %v", err) } // Need to check currently used version and version from constants, if differs then upgrade diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 6bd50ca9d70..2cf71ab9d50 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -308,13 +308,13 @@ func TestStaticPodControlPlane(t *testing.T) { defer os.RemoveAll(pathMgr.TempManifestDir()) defer os.RemoveAll(pathMgr.BackupManifestDir()) - tempCersDir, err := ioutil.TempDir("", "kubeadm-certs") + tempCertsDir, err := ioutil.TempDir("", "kubeadm-certs") if err != nil { t.Fatalf("couldn't create temporary certificates directory: %v", err) } - defer os.RemoveAll(tempCersDir) + defer os.RemoveAll(tempCertsDir) - oldcfg, err := getConfig("v1.7.0", tempCersDir) + oldcfg, err := getConfig("v1.7.0", tempCertsDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } @@ -337,6 +337,7 @@ func TestStaticPodControlPlane(t *testing.T) { t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err) } } + fmt.Printf("Wrote certs to %s\n", oldcfg.CertificatesDir) // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) @@ -353,7 +354,7 @@ func TestStaticPodControlPlane(t *testing.T) { t.Fatalf("couldn't read temp file: %v", err) } - newcfg, err := getConfig("v1.8.0", tempCersDir) + newcfg, err := getConfig("v1.8.0", tempCertsDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } From 509e9af5223de6849b7d99dbb53796f794876752 Mon Sep 17 00:00:00 2001 From: leigh schrandt Date: Fri, 23 Feb 2018 15:23:46 -0700 Subject: [PATCH 4/4] Update autogenerated docs --- docs/.generated_docs | 6 ++++++ .../kubeadm_alpha_phase_certs_apiserver-etcd-client.md | 3 +++ docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md | 3 +++ docs/admin/kubeadm_alpha_phase_certs_etcd-server.md | 3 +++ .../man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 | 3 +++ docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 | 3 +++ docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 | 3 +++ 7 files changed, 24 insertions(+) create mode 100644 docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md create mode 100644 docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md create mode 100644 docs/admin/kubeadm_alpha_phase_certs_etcd-server.md create mode 100644 docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 create mode 100644 docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 create mode 100644 docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 diff --git a/docs/.generated_docs b/docs/.generated_docs index 18e655dfcfc..f82b4b7d990 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -20,9 +20,12 @@ docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md docs/admin/kubeadm_alpha_phase_certs.md docs/admin/kubeadm_alpha_phase_certs_all.md +docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md docs/admin/kubeadm_alpha_phase_certs_apiserver-kubelet-client.md docs/admin/kubeadm_alpha_phase_certs_apiserver.md docs/admin/kubeadm_alpha_phase_certs_ca.md +docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md +docs/admin/kubeadm_alpha_phase_certs_etcd-server.md docs/admin/kubeadm_alpha_phase_certs_front-proxy-ca.md docs/admin/kubeadm_alpha_phase_certs_front-proxy-client.md docs/admin/kubeadm_alpha_phase_certs_sa.md @@ -83,9 +86,12 @@ docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 docs/man/man1/kubeadm-alpha-phase-certs-all.1 +docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 docs/man/man1/kubeadm-alpha-phase-certs-apiserver-kubelet-client.1 docs/man/man1/kubeadm-alpha-phase-certs-apiserver.1 docs/man/man1/kubeadm-alpha-phase-certs-ca.1 +docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 +docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 docs/man/man1/kubeadm-alpha-phase-certs-front-proxy-ca.1 docs/man/man1/kubeadm-alpha-phase-certs-front-proxy-client.1 docs/man/man1/kubeadm-alpha-phase-certs-sa.1 diff --git a/docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md b/docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md b/docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/admin/kubeadm_alpha_phase_certs_etcd-server.md b/docs/admin/kubeadm_alpha_phase_certs_etcd-server.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_alpha_phase_certs_etcd-server.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 b/docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file.