diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 983e9e8022d..2709df23e18 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -43,6 +43,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.AuthorizationModes = []string{"foo"} obj.CertificatesDir = "foo" obj.APIServerCertSANs = []string{"foo"} + obj.Etcd.ServerCertSANs = []string{"foo"} + obj.Etcd.PeerCertSANs = []string{"foo"} obj.Token = "foo" obj.CRISocket = "foo" obj.Etcd.Image = "foo" diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 70edf13f39a..cf6c9db3a1f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -178,6 +178,12 @@ type Etcd struct { Image string // SelfHosted holds configuration for self-hosting etcd. SelfHosted *SelfHostedEtcd + // ServerCertSANs sets extra Subject Alternative Names for the etcd server + // signing cert. This is currently used for the etcd static-pod. + ServerCertSANs []string + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer + // signing cert. This is currently used for the etcd static-pod. + PeerCertSANs []string } // SelfHostedEtcd describes options required to configure self-hosted etcd. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index 066dd7979cf..4d94ad7f208 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -170,6 +170,10 @@ type Etcd struct { Image string `json:"image"` // SelfHosted holds configuration for self-hosting etcd. SelfHosted *SelfHostedEtcd `json:"selfHosted,omitempty"` + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. + ServerCertSANs []string `json:"serverCertSANs,omitempty"` + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. + PeerCertSANs []string `json:"peerCertSANs,omitempty"` } // SelfHostedEtcd describes options required to configure self-hosted etcd. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go index 80c47bf9b90..1f5b4d2d0a0 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -122,6 +122,8 @@ func autoConvert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s co out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.Image = in.Image out.SelfHosted = (*kubeadm.SelfHostedEtcd)(unsafe.Pointer(in.SelfHosted)) + out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) + out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil } @@ -139,6 +141,8 @@ func autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s co out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.Image = in.Image out.SelfHosted = (*SelfHostedEtcd)(unsafe.Pointer(in.SelfHosted)) + out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) + out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go index 4eb4e008641..578c431af99 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go @@ -92,6 +92,16 @@ func (in *Etcd) DeepCopyInto(out *Etcd) { **out = **in } } + if in.ServerCertSANs != nil { + in, out := &in.ServerCertSANs, &out.ServerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PeerCertSANs != nil { + in, out := &in.PeerCertSANs, &out.PeerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index 27c31e0157a..3e8aa557515 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -72,7 +72,9 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateCloudProvider(c.CloudProvider, field.NewPath("cloudprovider"))...) allErrs = append(allErrs, ValidateAuthorizationModes(c.AuthorizationModes, field.NewPath("authorization-modes"))...) allErrs = append(allErrs, ValidateNetworking(&c.Networking, field.NewPath("networking"))...) - allErrs = append(allErrs, ValidateAPIServerCertSANs(c.APIServerCertSANs, field.NewPath("cert-altnames"))...) + allErrs = append(allErrs, ValidateCertSANs(c.APIServerCertSANs, field.NewPath("api-server-cert-altnames"))...) + allErrs = append(allErrs, ValidateCertSANs(c.Etcd.ServerCertSANs, field.NewPath("etcd-server-cert-altnames"))...) + allErrs = append(allErrs, ValidateCertSANs(c.Etcd.PeerCertSANs, field.NewPath("etcd-peer-cert-altnames"))...) allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificates-dir"))...) allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...) allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) @@ -228,8 +230,8 @@ func ValidateToken(t string, fldPath *field.Path) field.ErrorList { return allErrs } -// ValidateAPIServerCertSANs validates alternative names -func ValidateAPIServerCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { +// ValidateCertSANs validates alternative names +func ValidateCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} for _, altname := range altnames { if len(validation.IsDNS1123Subdomain(altname)) != 0 && net.ParseIP(altname) == nil { diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 331ad37609d..29262a998c1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -134,7 +134,7 @@ func TestValidateCloudProvider(t *testing.T) { } } -func TestValidateAPIServerCertSANs(t *testing.T) { +func TestValidateCertSANs(t *testing.T) { var tests = []struct { sans []string expected bool @@ -148,10 +148,10 @@ func TestValidateAPIServerCertSANs(t *testing.T) { {[]string{"my-hostname2", "my.other.subdomain", "2001:db8::10"}, true}, // supported } for _, rt := range tests { - actual := ValidateAPIServerCertSANs(rt.sans, nil) + actual := ValidateCertSANs(rt.sans, nil) if (len(actual) == 0) != rt.expected { t.Errorf( - "failed ValidateAPIServerCertSANs:\n\texpected: %t\n\t actual: %t", + "failed ValidateCertSANs:\n\texpected: %t\n\t actual: %t", rt.expected, (len(actual) == 0), ) diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 93775c57a21..30c6c6c5d92 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -92,6 +92,16 @@ func (in *Etcd) DeepCopyInto(out *Etcd) { **out = **in } } + if in.ServerCertSANs != nil { + in, out := &in.ServerCertSANs, &out.ServerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PeerCertSANs != nil { + in, out := &in.PeerCertSANs, &out.PeerCertSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index b63cf1bbdce..564b8754749 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -60,8 +60,8 @@ var ( apiServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the API server serving certificate and key and saves them into %s and %s files. - The certificate includes default subject alternative names and additional sans eventually provided by the user; - default sans are: , , kubernetes, kubernetes.default, kubernetes.default.svc, + The certificate includes default subject alternative names and additional SANs provided by the user; + default SANs are: , , kubernetes, kubernetes.default, kubernetes.default.svc, kubernetes.default.svc., (that is the .10 address in address space). If both files already exist, kubeadm skips the generation step and existing files will be used. @@ -74,6 +74,31 @@ var ( If both files already exist, kubeadm skips the generation step and existing files will be used. `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName) + etcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the etcd serving certificate and key and saves them into %s and %s files. + + The certificate includes default subject alternative names and additional SANs provided by the user; + default SANs are: localhost, 127.0.0.1. + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName) + + etcdPeerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the etcd peer certificate and key and saves them into %s and %s files. + + The certificate includes default subject alternative names and additional SANs provided by the user; + default SANs are: , . + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName) + + apiServerEtcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Generates the client certificate for the API server to connect to etcd securely and the respective key, + and saves them into %s and %s files. + + If both files already exist, kubeadm skips the generation step and existing files will be used. + `+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName) + saKeyLongDesc = fmt.Sprintf(normalizer.LongDesc(` Generates the private key for signing service account tokens along with its public key, and saves them into %s and %s files. @@ -157,6 +182,24 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command { long: apiServerKubeletCertLongDesc, cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, }, + { + use: "etcd-server", + short: "Generates etcd serving certificate and key", + long: etcdServerCertLongDesc, + cmdFunc: certsphase.CreateEtcdServerCertAndKeyFiles, + }, + { + use: "etcd-peer", + short: "Generates etcd peer certificate and key", + long: etcdPeerCertLongDesc, + cmdFunc: certsphase.CreateEtcdPeerCertAndKeyFiles, + }, + { + use: "apiserver-etcd-client", + short: "Generates client certificate for the API server to connect to etcd securely", + long: apiServerEtcdServerCertLongDesc, + cmdFunc: certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, + }, { use: "sa", short: "Generates a private key for signing service account tokens along with its public key", diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index d6e10000205..c2986240744 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -65,6 +65,33 @@ const ( // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" + // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdServerCertAndKeyBaseName = "etcd/server" + // EtcdServerCertName defines etcd's server certificate name + EtcdServerCertName = "etcd/server.crt" + // EtcdServerKeyName defines etcd's server key name + EtcdServerKeyName = "etcd/server.key" + // EtcdServerCertCommonName defines etcd's server certificate common name (CN) + EtcdServerCertCommonName = "kube-etcd" + + // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name + EtcdPeerCertAndKeyBaseName = "etcd/peer" + // EtcdPeerCertName defines etcd's peer certificate name + EtcdPeerCertName = "etcd/peer.crt" + // EtcdPeerKeyName defines etcd's peer key name + EtcdPeerKeyName = "etcd/peer.key" + // EtcdPeerCertCommonName defines etcd's peer certificate common name (CN) + EtcdPeerCertCommonName = "kube-etcd-peer" + + // APIServerEtcdClientCertAndKeyBaseName defines etcd client certificate and key base name + APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" + // APIServerEtcdClientCertName defines etcd client certificate name + APIServerEtcdClientCertName = "apiserver-etcd-client.crt" + // APIServerEtcdClientKeyName defines etcd client key name + APIServerEtcdClientKeyName = "apiserver-etcd-client.key" + // APIServerEtcdClientCertCommonName defines etcd client certificate common name (CN) + APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" + // ServiceAccountKeyBaseName defines SA key base name ServiceAccountKeyBaseName = "sa" // ServiceAccountPublicKeyName defines SA public key base name diff --git a/cmd/kubeadm/app/phases/certs/BUILD b/cmd/kubeadm/app/phases/certs/BUILD index 617c57d09c7..3ac7c0fab75 100644 --- a/cmd/kubeadm/app/phases/certs/BUILD +++ b/cmd/kubeadm/app/phases/certs/BUILD @@ -30,8 +30,6 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", - "//pkg/registry/core/service/ipallocator:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index 258c5b1da26..86583f4cf5d 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -20,16 +20,13 @@ import ( "crypto/rsa" "crypto/x509" "fmt" - "net" "os" "path/filepath" - "k8s.io/apimachinery/pkg/util/validation" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" - "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) // CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. @@ -40,6 +37,9 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { CreateCACertAndKeyfiles, CreateAPIServerCertAndKeyFiles, CreateAPIServerKubeletClientCertAndKeyFiles, + CreateEtcdServerCertAndKeyFiles, + CreateEtcdPeerCertAndKeyFiles, + CreateAPIServerEtcdClientCertAndKeyFiles, CreateServiceAccountKeyAndPublicKeyFiles, CreateFrontProxyCACertAndKeyFiles, CreateFrontProxyClientCertAndKeyFiles, @@ -79,7 +79,7 @@ func CreateCACertAndKeyfiles(cfg *kubeadmapi.MasterConfiguration) error { // It assumes the cluster CA certificate and key files should exists into the CertificatesDir func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } @@ -103,12 +103,12 @@ func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { // It assumes the cluster CA certificate and key files should exists into the CertificatesDir func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return err } - apiClientCert, apiClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) + apiKubeletClientCert, apiKubeletClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) if err != nil { return err } @@ -117,8 +117,80 @@ func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfigura cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, caCert, - apiClientCert, - apiClientKey, + apiKubeletClientCert, + apiKubeletClientKey, + ) +} + +// CreateEtcdServerCertAndKeyFiles create a new certificate and key file for etcd. +// If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + etcdServerCert, etcdServerKey, err := NewEtcdServerCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.EtcdServerCertAndKeyBaseName, + caCert, + etcdServerCert, + etcdServerKey, + ) +} + +// CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering. +// If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.EtcdPeerCertAndKeyBaseName, + caCert, + etcdPeerCert, + etcdPeerKey, + ) +} + +// CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd +// If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key file exist in the CertificatesDir +func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + apiEtcdClientCert, apiEtcdClientKey, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, + caCert, + apiEtcdClientCert, + apiEtcdClientKey, ) } @@ -163,7 +235,7 @@ func CreateFrontProxyCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) erro // It assumes the front proxy CAA certificate and key files should exists into the CertificatesDir func CreateFrontProxyClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { - frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName) + frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName) if err != nil { return err } @@ -196,7 +268,7 @@ func NewCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) { // NewAPIServerCertAndKey generate CA certificate for apiserver, signed by the given CA. func NewAPIServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { - altNames, err := getAltNames(cfg) + altNames, err := pkiutil.GetAPIServerAltNames(cfg) if err != nil { return nil, nil, fmt.Errorf("failure while composing altnames for API server: %v", err) } @@ -230,6 +302,64 @@ func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.Pr return apiClientCert, apiClientKey, nil } +// NewEtcdServerCertAndKey generate CA certificate for etcd, signed by the given CA. +func NewEtcdServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + altNames, err := pkiutil.GetEtcdAltNames(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failure while composing altnames for etcd: %v", err) + } + + config := certutil.Config{ + CommonName: kubeadmconstants.EtcdServerCertCommonName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + etcdServerCert, etcdServerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating etcd key and certificate: %v", err) + } + + return etcdServerCert, etcdServerKey, nil +} + +// NewEtcdPeerCertAndKey generate CA certificate for etcd peering, signed by the given CA. +func NewEtcdPeerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + altNames, err := pkiutil.GetEtcdPeerAltNames(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failure while composing altnames for etcd peering: %v", err) + } + + config := certutil.Config{ + CommonName: kubeadmconstants.EtcdPeerCertCommonName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + etcdPeerCert, etcdPeerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating etcd peer key and certificate: %v", err) + } + + return etcdPeerCert, etcdPeerKey, nil +} + +// NewAPIServerEtcdClientCertAndKey generate CA certificate for the apiservers to connect to etcd securely, signed by the given CA. +func NewAPIServerEtcdClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { + + config := certutil.Config{ + CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName, + Organization: []string{kubeadmconstants.MastersGroup}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config) + if err != nil { + return nil, nil, fmt.Errorf("failure while creating API server etcd client key and certificate: %v", err) + } + + return apiClientCert, apiClientKey, nil +} + // NewServiceAccountSigningKey generate public/private key pairs for signing service account tokens. func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) { @@ -268,8 +398,8 @@ func NewFrontProxyClientCertAndKey(frontProxyCACert *x509.Certificate, frontProx return frontProxyClientCert, frontProxyClientKey, nil } -// loadCertificateAuthorithy loads certificate authority -func loadCertificateAuthorithy(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) { +// loadCertificateAuthority loads certificate authority +func loadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) { // Checks if certificate authority exists in the PKI directory if !pkiutil.CertOrKeyExist(pkiDir, baseName) { return nil, nil, fmt.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir) @@ -503,55 +633,3 @@ func validatePrivatePublicKey(l certKeyLocation) error { } return nil } - -// getAltNames builds an AltNames object for to be used when generating apiserver certificate -func getAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { - - // advertise address - advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress) - if advertiseAddress == nil { - return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress) - } - - // internal IP address for the API server - _, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet) - if err != nil { - return nil, fmt.Errorf("error parsing CIDR %q: %v", cfg.Networking.ServiceSubnet, err) - } - - internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1) - if err != nil { - return nil, fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", svcSubnet.String(), err) - } - - // create AltNames with defaults DNSNames/IPs - altNames := &certutil.AltNames{ - DNSNames: []string{ - cfg.NodeName, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain), - }, - IPs: []net.IP{ - internalAPIServerVirtualIP, - advertiseAddress, - }, - } - - // adds additional SAN - for _, altname := range cfg.APIServerCertSANs { - if ip := net.ParseIP(altname); ip != nil { - altNames.IPs = append(altNames.IPs, ip) - } else if len(validation.IsDNS1123Subdomain(altname)) == 0 { - altNames.DNSNames = append(altNames.DNSNames, altname) - } - } - - // add api server dns advertise address - if len(cfg.API.ControlPlaneEndpoint) > 0 { - altNames.DNSNames = append(altNames.DNSNames, cfg.API.ControlPlaneEndpoint) - } - - return altNames, nil -} diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 1532c5e0403..55c1ea1f56d 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -258,53 +258,6 @@ func TestWriteKeyFilesIfNotExist(t *testing.T) { } } -func TestGetAltNames(t *testing.T) { - hostname := "valid-hostname" - advertiseIP := "1.2.3.4" - controlPlaneEndpoint := "api.k8s.io" - cfg := &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{AdvertiseAddress: advertiseIP, ControlPlaneEndpoint: controlPlaneEndpoint}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - NodeName: hostname, - APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95"}, - } - - altNames, err := getAltNames(cfg) - if err != nil { - t.Fatalf("failed calling getAltNames: %v", err) - } - - expectedDNSNames := []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", controlPlaneEndpoint} - for _, DNSName := range expectedDNSNames { - found := false - for _, val := range altNames.DNSNames { - if val == DNSName { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain DNSName %s", DNSName) - } - } - - expectedIPAddresses := []string{"10.96.0.1", advertiseIP, "10.1.245.94", "10.1.245.95"} - for _, IPAddress := range expectedIPAddresses { - found := false - for _, val := range altNames.IPs { - if val.Equal(net.ParseIP(IPAddress)) { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain IPAddress %s", IPAddress) - } - } -} - func TestNewCACertAndKey(t *testing.T) { caCert, _, err := NewCACertAndKey() if err != nil { @@ -322,7 +275,7 @@ func TestNewAPIServerCertAndKey(t *testing.T) { cfg := &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{AdvertiseAddress: addr}, Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - NodeName: "valid-hostname", + NodeName: hostname, } caCert, caKey, err := NewCACertAndKey() if err != nil { @@ -347,14 +300,93 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { t.Fatalf("failed creation of ca cert and key: %v", err) } - apiClientCert, _, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) + apiKubeletClientCert, _, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) if err != nil { t.Fatalf("failed creation of cert and key: %v", err) } - certstestutil.AssertCertificateIsSignedByCa(t, apiClientCert, caCert) - certstestutil.AssertCertificateHasClientAuthUsage(t, apiClientCert) - certstestutil.AssertCertificateHasOrganizations(t, apiClientCert, kubeadmconstants.MastersGroup) + certstestutil.AssertCertificateIsSignedByCa(t, apiKubeletClientCert, caCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, apiKubeletClientCert) + certstestutil.AssertCertificateHasOrganizations(t, apiKubeletClientCert, kubeadmconstants.MastersGroup) +} + +func TestNewEtcdServerCertAndKey(t *testing.T) { + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" + + cfg := &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ServerCertSANs: []string{ + proxy, + proxyIP, + }, + }, + } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdServerCert, _, err := NewEtcdServerCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdServerCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdServerCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdServerCert, "localhost", proxy) + certstestutil.AssertCertificateHasIPAddresses(t, etcdServerCert, net.ParseIP("127.0.0.1"), net.ParseIP(proxyIP)) +} + +func TestNewEtcdPeerCertAndKey(t *testing.T) { + hostname := "valid-hostname" + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" + + advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"} + for _, addr := range advertiseAddresses { + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: addr}, + NodeName: hostname, + Etcd: kubeadmapi.Etcd{ + PeerCertSANs: []string{ + proxy, + proxyIP, + }, + }, + } + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + etcdPeerCert, _, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, etcdPeerCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, etcdPeerCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, etcdPeerCert) + certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, proxy) + certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP(addr), net.ParseIP(proxyIP)) + } +} + +func TestNewAPIServerEtcdClientCertAndKey(t *testing.T) { + caCert, caKey, err := NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + apiEtcdClientCert, _, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey) + if err != nil { + t.Fatalf("failed creation of cert and key: %v", err) + } + + certstestutil.AssertCertificateIsSignedByCa(t, apiEtcdClientCert, caCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, apiEtcdClientCert) + certstestutil.AssertCertificateHasOrganizations(t, apiEtcdClientCert, kubeadmconstants.MastersGroup) } func TestNewNewServiceAccountSigningKey(t *testing.T) { @@ -551,6 +583,9 @@ func TestCreateCertificateFilesMethods(t *testing.T) { kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, + kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName, + kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName, + kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName, kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName, kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName, @@ -570,6 +605,21 @@ func TestCreateCertificateFilesMethods(t *testing.T) { createFunc: CreateAPIServerKubeletClientCertAndKeyFiles, expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateEtcdServerCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateEtcdPeerCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateAPIServerEtcdClientCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName}, + }, { createFunc: CreateServiceAccountKeyAndPublicKeyFiles, expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName}, diff --git a/cmd/kubeadm/app/phases/certs/doc.go b/cmd/kubeadm/app/phases/certs/doc.go index baf35b138be..da538747af7 100644 --- a/cmd/kubeadm/app/phases/certs/doc.go +++ b/cmd/kubeadm/app/phases/certs/doc.go @@ -23,7 +23,9 @@ package certs INPUTS: From MasterConfiguration .API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs - .APIServerCertSANs is needed for knowing which DNS names and IPs the API Server serving cert should be valid for + .APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN + .Etcd.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN + .Etcd.PeerCertSANs is an optional parameter for adding DNS names and IPs to the etcd peer cert SAN .Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has .Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to .CertificatesDir is required for knowing where all certificates should be stored @@ -36,6 +38,12 @@ package certs - apiserver.key - apiserver-kubelet-client.crt - apiserver-kubelet-client.key + - apiserver-etcd-client.crt + - apiserver-etcd-client.key + - etcd/server.crt + - etcd/server.key + - etcd/peer.crt + - etcd/peer.key - sa.pub - sa.key - front-proxy-ca.crt diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/BUILD b/cmd/kubeadm/app/phases/certs/pkiutil/BUILD index cb91095764b..f4c8a743178 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/BUILD +++ b/cmd/kubeadm/app/phases/certs/pkiutil/BUILD @@ -10,14 +10,23 @@ go_test( name = "go_default_test", srcs = ["pki_helpers_test.go"], embed = [":go_default_library"], - deps = ["//vendor/k8s.io/client-go/util/cert:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], ) go_library( name = "go_default_library", srcs = ["pki_helpers.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil", - deps = ["//vendor/k8s.io/client-go/util/cert:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//pkg/registry/core/service/ipallocator:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], ) filegroup( diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go index e6bcb27c955..996688ee685 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go @@ -20,11 +20,16 @@ import ( "crypto/rsa" "crypto/x509" "fmt" + "net" "os" "path/filepath" "time" + "k8s.io/apimachinery/pkg/util/validation" certutil "k8s.io/client-go/util/cert" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) // NewCertificateAuthority creates new certificate and private key for the certificate authority @@ -246,3 +251,106 @@ func pathForKey(pkiPath, name string) string { func pathForPublicKey(pkiPath, name string) string { return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name)) } + +// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate +func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { + // advertise address + advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress) + if advertiseAddress == nil { + return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress) + } + + // internal IP address for the API server + _, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet) + if err != nil { + return nil, fmt.Errorf("error parsing CIDR %q: %v", cfg.Networking.ServiceSubnet, err) + } + + internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1) + if err != nil { + return nil, fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", svcSubnet.String(), err) + } + + // create AltNames with defaults DNSNames/IPs + altNames := &certutil.AltNames{ + DNSNames: []string{ + cfg.NodeName, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain), + }, + IPs: []net.IP{ + internalAPIServerVirtualIP, + advertiseAddress, + }, + } + + // add api server dns advertise address + if len(cfg.API.ControlPlaneEndpoint) > 0 { + altNames.DNSNames = append(altNames.DNSNames, cfg.API.ControlPlaneEndpoint) + } + + appendSANsToAltNames(altNames, cfg.APIServerCertSANs, kubeadmconstants.APIServerCertName) + + return altNames, nil +} + +// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate. +// `localhost` is included in the SAN since this is the interface the etcd static pod listens on. +// Hostname and `API.AdvertiseAddress` are excluded since etcd does not listen on this interface by default. +// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`. +func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { + // create AltNames with defaults DNSNames/IPs + altNames := &certutil.AltNames{ + DNSNames: []string{"localhost"}, + IPs: []net.IP{net.IPv4(127, 0, 0, 1)}, + } + + appendSANsToAltNames(altNames, cfg.Etcd.ServerCertSANs, kubeadmconstants.EtcdServerCertName) + + return altNames, nil +} + +// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate. +// `localhost` is excluded from the SAN since etcd will not refer to itself as a peer. +// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one. +// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`. +func GetEtcdPeerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { + // advertise address + advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress) + if advertiseAddress == nil { + return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress) + } + + // create AltNames with defaults DNSNames/IPs + altNames := &certutil.AltNames{ + DNSNames: []string{cfg.NodeName}, + IPs: []net.IP{advertiseAddress}, + } + + appendSANsToAltNames(altNames, cfg.Etcd.PeerCertSANs, kubeadmconstants.EtcdPeerCertName) + + return altNames, nil +} + +// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert +// altNames is passed in with a pointer, and the struct is modified +// valid IP address strings are parsed and added to altNames.IPs as net.IP's +// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings +// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for +func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) { + for _, altname := range SANs { + if ip := net.ParseIP(altname); ip != nil { + altNames.IPs = append(altNames.IPs, ip) + } else if len(validation.IsDNS1123Subdomain(altname)) == 0 { + altNames.DNSNames = append(altNames.DNSNames, altname) + } else { + fmt.Printf( + "[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n", + altname, + certName, + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go index 34ac26ee14c..ba4057c6c92 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go @@ -21,10 +21,12 @@ import ( "crypto/rsa" "crypto/x509" "io/ioutil" + "net" "os" "testing" certutil "k8s.io/client-go/util/cert" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) func TestNewCertificateAuthority(t *testing.T) { @@ -432,3 +434,154 @@ func TestPathForPublicKey(t *testing.T) { t.Errorf("unexpected certificate path: %s", pubPath) } } + +func TestGetAPIServerAltNames(t *testing.T) { + hostname := "valid-hostname" + advertiseIP := "1.2.3.4" + controlPlaneEndpoint := "api.k8s.io" + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: advertiseIP, ControlPlaneEndpoint: controlPlaneEndpoint}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: hostname, + APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"}, + } + + altNames, err := GetAPIServerAltNames(cfg) + if err != nil { + t.Fatalf("failed calling GetAPIServerAltNames: %v", err) + } + + expectedDNSNames := []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", controlPlaneEndpoint} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{"10.96.0.1", advertiseIP, "10.1.245.94", "10.1.245.95"} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} + +func TestGetEtcdAltNames(t *testing.T) { + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" + cfg := &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ServerCertSANs: []string{ + proxy, + proxyIP, + "1.2.3.L", + "invalid,commas,in,DNS", + }, + }, + } + + altNames, err := GetEtcdAltNames(cfg) + if err != nil { + t.Fatalf("failed calling GetEtcdAltNames: %v", err) + } + + expectedDNSNames := []string{"localhost", proxy} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{"127.0.0.1", proxyIP} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} + +func TestGetEtcdPeerAltNames(t *testing.T) { + hostname := "valid-hostname" + proxy := "user-etcd-proxy" + proxyIP := "10.10.10.100" + advertiseIP := "1.2.3.4" + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: advertiseIP}, + NodeName: hostname, + Etcd: kubeadmapi.Etcd{ + PeerCertSANs: []string{ + proxy, + proxyIP, + "1.2.3.L", + "invalid,commas,in,DNS", + }, + }, + } + + altNames, err := GetEtcdPeerAltNames(cfg) + if err != nil { + t.Fatalf("failed calling GetEtcdPeerAltNames: %v", err) + } + + expectedDNSNames := []string{hostname, proxy} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{advertiseIP, proxyIP} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 8dbe1db716c..123763c2dce 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -190,21 +190,37 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServerExtraArgs)...) command = append(command, getAuthzParameters(cfg.AuthorizationModes)...) - // Check if the user decided to use an external etcd cluster + // If the user set endpoints for an external etcd cluster if len(cfg.Etcd.Endpoints) > 0 { command = append(command, fmt.Sprintf("--etcd-servers=%s", strings.Join(cfg.Etcd.Endpoints, ","))) - } else { - command = append(command, "--etcd-servers=http://127.0.0.1:2379") - } - // Is etcd secured? - if cfg.Etcd.CAFile != "" { - command = append(command, fmt.Sprintf("--etcd-cafile=%s", cfg.Etcd.CAFile)) - } - if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { - etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", cfg.Etcd.CertFile) - etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", cfg.Etcd.KeyFile) - command = append(command, etcdClientFileArg, etcdKeyFileArg) + // Use any user supplied etcd certificates + if cfg.Etcd.CAFile != "" { + command = append(command, fmt.Sprintf("--etcd-cafile=%s", cfg.Etcd.CAFile)) + } + if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", cfg.Etcd.CertFile) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", cfg.Etcd.KeyFile) + command = append(command, etcdClientFileArg, etcdKeyFileArg) + } + } else { + // Default to etcd static pod on localhost + etcdEndpointsArg := "--etcd-servers=https://127.0.0.1:2379" + etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)) + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)) + command = append(command, etcdEndpointsArg, etcdCAFileArg, etcdClientFileArg, etcdKeyFileArg) + + // Warn for unused user supplied variables + if cfg.Etcd.CAFile != "" { + fmt.Printf("[controlplane] WARNING: Configuration for %s CAFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CAFile, kubeadmconstants.Etcd) + } + if cfg.Etcd.CertFile != "" { + fmt.Printf("[controlplane] WARNING: Configuration for %s CertFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CertFile, kubeadmconstants.Etcd) + } + if cfg.Etcd.KeyFile != "" { + fmt.Printf("[controlplane] WARNING: Configuration for %s KeyFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.KeyFile, kubeadmconstants.Etcd) + } } if cfg.CloudProvider != "" { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index e12d700f10e..72110dfdb89 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -224,7 +224,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -258,7 +261,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -292,7 +298,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -327,9 +336,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -369,9 +379,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=4.3.2.1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -406,9 +417,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -443,9 +455,123 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + FeatureGates: map[string]bool{features.HighAvailability: true}, + Etcd: kubeadmapi.Etcd{Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, CAFile: "fuz", CertFile: "fiz", KeyFile: "faz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=https://8.6.4.1:2379,https://8.6.4.2:2379", + "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", + fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"}}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:2380", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{CAFile: "fuz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, { @@ -483,7 +609,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=2001:db8::1", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), "--audit-policy-file=/etc/kubernetes/audit/audit.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", @@ -522,7 +651,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--cloud-provider=gce", }, }, @@ -558,7 +690,10 @@ func TestGetAPIServerCommand(t *testing.T) { "--requestheader-allowed-names=front-proxy-client", "--authorization-mode=Node,RBAC", "--advertise-address=1.2.3.4", - "--etcd-servers=http://127.0.0.1:2379", + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=" + testCertsDir + "/ca.crt", + "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", + "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", "--cloud-provider=aws", }, }, diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 1de1e38adbe..d25503521f9 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -18,6 +18,7 @@ package etcd import ( "fmt" + "path/filepath" "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -28,7 +29,8 @@ import ( ) const ( - etcdVolumeName = "etcd" + etcdVolumeName = "etcd-data" + certsVolumeName = "k8s-certs" ) // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. @@ -50,7 +52,8 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { pathType := v1.HostPathDirectoryOrCreate etcdMounts := map[string]v1.Volume{ - etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir, &pathType), } return staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.Etcd, @@ -58,7 +61,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image), ImagePullPolicy: cfg.ImagePullPolicy, // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner - VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, + VolumeMounts: []v1.VolumeMount{ + staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false), + staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir, false), + }, LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP), }, etcdMounts) } @@ -66,9 +72,17 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { // getEtcdCommand builds the right etcd command from the given config object func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments := map[string]string{ - "listen-client-urls": "http://127.0.0.1:2379", - "advertise-client-urls": "http://127.0.0.1:2379", + "listen-client-urls": "https://127.0.0.1:2379", + "advertise-client-urls": "https://127.0.0.1:2379", "data-dir": cfg.Etcd.DataDir, + "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), + "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), + "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "client-cert-auth": "true", + "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), + "peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName), + "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "peer-client-cert-auth": "true", } command := []string{"etcd"} diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 569d2c6c5a1..22008bd38be 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -79,9 +79,17 @@ func TestGetEtcdCommand(t *testing.T) { }, expected: []string{ "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", + "--listen-client-urls=https://127.0.0.1:2379", + "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/var/lib/etcd", + "--cert-file=" + kubeadmconstants.EtcdServerCertName, + "--key-file=" + kubeadmconstants.EtcdServerKeyName, + "--trusted-ca-file=" + kubeadmconstants.CACertName, + "--client-cert-auth=true", + "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, + "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, + "--peer-trusted-ca-file=" + kubeadmconstants.CACertName, + "--peer-client-cert-auth=true", }, }, { @@ -89,16 +97,24 @@ func TestGetEtcdCommand(t *testing.T) { Etcd: kubeadmapi.Etcd{ DataDir: "/var/lib/etcd", ExtraArgs: map[string]string{ - "listen-client-urls": "http://10.0.1.10:2379", - "advertise-client-urls": "http://10.0.1.10:2379", + "listen-client-urls": "https://10.0.1.10:2379", + "advertise-client-urls": "https://10.0.1.10:2379", }, }, }, expected: []string{ "etcd", - "--listen-client-urls=http://10.0.1.10:2379", - "--advertise-client-urls=http://10.0.1.10:2379", + "--listen-client-urls=https://10.0.1.10:2379", + "--advertise-client-urls=https://10.0.1.10:2379", "--data-dir=/var/lib/etcd", + "--cert-file=" + kubeadmconstants.EtcdServerCertName, + "--key-file=" + kubeadmconstants.EtcdServerKeyName, + "--trusted-ca-file=" + kubeadmconstants.CACertName, + "--client-cert-auth=true", + "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, + "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, + "--peer-trusted-ca-file=" + kubeadmconstants.CACertName, + "--peer-client-cert-auth=true", }, }, { @@ -107,9 +123,17 @@ func TestGetEtcdCommand(t *testing.T) { }, expected: []string{ "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", + "--listen-client-urls=https://127.0.0.1:2379", + "--advertise-client-urls=https://127.0.0.1:2379", "--data-dir=/etc/foo", + "--cert-file=" + kubeadmconstants.EtcdServerCertName, + "--key-file=" + kubeadmconstants.EtcdServerKeyName, + "--trusted-ca-file=" + kubeadmconstants.CACertName, + "--client-cert-auth=true", + "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, + "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, + "--peer-trusted-ca-file=" + kubeadmconstants.CACertName, + "--peer-client-cert-auth=true", }, }, } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index dbce0222132..f79146a91f6 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -23,7 +23,8 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -133,6 +134,22 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP if component == constants.Etcd { recoverEtcd = true } + + // ensure etcd certs are generated for etcd and kube-apiserver + if component == constants.Etcd { + if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s certificate: %v", constants.Etcd, err) + } + if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s peer certificate: %v", constants.Etcd, err) + } + } + if component == constants.KubeAPIServer { + if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil { + return fmt.Errorf("failed to upgrade the %s %s-client certificate: %v", constants.KubeAPIServer, constants.Etcd, err) + } + } + // The old manifest is here; in the /etc/kubernetes/manifests/ currentManifestPath := pathMgr.RealManifestPath(component) // The new, upgraded manifest will be written here @@ -180,7 +197,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM if len(cfg.Etcd.Endpoints) != 0 { return false, fmt.Errorf("external etcd detected, won't try to change any etcd state") } - // Checking health state of etcd before proceeding with the upgrtade + // Checking health state of etcd before proceeding with the upgrade etcdCluster := util.LocalEtcdCluster{} etcdStatus, err := etcdCluster.GetEtcdClusterStatus() if err != nil { @@ -191,7 +208,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM backupEtcdDir := pathMgr.BackupEtcdDir() runningEtcdDir := cfg.Etcd.DataDir if err := util.CopyDir(runningEtcdDir, backupEtcdDir); err != nil { - return true, fmt.Errorf("fail to back up etcd data: %v", err) + return true, fmt.Errorf("failed to back up etcd data: %v", err) } // Need to check currently used version and version from constants, if differs then upgrade @@ -215,7 +232,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeName, constants.Etcd) if err != nil { - return true, fmt.Errorf("fail to get etcd pod's hash: %v", err) + return true, fmt.Errorf("failed to get etcd pod's hash: %v", err) } // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change @@ -227,7 +244,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM // Perform etcd upgrade using common to all control plane components function if err := upgradeComponent(constants.Etcd, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil { // Since etcd upgrade component failed, the old manifest has been restored - // now we need to check the heatlth of etcd cluster if it came back up with old manifest + // now we need to check the health of etcd cluster if it came back up with old manifest if _, err := etcdCluster.GetEtcdClusterStatus(); err != nil { // At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest if err := rollbackEtcdData(cfg, fmt.Errorf("etcd cluster is not healthy after upgrade: %v rolling back", err), pathMgr); err != nil { @@ -299,7 +316,7 @@ func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager // Write the updated static Pod manifests into the temporary directory fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir()) - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) if err != nil { return fmt.Errorf("error creating init static pod manifest files: %v", err) } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 5b131fbca1b..2cf71ab9d50 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -29,7 +29,8 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -49,7 +50,7 @@ apiServerExtraArgs: null authorizationModes: - Node - RBAC -certificatesDir: /etc/kubernetes/pki +certificatesDir: %s cloudProvider: "" controllerManagerExtraArgs: null etcd: @@ -60,6 +61,8 @@ etcd: extraArgs: null image: "" keyFile: "" + serverCertSANs: null + peerCertSANs: null featureFlags: null imageRepository: k8s.gcr.io kubernetesVersion: %s @@ -305,12 +308,39 @@ func TestStaticPodControlPlane(t *testing.T) { defer os.RemoveAll(pathMgr.TempManifestDir()) defer os.RemoveAll(pathMgr.BackupManifestDir()) - oldcfg, err := getConfig("v1.7.0") + tempCertsDir, err := ioutil.TempDir("", "kubeadm-certs") + if err != nil { + t.Fatalf("couldn't create temporary certificates directory: %v", err) + } + defer os.RemoveAll(tempCertsDir) + + oldcfg, err := getConfig("v1.7.0", tempCertsDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } + + // Initialize PKI minus any etcd certificates to simulate etcd PKI upgrade + certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{ + certsphase.CreateCACertAndKeyfiles, + certsphase.CreateAPIServerCertAndKeyFiles, + certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, + // certsphase.CreateEtcdServerCertAndKeyFiles, + // certsphase.CreateEtcdPeerCertAndKeyFiles, + // certsphase.CreateAPIServerEtcdClientCertAndKeyFiles, + certsphase.CreateServiceAccountKeyAndPublicKeyFiles, + certsphase.CreateFrontProxyCACertAndKeyFiles, + certsphase.CreateFrontProxyClientCertAndKeyFiles, + } + for _, action := range certActions { + err := action(oldcfg) + if err != nil { + t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err) + } + } + fmt.Printf("Wrote certs to %s\n", oldcfg.CertificatesDir) + // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) if err != nil { t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) } @@ -324,7 +354,7 @@ func TestStaticPodControlPlane(t *testing.T) { t.Fatalf("couldn't read temp file: %v", err) } - newcfg, err := getConfig("v1.8.0") + newcfg, err := getConfig("v1.8.0", tempCertsDir) if err != nil { t.Fatalf("couldn't create config: %v", err) } @@ -332,9 +362,10 @@ func TestStaticPodControlPlane(t *testing.T) { actualErr := StaticPodControlPlane(waiter, pathMgr, newcfg, false) if (actualErr != nil) != rt.expectedErr { t.Errorf( - "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t", + "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t\n\tactual error: %v", rt.expectedErr, (actualErr != nil), + actualErr, ) } @@ -365,10 +396,10 @@ func getAPIServerHash(dir string) (string, error) { return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil } -func getConfig(version string) (*kubeadmapi.MasterConfiguration, error) { +func getConfig(version string, certsDir string) (*kubeadmapi.MasterConfiguration, error) { externalcfg := &kubeadmapiext.MasterConfiguration{} internalcfg := &kubeadmapi.MasterConfiguration{} - if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, version)), externalcfg); err != nil { + if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, certsDir, version)), externalcfg); err != nil { return nil, fmt.Errorf("unable to decode config: %v", err) } legacyscheme.Scheme.Convert(externalcfg, internalcfg, nil) diff --git a/docs/.generated_docs b/docs/.generated_docs index 18e655dfcfc..f82b4b7d990 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -20,9 +20,12 @@ docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md docs/admin/kubeadm_alpha_phase_certs.md docs/admin/kubeadm_alpha_phase_certs_all.md +docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md docs/admin/kubeadm_alpha_phase_certs_apiserver-kubelet-client.md docs/admin/kubeadm_alpha_phase_certs_apiserver.md docs/admin/kubeadm_alpha_phase_certs_ca.md +docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md +docs/admin/kubeadm_alpha_phase_certs_etcd-server.md docs/admin/kubeadm_alpha_phase_certs_front-proxy-ca.md docs/admin/kubeadm_alpha_phase_certs_front-proxy-client.md docs/admin/kubeadm_alpha_phase_certs_sa.md @@ -83,9 +86,12 @@ docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 docs/man/man1/kubeadm-alpha-phase-certs-all.1 +docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 docs/man/man1/kubeadm-alpha-phase-certs-apiserver-kubelet-client.1 docs/man/man1/kubeadm-alpha-phase-certs-apiserver.1 docs/man/man1/kubeadm-alpha-phase-certs-ca.1 +docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 +docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 docs/man/man1/kubeadm-alpha-phase-certs-front-proxy-ca.1 docs/man/man1/kubeadm-alpha-phase-certs-front-proxy-client.1 docs/man/man1/kubeadm-alpha-phase-certs-sa.1 diff --git a/docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md b/docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_alpha_phase_certs_apiserver-etcd-client.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md b/docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_alpha_phase_certs_etcd-peer.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/admin/kubeadm_alpha_phase_certs_etcd-server.md b/docs/admin/kubeadm_alpha_phase_certs_etcd-server.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_alpha_phase_certs_etcd-server.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 b/docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-certs-apiserver-etcd-client.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-peer.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-certs-etcd-server.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file.