diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 19214f35d89..02946a04858 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -21,6 +21,7 @@ package app import ( "crypto/tls" + "fmt" "net" "net/http" "net/url" @@ -49,7 +50,6 @@ import ( generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver/authorizer" - genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/runtime/schema" @@ -85,20 +85,25 @@ func Run(s *options.ServerRunOptions) error { return err } - genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) - genericConfig := genericapiserver.NewConfig(). // create the new config - ApplyOptions(s.GenericServerRunOptions). // apply the options selected - ApplySecureServingOptions(s.SecureServing). - ApplyInsecureServingOptions(s.InsecureServing). - ApplyAuthenticationOptions(s.Authentication). - ApplyRBACSuperUser(s.Authorization.RBACSuperUser) - serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange) if err != nil { - glog.Fatalf("Error determining service IP ranges: %v", err) + return fmt.Errorf("error determining service IP ranges: %v", err) } - if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil { - glog.Fatalf("Failed to generate service certificate: %v", err) + + if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil { + return fmt.Errorf("error creating self-signed certificates: %v", err) + } + + genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) + + genericConfig, err := genericapiserver.NewConfig(). // create the new config + ApplyOptions(s.GenericServerRunOptions). // apply the options selected + ApplyInsecureServingOptions(s.InsecureServing). + ApplyAuthenticationOptions(s.Authentication). + ApplyRBACSuperUser(s.Authorization.RBACSuperUser). + ApplySecureServingOptions(s.SecureServing) + if err != nil { + return fmt.Errorf("failed to configure https: %s", err) } capabilities.Initialize(capabilities.Capabilities{ @@ -232,7 +237,7 @@ func Run(s *options.ServerRunOptions) error { } privilegedLoopbackToken := uuid.NewRandom().String() - selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken) + selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index cc4e4618dc7..76044259fd8 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -36,6 +36,8 @@ import ( // Install the testgroup API _ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install" + + "github.com/golang/glog" ) const ( @@ -93,20 +95,21 @@ func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error { if errs := serverOptions.InsecureServing.Validate("insecure-port"); len(errs) > 0 { return utilerrors.NewAggregate(errs) } + if err := serverOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(serverOptions.GenericServerRunOptions.AdvertiseAddress.String()); err != nil { + glog.Fatalf("Error creating self-signed certificates: %v", err) + } - config := genericapiserver.NewConfig(). + config, err := genericapiserver.NewConfig(). ApplyOptions(serverOptions.GenericServerRunOptions). - ApplySecureServingOptions(serverOptions.SecureServing). ApplyInsecureServingOptions(serverOptions.InsecureServing). ApplyAuthenticationOptions(serverOptions.Authentication). - Complete() - if err := config.MaybeGenerateServingCerts(); err != nil { - // this wasn't treated as fatal for this process before - fmt.Printf("Error creating cert: %v", err) + ApplySecureServingOptions(serverOptions.SecureServing) + if err != nil { + return fmt.Errorf("failed to configure https: %s", err) } config.Authorizer = authorizer.NewAlwaysAllowAuthorizer() - s, err := config.New() + s, err := config.Complete().New() if err != nil { return fmt.Errorf("Error in bringing up the server: %v", err) } diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index ba3d56bd74c..ee993699659 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -20,6 +20,7 @@ limitations under the License. package app import ( + "fmt" "strings" "time" @@ -37,7 +38,6 @@ import ( "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver/authorizer" - genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/generic" genericregistry "k8s.io/kubernetes/pkg/registry/generic/registry" @@ -73,16 +73,19 @@ func Run(s *options.ServerRunOptions) error { return err } - genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) - genericConfig := genericapiserver.NewConfig(). // create the new config - ApplyOptions(s.GenericServerRunOptions). // apply the options selected - ApplySecureServingOptions(s.SecureServing). - ApplyInsecureServingOptions(s.InsecureServing). - ApplyAuthenticationOptions(s.Authentication). - ApplyRBACSuperUser(s.Authorization.RBACSuperUser) + if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String()); err != nil { + return fmt.Errorf("error creating self-signed certificates: %v", err) + } - if err := genericConfig.MaybeGenerateServingCerts(); err != nil { - glog.Fatalf("Failed to generate service certificate: %v", err) + genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) + genericConfig, err := genericapiserver.NewConfig(). // create the new config + ApplyOptions(s.GenericServerRunOptions). // apply the options selected + ApplyInsecureServingOptions(s.InsecureServing). + ApplyAuthenticationOptions(s.Authentication). + ApplyRBACSuperUser(s.Authorization.RBACSuperUser). + ApplySecureServingOptions(s.SecureServing) + if err != nil { + return fmt.Errorf("failed to configure https: %s", err) } // TODO: register cluster federation resources here. @@ -130,7 +133,7 @@ func Run(s *options.ServerRunOptions) error { } privilegedLoopbackToken := uuid.NewRandom().String() - selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken) + selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index 2a9b1ad262c..06e8f7dbc53 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -17,12 +17,14 @@ limitations under the License. package genericapiserver import ( + "crypto/tls" + "encoding/pem" "fmt" "io" + "io/ioutil" "net" "net/http" "os" - "path" "regexp" goruntime "runtime" "sort" @@ -58,7 +60,6 @@ import ( "k8s.io/kubernetes/pkg/genericapiserver/routes" genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" "k8s.io/kubernetes/pkg/runtime" - certutil "k8s.io/kubernetes/pkg/util/cert" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/version" authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union" @@ -169,35 +170,21 @@ type ServingInfo struct { type SecureServingInfo struct { ServingInfo - // ServerCert is the TLS cert info for serving secure traffic - ServerCert GeneratableKeyCert - // SNICerts are named CertKeys for serving secure traffic with SNI support. - SNICerts []NamedCertKey + // Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is + // allowed to be in SNICerts. + Cert *tls.Certificate + + // CACert is an optional certificate authority used for the loopback connection of the Admission controllers. + // If this is nil, the certificate authority is extracted from Cert or a matching SNI certificate. + CACert *tls.Certificate + + // SNICerts are the TLS certificates by name used for SNI. + SNICerts map[string]*tls.Certificate + // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates ClientCA string } -type CertKey struct { - // CertFile is a file containing a PEM-encoded certificate - CertFile string - // KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile - KeyFile string -} - -type NamedCertKey struct { - CertKey - - // Names is a list of domain patterns: fully qualified domain names, possibly prefixed with - // wildcard segments. - Names []string -} - -type GeneratableKeyCert struct { - CertKey - // Generate indicates that the cert/key pair should be generated if its not present. - Generate bool -} - // NewConfig returns a Config struct with the default values func NewConfig() *Config { longRunningRE := regexp.MustCompile(options.DefaultLongRunningRequestRE) @@ -238,45 +225,69 @@ func NewConfig() *Config { return config.ApplyOptions(defaultOptions) } -func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) *Config { +func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) (*Config, error) { if secureServing == nil || secureServing.ServingOptions.BindPort <= 0 { - return c + return c, nil } secureServingInfo := &SecureServingInfo{ ServingInfo: ServingInfo{ BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)), }, - ServerCert: GeneratableKeyCert{ - CertKey: CertKey{ - CertFile: secureServing.ServerCert.CertKey.CertFile, - KeyFile: secureServing.ServerCert.CertKey.KeyFile, - }, - }, - SNICerts: []NamedCertKey{}, ClientCA: secureServing.ClientCA, } - if secureServing.ServerCert.CertKey.CertFile == "" && secureServing.ServerCert.CertKey.KeyFile == "" { - secureServingInfo.ServerCert.Generate = true - secureServingInfo.ServerCert.CertFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".crt") - secureServingInfo.ServerCert.KeyFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".key") + + serverCertFile, serverKeyFile := secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile + + // load main cert + if len(serverCertFile) != 0 || len(serverKeyFile) != 0 { + tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile) + if err != nil { + return nil, fmt.Errorf("unable to load server certificate: %v", err) + } + secureServingInfo.Cert = &tlsCert } - secureServingInfo.SNICerts = nil - for _, nkc := range secureServing.SNICertKeys { - secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{ - CertKey: CertKey{ - KeyFile: nkc.KeyFile, - CertFile: nkc.CertFile, - }, - Names: nkc.Names, + // optionally load CA cert + if len(secureServing.ServerCert.CACertFile) != 0 { + pemData, err := ioutil.ReadFile(secureServing.ServerCert.CACertFile) + if err != nil { + return nil, fmt.Errorf("failed to read certificate authority from %q: %v", secureServing.ServerCert.CACertFile, err) + } + block, pemData := pem.Decode(pemData) + if block == nil { + return nil, fmt.Errorf("no certificate found in certificate authority file %q", secureServing.ServerCert.CACertFile) + } + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("expected CERTIFICATE block in certiticate authority file %q, found: %s", secureServing.ServerCert.CACertFile, block.Type) + } + secureServingInfo.CACert = &tls.Certificate{ + Certificate: [][]byte{block.Bytes}, + } + } + + // load SNI certs + namedTlsCerts := make([]namedTlsCert, 0, len(secureServing.SNICertKeys)) + for _, nck := range secureServing.SNICertKeys { + tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile) + namedTlsCerts = append(namedTlsCerts, namedTlsCert{ + tlsCert: tlsCert, + names: nck.Names, }) + if err != nil { + return nil, fmt.Errorf("failed to load SNI cert and key: %v", err) + } + } + var err error + secureServingInfo.SNICerts, err = getNamedCertificateMap(namedTlsCerts) + if err != nil { + return nil, err } c.SecureServingInfo = secureServingInfo c.ReadWritePort = secureServing.ServingOptions.BindPort - return c + return c, nil } func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOptions) *Config { @@ -456,38 +467,6 @@ func (c completedConfig) New() (*GenericAPIServer, error) { return s, nil } -// MaybeGenerateServingCerts generates serving certificates if requested and needed. -func (c *Config) MaybeGenerateServingCerts(alternateIPs ...net.IP) error { - // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless - // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") - if c.SecureServingInfo != nil && c.SecureServingInfo.ServerCert.Generate { - canReadCertAndKey, err := certutil.CanReadCertAndKey(c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile) - if err != nil { - return err - } - if canReadCertAndKey { - return nil - } - // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? - alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} - - if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil { - return fmt.Errorf("unable to generate self signed cert: %v", err) - } else { - if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil { - return err - } - - if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil { - return err - } - glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile) - } - } - - return nil -} - func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) (secure, insecure http.Handler) { generic := func(handler http.Handler) http.Handler { handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") diff --git a/pkg/genericapiserver/config_selfclient.go b/pkg/genericapiserver/config_selfclient.go new file mode 100644 index 00000000000..17ab5d3da3b --- /dev/null +++ b/pkg/genericapiserver/config_selfclient.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "bytes" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net" + + "k8s.io/kubernetes/pkg/client/restclient" +) + +// NewSelfClientConfig returns a clientconfig which can be used to talk to this apiserver. +func NewSelfClientConfig(secureServingInfo *SecureServingInfo, insecureServingInfo *ServingInfo, token string) (*restclient.Config, error) { + if cfg, err := secureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil { + return cfg, err + } + if cfg, err := insecureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil { + return cfg, err + } + + return nil, errors.New("Unable to set url for apiserver local client") +} + +func (s *SecureServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) { + if s == nil || (s.Cert == nil && len(s.SNICerts) == 0) { + return nil, nil + } + + host, port, err := net.SplitHostPort(s.ServingInfo.BindAddress) + if err != nil { + // should never happen + return nil, fmt.Errorf("invalid secure bind address: %q", s.ServingInfo.BindAddress) + } + if host == "0.0.0.0" { + // compare MaybeDefaultWithSelfSignedCerts which adds "localhost" to the cert as alternateDNS + host = "localhost" + } + + clientConfig := &restclient.Config{ + // Increase QPS limits. The client is currently passed to all admission plugins, + // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 + // for more details. Once #22422 is fixed, we may want to remove it. + QPS: 50, + Burst: 100, + Host: "https://" + net.JoinHostPort(host, port), + BearerToken: token, + } + + // find certificate for host: either explicitly given, from the server cert bundle or one of the SNI certs + var derCA []byte + if s.CACert != nil { + derCA = s.CACert.Certificate[0] + } + if derCA == nil && s.Cert != nil { + x509Cert, err := x509.ParseCertificate(s.Cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse server certificate: %v", err) + } + + if (net.ParseIP(host) != nil && certMatchesIP(x509Cert, host)) || certMatchesName(x509Cert, host) { + derCA = s.Cert.Certificate[0] + } + } + if derCA == nil && net.ParseIP(host) == nil { + if cert, found := s.SNICerts[host]; found { + derCA = cert.Certificate[0] + } + } + if derCA == nil { + return nil, fmt.Errorf("failed to find certificate which matches %q", host) + } + pemCA := bytes.Buffer{} + if err := pem.Encode(&pemCA, &pem.Block{Type: "CERTIFICATE", Bytes: derCA}); err != nil { + return nil, err + } + clientConfig.CAData = pemCA.Bytes() + + return clientConfig, nil +} + +func (s *ServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) { + if s == nil { + return nil, nil + } + return &restclient.Config{ + Host: s.BindAddress, + // Increase QPS limits. The client is currently passed to all admission plugins, + // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 + // for more details. Once #22422 is fixed, we may want to remove it. + QPS: 50, + Burst: 100, + }, nil +} + +func certMatchesName(cert *x509.Certificate, name string) bool { + for _, certName := range cert.DNSNames { + if certName == name { + return true + } + } + + return false +} + +func certMatchesIP(cert *x509.Certificate, ip string) bool { + for _, certIP := range cert.IPAddresses { + if certIP.String() == ip { + return true + } + } + + return false +} diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index f16bcc7b527..65df7f219c0 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -75,7 +75,6 @@ type ServerRunOptions struct { // for testing). This is not actually exposed as a flag. DefaultStorageVersions string TargetRAMMB int - TLSCAFile string WatchCacheSizes []string } diff --git a/pkg/genericapiserver/options/serving.go b/pkg/genericapiserver/options/serving.go index 8487b9e35af..bcf880c09d5 100644 --- a/pkg/genericapiserver/options/serving.go +++ b/pkg/genericapiserver/options/serving.go @@ -17,14 +17,14 @@ limitations under the License. package options import ( - "errors" "fmt" "net" - "strconv" + "path" + "github.com/golang/glog" "github.com/spf13/pflag" - "k8s.io/kubernetes/pkg/client/restclient" + certutil "k8s.io/kubernetes/pkg/util/cert" "k8s.io/kubernetes/pkg/util/config" utilnet "k8s.io/kubernetes/pkg/util/net" ) @@ -43,14 +43,10 @@ type SecureServingOptions struct { SNICertKeys []config.NamedCertKey // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates ClientCA string - - // ServerCA is the certificate bundle for the signer of your serving certificate. Used for building a loopback - // connection to the API server for admission. - ServerCA string } type CertKey struct { - // CertFile is a file containing a PEM-encoded certificate + // CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain CertFile string // KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile KeyFile string @@ -59,6 +55,8 @@ type CertKey struct { type GeneratableKeyCert struct { CertKey CertKey + // CACertFile is an optional file containing the certificate chain for CertKey.CertFile + CACertFile string // CertDirectory is a directory that will contain the certificates. If the cert and key aren't specifically set // this will be used to derive a match with the "pair-name" CertDirectory string @@ -80,31 +78,6 @@ func NewSecureServingOptions() *SecureServingOptions { } } -func (s *SecureServingOptions) NewSelfClientConfig(token string) *restclient.Config { - if s == nil || s.ServingOptions.BindPort <= 0 || len(s.ServerCA) == 0 { - return nil - } - - clientConfig := &restclient.Config{ - // Increase QPS limits. The client is currently passed to all admission plugins, - // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 - // for more details. Once #22422 is fixed, we may want to remove it. - QPS: 50, - Burst: 100, - } - - // Use secure port if the ServerCA is specified - host := s.ServingOptions.BindAddress.String() - if host == "0.0.0.0" { - host = "localhost" - } - clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.ServingOptions.BindPort)) - clientConfig.CAFile = s.ServerCA - clientConfig.BearerToken = token - - return clientConfig -} - func (s *SecureServingOptions) Validate() []error { errors := []error{} if s == nil { @@ -138,6 +111,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") + fs.StringVar(&s.ServerCert.CACertFile, "tls-ca-file", s.ServerCert.CACertFile, "If set, this "+ + "certificate authority will used for secure access from Admission "+ + "Controllers. This must be a valid PEM-encoded CA bundle. Altneratively, the certificate authority "+ + "can be appended to the certificate provided by --tls-cert-file.") + fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+ "A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+ "domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+ @@ -151,18 +129,12 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { "If set, any request presenting a client certificate signed by one of "+ "the authorities in the client-ca-file is authenticated with an identity "+ "corresponding to the CommonName of the client certificate.") - - fs.StringVar(&s.ServerCA, "tls-ca-file", s.ServerCA, "If set, this "+ - "certificate authority will used for secure access from Admission "+ - "Controllers. This must be a valid PEM-encoded CA bundle.") - } func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress, "DEPRECATED: see --bind-address instead.") fs.MarkDeprecated("public-address-override", "see --bind-address instead.") - } func NewInsecureServingOptions() *ServingOptions { @@ -182,23 +154,6 @@ func (s ServingOptions) Validate(portArg string) []error { return errors } -func (s *ServingOptions) NewSelfClientConfig(token string) *restclient.Config { - if s == nil || s.BindPort <= 0 { - return nil - } - clientConfig := &restclient.Config{ - // Increase QPS limits. The client is currently passed to all admission plugins, - // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 - // for more details. Once #22422 is fixed, we may want to remove it. - QPS: 50, - Burst: 100, - } - - clientConfig.Host = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)) - - return clientConfig -} - func (s *ServingOptions) DefaultExternalAddress() (net.IP, error) { return utilnet.ChooseBindAddress(s.BindAddress) } @@ -224,14 +179,47 @@ func (s *ServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.MarkDeprecated("port", "see --insecure-port instead.") } -// Returns a clientconfig which can be used to talk to this apiserver. -func NewSelfClientConfig(secureServingOptions *SecureServingOptions, insecureServingOptions *ServingOptions, token string) (*restclient.Config, error) { - if cfg := secureServingOptions.NewSelfClientConfig(token); cfg != nil { - return cfg, nil - } - if cfg := insecureServingOptions.NewSelfClientConfig(token); cfg != nil { - return cfg, nil +func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateIPs ...net.IP) error { + keyCert := &s.ServerCert.CertKey + + if s == nil || len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 { + return nil } - return nil, errors.New("Unable to set url for apiserver local client") + keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt") + keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key") + + canReadCertAndKey, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile) + if err != nil { + return err + } + if !canReadCertAndKey { + // TODO: It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless + // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") + // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? + alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} + + // add either the bind address or localhost to the valid alternates + bindIP := s.ServingOptions.BindAddress.String() + if bindIP == "0.0.0.0" { + alternateDNS = append(alternateDNS, "localhost") + } else { + alternateIPs = append(alternateIPs, s.ServingOptions.BindAddress) + } + + if cert, key, err := certutil.GenerateSelfSignedCertKey(publicAddress, alternateIPs, alternateDNS); err != nil { + return fmt.Errorf("unable to generate self signed cert: %v", err) + } else { + if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil { + return err + } + + if err := certutil.WriteKey(keyCert.KeyFile, key); err != nil { + return err + } + glog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile) + } + } + + return nil } diff --git a/pkg/genericapiserver/serve.go b/pkg/genericapiserver/serve.go index bcdf1921288..1a17eca3ea1 100644 --- a/pkg/genericapiserver/serve.go +++ b/pkg/genericapiserver/serve.go @@ -42,17 +42,12 @@ const ( // be loaded or the initial listen call fails. The actual server loop (stoppable by closing // stopCh) runs in a go routine, i.e. serveSecurely does not block. func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { - namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts) - if err != nil { - return fmt.Errorf("unable to load SNI certificates: %v", err) - } - secureServer := &http.Server{ Addr: s.SecureServingInfo.BindAddress, Handler: s.Handler, MaxHeaderBytes: 1 << 20, TLSConfig: &tls.Config{ - NameToCertificate: namedCerts, + NameToCertificate: s.SecureServingInfo.SNICerts, // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage @@ -62,19 +57,15 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { }, } - if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 { - secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1) - secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile) - if err != nil { - return fmt.Errorf("unable to load server certificate: %v", err) - } + if s.SecureServingInfo.Cert != nil { + secureServer.TLSConfig.Certificates = []tls.Certificate{*s.SecureServingInfo.Cert} } // append all named certs. Otherwise, the go tls stack will think no SNI processing // is necessary because there is only one cert anyway. // Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI // cert will become the default cert. That's what we expect anyway. - for _, c := range namedCerts { + for _, c := range s.SecureServingInfo.SNICerts { secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c) } @@ -91,6 +82,7 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { } glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress) + var err error s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh) return err } @@ -186,48 +178,40 @@ func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int return tcpAddr.Port, nil } -// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in -// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded. -// Returns nil if len(namedCertKeys) == 0 -func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) { - if len(namedCertKeys) == 0 { - return nil, nil - } +type namedTlsCert struct { + tlsCert tls.Certificate - // load keys - tlsCerts := make([]tls.Certificate, len(namedCertKeys)) - for i := range namedCertKeys { - var err error - nkc := &namedCertKeys[i] - tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile) - if err != nil { - return nil, err - } - } + // names is a list of domain patterns: fully qualified domain names, possibly prefixed with + // wildcard segments. + names []string +} +// getNamedCertificateMap returns a map of *tls.Certificate by name. It's is +// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs +// cannot be loaded. Returns nil if len(certs) == 0 +func getNamedCertificateMap(certs []namedTlsCert) (map[string]*tls.Certificate, error) { // register certs with implicit names first, reverse order such that earlier trump over the later - tlsCertsByName := map[string]*tls.Certificate{} - for i := len(namedCertKeys) - 1; i >= 0; i-- { - nkc := &namedCertKeys[i] - if len(nkc.Names) > 0 { + byName := map[string]*tls.Certificate{} + for i := len(certs) - 1; i >= 0; i-- { + if len(certs[i].names) > 0 { continue } - cert := &tlsCerts[i] + cert := &certs[i].tlsCert // read names from certificate common names and DNS names if len(cert.Certificate) == 0 { - return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile) + return nil, fmt.Errorf("empty SNI certificate, skipping") } x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) if err != nil { - return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err) + return nil, fmt.Errorf("parse error for SNI certificate: %v", err) } cn := x509Cert.Subject.CommonName if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 { - tlsCertsByName[cn] = cert + byName[cn] = cert } for _, san := range x509Cert.DNSNames { - tlsCertsByName[san] = cert + byName[san] = cert } // intentionally all IPs in the cert are ignored as SNI forbids passing IPs // to select a cert. Before go 1.6 the tls happily passed IPs as SNI values. @@ -235,17 +219,14 @@ func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certi // register certs with explicit names last, overwriting every of the implicit ones, // again in reverse order. - for i := len(namedCertKeys) - 1; i >= 0; i-- { - nkc := &namedCertKeys[i] - if len(nkc.Names) == 0 { - continue - } - for _, name := range nkc.Names { - tlsCertsByName[name] = &tlsCerts[i] + for i := len(certs) - 1; i >= 0; i-- { + namedCert := &certs[i] + for _, name := range namedCert.names { + byName[name] = &certs[i].tlsCert } } - return tlsCertsByName, nil + return byName, nil } // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted diff --git a/pkg/genericapiserver/serve_test.go b/pkg/genericapiserver/serve_test.go index aac9499a6ff..bbdb6041c65 100644 --- a/pkg/genericapiserver/serve_test.go +++ b/pkg/genericapiserver/serve_test.go @@ -20,15 +20,18 @@ import ( "crypto/tls" "crypto/x509" "encoding/base64" + "encoding/pem" "fmt" "io/ioutil" "net" "os" "testing" - utilcert "k8s.io/kubernetes/pkg/util/cert" - "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/pkg/genericapiserver/options" + utilcert "k8s.io/kubernetes/pkg/util/cert" + "k8s.io/kubernetes/pkg/util/config" ) type TestCertSpec struct { @@ -41,47 +44,6 @@ type NamedTestCertSpec struct { explicitNames []string // as --tls-sni-cert-key explicit names } -func createTestCerts(spec TestCertSpec) (certFilePath, keyFilePath string, err error) { - var ips []net.IP - for _, ip := range spec.ips { - ips = append(ips, net.ParseIP(ip)) - } - - certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, ips, spec.names) - if err != nil { - return "", "", err - } - - certFile, err := ioutil.TempFile(os.TempDir(), "cert") - if err != nil { - return "", "", err - } - - keyFile, err := ioutil.TempFile(os.TempDir(), "key") - if err != nil { - os.Remove(certFile.Name()) - return "", "", err - } - - _, err = certFile.Write(certPem) - if err != nil { - os.Remove(certFile.Name()) - os.Remove(keyFile.Name()) - return "", "", err - } - certFile.Close() - - _, err = keyFile.Write(keyPem) - if err != nil { - os.Remove(certFile.Name()) - os.Remove(keyFile.Name()) - return "", "", err - } - keyFile.Close() - - return certFile.Name(), keyFile.Name(), nil -} - func TestGetNamedCertificateMap(t *testing.T) { tests := []struct { certs []NamedTestCertSpec @@ -225,26 +187,21 @@ func TestGetNamedCertificateMap(t *testing.T) { NextTest: for i, test := range tests { - var namedCertKeys []NamedCertKey + var namedTLSCerts []namedTlsCert bySignature := map[string]int{} // index in test.certs by cert signature for j, c := range test.certs { - certFile, keyFile, err := createTestCerts(c.TestCertSpec) + cert, err := createTestTLSCerts(c.TestCertSpec) if err != nil { t.Errorf("%d - failed to create cert %d: %v", i, j, err) continue NextTest } - defer os.Remove(certFile) - defer os.Remove(keyFile) - namedCertKeys = append(namedCertKeys, NamedCertKey{ - CertKey: CertKey{ - KeyFile: keyFile, - CertFile: certFile, - }, - Names: c.explicitNames, + namedTLSCerts = append(namedTLSCerts, namedTlsCert{ + tlsCert: cert, + names: c.explicitNames, }) - sig, err := certFileSignature(certFile, keyFile) + sig, err := certSignature(cert) if err != nil { t.Errorf("%d - failed to get signature for %d: %v", i, j, err) continue NextTest @@ -252,7 +209,7 @@ NextTest: bySignature[sig] = j } - certMap, err := getNamedCertificateMap(namedCertKeys) + certMap, err := getNamedCertificateMap(namedTLSCerts) if err == nil && len(test.errorString) != 0 { t.Errorf("%d - expected no error, got: %v", i, err) } else if err != nil && err.Error() != test.errorString { @@ -363,19 +320,30 @@ func TestServerRunWithSNI(t *testing.T) { }, } + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + NextTest: for i, test := range tests { // create server cert - serverCertFile, serverKeyFile, err := createTestCerts(test.Cert) + serverCertBundleFile, serverKeyFile, err := createTestCertFiles(tempDir, test.Cert) if err != nil { t.Errorf("%d - failed to create server cert: %v", i, err) + continue NextTest } - defer os.Remove(serverCertFile) - defer os.Remove(serverKeyFile) + ca, err := caCertFromBundle(serverCertBundleFile) + if err != nil { + t.Errorf("%d - failed to extract ca cert from server cert bundle: %v", i, err) + continue NextTest + } + caCerts := []*x509.Certificate{ca} // create SNI certs - var namedCertKeys []NamedCertKey - serverSig, err := certFileSignature(serverCertFile, serverKeyFile) + var namedCertKeys []config.NamedCertKey + serverSig, err := certFileSignature(serverCertBundleFile, serverKeyFile) if err != nil { t.Errorf("%d - failed to get server cert signature: %v", i, err) continue NextTest @@ -384,24 +352,27 @@ NextTest: serverSig: -1, } for j, c := range test.SNICerts { - certFile, keyFile, err := createTestCerts(c.TestCertSpec) + certBundleFile, keyFile, err := createTestCertFiles(tempDir, c.TestCertSpec) if err != nil { t.Errorf("%d - failed to create SNI cert %d: %v", i, j, err) continue NextTest } - defer os.Remove(certFile) - defer os.Remove(keyFile) - namedCertKeys = append(namedCertKeys, NamedCertKey{ - CertKey: CertKey{ - KeyFile: keyFile, - CertFile: certFile, - }, - Names: c.explicitNames, + namedCertKeys = append(namedCertKeys, config.NamedCertKey{ + KeyFile: keyFile, + CertFile: certBundleFile, + Names: c.explicitNames, }) + ca, err := caCertFromBundle(certBundleFile) + if err != nil { + t.Errorf("%d - failed to extract ca cert from SNI cert %d: %v", i, j, err) + continue NextTest + } + caCerts = append(caCerts, ca) + // store index in namedCertKeys with the signature as the key - sig, err := certFileSignature(certFile, keyFile) + sig, err := certFileSignature(certBundleFile, keyFile) if err != nil { t.Errorf("%d - failed get SNI cert %d signature: %v", i, j, err) continue NextTest @@ -416,17 +387,22 @@ NextTest: defer etcdserver.Terminate(t) config.EnableIndex = true - config.SecureServingInfo = &SecureServingInfo{ - ServingInfo: ServingInfo{ - BindAddress: "localhost:0", + _, err = config.ApplySecureServingOptions(&options.SecureServingOptions{ + ServingOptions: options.ServingOptions{ + BindAddress: net.ParseIP("127.0.0.1"), + BindPort: 6443, }, - ServerCert: GeneratableKeyCert{ - CertKey: CertKey{ - CertFile: serverCertFile, + ServerCert: options.GeneratableKeyCert{ + CertKey: options.CertKey{ + CertFile: serverCertBundleFile, KeyFile: serverKeyFile, }, }, - SNICerts: namedCertKeys, + SNICertKeys: namedCertKeys, + }) + if err != nil { + t.Errorf("%d - failed applying the SecureServingOptions: %v", i, err) + continue NextTest } config.InsecureServingInfo = nil @@ -436,27 +412,18 @@ NextTest: continue NextTest } + // patch in a 0-port to enable auto port allocation + s.SecureServingInfo.BindAddress = "127.0.0.1:0" + if err := s.serveSecurely(stopCh); err != nil { t.Errorf("%d - failed running the server: %v", i, err) continue NextTest } - // load certificates into a pool + // load ca certificates into a pool roots := x509.NewCertPool() - certFiles := []string{serverCertFile} - for _, c := range namedCertKeys { - certFiles = append(certFiles, c.CertFile) - } - for _, certFile := range certFiles { - bs, err := ioutil.ReadFile(certFile) - if err != nil { - t.Errorf("%d - error reading %q: %v", i, certFile, err) - continue NextTest - } - if ok := roots.AppendCertsFromPEM(bs); !ok { - t.Errorf("%d - error adding cert %q to the pool", i, certFile) - continue NextTest - } + for _, caCert := range caCerts { + roots.AddCert(caCert) } // try to dial @@ -485,6 +452,77 @@ NextTest: } } +func parseIPList(ips []string) []net.IP { + var netIPs []net.IP + for _, ip := range ips { + netIPs = append(netIPs, net.ParseIP(ip)) + } + return netIPs +} + +func createTestTLSCerts(spec TestCertSpec) (tlsCert tls.Certificate, err error) { + certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names) + if err != nil { + return tlsCert, err + } + + tlsCert, err = tls.X509KeyPair(certPem, keyPem) + return tlsCert, err +} + +func createTestCertFiles(dir string, spec TestCertSpec) (certFilePath, keyFilePath string, err error) { + certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names) + if err != nil { + return "", "", err + } + + certFile, err := ioutil.TempFile(dir, "cert") + if err != nil { + return "", "", err + } + + keyFile, err := ioutil.TempFile(dir, "key") + if err != nil { + return "", "", err + } + + _, err = certFile.Write(certPem) + if err != nil { + return "", "", err + } + certFile.Close() + + _, err = keyFile.Write(keyPem) + if err != nil { + return "", "", err + } + keyFile.Close() + + return certFile.Name(), keyFile.Name(), nil +} + +func caCertFromBundle(bundlePath string) (*x509.Certificate, error) { + pemData, err := ioutil.ReadFile(bundlePath) + if err != nil { + return nil, err + } + + // fetch last block + var block *pem.Block + for { + var nextBlock *pem.Block + nextBlock, pemData = pem.Decode(pemData) + if nextBlock == nil { + if block == nil { + return nil, fmt.Errorf("no certificate found in %q", bundlePath) + + } + return x509.ParseCertificate(block.Bytes) + } + block = nextBlock + } +} + func x509CertSignature(cert *x509.Certificate) string { return base64.StdEncoding.EncodeToString(cert.Signature) } @@ -494,13 +532,13 @@ func certFileSignature(certFile, keyFile string) (string, error) { if err != nil { return "", err } + return certSignature(cert) +} +func certSignature(cert tls.Certificate) (string, error) { x509Certs, err := x509.ParseCertificates(cert.Certificate[0]) if err != nil { return "", err } - if len(x509Certs) == 0 { - return "", fmt.Errorf("expected at least one cert after reparsing cert %q", certFile) - } return x509CertSignature(x509Certs[0]), nil }