From 5b1d45bc150f7e007cf842bcab08344e92a8ed38 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 1 Dec 2016 17:21:49 +0100 Subject: [PATCH 1/4] Stratify certificate loading and self-sign cert generation This removes all dependencies on Config during cert generation, only operating on ServerRunOptions. This way we get rid of the repeated call of Config.Complete and cleanly stratify the GenericApiServer bootstrapping. --- cmd/kube-apiserver/app/server.go | 31 ++- examples/apiserver/apiserver.go | 17 +- .../cmd/federation-apiserver/app/server.go | 25 +- pkg/genericapiserver/config.go | 141 +++++------ pkg/genericapiserver/config_selfclient.go | 131 ++++++++++ .../options/server_run_options.go | 1 - pkg/genericapiserver/options/serving.go | 116 ++++----- pkg/genericapiserver/serve.go | 75 +++--- pkg/genericapiserver/serve_test.go | 228 ++++++++++-------- 9 files changed, 446 insertions(+), 319 deletions(-) create mode 100644 pkg/genericapiserver/config_selfclient.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 19214f35d89..02946a04858 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -21,6 +21,7 @@ package app import ( "crypto/tls" + "fmt" "net" "net/http" "net/url" @@ -49,7 +50,6 @@ import ( generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver/authorizer" - genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/runtime/schema" @@ -85,20 +85,25 @@ func Run(s *options.ServerRunOptions) error { return err } - genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) - genericConfig := genericapiserver.NewConfig(). // create the new config - ApplyOptions(s.GenericServerRunOptions). // apply the options selected - ApplySecureServingOptions(s.SecureServing). - ApplyInsecureServingOptions(s.InsecureServing). - ApplyAuthenticationOptions(s.Authentication). - ApplyRBACSuperUser(s.Authorization.RBACSuperUser) - serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange) if err != nil { - glog.Fatalf("Error determining service IP ranges: %v", err) + return fmt.Errorf("error determining service IP ranges: %v", err) } - if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil { - glog.Fatalf("Failed to generate service certificate: %v", err) + + if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil { + return fmt.Errorf("error creating self-signed certificates: %v", err) + } + + genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) + + genericConfig, err := genericapiserver.NewConfig(). // create the new config + ApplyOptions(s.GenericServerRunOptions). // apply the options selected + ApplyInsecureServingOptions(s.InsecureServing). + ApplyAuthenticationOptions(s.Authentication). + ApplyRBACSuperUser(s.Authorization.RBACSuperUser). + ApplySecureServingOptions(s.SecureServing) + if err != nil { + return fmt.Errorf("failed to configure https: %s", err) } capabilities.Initialize(capabilities.Capabilities{ @@ -232,7 +237,7 @@ func Run(s *options.ServerRunOptions) error { } privilegedLoopbackToken := uuid.NewRandom().String() - selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken) + selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index cc4e4618dc7..76044259fd8 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -36,6 +36,8 @@ import ( // Install the testgroup API _ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install" + + "github.com/golang/glog" ) const ( @@ -93,20 +95,21 @@ func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error { if errs := serverOptions.InsecureServing.Validate("insecure-port"); len(errs) > 0 { return utilerrors.NewAggregate(errs) } + if err := serverOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(serverOptions.GenericServerRunOptions.AdvertiseAddress.String()); err != nil { + glog.Fatalf("Error creating self-signed certificates: %v", err) + } - config := genericapiserver.NewConfig(). + config, err := genericapiserver.NewConfig(). ApplyOptions(serverOptions.GenericServerRunOptions). - ApplySecureServingOptions(serverOptions.SecureServing). ApplyInsecureServingOptions(serverOptions.InsecureServing). ApplyAuthenticationOptions(serverOptions.Authentication). - Complete() - if err := config.MaybeGenerateServingCerts(); err != nil { - // this wasn't treated as fatal for this process before - fmt.Printf("Error creating cert: %v", err) + ApplySecureServingOptions(serverOptions.SecureServing) + if err != nil { + return fmt.Errorf("failed to configure https: %s", err) } config.Authorizer = authorizer.NewAlwaysAllowAuthorizer() - s, err := config.New() + s, err := config.Complete().New() if err != nil { return fmt.Errorf("Error in bringing up the server: %v", err) } diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index ba3d56bd74c..ee993699659 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -20,6 +20,7 @@ limitations under the License. package app import ( + "fmt" "strings" "time" @@ -37,7 +38,6 @@ import ( "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver/authorizer" - genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/generic" genericregistry "k8s.io/kubernetes/pkg/registry/generic/registry" @@ -73,16 +73,19 @@ func Run(s *options.ServerRunOptions) error { return err } - genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) - genericConfig := genericapiserver.NewConfig(). // create the new config - ApplyOptions(s.GenericServerRunOptions). // apply the options selected - ApplySecureServingOptions(s.SecureServing). - ApplyInsecureServingOptions(s.InsecureServing). - ApplyAuthenticationOptions(s.Authentication). - ApplyRBACSuperUser(s.Authorization.RBACSuperUser) + if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String()); err != nil { + return fmt.Errorf("error creating self-signed certificates: %v", err) + } - if err := genericConfig.MaybeGenerateServingCerts(); err != nil { - glog.Fatalf("Failed to generate service certificate: %v", err) + genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) + genericConfig, err := genericapiserver.NewConfig(). // create the new config + ApplyOptions(s.GenericServerRunOptions). // apply the options selected + ApplyInsecureServingOptions(s.InsecureServing). + ApplyAuthenticationOptions(s.Authentication). + ApplyRBACSuperUser(s.Authorization.RBACSuperUser). + ApplySecureServingOptions(s.SecureServing) + if err != nil { + return fmt.Errorf("failed to configure https: %s", err) } // TODO: register cluster federation resources here. @@ -130,7 +133,7 @@ func Run(s *options.ServerRunOptions) error { } privilegedLoopbackToken := uuid.NewRandom().String() - selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken) + selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index 2a9b1ad262c..06e8f7dbc53 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -17,12 +17,14 @@ limitations under the License. package genericapiserver import ( + "crypto/tls" + "encoding/pem" "fmt" "io" + "io/ioutil" "net" "net/http" "os" - "path" "regexp" goruntime "runtime" "sort" @@ -58,7 +60,6 @@ import ( "k8s.io/kubernetes/pkg/genericapiserver/routes" genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" "k8s.io/kubernetes/pkg/runtime" - certutil "k8s.io/kubernetes/pkg/util/cert" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/version" authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union" @@ -169,35 +170,21 @@ type ServingInfo struct { type SecureServingInfo struct { ServingInfo - // ServerCert is the TLS cert info for serving secure traffic - ServerCert GeneratableKeyCert - // SNICerts are named CertKeys for serving secure traffic with SNI support. - SNICerts []NamedCertKey + // Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is + // allowed to be in SNICerts. + Cert *tls.Certificate + + // CACert is an optional certificate authority used for the loopback connection of the Admission controllers. + // If this is nil, the certificate authority is extracted from Cert or a matching SNI certificate. + CACert *tls.Certificate + + // SNICerts are the TLS certificates by name used for SNI. + SNICerts map[string]*tls.Certificate + // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates ClientCA string } -type CertKey struct { - // CertFile is a file containing a PEM-encoded certificate - CertFile string - // KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile - KeyFile string -} - -type NamedCertKey struct { - CertKey - - // Names is a list of domain patterns: fully qualified domain names, possibly prefixed with - // wildcard segments. - Names []string -} - -type GeneratableKeyCert struct { - CertKey - // Generate indicates that the cert/key pair should be generated if its not present. - Generate bool -} - // NewConfig returns a Config struct with the default values func NewConfig() *Config { longRunningRE := regexp.MustCompile(options.DefaultLongRunningRequestRE) @@ -238,45 +225,69 @@ func NewConfig() *Config { return config.ApplyOptions(defaultOptions) } -func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) *Config { +func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) (*Config, error) { if secureServing == nil || secureServing.ServingOptions.BindPort <= 0 { - return c + return c, nil } secureServingInfo := &SecureServingInfo{ ServingInfo: ServingInfo{ BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)), }, - ServerCert: GeneratableKeyCert{ - CertKey: CertKey{ - CertFile: secureServing.ServerCert.CertKey.CertFile, - KeyFile: secureServing.ServerCert.CertKey.KeyFile, - }, - }, - SNICerts: []NamedCertKey{}, ClientCA: secureServing.ClientCA, } - if secureServing.ServerCert.CertKey.CertFile == "" && secureServing.ServerCert.CertKey.KeyFile == "" { - secureServingInfo.ServerCert.Generate = true - secureServingInfo.ServerCert.CertFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".crt") - secureServingInfo.ServerCert.KeyFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".key") + + serverCertFile, serverKeyFile := secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile + + // load main cert + if len(serverCertFile) != 0 || len(serverKeyFile) != 0 { + tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile) + if err != nil { + return nil, fmt.Errorf("unable to load server certificate: %v", err) + } + secureServingInfo.Cert = &tlsCert } - secureServingInfo.SNICerts = nil - for _, nkc := range secureServing.SNICertKeys { - secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{ - CertKey: CertKey{ - KeyFile: nkc.KeyFile, - CertFile: nkc.CertFile, - }, - Names: nkc.Names, + // optionally load CA cert + if len(secureServing.ServerCert.CACertFile) != 0 { + pemData, err := ioutil.ReadFile(secureServing.ServerCert.CACertFile) + if err != nil { + return nil, fmt.Errorf("failed to read certificate authority from %q: %v", secureServing.ServerCert.CACertFile, err) + } + block, pemData := pem.Decode(pemData) + if block == nil { + return nil, fmt.Errorf("no certificate found in certificate authority file %q", secureServing.ServerCert.CACertFile) + } + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("expected CERTIFICATE block in certiticate authority file %q, found: %s", secureServing.ServerCert.CACertFile, block.Type) + } + secureServingInfo.CACert = &tls.Certificate{ + Certificate: [][]byte{block.Bytes}, + } + } + + // load SNI certs + namedTlsCerts := make([]namedTlsCert, 0, len(secureServing.SNICertKeys)) + for _, nck := range secureServing.SNICertKeys { + tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile) + namedTlsCerts = append(namedTlsCerts, namedTlsCert{ + tlsCert: tlsCert, + names: nck.Names, }) + if err != nil { + return nil, fmt.Errorf("failed to load SNI cert and key: %v", err) + } + } + var err error + secureServingInfo.SNICerts, err = getNamedCertificateMap(namedTlsCerts) + if err != nil { + return nil, err } c.SecureServingInfo = secureServingInfo c.ReadWritePort = secureServing.ServingOptions.BindPort - return c + return c, nil } func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOptions) *Config { @@ -456,38 +467,6 @@ func (c completedConfig) New() (*GenericAPIServer, error) { return s, nil } -// MaybeGenerateServingCerts generates serving certificates if requested and needed. -func (c *Config) MaybeGenerateServingCerts(alternateIPs ...net.IP) error { - // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless - // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") - if c.SecureServingInfo != nil && c.SecureServingInfo.ServerCert.Generate { - canReadCertAndKey, err := certutil.CanReadCertAndKey(c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile) - if err != nil { - return err - } - if canReadCertAndKey { - return nil - } - // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? - alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} - - if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil { - return fmt.Errorf("unable to generate self signed cert: %v", err) - } else { - if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil { - return err - } - - if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil { - return err - } - glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile) - } - } - - return nil -} - func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) (secure, insecure http.Handler) { generic := func(handler http.Handler) http.Handler { handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") diff --git a/pkg/genericapiserver/config_selfclient.go b/pkg/genericapiserver/config_selfclient.go new file mode 100644 index 00000000000..17ab5d3da3b --- /dev/null +++ b/pkg/genericapiserver/config_selfclient.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "bytes" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net" + + "k8s.io/kubernetes/pkg/client/restclient" +) + +// NewSelfClientConfig returns a clientconfig which can be used to talk to this apiserver. +func NewSelfClientConfig(secureServingInfo *SecureServingInfo, insecureServingInfo *ServingInfo, token string) (*restclient.Config, error) { + if cfg, err := secureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil { + return cfg, err + } + if cfg, err := insecureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil { + return cfg, err + } + + return nil, errors.New("Unable to set url for apiserver local client") +} + +func (s *SecureServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) { + if s == nil || (s.Cert == nil && len(s.SNICerts) == 0) { + return nil, nil + } + + host, port, err := net.SplitHostPort(s.ServingInfo.BindAddress) + if err != nil { + // should never happen + return nil, fmt.Errorf("invalid secure bind address: %q", s.ServingInfo.BindAddress) + } + if host == "0.0.0.0" { + // compare MaybeDefaultWithSelfSignedCerts which adds "localhost" to the cert as alternateDNS + host = "localhost" + } + + clientConfig := &restclient.Config{ + // Increase QPS limits. The client is currently passed to all admission plugins, + // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 + // for more details. Once #22422 is fixed, we may want to remove it. + QPS: 50, + Burst: 100, + Host: "https://" + net.JoinHostPort(host, port), + BearerToken: token, + } + + // find certificate for host: either explicitly given, from the server cert bundle or one of the SNI certs + var derCA []byte + if s.CACert != nil { + derCA = s.CACert.Certificate[0] + } + if derCA == nil && s.Cert != nil { + x509Cert, err := x509.ParseCertificate(s.Cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse server certificate: %v", err) + } + + if (net.ParseIP(host) != nil && certMatchesIP(x509Cert, host)) || certMatchesName(x509Cert, host) { + derCA = s.Cert.Certificate[0] + } + } + if derCA == nil && net.ParseIP(host) == nil { + if cert, found := s.SNICerts[host]; found { + derCA = cert.Certificate[0] + } + } + if derCA == nil { + return nil, fmt.Errorf("failed to find certificate which matches %q", host) + } + pemCA := bytes.Buffer{} + if err := pem.Encode(&pemCA, &pem.Block{Type: "CERTIFICATE", Bytes: derCA}); err != nil { + return nil, err + } + clientConfig.CAData = pemCA.Bytes() + + return clientConfig, nil +} + +func (s *ServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) { + if s == nil { + return nil, nil + } + return &restclient.Config{ + Host: s.BindAddress, + // Increase QPS limits. The client is currently passed to all admission plugins, + // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 + // for more details. Once #22422 is fixed, we may want to remove it. + QPS: 50, + Burst: 100, + }, nil +} + +func certMatchesName(cert *x509.Certificate, name string) bool { + for _, certName := range cert.DNSNames { + if certName == name { + return true + } + } + + return false +} + +func certMatchesIP(cert *x509.Certificate, ip string) bool { + for _, certIP := range cert.IPAddresses { + if certIP.String() == ip { + return true + } + } + + return false +} diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index f16bcc7b527..65df7f219c0 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -75,7 +75,6 @@ type ServerRunOptions struct { // for testing). This is not actually exposed as a flag. DefaultStorageVersions string TargetRAMMB int - TLSCAFile string WatchCacheSizes []string } diff --git a/pkg/genericapiserver/options/serving.go b/pkg/genericapiserver/options/serving.go index 8487b9e35af..bcf880c09d5 100644 --- a/pkg/genericapiserver/options/serving.go +++ b/pkg/genericapiserver/options/serving.go @@ -17,14 +17,14 @@ limitations under the License. package options import ( - "errors" "fmt" "net" - "strconv" + "path" + "github.com/golang/glog" "github.com/spf13/pflag" - "k8s.io/kubernetes/pkg/client/restclient" + certutil "k8s.io/kubernetes/pkg/util/cert" "k8s.io/kubernetes/pkg/util/config" utilnet "k8s.io/kubernetes/pkg/util/net" ) @@ -43,14 +43,10 @@ type SecureServingOptions struct { SNICertKeys []config.NamedCertKey // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates ClientCA string - - // ServerCA is the certificate bundle for the signer of your serving certificate. Used for building a loopback - // connection to the API server for admission. - ServerCA string } type CertKey struct { - // CertFile is a file containing a PEM-encoded certificate + // CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain CertFile string // KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile KeyFile string @@ -59,6 +55,8 @@ type CertKey struct { type GeneratableKeyCert struct { CertKey CertKey + // CACertFile is an optional file containing the certificate chain for CertKey.CertFile + CACertFile string // CertDirectory is a directory that will contain the certificates. If the cert and key aren't specifically set // this will be used to derive a match with the "pair-name" CertDirectory string @@ -80,31 +78,6 @@ func NewSecureServingOptions() *SecureServingOptions { } } -func (s *SecureServingOptions) NewSelfClientConfig(token string) *restclient.Config { - if s == nil || s.ServingOptions.BindPort <= 0 || len(s.ServerCA) == 0 { - return nil - } - - clientConfig := &restclient.Config{ - // Increase QPS limits. The client is currently passed to all admission plugins, - // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 - // for more details. Once #22422 is fixed, we may want to remove it. - QPS: 50, - Burst: 100, - } - - // Use secure port if the ServerCA is specified - host := s.ServingOptions.BindAddress.String() - if host == "0.0.0.0" { - host = "localhost" - } - clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.ServingOptions.BindPort)) - clientConfig.CAFile = s.ServerCA - clientConfig.BearerToken = token - - return clientConfig -} - func (s *SecureServingOptions) Validate() []error { errors := []error{} if s == nil { @@ -138,6 +111,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") + fs.StringVar(&s.ServerCert.CACertFile, "tls-ca-file", s.ServerCert.CACertFile, "If set, this "+ + "certificate authority will used for secure access from Admission "+ + "Controllers. This must be a valid PEM-encoded CA bundle. Altneratively, the certificate authority "+ + "can be appended to the certificate provided by --tls-cert-file.") + fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+ "A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+ "domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+ @@ -151,18 +129,12 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { "If set, any request presenting a client certificate signed by one of "+ "the authorities in the client-ca-file is authenticated with an identity "+ "corresponding to the CommonName of the client certificate.") - - fs.StringVar(&s.ServerCA, "tls-ca-file", s.ServerCA, "If set, this "+ - "certificate authority will used for secure access from Admission "+ - "Controllers. This must be a valid PEM-encoded CA bundle.") - } func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress, "DEPRECATED: see --bind-address instead.") fs.MarkDeprecated("public-address-override", "see --bind-address instead.") - } func NewInsecureServingOptions() *ServingOptions { @@ -182,23 +154,6 @@ func (s ServingOptions) Validate(portArg string) []error { return errors } -func (s *ServingOptions) NewSelfClientConfig(token string) *restclient.Config { - if s == nil || s.BindPort <= 0 { - return nil - } - clientConfig := &restclient.Config{ - // Increase QPS limits. The client is currently passed to all admission plugins, - // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 - // for more details. Once #22422 is fixed, we may want to remove it. - QPS: 50, - Burst: 100, - } - - clientConfig.Host = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)) - - return clientConfig -} - func (s *ServingOptions) DefaultExternalAddress() (net.IP, error) { return utilnet.ChooseBindAddress(s.BindAddress) } @@ -224,14 +179,47 @@ func (s *ServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.MarkDeprecated("port", "see --insecure-port instead.") } -// Returns a clientconfig which can be used to talk to this apiserver. -func NewSelfClientConfig(secureServingOptions *SecureServingOptions, insecureServingOptions *ServingOptions, token string) (*restclient.Config, error) { - if cfg := secureServingOptions.NewSelfClientConfig(token); cfg != nil { - return cfg, nil - } - if cfg := insecureServingOptions.NewSelfClientConfig(token); cfg != nil { - return cfg, nil +func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateIPs ...net.IP) error { + keyCert := &s.ServerCert.CertKey + + if s == nil || len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 { + return nil } - return nil, errors.New("Unable to set url for apiserver local client") + keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt") + keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key") + + canReadCertAndKey, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile) + if err != nil { + return err + } + if !canReadCertAndKey { + // TODO: It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless + // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") + // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? + alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} + + // add either the bind address or localhost to the valid alternates + bindIP := s.ServingOptions.BindAddress.String() + if bindIP == "0.0.0.0" { + alternateDNS = append(alternateDNS, "localhost") + } else { + alternateIPs = append(alternateIPs, s.ServingOptions.BindAddress) + } + + if cert, key, err := certutil.GenerateSelfSignedCertKey(publicAddress, alternateIPs, alternateDNS); err != nil { + return fmt.Errorf("unable to generate self signed cert: %v", err) + } else { + if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil { + return err + } + + if err := certutil.WriteKey(keyCert.KeyFile, key); err != nil { + return err + } + glog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile) + } + } + + return nil } diff --git a/pkg/genericapiserver/serve.go b/pkg/genericapiserver/serve.go index bcdf1921288..1a17eca3ea1 100644 --- a/pkg/genericapiserver/serve.go +++ b/pkg/genericapiserver/serve.go @@ -42,17 +42,12 @@ const ( // be loaded or the initial listen call fails. The actual server loop (stoppable by closing // stopCh) runs in a go routine, i.e. serveSecurely does not block. func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { - namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts) - if err != nil { - return fmt.Errorf("unable to load SNI certificates: %v", err) - } - secureServer := &http.Server{ Addr: s.SecureServingInfo.BindAddress, Handler: s.Handler, MaxHeaderBytes: 1 << 20, TLSConfig: &tls.Config{ - NameToCertificate: namedCerts, + NameToCertificate: s.SecureServingInfo.SNICerts, // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage @@ -62,19 +57,15 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { }, } - if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 { - secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1) - secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile) - if err != nil { - return fmt.Errorf("unable to load server certificate: %v", err) - } + if s.SecureServingInfo.Cert != nil { + secureServer.TLSConfig.Certificates = []tls.Certificate{*s.SecureServingInfo.Cert} } // append all named certs. Otherwise, the go tls stack will think no SNI processing // is necessary because there is only one cert anyway. // Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI // cert will become the default cert. That's what we expect anyway. - for _, c := range namedCerts { + for _, c := range s.SecureServingInfo.SNICerts { secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c) } @@ -91,6 +82,7 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { } glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress) + var err error s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh) return err } @@ -186,48 +178,40 @@ func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int return tcpAddr.Port, nil } -// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in -// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded. -// Returns nil if len(namedCertKeys) == 0 -func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) { - if len(namedCertKeys) == 0 { - return nil, nil - } +type namedTlsCert struct { + tlsCert tls.Certificate - // load keys - tlsCerts := make([]tls.Certificate, len(namedCertKeys)) - for i := range namedCertKeys { - var err error - nkc := &namedCertKeys[i] - tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile) - if err != nil { - return nil, err - } - } + // names is a list of domain patterns: fully qualified domain names, possibly prefixed with + // wildcard segments. + names []string +} +// getNamedCertificateMap returns a map of *tls.Certificate by name. It's is +// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs +// cannot be loaded. Returns nil if len(certs) == 0 +func getNamedCertificateMap(certs []namedTlsCert) (map[string]*tls.Certificate, error) { // register certs with implicit names first, reverse order such that earlier trump over the later - tlsCertsByName := map[string]*tls.Certificate{} - for i := len(namedCertKeys) - 1; i >= 0; i-- { - nkc := &namedCertKeys[i] - if len(nkc.Names) > 0 { + byName := map[string]*tls.Certificate{} + for i := len(certs) - 1; i >= 0; i-- { + if len(certs[i].names) > 0 { continue } - cert := &tlsCerts[i] + cert := &certs[i].tlsCert // read names from certificate common names and DNS names if len(cert.Certificate) == 0 { - return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile) + return nil, fmt.Errorf("empty SNI certificate, skipping") } x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) if err != nil { - return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err) + return nil, fmt.Errorf("parse error for SNI certificate: %v", err) } cn := x509Cert.Subject.CommonName if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 { - tlsCertsByName[cn] = cert + byName[cn] = cert } for _, san := range x509Cert.DNSNames { - tlsCertsByName[san] = cert + byName[san] = cert } // intentionally all IPs in the cert are ignored as SNI forbids passing IPs // to select a cert. Before go 1.6 the tls happily passed IPs as SNI values. @@ -235,17 +219,14 @@ func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certi // register certs with explicit names last, overwriting every of the implicit ones, // again in reverse order. - for i := len(namedCertKeys) - 1; i >= 0; i-- { - nkc := &namedCertKeys[i] - if len(nkc.Names) == 0 { - continue - } - for _, name := range nkc.Names { - tlsCertsByName[name] = &tlsCerts[i] + for i := len(certs) - 1; i >= 0; i-- { + namedCert := &certs[i] + for _, name := range namedCert.names { + byName[name] = &certs[i].tlsCert } } - return tlsCertsByName, nil + return byName, nil } // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted diff --git a/pkg/genericapiserver/serve_test.go b/pkg/genericapiserver/serve_test.go index aac9499a6ff..bbdb6041c65 100644 --- a/pkg/genericapiserver/serve_test.go +++ b/pkg/genericapiserver/serve_test.go @@ -20,15 +20,18 @@ import ( "crypto/tls" "crypto/x509" "encoding/base64" + "encoding/pem" "fmt" "io/ioutil" "net" "os" "testing" - utilcert "k8s.io/kubernetes/pkg/util/cert" - "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/pkg/genericapiserver/options" + utilcert "k8s.io/kubernetes/pkg/util/cert" + "k8s.io/kubernetes/pkg/util/config" ) type TestCertSpec struct { @@ -41,47 +44,6 @@ type NamedTestCertSpec struct { explicitNames []string // as --tls-sni-cert-key explicit names } -func createTestCerts(spec TestCertSpec) (certFilePath, keyFilePath string, err error) { - var ips []net.IP - for _, ip := range spec.ips { - ips = append(ips, net.ParseIP(ip)) - } - - certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, ips, spec.names) - if err != nil { - return "", "", err - } - - certFile, err := ioutil.TempFile(os.TempDir(), "cert") - if err != nil { - return "", "", err - } - - keyFile, err := ioutil.TempFile(os.TempDir(), "key") - if err != nil { - os.Remove(certFile.Name()) - return "", "", err - } - - _, err = certFile.Write(certPem) - if err != nil { - os.Remove(certFile.Name()) - os.Remove(keyFile.Name()) - return "", "", err - } - certFile.Close() - - _, err = keyFile.Write(keyPem) - if err != nil { - os.Remove(certFile.Name()) - os.Remove(keyFile.Name()) - return "", "", err - } - keyFile.Close() - - return certFile.Name(), keyFile.Name(), nil -} - func TestGetNamedCertificateMap(t *testing.T) { tests := []struct { certs []NamedTestCertSpec @@ -225,26 +187,21 @@ func TestGetNamedCertificateMap(t *testing.T) { NextTest: for i, test := range tests { - var namedCertKeys []NamedCertKey + var namedTLSCerts []namedTlsCert bySignature := map[string]int{} // index in test.certs by cert signature for j, c := range test.certs { - certFile, keyFile, err := createTestCerts(c.TestCertSpec) + cert, err := createTestTLSCerts(c.TestCertSpec) if err != nil { t.Errorf("%d - failed to create cert %d: %v", i, j, err) continue NextTest } - defer os.Remove(certFile) - defer os.Remove(keyFile) - namedCertKeys = append(namedCertKeys, NamedCertKey{ - CertKey: CertKey{ - KeyFile: keyFile, - CertFile: certFile, - }, - Names: c.explicitNames, + namedTLSCerts = append(namedTLSCerts, namedTlsCert{ + tlsCert: cert, + names: c.explicitNames, }) - sig, err := certFileSignature(certFile, keyFile) + sig, err := certSignature(cert) if err != nil { t.Errorf("%d - failed to get signature for %d: %v", i, j, err) continue NextTest @@ -252,7 +209,7 @@ NextTest: bySignature[sig] = j } - certMap, err := getNamedCertificateMap(namedCertKeys) + certMap, err := getNamedCertificateMap(namedTLSCerts) if err == nil && len(test.errorString) != 0 { t.Errorf("%d - expected no error, got: %v", i, err) } else if err != nil && err.Error() != test.errorString { @@ -363,19 +320,30 @@ func TestServerRunWithSNI(t *testing.T) { }, } + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + NextTest: for i, test := range tests { // create server cert - serverCertFile, serverKeyFile, err := createTestCerts(test.Cert) + serverCertBundleFile, serverKeyFile, err := createTestCertFiles(tempDir, test.Cert) if err != nil { t.Errorf("%d - failed to create server cert: %v", i, err) + continue NextTest } - defer os.Remove(serverCertFile) - defer os.Remove(serverKeyFile) + ca, err := caCertFromBundle(serverCertBundleFile) + if err != nil { + t.Errorf("%d - failed to extract ca cert from server cert bundle: %v", i, err) + continue NextTest + } + caCerts := []*x509.Certificate{ca} // create SNI certs - var namedCertKeys []NamedCertKey - serverSig, err := certFileSignature(serverCertFile, serverKeyFile) + var namedCertKeys []config.NamedCertKey + serverSig, err := certFileSignature(serverCertBundleFile, serverKeyFile) if err != nil { t.Errorf("%d - failed to get server cert signature: %v", i, err) continue NextTest @@ -384,24 +352,27 @@ NextTest: serverSig: -1, } for j, c := range test.SNICerts { - certFile, keyFile, err := createTestCerts(c.TestCertSpec) + certBundleFile, keyFile, err := createTestCertFiles(tempDir, c.TestCertSpec) if err != nil { t.Errorf("%d - failed to create SNI cert %d: %v", i, j, err) continue NextTest } - defer os.Remove(certFile) - defer os.Remove(keyFile) - namedCertKeys = append(namedCertKeys, NamedCertKey{ - CertKey: CertKey{ - KeyFile: keyFile, - CertFile: certFile, - }, - Names: c.explicitNames, + namedCertKeys = append(namedCertKeys, config.NamedCertKey{ + KeyFile: keyFile, + CertFile: certBundleFile, + Names: c.explicitNames, }) + ca, err := caCertFromBundle(certBundleFile) + if err != nil { + t.Errorf("%d - failed to extract ca cert from SNI cert %d: %v", i, j, err) + continue NextTest + } + caCerts = append(caCerts, ca) + // store index in namedCertKeys with the signature as the key - sig, err := certFileSignature(certFile, keyFile) + sig, err := certFileSignature(certBundleFile, keyFile) if err != nil { t.Errorf("%d - failed get SNI cert %d signature: %v", i, j, err) continue NextTest @@ -416,17 +387,22 @@ NextTest: defer etcdserver.Terminate(t) config.EnableIndex = true - config.SecureServingInfo = &SecureServingInfo{ - ServingInfo: ServingInfo{ - BindAddress: "localhost:0", + _, err = config.ApplySecureServingOptions(&options.SecureServingOptions{ + ServingOptions: options.ServingOptions{ + BindAddress: net.ParseIP("127.0.0.1"), + BindPort: 6443, }, - ServerCert: GeneratableKeyCert{ - CertKey: CertKey{ - CertFile: serverCertFile, + ServerCert: options.GeneratableKeyCert{ + CertKey: options.CertKey{ + CertFile: serverCertBundleFile, KeyFile: serverKeyFile, }, }, - SNICerts: namedCertKeys, + SNICertKeys: namedCertKeys, + }) + if err != nil { + t.Errorf("%d - failed applying the SecureServingOptions: %v", i, err) + continue NextTest } config.InsecureServingInfo = nil @@ -436,27 +412,18 @@ NextTest: continue NextTest } + // patch in a 0-port to enable auto port allocation + s.SecureServingInfo.BindAddress = "127.0.0.1:0" + if err := s.serveSecurely(stopCh); err != nil { t.Errorf("%d - failed running the server: %v", i, err) continue NextTest } - // load certificates into a pool + // load ca certificates into a pool roots := x509.NewCertPool() - certFiles := []string{serverCertFile} - for _, c := range namedCertKeys { - certFiles = append(certFiles, c.CertFile) - } - for _, certFile := range certFiles { - bs, err := ioutil.ReadFile(certFile) - if err != nil { - t.Errorf("%d - error reading %q: %v", i, certFile, err) - continue NextTest - } - if ok := roots.AppendCertsFromPEM(bs); !ok { - t.Errorf("%d - error adding cert %q to the pool", i, certFile) - continue NextTest - } + for _, caCert := range caCerts { + roots.AddCert(caCert) } // try to dial @@ -485,6 +452,77 @@ NextTest: } } +func parseIPList(ips []string) []net.IP { + var netIPs []net.IP + for _, ip := range ips { + netIPs = append(netIPs, net.ParseIP(ip)) + } + return netIPs +} + +func createTestTLSCerts(spec TestCertSpec) (tlsCert tls.Certificate, err error) { + certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names) + if err != nil { + return tlsCert, err + } + + tlsCert, err = tls.X509KeyPair(certPem, keyPem) + return tlsCert, err +} + +func createTestCertFiles(dir string, spec TestCertSpec) (certFilePath, keyFilePath string, err error) { + certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names) + if err != nil { + return "", "", err + } + + certFile, err := ioutil.TempFile(dir, "cert") + if err != nil { + return "", "", err + } + + keyFile, err := ioutil.TempFile(dir, "key") + if err != nil { + return "", "", err + } + + _, err = certFile.Write(certPem) + if err != nil { + return "", "", err + } + certFile.Close() + + _, err = keyFile.Write(keyPem) + if err != nil { + return "", "", err + } + keyFile.Close() + + return certFile.Name(), keyFile.Name(), nil +} + +func caCertFromBundle(bundlePath string) (*x509.Certificate, error) { + pemData, err := ioutil.ReadFile(bundlePath) + if err != nil { + return nil, err + } + + // fetch last block + var block *pem.Block + for { + var nextBlock *pem.Block + nextBlock, pemData = pem.Decode(pemData) + if nextBlock == nil { + if block == nil { + return nil, fmt.Errorf("no certificate found in %q", bundlePath) + + } + return x509.ParseCertificate(block.Bytes) + } + block = nextBlock + } +} + func x509CertSignature(cert *x509.Certificate) string { return base64.StdEncoding.EncodeToString(cert.Signature) } @@ -494,13 +532,13 @@ func certFileSignature(certFile, keyFile string) (string, error) { if err != nil { return "", err } + return certSignature(cert) +} +func certSignature(cert tls.Certificate) (string, error) { x509Certs, err := x509.ParseCertificates(cert.Certificate[0]) if err != nil { return "", err } - if len(x509Certs) == 0 { - return "", fmt.Errorf("expected at least one cert after reparsing cert %q", certFile) - } return x509CertSignature(x509Certs[0]), nil } From d647d0ee8bea552310472021370da933052319d3 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 2 Dec 2016 21:07:16 +0100 Subject: [PATCH 2/4] Make failed secure loopback client non-fatal if insecure port is open --- pkg/genericapiserver/config_selfclient.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/genericapiserver/config_selfclient.go b/pkg/genericapiserver/config_selfclient.go index 17ab5d3da3b..6a1d86df377 100644 --- a/pkg/genericapiserver/config_selfclient.go +++ b/pkg/genericapiserver/config_selfclient.go @@ -25,12 +25,19 @@ import ( "net" "k8s.io/kubernetes/pkg/client/restclient" + + "github.com/golang/glog" ) // NewSelfClientConfig returns a clientconfig which can be used to talk to this apiserver. func NewSelfClientConfig(secureServingInfo *SecureServingInfo, insecureServingInfo *ServingInfo, token string) (*restclient.Config, error) { if cfg, err := secureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil { - return cfg, err + if insecureServingInfo == nil { + // be fatal if insecure port is not available + return cfg, err + } else { + glog.Warningf("Failed to create secure local client, falling back to insecure local connection: %v", err) + } } if cfg, err := insecureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil { return cfg, err From a4cf364dbd32035e66a93908ab951d5bc845c219 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 5 Dec 2016 14:57:53 +0100 Subject: [PATCH 3/4] Replace glog.Fatals with "return fmt.Errorf" in apiservers --- cmd/kube-apiserver/app/server.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 02946a04858..8d34964def2 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -125,7 +125,7 @@ func Run(s *options.ServerRunOptions) error { var installSSH genericapiserver.InstallSSHKey cloud, err := cloudprovider.InitCloudProvider(s.GenericServerRunOptions.CloudProvider, s.GenericServerRunOptions.CloudConfigFile) if err != nil { - glog.Fatalf("Cloud provider could not be initialized: %v", err) + return fmt.Errorf("cloud provider could not be initialized: %v", err) } if cloud != nil { if instances, supported := cloud.Instances(); supported { @@ -133,7 +133,7 @@ func Run(s *options.ServerRunOptions) error { } } if s.KubeletConfig.Port == 0 { - glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") + return fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified") } // Set up the tunneler // TODO(cjcullen): If we want this to handle per-kubelet ports or other @@ -178,7 +178,7 @@ func Run(s *options.ServerRunOptions) error { storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion() if err != nil { - glog.Fatalf("error generating storage version map: %s", err) + return fmt.Errorf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, @@ -187,7 +187,7 @@ func Run(s *options.ServerRunOptions) error { []schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")}, master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig) if err != nil { - glog.Fatalf("error in initializing storage factory: %s", err) + return fmt.Errorf("error in initializing storage factory: %s", err) } storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) @@ -226,20 +226,20 @@ func Run(s *options.ServerRunOptions) error { // go directly to etcd to avoid recursive auth insanity storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts")) if err != nil { - glog.Fatalf("Unable to get serviceaccounts storage: %v", err) + return fmt.Errorf("unable to get serviceaccounts storage: %v", err) } authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets"))) } apiAuthenticator, securityDefinitions, err := authenticator.New(authenticatorConfig) if err != nil { - glog.Fatalf("Invalid Authentication Config: %v", err) + return fmt.Errorf("invalid Authentication Config: %v", err) } privilegedLoopbackToken := uuid.NewRandom().String() selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken) if err != nil { - glog.Fatalf("Failed to create clientset: %v", err) + return fmt.Errorf("failed to create clientset: %v", err) } client, err := internalclientset.NewForConfig(selfClientConfig) if err != nil { @@ -250,14 +250,14 @@ func Run(s *options.ServerRunOptions) error { authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers) apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationConfig) if err != nil { - glog.Fatalf("Invalid Authorization Config: %v", err) + return fmt.Errorf("invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",") pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer) admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer) if err != nil { - glog.Fatalf("Failed to initialize plugins: %v", err) + return fmt.Errorf("failed to initialize plugins: %v", err) } proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ From 3f01c37b9dc201e191c2e5d5e3b3e08865d20e49 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 1 Dec 2016 17:37:22 +0100 Subject: [PATCH 4/4] Update generated files --- cmd/kube-apiserver/app/BUILD | 1 - federation/cmd/federation-apiserver/app/BUILD | 1 - pkg/genericapiserver/BUILD | 2 ++ pkg/genericapiserver/options/BUILD | 3 ++- test/test_owners.csv | 8 ++++++-- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index b17e4ec7f4e..d6567b8d524 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -35,7 +35,6 @@ go_library( "//pkg/generated/openapi:go_default_library", "//pkg/genericapiserver:go_default_library", "//pkg/genericapiserver/authorizer:go_default_library", - "//pkg/genericapiserver/options:go_default_library", "//pkg/master:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/runtime/schema:go_default_library", diff --git a/federation/cmd/federation-apiserver/app/BUILD b/federation/cmd/federation-apiserver/app/BUILD index 46fa167c6f1..848e0c11e04 100644 --- a/federation/cmd/federation-apiserver/app/BUILD +++ b/federation/cmd/federation-apiserver/app/BUILD @@ -42,7 +42,6 @@ go_library( "//pkg/generated/openapi:go_default_library", "//pkg/genericapiserver:go_default_library", "//pkg/genericapiserver/authorizer:go_default_library", - "//pkg/genericapiserver/options:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/configmap/etcd:go_default_library", "//pkg/registry/core/event/etcd:go_default_library", diff --git a/pkg/genericapiserver/BUILD b/pkg/genericapiserver/BUILD index 607fd84e9b1..78ffec842c6 100644 --- a/pkg/genericapiserver/BUILD +++ b/pkg/genericapiserver/BUILD @@ -14,6 +14,7 @@ go_library( name = "go_default_library", srcs = [ "config.go", + "config_selfclient.go", "default_storage_factory_builder.go", "discovery.go", "doc.go", @@ -114,6 +115,7 @@ go_test( "//pkg/storage/storagebackend:go_default_library", "//pkg/util/cert:go_default_library", "//pkg/util/clock:go_default_library", + "//pkg/util/config:go_default_library", "//pkg/util/net:go_default_library", "//pkg/util/sets:go_default_library", "//pkg/version:go_default_library", diff --git a/pkg/genericapiserver/options/BUILD b/pkg/genericapiserver/options/BUILD index 7280966c8e2..8cb7315fb3b 100644 --- a/pkg/genericapiserver/options/BUILD +++ b/pkg/genericapiserver/options/BUILD @@ -28,14 +28,15 @@ go_library( "//pkg/apiserver/authenticator:go_default_library", "//pkg/client/clientset_generated/release_1_5/typed/authentication/v1beta1:go_default_library", "//pkg/client/clientset_generated/release_1_5/typed/authorization/v1beta1:go_default_library", - "//pkg/client/restclient:go_default_library", "//pkg/client/unversioned/clientcmd:go_default_library", "//pkg/controller/informers:go_default_library", "//pkg/genericapiserver/authorizer:go_default_library", "//pkg/runtime/schema:go_default_library", "//pkg/storage/storagebackend:go_default_library", + "//pkg/util/cert:go_default_library", "//pkg/util/config:go_default_library", "//pkg/util/net:go_default_library", + "//vendor:github.com/golang/glog", "//vendor:github.com/spf13/pflag", ], ) diff --git a/test/test_owners.csv b/test/test_owners.csv index 0acf7251f28..998ca3685d4 100644 --- a/test/test_owners.csv +++ b/test/test_owners.csv @@ -140,6 +140,7 @@ Federated Services DNS non-local federated service missing local service should Federated Services DNS non-local federated service should be able to discover a non-local federated service,jlowdermilk,1 Federated Services DNS should be able to discover a federated service,derekwaynecarr,1 Federated Services Service creation should create matching services in underlying clusters,jbeda,1 +Federated Services Service creation should not be deleted from underlying clusters when it is deleted,sttts,0 Federated Services Service creation should succeed,rmmh,1 Federated ingresses Federated Ingresses Ingress connectivity and DNS should be able to connect to a federated ingress via its load balancer,rmmh,1 Federated ingresses Federated Ingresses should be created and deleted successfully,dchen1107,1 @@ -444,7 +445,7 @@ Services should be able to create a functioning NodePort service,bprashanth,0 Services should be able to up and down services,bprashanth,0 Services should check NodePort out-of-range,bprashanth,0 Services should create endpoints for unready pods,maisem,0 -Serivces should only allow access from service loadbalancer source ranges,freehan,0 +Services should only allow access from service loadbalancer source ranges,sttts,0 Services should preserve source pod IP for traffic thru service cluster IP,Random-Liu,1 Services should prevent NodePort collisions,bprashanth,0 Services should provide secure master service,bprashanth,0 @@ -546,7 +547,6 @@ k8s.io/kubernetes/pkg/api/errors,yifan-gu,1 k8s.io/kubernetes/pkg/api/events,jlowdermilk,1 k8s.io/kubernetes/pkg/api/install,timothysc,1 k8s.io/kubernetes/pkg/api/meta,fabioy,1 -k8s.io/kubernetes/pkg/api/pod,piosz,1 k8s.io/kubernetes/pkg/api/resource,smarterclayton,1 k8s.io/kubernetes/pkg/api/service,spxtr,1 k8s.io/kubernetes/pkg/api/testapi,caesarxuchao,1 @@ -554,6 +554,9 @@ k8s.io/kubernetes/pkg/api/unversioned,kevin-wangzefeng,1 k8s.io/kubernetes/pkg/api/unversioned/validation,brendandburns,1 k8s.io/kubernetes/pkg/api/util,ghodss,1 k8s.io/kubernetes/pkg/api/v1,vulpecula,1 +k8s.io/kubernetes/pkg/api/v1/endpoints,sttts,0 +k8s.io/kubernetes/pkg/api/v1/pod,sttts,0 +k8s.io/kubernetes/pkg/api/v1/service,sttts,0 k8s.io/kubernetes/pkg/api/validation,smarterclayton,1 k8s.io/kubernetes/pkg/api/validation/path,luxas,1 k8s.io/kubernetes/pkg/apimachinery,gmarek,1 @@ -786,6 +789,7 @@ k8s.io/kubernetes/pkg/registry/policy/poddisruptionbudget/etcd,xiang90,1 k8s.io/kubernetes/pkg/registry/storage/storageclass,brendandburns,1 k8s.io/kubernetes/pkg/registry/storage/storageclass/etcd,eparis,1 k8s.io/kubernetes/pkg/runtime,wojtek-t,0 +k8s.io/kubernetes/pkg/runtime/schema,sttts,0 k8s.io/kubernetes/pkg/runtime/serializer,wojtek-t,0 k8s.io/kubernetes/pkg/runtime/serializer/json,wojtek-t,0 k8s.io/kubernetes/pkg/runtime/serializer/protobuf,wojtek-t,0