diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 0bdb71e485a..9f554210f09 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -335,6 +335,6 @@ func Run(s *options.ServerRunOptions) error { } sharedInformers.Start(wait.NeverStop) - m.GenericAPIServer.PrepareRun().Run() + m.GenericAPIServer.PrepareRun().Run(wait.NeverStop) return nil } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 506bd0344df..874bca6577a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -502,9 +502,19 @@ func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt") kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key") if !certutil.CanReadCertOrKey(kc.TLSCertFile, kc.TLSPrivateKeyFile) { - if err := certutil.GenerateSelfSignedCert(nodeutil.GetHostname(kc.HostnameOverride), kc.TLSCertFile, kc.TLSPrivateKeyFile, nil, nil); err != nil { + cert, key, err := certutil.GenerateSelfSignedCertKey(nodeutil.GetHostname(kc.HostnameOverride), nil, nil) + if err != nil { return nil, fmt.Errorf("unable to generate self signed cert: %v", err) } + + if err := certutil.WriteCert(kc.TLSCertFile, cert); err != nil { + return nil, err + } + + if err := certutil.WriteKey(kc.TLSPrivateKeyFile, key); err != nil { + return nil, err + } + glog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile) } } diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index d1ff0d80b7b..72d1f77f962 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -60,7 +60,7 @@ func NewServerRunOptions() *genericoptions.ServerRunOptions { return serverOptions } -func Run(serverOptions *genericoptions.ServerRunOptions) error { +func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error { // Set ServiceClusterIPRange _, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") serverOptions.ServiceClusterIPRange = *serviceClusterIPRange @@ -105,6 +105,6 @@ func Run(serverOptions *genericoptions.ServerRunOptions) error { if err := s.InstallAPIGroup(&apiGroupInfo); err != nil { return fmt.Errorf("Error in installing API: %v", err) } - s.PrepareRun().Run() + s.PrepareRun().Run(stopCh) return nil } diff --git a/examples/apiserver/server/main.go b/examples/apiserver/server/main.go index a145adba68b..5c69edf708f 100644 --- a/examples/apiserver/server/main.go +++ b/examples/apiserver/server/main.go @@ -19,6 +19,7 @@ package main import ( "k8s.io/kubernetes/examples/apiserver" "k8s.io/kubernetes/pkg/util/flag" + "k8s.io/kubernetes/pkg/util/wait" "github.com/golang/glog" "github.com/spf13/pflag" @@ -32,7 +33,7 @@ func main() { serverRunOptions.AddEtcdStorageFlags(pflag.CommandLine) flag.InitFlags() - if err := apiserver.Run(serverRunOptions); err != nil { + if err := apiserver.Run(serverRunOptions, wait.NeverStop); err != nil { glog.Fatalf("Error in bringing up the server: %v", err) } } diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index 3ee53be4002..a1bd1e3bf68 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -225,7 +225,7 @@ func Run(s *options.ServerRunOptions) error { installExtensionsAPIs(m, restOptionsFactory) sharedInformers.Start(wait.NeverStop) - m.PrepareRun().Run() + m.PrepareRun().Run(wait.NeverStop) return nil } diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index f3c1211e9da..5a2c7edb66d 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -555,6 +555,7 @@ tls-ca-file tls-cert-file tls-private-key-file to-version +tls-sni-cert-key token-auth-file ttl-keys-prefix ttl-secs diff --git a/pkg/genericapiserver/BUILD b/pkg/genericapiserver/BUILD index 6fcf55a8e90..64b46a2c188 100644 --- a/pkg/genericapiserver/BUILD +++ b/pkg/genericapiserver/BUILD @@ -21,6 +21,7 @@ go_library( "resource_config.go", "resource_encoding_config.go", "reststorage_interfaces.go", + "serve.go", "storage_factory.go", "tunneler.go", ], @@ -66,6 +67,7 @@ go_library( "//vendor:github.com/emicklei/go-restful", "//vendor:github.com/go-openapi/spec", "//vendor:github.com/golang/glog", + "//vendor:github.com/pkg/errors", "//vendor:github.com/prometheus/client_golang/prometheus", "//vendor:gopkg.in/natefinch/lumberjack.v2", ], @@ -77,6 +79,7 @@ go_test( "default_storage_factory_builder_test.go", "genericapiserver_test.go", "resource_config_test.go", + "serve_test.go", "server_run_options_test.go", "storage_factory_test.go", "tunneler_test.go", @@ -100,6 +103,7 @@ go_test( "//pkg/registry/core/service/ipallocator:go_default_library", "//pkg/storage/etcd/testing:go_default_library", "//pkg/storage/storagebackend:go_default_library", + "//pkg/util/cert:go_default_library", "//pkg/util/clock:go_default_library", "//pkg/util/sets:go_default_library", "//pkg/version:go_default_library", diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index 0a3e7c20a53..44916181667 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -110,7 +110,7 @@ type Config struct { // same value for this field. (Numbers > 1 currently untested.) MasterCount int - SecureServingInfo *ServingInfo + SecureServingInfo *SecureServingInfo InsecureServingInfo *ServingInfo // The port on PublicAddress where a read-write server will be installed. @@ -177,17 +177,36 @@ type Config struct { type ServingInfo struct { // BindAddress is the ip:port to serve on BindAddress string +} + +type SecureServingInfo struct { + ServingInfo + // ServerCert is the TLS cert info for serving secure traffic - ServerCert CertInfo + ServerCert GeneratableKeyCert + // SNICerts are named CertKeys for serving secure traffic with SNI support. + SNICerts []NamedCertKey // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates ClientCA string } -type CertInfo struct { +type CertKey struct { // CertFile is a file containing a PEM-encoded certificate CertFile string // KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile KeyFile string +} + +type NamedCertKey struct { + CertKey + + // Names is a list of domain patterns: fully qualified domain names, possibly prefixed with + // wildcard segments. + Names []string +} + +type GeneratableKeyCert struct { + CertKey // Generate indicates that the cert/key pair should be generated if its not present. Generate bool } @@ -248,12 +267,17 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { } if options.SecurePort > 0 { - secureServingInfo := &ServingInfo{ - BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)), - ServerCert: CertInfo{ - CertFile: options.TLSCertFile, - KeyFile: options.TLSPrivateKeyFile, + secureServingInfo := &SecureServingInfo{ + ServingInfo: ServingInfo{ + BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)), }, + ServerCert: GeneratableKeyCert{ + CertKey: CertKey{ + CertFile: options.TLSCertFile, + KeyFile: options.TLSPrivateKeyFile, + }, + }, + SNICerts: []NamedCertKey{}, ClientCA: options.ClientCAFile, } if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" { @@ -262,6 +286,17 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { secureServingInfo.ServerCert.KeyFile = path.Join(options.CertDirectory, "apiserver.key") } + secureServingInfo.SNICerts = nil + for _, nkc := range options.SNICertKeys { + secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{ + CertKey: CertKey{ + KeyFile: nkc.KeyFile, + CertFile: nkc.CertFile, + }, + Names: nkc.Names, + }) + } + c.SecureServingInfo = secureServingInfo c.ReadWritePort = options.SecurePort } @@ -434,9 +469,16 @@ func (c completedConfig) MaybeGenerateServingCerts() error { alternateIPs := []net.IP{c.ServiceReadWriteIP} alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} - if err := certutil.GenerateSelfSignedCert(c.PublicAddress.String(), c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile, alternateIPs, alternateDNS); err != nil { - return fmt.Errorf("Unable to generate self signed cert: %v", err) + if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil { + return fmt.Errorf("unable to generate self signed cert: %v", err) } else { + if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil { + return err + } + + if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil { + return err + } glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile) } } diff --git a/pkg/genericapiserver/genericapiserver.go b/pkg/genericapiserver/genericapiserver.go index b3d9f93f53c..e824c27fc6f 100644 --- a/pkg/genericapiserver/genericapiserver.go +++ b/pkg/genericapiserver/genericapiserver.go @@ -17,7 +17,6 @@ limitations under the License. package genericapiserver import ( - "crypto/tls" "fmt" "mime" "net" @@ -44,9 +43,7 @@ import ( "k8s.io/kubernetes/pkg/genericapiserver/openapi/common" "k8s.io/kubernetes/pkg/genericapiserver/routes" "k8s.io/kubernetes/pkg/runtime" - certutil "k8s.io/kubernetes/pkg/util/cert" utilnet "k8s.io/kubernetes/pkg/util/net" - utilruntime "k8s.io/kubernetes/pkg/util/runtime" "k8s.io/kubernetes/pkg/util/sets" ) @@ -112,9 +109,12 @@ type GenericAPIServer struct { // The registered APIs HandlerContainer *genericmux.APIContainer - SecureServingInfo *ServingInfo + SecureServingInfo *SecureServingInfo InsecureServingInfo *ServingInfo + // numerical ports, set after listening + effectiveSecurePort, effectiveInsecurePort int + // ExternalAddress is the address (hostname or IP and port) that should be used in // external (public internet) URLs for this GenericAPIServer. ExternalAddress string @@ -180,7 +180,6 @@ type preparedGenericAPIServer struct { // PrepareRun does post API installation setup steps. func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer { - // install APIs which depend on other APIs to be installed if s.enableSwaggerSupport { routes.Swagger{ExternalAddress: s.ExternalAddress}.Install(s.HandlerContainer) } @@ -192,76 +191,18 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer { return preparedGenericAPIServer{s} } -func (s preparedGenericAPIServer) Run() { +// Run spawns the http servers (secure and insecure). It only returns if stopCh is closed +// or one of the ports cannot be listened on initially. +func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) { if s.SecureServingInfo != nil && s.Handler != nil { - secureServer := &http.Server{ - Addr: s.SecureServingInfo.BindAddress, - Handler: s.Handler, - MaxHeaderBytes: 1 << 20, - TLSConfig: &tls.Config{ - // Can't use SSLv3 because of POODLE and BEAST - // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher - // Can't use TLSv1.1 because of RC4 cipher usage - MinVersion: tls.VersionTLS12, - }, + if err := s.serveSecurely(stopCh); err != nil { + glog.Fatal(err) } - - if len(s.SecureServingInfo.ClientCA) > 0 { - clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA) - if err != nil { - glog.Fatalf("Unable to load client CA file: %v", err) - } - // Populate PeerCertificates in requests, but don't reject connections without certificates - // This allows certificates to be validated by authenticators, while still allowing other auth types - secureServer.TLSConfig.ClientAuth = tls.RequestClientCert - // Specify allowed CAs for client certificates - secureServer.TLSConfig.ClientCAs = clientCAs - // "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server - secureServer.TLSConfig.NextProtos = []string{"h2"} - - } - - glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress) - go func() { - defer utilruntime.HandleCrash() - - for { - if err := secureServer.ListenAndServeTLS(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile); err != nil { - glog.Errorf("Unable to listen for secure (%v); will try again.", err) - } - time.Sleep(15 * time.Second) - } - }() } if s.InsecureServingInfo != nil && s.InsecureHandler != nil { - insecureServer := &http.Server{ - Addr: s.InsecureServingInfo.BindAddress, - Handler: s.InsecureHandler, - MaxHeaderBytes: 1 << 20, - } - glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress) - go func() { - defer utilruntime.HandleCrash() - - for { - if err := insecureServer.ListenAndServe(); err != nil { - glog.Errorf("Unable to listen for insecure (%v); will try again.", err) - } - time.Sleep(15 * time.Second) - } - }() - } - - // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) per port - if s.SecureServingInfo != nil { - if err := waitForSuccessfulDial(true, "tcp", s.SecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil { - glog.Fatalf("Secure server never started: %v", err) - } - } - if s.InsecureServingInfo != nil { - if err := waitForSuccessfulDial(false, "tcp", s.InsecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil { - glog.Fatalf("Insecure server never started: %v", err) + if err := s.serveInsecurely(stopCh); err != nil { + glog.Fatal(err) } } @@ -272,7 +213,7 @@ func (s preparedGenericAPIServer) Run() { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } - select {} + <-stopCh } // installAPIResources is a private method for installing the REST storage backing each api groupversionresource @@ -471,27 +412,3 @@ func NewDefaultAPIGroupInfo(group string) APIGroupInfo { NegotiatedSerializer: api.Codecs, } } - -// waitForSuccessfulDial attempts to connect to the given address, closing and returning nil on the first successful connection. -func waitForSuccessfulDial(https bool, network, address string, timeout, interval time.Duration, retries int) error { - var ( - conn net.Conn - err error - ) - for i := 0; i <= retries; i++ { - dialer := net.Dialer{Timeout: timeout} - if https { - conn, err = tls.DialWithDialer(&dialer, network, address, &tls.Config{InsecureSkipVerify: true}) - } else { - conn, err = dialer.Dial(network, address) - } - if err != nil { - glog.V(5).Infof("Got error %#v, trying again: %#v\n", err, address) - time.Sleep(interval) - continue - } - conn.Close() - return nil - } - return err -} diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index 79c4d25bb69..f80a36a0e07 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -118,6 +118,7 @@ type ServerRunOptions struct { TLSCAFile string TLSCertFile string TLSPrivateKeyFile string + SNICertKeys []config.NamedCertKey TokenAuthFile string EnableAnyToken bool WatchCacheSizes []string @@ -488,13 +489,22 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "Controllers. This must be a valid PEM-encoded CA bundle.") fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+ - "File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ "after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+ "--tls-private-key-file are not provided, a self-signed certificate and key "+ "are generated for the public address and saved to /var/run/kubernetes.") fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile, - "File containing x509 private key matching --tls-cert-file.") + "File containing the default x509 private key matching --tls-cert-file.") + + fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+ + "A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+ + "domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+ + "segments. If no domain patterns are provided, the names of the certificate are "+ + "extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+ + "trump over extracted names. For multiple key/certificate pairs, use the "+ + "--tls-sni-cert-key multiple times. "+ + "Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".") fs.StringVar(&s.TokenAuthFile, "token-auth-file", s.TokenAuthFile, ""+ "If set, the file that will be used to secure the secure port of the API server "+ diff --git a/pkg/genericapiserver/serve.go b/pkg/genericapiserver/serve.go new file mode 100644 index 00000000000..e595bf2812a --- /dev/null +++ b/pkg/genericapiserver/serve.go @@ -0,0 +1,262 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/http" + "sync" + "time" + + certutil "k8s.io/kubernetes/pkg/util/cert" + utilruntime "k8s.io/kubernetes/pkg/util/runtime" + + "github.com/golang/glog" + "github.com/pkg/errors" +) + +const ( + defaultKeepAlivePeriod = 3 * time.Minute +) + +// serveSecurely runs the secure http server. It fails only if certificates cannot +// be loaded or the initial listen call fails. The actual server loop (stoppable by closing +// stopCh) runs in a go routine, i.e. serveSecurely does not block. +func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error { + namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts) + if err != nil { + return fmt.Errorf("unable to load SNI certificates: %v", err) + } + + secureServer := &http.Server{ + Addr: s.SecureServingInfo.BindAddress, + Handler: s.Handler, + MaxHeaderBytes: 1 << 20, + TLSConfig: &tls.Config{ + NameToCertificate: namedCerts, + // Can't use SSLv3 because of POODLE and BEAST + // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher + // Can't use TLSv1.1 because of RC4 cipher usage + MinVersion: tls.VersionTLS12, + // enable HTTP2 for go's 1.7 HTTP Server + NextProtos: []string{"h2", "http/1.1"}, + }, + } + + if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 { + secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1) + secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile) + if err != nil { + return fmt.Errorf("unable to load server certificate: %v", err) + } + } + + // append all named certs. Otherwise, the go tls stack will think no SNI processing + // is necessary because there is only one cert anyway. + // Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI + // cert will become the default cert. That's what we expect anyway. + for _, c := range namedCerts { + secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c) + } + + if len(s.SecureServingInfo.ClientCA) > 0 { + clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA) + if err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) + } + // Populate PeerCertificates in requests, but don't reject connections without certificates + // This allows certificates to be validated by authenticators, while still allowing other auth types + secureServer.TLSConfig.ClientAuth = tls.RequestClientCert + // Specify allowed CAs for client certificates + secureServer.TLSConfig.ClientCAs = clientCAs + } + + glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress) + s.effectiveSecurePort, err = runServer(secureServer, stopCh) + return err +} + +// serveInsecurely run the insecure http server. It fails only if the initial listen +// call fails. The actual server loop (stoppable by closing stopCh) runs in a go +// routine, i.e. serveInsecurely does not block. +func (s *GenericAPIServer) serveInsecurely(stopCh <-chan struct{}) error { + insecureServer := &http.Server{ + Addr: s.InsecureServingInfo.BindAddress, + Handler: s.InsecureHandler, + MaxHeaderBytes: 1 << 20, + } + glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress) + var err error + s.effectiveInsecurePort, err = runServer(insecureServer, stopCh) + return err +} + +// runServer listens on the given port, then spawns a go-routine continuously serving +// until the stopCh is closed. The port is returned. This function does not block. +func runServer(server *http.Server, stopCh <-chan struct{}) (int, error) { + if len(server.Addr) == 0 { + return 0, errors.New("address cannot be empty") + } + + // first listen is synchronous (fail early!) + ln, err := net.Listen("tcp", server.Addr) + if err != nil { + return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err) + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + lock := sync.Mutex{} // to avoid we close an old listener during a listen retry + go func() { + <-stopCh + lock.Lock() + defer lock.Unlock() + ln.Close() + }() + + go func() { + defer utilruntime.HandleCrash() + + for { + var listener net.Listener + listener = tcpKeepAliveListener{ln.(*net.TCPListener)} + if server.TLSConfig != nil { + listener = tls.NewListener(listener, server.TLSConfig) + } + + err := server.Serve(listener) + glog.Errorf("Error serving %v (%v); will try again.", server.Addr, err) + + // listen again + func() { + lock.Lock() + defer lock.Unlock() + for { + time.Sleep(15 * time.Second) + + ln, err = net.Listen("tcp", server.Addr) + if err == nil { + return + } + select { + case <-stopCh: + return + default: + } + glog.Errorf("Error listening on %v (%v); will try again.", server.Addr, err) + } + }() + + select { + case <-stopCh: + return + default: + } + } + }() + + return tcpAddr.Port, nil +} + +// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in +// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded. +// Returns nil if len(namedCertKeys) == 0 +func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) { + if len(namedCertKeys) == 0 { + return nil, nil + } + + // load keys + tlsCerts := make([]tls.Certificate, len(namedCertKeys)) + for i := range namedCertKeys { + var err error + nkc := &namedCertKeys[i] + tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile) + if err != nil { + return nil, err + } + } + + // register certs with implicit names first, reverse order such that earlier trump over the later + tlsCertsByName := map[string]*tls.Certificate{} + for i := len(namedCertKeys) - 1; i >= 0; i-- { + nkc := &namedCertKeys[i] + if len(nkc.Names) > 0 { + continue + } + cert := &tlsCerts[i] + + // read names from certificate common names and DNS names + if len(cert.Certificate) == 0 { + return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile) + } + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err) + } + if len(x509Cert.Subject.CommonName) > 0 { + tlsCertsByName[x509Cert.Subject.CommonName] = cert + } + for _, san := range x509Cert.DNSNames { + tlsCertsByName[san] = cert + } + // intentionally all IPs in the cert are ignored as SNI forbids passing IPs + // to select a cert. Before go 1.6 the tls happily passed IPs as SNI values. + } + + // register certs with explicit names last, overwriting every of the implicit ones, + // again in reverse order. + for i := len(namedCertKeys) - 1; i >= 0; i-- { + nkc := &namedCertKeys[i] + if len(nkc.Names) == 0 { + continue + } + for _, name := range nkc.Names { + tlsCertsByName[name] = &tlsCerts[i] + } + } + + return tlsCertsByName, nil +} + +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by ListenAndServe and ListenAndServeTLS so +// dead TCP connections (e.g. closing laptop mid-download) eventually +// go away. +// +// Copied from Go 1.7.2 net/http/server.go +type tcpKeepAliveListener struct { + *net.TCPListener +} + +func (ln tcpKeepAliveListener) Accept() (net.Conn, error) { + tc, err := ln.AcceptTCP() + if err != nil { + return nil, err + } + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(defaultKeepAlivePeriod) + return tc, nil +} diff --git a/pkg/genericapiserver/serve_test.go b/pkg/genericapiserver/serve_test.go new file mode 100644 index 00000000000..aac9499a6ff --- /dev/null +++ b/pkg/genericapiserver/serve_test.go @@ -0,0 +1,506 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "io/ioutil" + "net" + "os" + "testing" + + utilcert "k8s.io/kubernetes/pkg/util/cert" + + "github.com/stretchr/testify/assert" +) + +type TestCertSpec struct { + host string + names, ips []string // in certificate +} + +type NamedTestCertSpec struct { + TestCertSpec + explicitNames []string // as --tls-sni-cert-key explicit names +} + +func createTestCerts(spec TestCertSpec) (certFilePath, keyFilePath string, err error) { + var ips []net.IP + for _, ip := range spec.ips { + ips = append(ips, net.ParseIP(ip)) + } + + certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, ips, spec.names) + if err != nil { + return "", "", err + } + + certFile, err := ioutil.TempFile(os.TempDir(), "cert") + if err != nil { + return "", "", err + } + + keyFile, err := ioutil.TempFile(os.TempDir(), "key") + if err != nil { + os.Remove(certFile.Name()) + return "", "", err + } + + _, err = certFile.Write(certPem) + if err != nil { + os.Remove(certFile.Name()) + os.Remove(keyFile.Name()) + return "", "", err + } + certFile.Close() + + _, err = keyFile.Write(keyPem) + if err != nil { + os.Remove(certFile.Name()) + os.Remove(keyFile.Name()) + return "", "", err + } + keyFile.Close() + + return certFile.Name(), keyFile.Name(), nil +} + +func TestGetNamedCertificateMap(t *testing.T) { + tests := []struct { + certs []NamedTestCertSpec + explicitNames []string + expected map[string]int // name to certs[*] index + errorString string + }{ + { + // empty certs + expected: map[string]int{}, + }, + { + // only one cert + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + }, + }, + }, + expected: map[string]int{ + "test.com": 0, + }, + }, + { + // ips are ignored + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + ips: []string{"1.2.3.4"}, + }, + }, + }, + expected: map[string]int{ + "test.com": 0, + }, + }, + { + // two certs with the same name + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + }, + }, + { + TestCertSpec: TestCertSpec{ + host: "test.com", + }, + }, + }, + expected: map[string]int{ + "test.com": 0, + }, + }, + { + // two certs with different names + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test2.com", + }, + }, + { + TestCertSpec: TestCertSpec{ + host: "test1.com", + }, + }, + }, + expected: map[string]int{ + "test1.com": 1, + "test2.com": 0, + }, + }, + { + // two certs with the same name, explicit trumps + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + }, + }, + { + TestCertSpec: TestCertSpec{ + host: "test.com", + }, + explicitNames: []string{"test.com"}, + }, + }, + expected: map[string]int{ + "test.com": 1, + }, + }, + { + // certs with partial overlap; ips are ignored + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "a", + names: []string{"a.test.com", "test.com"}, + }, + }, + { + TestCertSpec: TestCertSpec{ + host: "b", + names: []string{"b.test.com", "test.com"}, + }, + }, + }, + expected: map[string]int{ + "a": 0, "b": 1, + "a.test.com": 0, "b.test.com": 1, + "test.com": 0, + }, + }, + { + // wildcards + certs: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "a", + names: []string{"a.test.com", "test.com"}, + }, + explicitNames: []string{"*.test.com", "test.com"}, + }, + { + TestCertSpec: TestCertSpec{ + host: "b", + names: []string{"b.test.com", "test.com"}, + }, + explicitNames: []string{"dev.test.com", "test.com"}, + }}, + expected: map[string]int{ + "test.com": 0, + "*.test.com": 0, + "dev.test.com": 1, + }, + }, + } + +NextTest: + for i, test := range tests { + var namedCertKeys []NamedCertKey + bySignature := map[string]int{} // index in test.certs by cert signature + for j, c := range test.certs { + certFile, keyFile, err := createTestCerts(c.TestCertSpec) + if err != nil { + t.Errorf("%d - failed to create cert %d: %v", i, j, err) + continue NextTest + } + defer os.Remove(certFile) + defer os.Remove(keyFile) + + namedCertKeys = append(namedCertKeys, NamedCertKey{ + CertKey: CertKey{ + KeyFile: keyFile, + CertFile: certFile, + }, + Names: c.explicitNames, + }) + + sig, err := certFileSignature(certFile, keyFile) + if err != nil { + t.Errorf("%d - failed to get signature for %d: %v", i, j, err) + continue NextTest + } + bySignature[sig] = j + } + + certMap, err := getNamedCertificateMap(namedCertKeys) + if err == nil && len(test.errorString) != 0 { + t.Errorf("%d - expected no error, got: %v", i, err) + } else if err != nil && err.Error() != test.errorString { + t.Errorf("%d - expected error %q, got: %v", i, test.errorString, err) + } else { + got := map[string]int{} + for name, cert := range certMap { + x509Certs, err := x509.ParseCertificates(cert.Certificate[0]) + assert.NoError(t, err, "%d - invalid certificate for %q", i, name) + assert.True(t, len(x509Certs) > 0, "%d - expected at least one x509 cert in tls cert for %q", i, name) + got[name] = bySignature[x509CertSignature(x509Certs[0])] + } + + assert.EqualValues(t, test.expected, got, "%d - wrong certificate map", i) + } + } +} + +func TestServerRunWithSNI(t *testing.T) { + tests := []struct { + Cert TestCertSpec + SNICerts []NamedTestCertSpec + ExpectedCertIndex int + + // passed in the client hello info, "localhost" if unset + ServerName string + }{ + { + // only one cert + Cert: TestCertSpec{ + host: "localhost", + }, + ExpectedCertIndex: -1, + }, + { + // cert with multiple alternate names + Cert: TestCertSpec{ + host: "localhost", + names: []string{"test.com"}, + ips: []string{"127.0.0.1"}, + }, + ExpectedCertIndex: -1, + ServerName: "test.com", + }, + { + // one SNI and the default cert with the same name + Cert: TestCertSpec{ + host: "localhost", + }, + SNICerts: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "localhost", + }, + }, + }, + ExpectedCertIndex: 0, + }, + { + // matching SNI cert + Cert: TestCertSpec{ + host: "localhost", + }, + SNICerts: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + }, + }, + }, + ExpectedCertIndex: 0, + ServerName: "test.com", + }, + { + // matching IP in SNI cert and the server cert. But IPs must not be + // passed via SNI. Hence, the ServerName in the HELLO packet is empty + // and the server should select the non-SNI cert. + Cert: TestCertSpec{ + host: "localhost", + ips: []string{"10.0.0.1"}, + }, + SNICerts: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + ips: []string{"10.0.0.1"}, + }, + }, + }, + ExpectedCertIndex: -1, + ServerName: "10.0.0.1", + }, + { + // wildcards + Cert: TestCertSpec{ + host: "localhost", + }, + SNICerts: []NamedTestCertSpec{ + { + TestCertSpec: TestCertSpec{ + host: "test.com", + names: []string{"*.test.com"}, + }, + }, + }, + ExpectedCertIndex: 0, + ServerName: "www.test.com", + }, + } + +NextTest: + for i, test := range tests { + // create server cert + serverCertFile, serverKeyFile, err := createTestCerts(test.Cert) + if err != nil { + t.Errorf("%d - failed to create server cert: %v", i, err) + } + defer os.Remove(serverCertFile) + defer os.Remove(serverKeyFile) + + // create SNI certs + var namedCertKeys []NamedCertKey + serverSig, err := certFileSignature(serverCertFile, serverKeyFile) + if err != nil { + t.Errorf("%d - failed to get server cert signature: %v", i, err) + continue NextTest + } + signatures := map[string]int{ + serverSig: -1, + } + for j, c := range test.SNICerts { + certFile, keyFile, err := createTestCerts(c.TestCertSpec) + if err != nil { + t.Errorf("%d - failed to create SNI cert %d: %v", i, j, err) + continue NextTest + } + defer os.Remove(certFile) + defer os.Remove(keyFile) + + namedCertKeys = append(namedCertKeys, NamedCertKey{ + CertKey: CertKey{ + KeyFile: keyFile, + CertFile: certFile, + }, + Names: c.explicitNames, + }) + + // store index in namedCertKeys with the signature as the key + sig, err := certFileSignature(certFile, keyFile) + if err != nil { + t.Errorf("%d - failed get SNI cert %d signature: %v", i, j, err) + continue NextTest + } + signatures[sig] = j + } + + stopCh := make(chan struct{}) + + // launch server + etcdserver, config, _ := setUp(t) + defer etcdserver.Terminate(t) + + config.EnableIndex = true + config.SecureServingInfo = &SecureServingInfo{ + ServingInfo: ServingInfo{ + BindAddress: "localhost:0", + }, + ServerCert: GeneratableKeyCert{ + CertKey: CertKey{ + CertFile: serverCertFile, + KeyFile: serverKeyFile, + }, + }, + SNICerts: namedCertKeys, + } + config.InsecureServingInfo = nil + + s, err := config.Complete().New() + if err != nil { + t.Errorf("%d - failed creating the server: %v", i, err) + continue NextTest + } + + if err := s.serveSecurely(stopCh); err != nil { + t.Errorf("%d - failed running the server: %v", i, err) + continue NextTest + } + + // load certificates into a pool + roots := x509.NewCertPool() + certFiles := []string{serverCertFile} + for _, c := range namedCertKeys { + certFiles = append(certFiles, c.CertFile) + } + for _, certFile := range certFiles { + bs, err := ioutil.ReadFile(certFile) + if err != nil { + t.Errorf("%d - error reading %q: %v", i, certFile, err) + continue NextTest + } + if ok := roots.AppendCertsFromPEM(bs); !ok { + t.Errorf("%d - error adding cert %q to the pool", i, certFile) + continue NextTest + } + } + + // try to dial + addr := fmt.Sprintf("localhost:%d", s.effectiveSecurePort) + t.Logf("Dialing %s as %q", addr, test.ServerName) + conn, err := tls.Dial("tcp", addr, &tls.Config{ + RootCAs: roots, + ServerName: test.ServerName, // used for SNI in the client HELLO packet + }) + if err != nil { + t.Errorf("%d - failed to connect: %v", i, err) + continue NextTest + } + + // check returned server certificate + sig := x509CertSignature(conn.ConnectionState().PeerCertificates[0]) + gotCertIndex, found := signatures[sig] + if !found { + t.Errorf("%d - unknown signature returned from server: %s", i, sig) + } + if gotCertIndex != test.ExpectedCertIndex { + t.Errorf("%d - expected cert index %d, got cert index %d", i, test.ExpectedCertIndex, gotCertIndex) + } + + conn.Close() + } +} + +func x509CertSignature(cert *x509.Certificate) string { + return base64.StdEncoding.EncodeToString(cert.Signature) +} + +func certFileSignature(certFile, keyFile string) (string, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return "", err + } + + x509Certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return "", err + } + if len(x509Certs) == 0 { + return "", fmt.Errorf("expected at least one cert after reparsing cert %q", certFile) + } + return x509CertSignature(x509Certs[0]), nil +} diff --git a/pkg/util/cert/cert.go b/pkg/util/cert/cert.go index 32e968bb60e..ee071e9b612 100644 --- a/pkg/util/cert/cert.go +++ b/pkg/util/cert/cert.go @@ -126,22 +126,19 @@ func MakeEllipticPrivateKeyPEM() ([]byte, error) { return pem.EncodeToMemory(privateKeyPemBlock), nil } -// GenerateSelfSignedCert creates a self-signed certificate and key for the given host. +// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host. // Host may be an IP or a DNS name // You may also specify additional subject alt names (either ip or dns names) for the certificate -// The certificate will be created with file mode 0644. The key will be created with file mode 0600. -// If the certificate or key files already exist, they will be overwritten. -// Any parent directories of the certPath or keyPath will be created as needed with file mode 0755. -func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.IP, alternateDNS []string) error { +func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) { priv, err := rsa.GenerateKey(cryptorand.Reader, 2048) if err != nil { - return err + return nil, nil, err } template := x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{ - CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()), + CommonName: host, }, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour * 24 * 365), @@ -163,30 +160,22 @@ func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.I derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, &template, &priv.PublicKey, priv) if err != nil { - return err + return nil, nil, err } // Generate cert certBuffer := bytes.Buffer{} if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { - return err + return nil, nil, err } // Generate key keyBuffer := bytes.Buffer{} if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { - return err + return nil, nil, err } - if err := WriteCert(certPath, certBuffer.Bytes()); err != nil { - return err - } - - if err := WriteKey(keyPath, keyBuffer.Bytes()); err != nil { - return err - } - - return nil + return certBuffer.Bytes(), keyBuffer.Bytes(), nil } // FormatBytesCert receives byte array certificate and formats in human-readable format diff --git a/pkg/util/config/BUILD b/pkg/util/config/BUILD index 4ce8146ba14..30d4214b415 100644 --- a/pkg/util/config/BUILD +++ b/pkg/util/config/BUILD @@ -17,6 +17,7 @@ go_library( "configuration_map.go", "doc.go", "feature_gate.go", + "namedcertkey_flag.go", ], tags = ["automanaged"], deps = [ @@ -31,6 +32,7 @@ go_test( srcs = [ "config_test.go", "feature_gate_test.go", + "namedcertkey_flag_test.go", ], library = "go_default_library", tags = ["automanaged"], diff --git a/pkg/util/config/namedcertkey_flag.go b/pkg/util/config/namedcertkey_flag.go new file mode 100644 index 00000000000..39f20f681f7 --- /dev/null +++ b/pkg/util/config/namedcertkey_flag.go @@ -0,0 +1,113 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "errors" + "flag" + "strings" +) + +// NamedCertKey is a flag value parsing "certfile,keyfile" and "certfile,keyfile:name,name,name". +type NamedCertKey struct { + Names []string + CertFile, KeyFile string +} + +var _ flag.Value = &NamedCertKey{} + +func (nkc *NamedCertKey) String() string { + s := nkc.CertFile + "," + nkc.KeyFile + if len(nkc.Names) > 0 { + s = s + ":" + strings.Join(nkc.Names, ",") + } + return s +} + +func (nkc *NamedCertKey) Set(value string) error { + cs := strings.SplitN(value, ":", 2) + var keycert string + if len(cs) == 2 { + var names string + keycert, names = strings.TrimSpace(cs[0]), strings.TrimSpace(cs[1]) + if names == "" { + return errors.New("empty names list is not allowed") + } + nkc.Names = nil + for _, name := range strings.Split(names, ",") { + nkc.Names = append(nkc.Names, strings.TrimSpace(name)) + } + } else { + nkc.Names = nil + keycert = strings.TrimSpace(cs[0]) + } + cs = strings.Split(keycert, ",") + if len(cs) != 2 { + return errors.New("expected comma separated certificate and key file paths") + } + nkc.CertFile = strings.TrimSpace(cs[0]) + nkc.KeyFile = strings.TrimSpace(cs[1]) + return nil +} + +func (*NamedCertKey) Type() string { + return "namedCertKey" +} + +// NamedCertKeyArray is a flag value parsing NamedCertKeys, each passed with its own +// flag instance (in contrast to comma separated slices). +type NamedCertKeyArray struct { + value *[]NamedCertKey + changed bool +} + +var _ flag.Value = &NamedCertKey{} + +// NewNamedKeyCertArray creates a new NamedCertKeyArray with the internal value +// pointing to p. +func NewNamedCertKeyArray(p *[]NamedCertKey) *NamedCertKeyArray { + return &NamedCertKeyArray{ + value: p, + } +} + +func (a *NamedCertKeyArray) Set(val string) error { + nkc := NamedCertKey{} + err := nkc.Set(val) + if err != nil { + return err + } + if !a.changed { + *a.value = []NamedCertKey{nkc} + a.changed = true + } else { + *a.value = append(*a.value, nkc) + } + return nil +} + +func (a *NamedCertKeyArray) Type() string { + return "namedCertKey" +} + +func (a *NamedCertKeyArray) String() string { + nkcs := make([]string, 0, len(*a.value)) + for i := range *a.value { + nkcs = append(nkcs, (*a.value)[i].String()) + } + return "[" + strings.Join(nkcs, ";") + "]" +} diff --git a/pkg/util/config/namedcertkey_flag_test.go b/pkg/util/config/namedcertkey_flag_test.go new file mode 100644 index 00000000000..5aa48230b6f --- /dev/null +++ b/pkg/util/config/namedcertkey_flag_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "github.com/spf13/pflag" + "reflect" + "strings" + "testing" +) + +func TestNamedCertKeyArrayFlag(t *testing.T) { + tests := []struct { + args []string + def []NamedCertKey + expected []NamedCertKey + parseError string + }{ + { + args: []string{}, + expected: nil, + }, + { + args: []string{"foo.crt,foo.key"}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + }}, + }, + { + args: []string{" foo.crt , foo.key "}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + }}, + }, + { + args: []string{"foo.crt,foo.key:abc"}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + Names: []string{"abc"}, + }}, + }, + { + args: []string{"foo.crt,foo.key: abc "}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + Names: []string{"abc"}, + }}, + }, + { + args: []string{"foo.crt,foo.key:"}, + parseError: "empty names list is not allowed", + }, + { + args: []string{""}, + parseError: "expected comma separated certificate and key file paths", + }, + { + args: []string{" "}, + parseError: "expected comma separated certificate and key file paths", + }, + { + args: []string{"a,b,c"}, + parseError: "expected comma separated certificate and key file paths", + }, + { + args: []string{"foo.crt,foo.key:abc,def,ghi"}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + Names: []string{"abc", "def", "ghi"}, + }}, + }, + { + args: []string{"foo.crt,foo.key:*.*.*"}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + Names: []string{"*.*.*"}, + }}, + }, + { + args: []string{"foo.crt,foo.key", "bar.crt,bar.key"}, + expected: []NamedCertKey{{ + KeyFile: "foo.key", + CertFile: "foo.crt", + }, { + KeyFile: "bar.key", + CertFile: "bar.crt", + }}, + }, + } + for i, test := range tests { + fs := pflag.NewFlagSet("testNamedCertKeyArray", pflag.ContinueOnError) + var nkcs []NamedCertKey + for _, d := range test.def { + nkcs = append(nkcs, d) + } + fs.Var(NewNamedCertKeyArray(&nkcs), "tls-sni-cert-key", "usage") + + args := []string{} + for _, a := range test.args { + args = append(args, fmt.Sprintf("--tls-sni-cert-key=%s", a)) + } + + err := fs.Parse(args) + if test.parseError != "" { + if err == nil { + t.Errorf("%d: expected error %q, got nil", i, test.parseError) + } else if !strings.Contains(err.Error(), test.parseError) { + t.Errorf("%d: expected error %q, got %q", i, test.parseError, err) + } + } else if err != nil { + t.Errorf("%d: expected nil error, got %v", i, err) + } + if !reflect.DeepEqual(nkcs, test.expected) { + t.Errorf("%d: expected %+v, got %+v", i, test.expected, nkcs) + } + } +} diff --git a/test/integration/discoverysummarizer/discoverysummarizer_test.go b/test/integration/discoverysummarizer/discoverysummarizer_test.go index 2ee06618218..353cd8733f1 100644 --- a/test/integration/discoverysummarizer/discoverysummarizer_test.go +++ b/test/integration/discoverysummarizer/discoverysummarizer_test.go @@ -65,12 +65,13 @@ func runDiscoverySummarizer(t *testing.T) string { return serverURL } -func runAPIServer(t *testing.T) string { +func runAPIServer(t *testing.T, stopCh <-chan struct{}) string { serverRunOptions := apiserver.NewServerRunOptions() - // Change the port, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel. - serverRunOptions.InsecurePort = 8083 + // Change the ports, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel. + serverRunOptions.SecurePort = 6443 + 3 + serverRunOptions.InsecurePort = 8080 + 3 go func() { - if err := apiserver.Run(serverRunOptions); err != nil { + if err := apiserver.Run(serverRunOptions, stopCh); err != nil { t.Fatalf("Error in bringing up the example apiserver: %v", err) } }() @@ -98,7 +99,9 @@ func TestRunDiscoverySummarizer(t *testing.T) { testResponse(t, discoveryURL, "/randomPath", http.StatusNotFound) // Run the APIServer now to test the good case. - runAPIServer(t) + stopCh := make(chan struct{}) + runAPIServer(t, stopCh) + defer close(stopCh) // Test /api path. // There is no server running at that URL, so we will get a 500. diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 8bed495c60e..858d2ee390e 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -42,11 +42,13 @@ var groupVersionForDiscovery = unversioned.GroupVersionForDiscovery{ func TestRunServer(t *testing.T) { serverIP := fmt.Sprintf("http://localhost:%d", apiserver.InsecurePort) + stopCh := make(chan struct{}) go func() { - if err := apiserver.Run(apiserver.NewServerRunOptions()); err != nil { + if err := apiserver.Run(apiserver.NewServerRunOptions(), stopCh); err != nil { t.Fatalf("Error in bringing up the server: %v", err) } }() + defer close(stopCh) if err := waitForApiserverUp(serverIP); err != nil { t.Fatalf("%v", err) } @@ -58,14 +60,16 @@ func TestRunServer(t *testing.T) { func TestRunSecureServer(t *testing.T) { serverIP := fmt.Sprintf("https://localhost:%d", apiserver.SecurePort) + stopCh := make(chan struct{}) go func() { options := apiserver.NewServerRunOptions() options.InsecurePort = 0 options.SecurePort = apiserver.SecurePort - if err := apiserver.Run(options); err != nil { + if err := apiserver.Run(options, stopCh); err != nil { t.Fatalf("Error in bringing up the server: %v", err) } }() + defer close(stopCh) if err := waitForApiserverUp(serverIP); err != nil { t.Fatalf("%v", err) } diff --git a/test/integration/federation/server_test.go b/test/integration/federation/server_test.go index 0bd5e56e698..7c8b12b1d1a 100644 --- a/test/integration/federation/server_test.go +++ b/test/integration/federation/server_test.go @@ -77,7 +77,8 @@ func TestLongRunningRequestRegexp(t *testing.T) { } } -var insecurePort = 8082 +var securePort = 6443 + 2 +var insecurePort = 8080 + 2 var serverIP = fmt.Sprintf("http://localhost:%v", insecurePort) var groupVersions = []unversioned.GroupVersion{ fed_v1b1.SchemeGroupVersion, @@ -86,6 +87,7 @@ var groupVersions = []unversioned.GroupVersion{ func TestRun(t *testing.T) { s := options.NewServerRunOptions() + s.GenericServerRunOptions.SecurePort = securePort s.GenericServerRunOptions.InsecurePort = insecurePort _, ipNet, _ := net.ParseCIDR("10.10.10.0/24") s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet