github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #35109 from sttts/sttts-sni

Browse Source 
Automatic merge from submit-queue

Add SNI support to the apiserver

This PR adds the `--tls-sni-key-cert` flag to the apiserver. It can be passed multiple times in the following ways:

``` shell
$ apiserver \
    --tls-sni-cert-key '*.example.com,example.com: example.key,example.crt' \
    --tls-sni-cert-key 'foo.key,foo.crt'
```

The first variant explicitly sets the accepted domain names, the second variant reads the common names and DNS names from the certificate itself.

If no domain name matches, the existing certificate (`--tls-cert-file`) is used.

``` golang
    fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
        "A pair of x509 certificate and private key file paths, optionally prefixed with a list of "+
        "domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
        "segments. If no domain patterns are provided, the names of the certificate are "+
        "extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
        "trump over extracted names. For multiple key/certificate pairs, use the "+
        "--tls-sni-key-cert multiple times. "+
        "Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".")
```

``` release-note
Add SNI support to the apiserver

Pass multiple certificates and domain name patterns with `--tls-sni-cert-key` and the right certificate will be chosen depending on the url the client is using.
```
This commit is contained in:
Kubernetes Submit Queue 2016-11-01 04:25:54 -07:00 committed by GitHub
commit 6babfb6ccc
19 changed files with 1144 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-apiserver/app/server.go
View File

@ -335,6 +335,6 @@ func Run(s *options.ServerRunOptions) error {
}
sharedInformers.Start(wait.NeverStop)
m.GenericAPIServer.PrepareRun().Run()
m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}

12
cmd/kubelet/app/server.go
View File

@ -502,9 +502,19 @@ func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions
kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt")
kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key")
if !certutil.CanReadCertOrKey(kc.TLSCertFile, kc.TLSPrivateKeyFile) {
if err := certutil.GenerateSelfSignedCert(nodeutil.GetHostname(kc.HostnameOverride), kc.TLSCertFile, kc.TLSPrivateKeyFile, nil, nil); err != nil {
cert, key, err := certutil.GenerateSelfSignedCertKey(nodeutil.GetHostname(kc.HostnameOverride), nil, nil)
if err != nil {
return nil, fmt.Errorf("unable to generate self signed cert: %v", err)
}
if err := certutil.WriteCert(kc.TLSCertFile, cert); err != nil {
return nil, err
}
if err := certutil.WriteKey(kc.TLSPrivateKeyFile, key); err != nil {
return nil, err
}
glog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile)
}
}

4
examples/apiserver/apiserver.go
View File

@ -60,7 +60,7 @@ func NewServerRunOptions() *genericoptions.ServerRunOptions {
return serverOptions
}
func Run(serverOptions *genericoptions.ServerRunOptions) error {
func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error {
// Set ServiceClusterIPRange
_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
@ -105,6 +105,6 @@ func Run(serverOptions *genericoptions.ServerRunOptions) error {
if err := s.InstallAPIGroup(&apiGroupInfo); err != nil {
return fmt.Errorf("Error in installing API: %v", err)
}
s.PrepareRun().Run()
s.PrepareRun().Run(stopCh)
return nil
}

3
examples/apiserver/server/main.go
View File

@ -19,6 +19,7 @@ package main
import (
"k8s.io/kubernetes/examples/apiserver"
"k8s.io/kubernetes/pkg/util/flag"
"k8s.io/kubernetes/pkg/util/wait"
"github.com/golang/glog"
"github.com/spf13/pflag"
@ -32,7 +33,7 @@ func main() {
serverRunOptions.AddEtcdStorageFlags(pflag.CommandLine)
flag.InitFlags()
if err := apiserver.Run(serverRunOptions); err != nil {
if err := apiserver.Run(serverRunOptions, wait.NeverStop); err != nil {
glog.Fatalf("Error in bringing up the server: %v", err)
}
}

2
federation/cmd/federation-apiserver/app/server.go
View File

@ -225,7 +225,7 @@ func Run(s *options.ServerRunOptions) error {
installExtensionsAPIs(m, restOptionsFactory)
sharedInformers.Start(wait.NeverStop)
m.PrepareRun().Run()
m.PrepareRun().Run(wait.NeverStop)
return nil
}

1
hack/verify-flags/known-flags.txt
View File

@ -555,6 +555,7 @@ tls-ca-file
tls-cert-file
tls-private-key-file
to-version
tls-sni-cert-key
token-auth-file
ttl-keys-prefix
ttl-secs

4
pkg/genericapiserver/BUILD
View File

@ -21,6 +21,7 @@ go_library(
"resource_config.go",
"resource_encoding_config.go",
"reststorage_interfaces.go",
"serve.go",
"storage_factory.go",
"tunneler.go",
],
@ -66,6 +67,7 @@ go_library(
"//vendor:github.com/emicklei/go-restful",
"//vendor:github.com/go-openapi/spec",
"//vendor:github.com/golang/glog",
"//vendor:github.com/pkg/errors",
"//vendor:github.com/prometheus/client_golang/prometheus",
"//vendor:gopkg.in/natefinch/lumberjack.v2",
],
@ -77,6 +79,7 @@ go_test(
"default_storage_factory_builder_test.go",
"genericapiserver_test.go",
"resource_config_test.go",
"serve_test.go",
"server_run_options_test.go",
"storage_factory_test.go",
"tunneler_test.go",
@ -100,6 +103,7 @@ go_test(
"//pkg/registry/core/service/ipallocator:go_default_library",
"//pkg/storage/etcd/testing:go_default_library",
"//pkg/storage/storagebackend:go_default_library",
"//pkg/util/cert:go_default_library",
"//pkg/util/clock:go_default_library",
"//pkg/util/sets:go_default_library",
"//pkg/version:go_default_library",

62
pkg/genericapiserver/config.go
View File

@ -110,7 +110,7 @@ type Config struct {
// same value for this field. (Numbers > 1 currently untested.)
MasterCount int
SecureServingInfo *ServingInfo
SecureServingInfo *SecureServingInfo
InsecureServingInfo *ServingInfo
// The port on PublicAddress where a read-write server will be installed.
@ -177,17 +177,36 @@ type Config struct {
type ServingInfo struct {
// BindAddress is the ip:port to serve on
BindAddress string
}
type SecureServingInfo struct {
ServingInfo
// ServerCert is the TLS cert info for serving secure traffic
ServerCert CertInfo
ServerCert GeneratableKeyCert
// SNICerts are named CertKeys for serving secure traffic with SNI support.
SNICerts []NamedCertKey
// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
ClientCA string
}
type CertInfo struct {
type CertKey struct {
// CertFile is a file containing a PEM-encoded certificate
CertFile string
// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
KeyFile string
}
type NamedCertKey struct {
CertKey
// Names is a list of domain patterns: fully qualified domain names, possibly prefixed with
// wildcard segments.
Names []string
}
type GeneratableKeyCert struct {
CertKey
// Generate indicates that the cert/key pair should be generated if its not present.
Generate bool
}
@ -248,12 +267,17 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
}
if options.SecurePort > 0 {
secureServingInfo := &ServingInfo{
BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)),
ServerCert: CertInfo{
CertFile: options.TLSCertFile,
KeyFile: options.TLSPrivateKeyFile,
secureServingInfo := &SecureServingInfo{
ServingInfo: ServingInfo{
BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)),
},
ServerCert: GeneratableKeyCert{
CertKey: CertKey{
CertFile: options.TLSCertFile,
KeyFile: options.TLSPrivateKeyFile,
},
},
SNICerts: []NamedCertKey{},
ClientCA: options.ClientCAFile,
}
if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" {
@ -262,6 +286,17 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
secureServingInfo.ServerCert.KeyFile = path.Join(options.CertDirectory, "apiserver.key")
}
secureServingInfo.SNICerts = nil
for _, nkc := range options.SNICertKeys {
secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{
CertKey: CertKey{
KeyFile: nkc.KeyFile,
CertFile: nkc.CertFile,
},
Names: nkc.Names,
})
}
c.SecureServingInfo = secureServingInfo
c.ReadWritePort = options.SecurePort
}
@ -434,9 +469,16 @@ func (c completedConfig) MaybeGenerateServingCerts() error {
alternateIPs := []net.IP{c.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
if err := certutil.GenerateSelfSignedCert(c.PublicAddress.String(), c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile, alternateIPs, alternateDNS); err != nil {
return fmt.Errorf("Unable to generate self signed cert: %v", err)
if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil {
return fmt.Errorf("unable to generate self signed cert: %v", err)
} else {
if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil {
return err
}
if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil {
return err
}
glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile)
}
}

107
pkg/genericapiserver/genericapiserver.go
View File

@ -17,7 +17,6 @@ limitations under the License.
package genericapiserver
import (
"crypto/tls"
"fmt"
"mime"
"net"
@ -44,9 +43,7 @@ import (
"k8s.io/kubernetes/pkg/genericapiserver/openapi/common"
"k8s.io/kubernetes/pkg/genericapiserver/routes"
"k8s.io/kubernetes/pkg/runtime"
certutil "k8s.io/kubernetes/pkg/util/cert"
utilnet "k8s.io/kubernetes/pkg/util/net"
utilruntime "k8s.io/kubernetes/pkg/util/runtime"
"k8s.io/kubernetes/pkg/util/sets"
)
@ -112,9 +109,12 @@ type GenericAPIServer struct {
// The registered APIs
HandlerContainer *genericmux.APIContainer
SecureServingInfo *ServingInfo
SecureServingInfo *SecureServingInfo
InsecureServingInfo *ServingInfo
// numerical ports, set after listening
effectiveSecurePort, effectiveInsecurePort int
// ExternalAddress is the address (hostname or IP and port) that should be used in
// external (public internet) URLs for this GenericAPIServer.
ExternalAddress string
@ -180,7 +180,6 @@ type preparedGenericAPIServer struct {
// PrepareRun does post API installation setup steps.
func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
// install APIs which depend on other APIs to be installed
if s.enableSwaggerSupport {
routes.Swagger{ExternalAddress: s.ExternalAddress}.Install(s.HandlerContainer)
}
@ -192,76 +191,18 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
return preparedGenericAPIServer{s}
}
func (s preparedGenericAPIServer) Run() {
// Run spawns the http servers (secure and insecure). It only returns if stopCh is closed
// or one of the ports cannot be listened on initially.
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
if s.SecureServingInfo != nil && s.Handler != nil {
secureServer := &http.Server{
Addr: s.SecureServingInfo.BindAddress,
Handler: s.Handler,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
},
if err := s.serveSecurely(stopCh); err != nil {
glog.Fatal(err)
}
if len(s.SecureServingInfo.ClientCA) > 0 {
clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
// "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server
secureServer.TLSConfig.NextProtos = []string{"h2"}
}
glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
go func() {
defer utilruntime.HandleCrash()
for {
if err := secureServer.ListenAndServeTLS(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
if s.InsecureServingInfo != nil && s.InsecureHandler != nil {
insecureServer := &http.Server{
Addr: s.InsecureServingInfo.BindAddress,
Handler: s.InsecureHandler,
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
go func() {
defer utilruntime.HandleCrash()
for {
if err := insecureServer.ListenAndServe(); err != nil {
glog.Errorf("Unable to listen for insecure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) per port
if s.SecureServingInfo != nil {
if err := waitForSuccessfulDial(true, "tcp", s.SecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
glog.Fatalf("Secure server never started: %v", err)
}
}
if s.InsecureServingInfo != nil {
if err := waitForSuccessfulDial(false, "tcp", s.InsecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
glog.Fatalf("Insecure server never started: %v", err)
if err := s.serveInsecurely(stopCh); err != nil {
glog.Fatal(err)
}
}
@ -272,7 +213,7 @@ func (s preparedGenericAPIServer) Run() {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
select {}
<-stopCh
}
// installAPIResources is a private method for installing the REST storage backing each api groupversionresource
@ -471,27 +412,3 @@ func NewDefaultAPIGroupInfo(group string) APIGroupInfo {
NegotiatedSerializer: api.Codecs,
}
}
// waitForSuccessfulDial attempts to connect to the given address, closing and returning nil on the first successful connection.
func waitForSuccessfulDial(https bool, network, address string, timeout, interval time.Duration, retries int) error {
var (
conn net.Conn
err error
)
for i := 0; i <= retries; i++ {
dialer := net.Dialer{Timeout: timeout}
if https {
conn, err = tls.DialWithDialer(&dialer, network, address, &tls.Config{InsecureSkipVerify: true})
} else {
conn, err = dialer.Dial(network, address)
}
if err != nil {
glog.V(5).Infof("Got error %#v, trying again: %#v\n", err, address)
time.Sleep(interval)
continue
}
conn.Close()
return nil
}
return err
}

14
pkg/genericapiserver/options/server_run_options.go
View File

@ -118,6 +118,7 @@ type ServerRunOptions struct {
TLSCAFile string
TLSCertFile string
TLSPrivateKeyFile string
SNICertKeys []config.NamedCertKey
TokenAuthFile string
EnableAnyToken bool
WatchCacheSizes []string
@ -488,13 +489,22 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"Controllers. This must be a valid PEM-encoded CA bundle.")
fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+
"File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
"--tls-private-key-file are not provided, a self-signed certificate and key "+
"are generated for the public address and saved to /var/run/kubernetes.")
fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile,
"File containing x509 private key matching --tls-cert-file.")
"File containing the default x509 private key matching --tls-cert-file.")
fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
"segments. If no domain patterns are provided, the names of the certificate are "+
"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
"trump over extracted names. For multiple key/certificate pairs, use the "+
"--tls-sni-cert-key multiple times. "+
"Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".")
fs.StringVar(&s.TokenAuthFile, "token-auth-file", s.TokenAuthFile, ""+
"If set, the file that will be used to secure the secure port of the API server "+

262
pkg/genericapiserver/serve.go Normal file
View File

@ -0,0 +1,262 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package genericapiserver
import (
"crypto/tls"
"crypto/x509"
"fmt"
"net"
"net/http"
"sync"
"time"
certutil "k8s.io/kubernetes/pkg/util/cert"
utilruntime "k8s.io/kubernetes/pkg/util/runtime"
"github.com/golang/glog"
"github.com/pkg/errors"
)
const (
defaultKeepAlivePeriod = 3 * time.Minute
)
// serveSecurely runs the secure http server. It fails only if certificates cannot
// be loaded or the initial listen call fails. The actual server loop (stoppable by closing
// stopCh) runs in a go routine, i.e. serveSecurely does not block.
func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts)
if err != nil {
return fmt.Errorf("unable to load SNI certificates: %v", err)
}
secureServer := &http.Server{
Addr: s.SecureServingInfo.BindAddress,
Handler: s.Handler,
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
NameToCertificate: namedCerts,
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
// enable HTTP2 for go's 1.7 HTTP Server
NextProtos: []string{"h2", "http/1.1"},
},
}
if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 {
secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile)
if err != nil {
return fmt.Errorf("unable to load server certificate: %v", err)
}
}
// append all named certs. Otherwise, the go tls stack will think no SNI processing
// is necessary because there is only one cert anyway.
// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
// cert will become the default cert. That's what we expect anyway.
for _, c := range namedCerts {
secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
}
if len(s.SecureServingInfo.ClientCA) > 0 {
clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
if err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
s.effectiveSecurePort, err = runServer(secureServer, stopCh)
return err
}
// serveInsecurely run the insecure http server. It fails only if the initial listen
// call fails. The actual server loop (stoppable by closing stopCh) runs in a go
// routine, i.e. serveInsecurely does not block.
func (s *GenericAPIServer) serveInsecurely(stopCh <-chan struct{}) error {
insecureServer := &http.Server{
Addr: s.InsecureServingInfo.BindAddress,
Handler: s.InsecureHandler,
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
var err error
s.effectiveInsecurePort, err = runServer(insecureServer, stopCh)
return err
}
// runServer listens on the given port, then spawns a go-routine continuously serving
// until the stopCh is closed. The port is returned. This function does not block.
func runServer(server *http.Server, stopCh <-chan struct{}) (int, error) {
if len(server.Addr) == 0 {
return 0, errors.New("address cannot be empty")
}
// first listen is synchronous (fail early!)
ln, err := net.Listen("tcp", server.Addr)
if err != nil {
return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
}
// get port
tcpAddr, ok := ln.Addr().(*net.TCPAddr)
if !ok {
ln.Close()
return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
}
lock := sync.Mutex{} // to avoid we close an old listener during a listen retry
go func() {
<-stopCh
lock.Lock()
defer lock.Unlock()
ln.Close()
}()
go func() {
defer utilruntime.HandleCrash()
for {
var listener net.Listener
listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
if server.TLSConfig != nil {
listener = tls.NewListener(listener, server.TLSConfig)
}
err := server.Serve(listener)
glog.Errorf("Error serving %v (%v); will try again.", server.Addr, err)
// listen again
func() {
lock.Lock()
defer lock.Unlock()
for {
time.Sleep(15 * time.Second)
ln, err = net.Listen("tcp", server.Addr)
if err == nil {
return
}
select {
case <-stopCh:
return
default:
}
glog.Errorf("Error listening on %v (%v); will try again.", server.Addr, err)
}
}()
select {
case <-stopCh:
return
default:
}
}
}()
return tcpAddr.Port, nil
}
// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in
// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded.
// Returns nil if len(namedCertKeys) == 0
func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) {
if len(namedCertKeys) == 0 {
return nil, nil
}
// load keys
tlsCerts := make([]tls.Certificate, len(namedCertKeys))
for i := range namedCertKeys {
var err error
nkc := &namedCertKeys[i]
tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile)
if err != nil {
return nil, err
}
}
// register certs with implicit names first, reverse order such that earlier trump over the later
tlsCertsByName := map[string]*tls.Certificate{}
for i := len(namedCertKeys) - 1; i >= 0; i-- {
nkc := &namedCertKeys[i]
if len(nkc.Names) > 0 {
continue
}
cert := &tlsCerts[i]
// read names from certificate common names and DNS names
if len(cert.Certificate) == 0 {
return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile)
}
x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err)
}
if len(x509Cert.Subject.CommonName) > 0 {
tlsCertsByName[x509Cert.Subject.CommonName] = cert
}
for _, san := range x509Cert.DNSNames {
tlsCertsByName[san] = cert
}
// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
}
// register certs with explicit names last, overwriting every of the implicit ones,
// again in reverse order.
for i := len(namedCertKeys) - 1; i >= 0; i-- {
nkc := &namedCertKeys[i]
if len(nkc.Names) == 0 {
continue
}
for _, name := range nkc.Names {
tlsCertsByName[name] = &tlsCerts[i]
}
}
return tlsCertsByName, nil
}
// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe and ListenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away.
//
// Copied from Go 1.7.2 net/http/server.go
type tcpKeepAliveListener struct {
*net.TCPListener
}
func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
tc, err := ln.AcceptTCP()
if err != nil {
return nil, err
}
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
return tc, nil
}

506
pkg/genericapiserver/serve_test.go Normal file
View File

@ -0,0 +1,506 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package genericapiserver
import (
"crypto/tls"
"crypto/x509"
"encoding/base64"
"fmt"
"io/ioutil"
"net"
"os"
"testing"
utilcert "k8s.io/kubernetes/pkg/util/cert"
"github.com/stretchr/testify/assert"
)
type TestCertSpec struct {
host string
names, ips []string // in certificate
}
type NamedTestCertSpec struct {
TestCertSpec
explicitNames []string // as --tls-sni-cert-key explicit names
}
func createTestCerts(spec TestCertSpec) (certFilePath, keyFilePath string, err error) {
var ips []net.IP
for _, ip := range spec.ips {
ips = append(ips, net.ParseIP(ip))
}
certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, ips, spec.names)
if err != nil {
return "", "", err
}
certFile, err := ioutil.TempFile(os.TempDir(), "cert")
if err != nil {
return "", "", err
}
keyFile, err := ioutil.TempFile(os.TempDir(), "key")
if err != nil {
os.Remove(certFile.Name())
return "", "", err
}
_, err = certFile.Write(certPem)
if err != nil {
os.Remove(certFile.Name())
os.Remove(keyFile.Name())
return "", "", err
}
certFile.Close()
_, err = keyFile.Write(keyPem)
if err != nil {
os.Remove(certFile.Name())
os.Remove(keyFile.Name())
return "", "", err
}
keyFile.Close()
return certFile.Name(), keyFile.Name(), nil
}
func TestGetNamedCertificateMap(t *testing.T) {
tests := []struct {
certs []NamedTestCertSpec
explicitNames []string
expected map[string]int // name to certs[*] index
errorString string
}{
{
// empty certs
expected: map[string]int{},
},
{
// only one cert
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
},
expected: map[string]int{
"test.com": 0,
},
},
{
// ips are ignored
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
ips: []string{"1.2.3.4"},
},
},
},
expected: map[string]int{
"test.com": 0,
},
},
{
// two certs with the same name
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
},
expected: map[string]int{
"test.com": 0,
},
},
{
// two certs with different names
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test2.com",
},
},
{
TestCertSpec: TestCertSpec{
host: "test1.com",
},
},
},
expected: map[string]int{
"test1.com": 1,
"test2.com": 0,
},
},
{
// two certs with the same name, explicit trumps
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
explicitNames: []string{"test.com"},
},
},
expected: map[string]int{
"test.com": 1,
},
},
{
// certs with partial overlap; ips are ignored
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "a",
names: []string{"a.test.com", "test.com"},
},
},
{
TestCertSpec: TestCertSpec{
host: "b",
names: []string{"b.test.com", "test.com"},
},
},
},
expected: map[string]int{
"a": 0, "b": 1,
"a.test.com": 0, "b.test.com": 1,
"test.com": 0,
},
},
{
// wildcards
certs: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "a",
names: []string{"a.test.com", "test.com"},
},
explicitNames: []string{"*.test.com", "test.com"},
},
{
TestCertSpec: TestCertSpec{
host: "b",
names: []string{"b.test.com", "test.com"},
},
explicitNames: []string{"dev.test.com", "test.com"},
}},
expected: map[string]int{
"test.com": 0,
"*.test.com": 0,
"dev.test.com": 1,
},
},
}
NextTest:
for i, test := range tests {
var namedCertKeys []NamedCertKey
bySignature := map[string]int{} // index in test.certs by cert signature
for j, c := range test.certs {
certFile, keyFile, err := createTestCerts(c.TestCertSpec)
if err != nil {
t.Errorf("%d - failed to create cert %d: %v", i, j, err)
continue NextTest
}
defer os.Remove(certFile)
defer os.Remove(keyFile)
namedCertKeys = append(namedCertKeys, NamedCertKey{
CertKey: CertKey{
KeyFile: keyFile,
CertFile: certFile,
},
Names: c.explicitNames,
})
sig, err := certFileSignature(certFile, keyFile)
if err != nil {
t.Errorf("%d - failed to get signature for %d: %v", i, j, err)
continue NextTest
}
bySignature[sig] = j
}
certMap, err := getNamedCertificateMap(namedCertKeys)
if err == nil && len(test.errorString) != 0 {
t.Errorf("%d - expected no error, got: %v", i, err)
} else if err != nil && err.Error() != test.errorString {
t.Errorf("%d - expected error %q, got: %v", i, test.errorString, err)
} else {
got := map[string]int{}
for name, cert := range certMap {
x509Certs, err := x509.ParseCertificates(cert.Certificate[0])
assert.NoError(t, err, "%d - invalid certificate for %q", i, name)
assert.True(t, len(x509Certs) > 0, "%d - expected at least one x509 cert in tls cert for %q", i, name)
got[name] = bySignature[x509CertSignature(x509Certs[0])]
}
assert.EqualValues(t, test.expected, got, "%d - wrong certificate map", i)
}
}
}
func TestServerRunWithSNI(t *testing.T) {
tests := []struct {
Cert TestCertSpec
SNICerts []NamedTestCertSpec
ExpectedCertIndex int
// passed in the client hello info, "localhost" if unset
ServerName string
}{
{
// only one cert
Cert: TestCertSpec{
host: "localhost",
},
ExpectedCertIndex: -1,
},
{
// cert with multiple alternate names
Cert: TestCertSpec{
host: "localhost",
names: []string{"test.com"},
ips: []string{"127.0.0.1"},
},
ExpectedCertIndex: -1,
ServerName: "test.com",
},
{
// one SNI and the default cert with the same name
Cert: TestCertSpec{
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "localhost",
},
},
},
ExpectedCertIndex: 0,
},
{
// matching SNI cert
Cert: TestCertSpec{
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
},
ExpectedCertIndex: 0,
ServerName: "test.com",
},
{
// matching IP in SNI cert and the server cert. But IPs must not be
// passed via SNI. Hence, the ServerName in the HELLO packet is empty
// and the server should select the non-SNI cert.
Cert: TestCertSpec{
host: "localhost",
ips: []string{"10.0.0.1"},
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
ips: []string{"10.0.0.1"},
},
},
},
ExpectedCertIndex: -1,
ServerName: "10.0.0.1",
},
{
// wildcards
Cert: TestCertSpec{
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
names: []string{"*.test.com"},
},
},
},
ExpectedCertIndex: 0,
ServerName: "www.test.com",
},
}
NextTest:
for i, test := range tests {
// create server cert
serverCertFile, serverKeyFile, err := createTestCerts(test.Cert)
if err != nil {
t.Errorf("%d - failed to create server cert: %v", i, err)
}
defer os.Remove(serverCertFile)
defer os.Remove(serverKeyFile)
// create SNI certs
var namedCertKeys []NamedCertKey
serverSig, err := certFileSignature(serverCertFile, serverKeyFile)
if err != nil {
t.Errorf("%d - failed to get server cert signature: %v", i, err)
continue NextTest
}
signatures := map[string]int{
serverSig: -1,
}
for j, c := range test.SNICerts {
certFile, keyFile, err := createTestCerts(c.TestCertSpec)
if err != nil {
t.Errorf("%d - failed to create SNI cert %d: %v", i, j, err)
continue NextTest
}
defer os.Remove(certFile)
defer os.Remove(keyFile)
namedCertKeys = append(namedCertKeys, NamedCertKey{
CertKey: CertKey{
KeyFile: keyFile,
CertFile: certFile,
},
Names: c.explicitNames,
})
// store index in namedCertKeys with the signature as the key
sig, err := certFileSignature(certFile, keyFile)
if err != nil {
t.Errorf("%d - failed get SNI cert %d signature: %v", i, j, err)
continue NextTest
}
signatures[sig] = j
}
stopCh := make(chan struct{})
// launch server
etcdserver, config, _ := setUp(t)
defer etcdserver.Terminate(t)
config.EnableIndex = true
config.SecureServingInfo = &SecureServingInfo{
ServingInfo: ServingInfo{
BindAddress: "localhost:0",
},
ServerCert: GeneratableKeyCert{
CertKey: CertKey{
CertFile: serverCertFile,
KeyFile: serverKeyFile,
},
},
SNICerts: namedCertKeys,
}
config.InsecureServingInfo = nil
s, err := config.Complete().New()
if err != nil {
t.Errorf("%d - failed creating the server: %v", i, err)
continue NextTest
}
if err := s.serveSecurely(stopCh); err != nil {
t.Errorf("%d - failed running the server: %v", i, err)
continue NextTest
}
// load certificates into a pool
roots := x509.NewCertPool()
certFiles := []string{serverCertFile}
for _, c := range namedCertKeys {
certFiles = append(certFiles, c.CertFile)
}
for _, certFile := range certFiles {
bs, err := ioutil.ReadFile(certFile)
if err != nil {
t.Errorf("%d - error reading %q: %v", i, certFile, err)
continue NextTest
}
if ok := roots.AppendCertsFromPEM(bs); !ok {
t.Errorf("%d - error adding cert %q to the pool", i, certFile)
continue NextTest
}
}
// try to dial
addr := fmt.Sprintf("localhost:%d", s.effectiveSecurePort)
t.Logf("Dialing %s as %q", addr, test.ServerName)
conn, err := tls.Dial("tcp", addr, &tls.Config{
RootCAs: roots,
ServerName: test.ServerName, // used for SNI in the client HELLO packet
})
if err != nil {
t.Errorf("%d - failed to connect: %v", i, err)
continue NextTest
}
// check returned server certificate
sig := x509CertSignature(conn.ConnectionState().PeerCertificates[0])
gotCertIndex, found := signatures[sig]
if !found {
t.Errorf("%d - unknown signature returned from server: %s", i, sig)
}
if gotCertIndex != test.ExpectedCertIndex {
t.Errorf("%d - expected cert index %d, got cert index %d", i, test.ExpectedCertIndex, gotCertIndex)
}
conn.Close()
}
}
func x509CertSignature(cert *x509.Certificate) string {
return base64.StdEncoding.EncodeToString(cert.Signature)
}
func certFileSignature(certFile, keyFile string) (string, error) {
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return "", err
}
x509Certs, err := x509.ParseCertificates(cert.Certificate[0])
if err != nil {
return "", err
}
if len(x509Certs) == 0 {
return "", fmt.Errorf("expected at least one cert after reparsing cert %q", certFile)
}
return x509CertSignature(x509Certs[0]), nil
}

27
pkg/util/cert/cert.go
View File

@ -126,22 +126,19 @@ func MakeEllipticPrivateKeyPEM() ([]byte, error) {
return pem.EncodeToMemory(privateKeyPemBlock), nil
}
// GenerateSelfSignedCert creates a self-signed certificate and key for the given host.
// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host.
// Host may be an IP or a DNS name
// You may also specify additional subject alt names (either ip or dns names) for the certificate
// The certificate will be created with file mode 0644. The key will be created with file mode 0600.
// If the certificate or key files already exist, they will be overwritten.
// Any parent directories of the certPath or keyPath will be created as needed with file mode 0755.
func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.IP, alternateDNS []string) error {
func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
if err != nil {
return err
return nil, nil, err
}
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
CommonName: host,
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24 * 365),
@ -163,30 +160,22 @@ func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.I
derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return err
return nil, nil, err
}
// Generate cert
certBuffer := bytes.Buffer{}
if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
return err
return nil, nil, err
}
// Generate key
keyBuffer := bytes.Buffer{}
if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
return err
return nil, nil, err
}
if err := WriteCert(certPath, certBuffer.Bytes()); err != nil {
return err
}
if err := WriteKey(keyPath, keyBuffer.Bytes()); err != nil {
return err
}
return nil
return certBuffer.Bytes(), keyBuffer.Bytes(), nil
}
// FormatBytesCert receives byte array certificate and formats in human-readable format

2
pkg/util/config/BUILD
View File

@ -17,6 +17,7 @@ go_library(
"configuration_map.go",
"doc.go",
"feature_gate.go",
"namedcertkey_flag.go",
],
tags = ["automanaged"],
deps = [
@ -31,6 +32,7 @@ go_test(
srcs = [
"config_test.go",
"feature_gate_test.go",
"namedcertkey_flag_test.go",
],
library = "go_default_library",
tags = ["automanaged"],

113
pkg/util/config/namedcertkey_flag.go Normal file
View File

@ -0,0 +1,113 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"errors"
"flag"
"strings"
)
// NamedCertKey is a flag value parsing "certfile,keyfile" and "certfile,keyfile:name,name,name".
type NamedCertKey struct {
Names []string
CertFile, KeyFile string
}
var _ flag.Value = &NamedCertKey{}
func (nkc *NamedCertKey) String() string {
s := nkc.CertFile + "," + nkc.KeyFile
if len(nkc.Names) > 0 {
s = s + ":" + strings.Join(nkc.Names, ",")
}
return s
}
func (nkc *NamedCertKey) Set(value string) error {
cs := strings.SplitN(value, ":", 2)
var keycert string
if len(cs) == 2 {
var names string
keycert, names = strings.TrimSpace(cs[0]), strings.TrimSpace(cs[1])
if names == "" {
return errors.New("empty names list is not allowed")
}
nkc.Names = nil
for _, name := range strings.Split(names, ",") {
nkc.Names = append(nkc.Names, strings.TrimSpace(name))
}
} else {
nkc.Names = nil
keycert = strings.TrimSpace(cs[0])
}
cs = strings.Split(keycert, ",")
if len(cs) != 2 {
return errors.New("expected comma separated certificate and key file paths")
}
nkc.CertFile = strings.TrimSpace(cs[0])
nkc.KeyFile = strings.TrimSpace(cs[1])
return nil
}
func (*NamedCertKey) Type() string {
return "namedCertKey"
}
// NamedCertKeyArray is a flag value parsing NamedCertKeys, each passed with its own
// flag instance (in contrast to comma separated slices).
type NamedCertKeyArray struct {
value *[]NamedCertKey
changed bool
}
var _ flag.Value = &NamedCertKey{}
// NewNamedKeyCertArray creates a new NamedCertKeyArray with the internal value
// pointing to p.
func NewNamedCertKeyArray(p *[]NamedCertKey) *NamedCertKeyArray {
return &NamedCertKeyArray{
value: p,
}
}
func (a *NamedCertKeyArray) Set(val string) error {
nkc := NamedCertKey{}
err := nkc.Set(val)
if err != nil {
return err
}
if !a.changed {
*a.value = []NamedCertKey{nkc}
a.changed = true
} else {
*a.value = append(*a.value, nkc)
}
return nil
}
func (a *NamedCertKeyArray) Type() string {
return "namedCertKey"
}
func (a *NamedCertKeyArray) String() string {
nkcs := make([]string, 0, len(*a.value))
for i := range *a.value {
nkcs = append(nkcs, (*a.value)[i].String())
}
return "[" + strings.Join(nkcs, ";") + "]"
}

138
pkg/util/config/namedcertkey_flag_test.go Normal file
View File

@ -0,0 +1,138 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
"github.com/spf13/pflag"
"reflect"
"strings"
"testing"
)
func TestNamedCertKeyArrayFlag(t *testing.T) {
tests := []struct {
args []string
def []NamedCertKey
expected []NamedCertKey
parseError string
}{
{
args: []string{},
expected: nil,
},
{
args: []string{"foo.crt,foo.key"},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
}},
},
{
args: []string{" foo.crt , foo.key "},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
}},
},
{
args: []string{"foo.crt,foo.key:abc"},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
Names: []string{"abc"},
}},
},
{
args: []string{"foo.crt,foo.key: abc "},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
Names: []string{"abc"},
}},
},
{
args: []string{"foo.crt,foo.key:"},
parseError: "empty names list is not allowed",
},
{
args: []string{""},
parseError: "expected comma separated certificate and key file paths",
},
{
args: []string{" "},
parseError: "expected comma separated certificate and key file paths",
},
{
args: []string{"a,b,c"},
parseError: "expected comma separated certificate and key file paths",
},
{
args: []string{"foo.crt,foo.key:abc,def,ghi"},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
Names: []string{"abc", "def", "ghi"},
}},
},
{
args: []string{"foo.crt,foo.key:*.*.*"},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
Names: []string{"*.*.*"},
}},
},
{
args: []string{"foo.crt,foo.key", "bar.crt,bar.key"},
expected: []NamedCertKey{{
KeyFile: "foo.key",
CertFile: "foo.crt",
}, {
KeyFile: "bar.key",
CertFile: "bar.crt",
}},
},
}
for i, test := range tests {
fs := pflag.NewFlagSet("testNamedCertKeyArray", pflag.ContinueOnError)
var nkcs []NamedCertKey
for _, d := range test.def {
nkcs = append(nkcs, d)
}
fs.Var(NewNamedCertKeyArray(&nkcs), "tls-sni-cert-key", "usage")
args := []string{}
for _, a := range test.args {
args = append(args, fmt.Sprintf("--tls-sni-cert-key=%s", a))
}
err := fs.Parse(args)
if test.parseError != "" {
if err == nil {
t.Errorf("%d: expected error %q, got nil", i, test.parseError)
} else if !strings.Contains(err.Error(), test.parseError) {
t.Errorf("%d: expected error %q, got %q", i, test.parseError, err)
}
} else if err != nil {
t.Errorf("%d: expected nil error, got %v", i, err)
}
if !reflect.DeepEqual(nkcs, test.expected) {
t.Errorf("%d: expected %+v, got %+v", i, test.expected, nkcs)
}
}
}

13
test/integration/discoverysummarizer/discoverysummarizer_test.go
View File

@ -65,12 +65,13 @@ func runDiscoverySummarizer(t *testing.T) string {
return serverURL
}
func runAPIServer(t *testing.T) string {
func runAPIServer(t *testing.T, stopCh <-chan struct{}) string {
serverRunOptions := apiserver.NewServerRunOptions()
// Change the port, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel.
serverRunOptions.InsecurePort = 8083
// Change the ports, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel.
serverRunOptions.SecurePort = 6443 + 3
serverRunOptions.InsecurePort = 8080 + 3
go func() {
if err := apiserver.Run(serverRunOptions); err != nil {
if err := apiserver.Run(serverRunOptions, stopCh); err != nil {
t.Fatalf("Error in bringing up the example apiserver: %v", err)
}
}()
@ -98,7 +99,9 @@ func TestRunDiscoverySummarizer(t *testing.T) {
testResponse(t, discoveryURL, "/randomPath", http.StatusNotFound)
// Run the APIServer now to test the good case.
runAPIServer(t)
stopCh := make(chan struct{})
runAPIServer(t, stopCh)
defer close(stopCh)
// Test /api path.
// There is no server running at that URL, so we will get a 500.

8
test/integration/examples/apiserver_test.go
View File

@ -42,11 +42,13 @@ var groupVersionForDiscovery = unversioned.GroupVersionForDiscovery{
func TestRunServer(t *testing.T) {
serverIP := fmt.Sprintf("http://localhost:%d", apiserver.InsecurePort)
stopCh := make(chan struct{})
go func() {
if err := apiserver.Run(apiserver.NewServerRunOptions()); err != nil {
if err := apiserver.Run(apiserver.NewServerRunOptions(), stopCh); err != nil {
t.Fatalf("Error in bringing up the server: %v", err)
}
}()
defer close(stopCh)
if err := waitForApiserverUp(serverIP); err != nil {
t.Fatalf("%v", err)
}
@ -58,14 +60,16 @@ func TestRunServer(t *testing.T) {
func TestRunSecureServer(t *testing.T) {
serverIP := fmt.Sprintf("https://localhost:%d", apiserver.SecurePort)
stopCh := make(chan struct{})
go func() {
options := apiserver.NewServerRunOptions()
options.InsecurePort = 0
options.SecurePort = apiserver.SecurePort
if err := apiserver.Run(options); err != nil {
if err := apiserver.Run(options, stopCh); err != nil {
t.Fatalf("Error in bringing up the server: %v", err)
}
}()
defer close(stopCh)
if err := waitForApiserverUp(serverIP); err != nil {
t.Fatalf("%v", err)
}

4
test/integration/federation/server_test.go
View File

@ -77,7 +77,8 @@ func TestLongRunningRequestRegexp(t *testing.T) {
}
}
var insecurePort = 8082
var securePort = 6443 + 2
var insecurePort = 8080 + 2
var serverIP = fmt.Sprintf("http://localhost:%v", insecurePort)
var groupVersions = []unversioned.GroupVersion{
fed_v1b1.SchemeGroupVersion,
@ -86,6 +87,7 @@ var groupVersions = []unversioned.GroupVersion{
func TestRun(t *testing.T) {
s := options.NewServerRunOptions()
s.GenericServerRunOptions.SecurePort = securePort
s.GenericServerRunOptions.InsecurePort = insecurePort
_, ipNet, _ := net.ParseCIDR("10.10.10.0/24")
s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet