Merge pull request #37830 from sttts/sttts-stratify-cert-generation

Automatic merge from submit-queue

Stratify apiserver cert generation

- move self-signed cert generation to `SecureServingOptions.MaybeDefaultWithSelfSignedCerts`
- make cert generation only depend on `ServerRunOptions`, not on an unfinished `Config` (this breaks the chicken-egg problem of a finished config in https://github.com/kubernetes/kubernetes/pull/35387#pullrequestreview-5368176)
- move loopback client config code into `config_selfclient.go`

Replaces https://github.com/kubernetes/kubernetes/pull/35387#event-833649341 by getting rid of duplicated `Complete`.

cmd/kube-apiserver/app/BUILD

@@ -35,7 +35,6 @@ go_library(
         "//pkg/generated/openapi:go_default_library",
         "//pkg/genericapiserver:go_default_library",
         "//pkg/genericapiserver/authorizer:go_default_library",
-        "//pkg/genericapiserver/options:go_default_library",
         "//pkg/master:go_default_library",
         "//pkg/registry/cachesize:go_default_library",
         "//pkg/runtime/schema:go_default_library",

cmd/kube-apiserver/app/server.go

@@ -21,6 +21,7 @@ package app
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -49,7 +50,6 @@ import (
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
-	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	"k8s.io/kubernetes/pkg/master"
 	"k8s.io/kubernetes/pkg/registry/cachesize"
 	"k8s.io/kubernetes/pkg/runtime/schema"
@@ -85,20 +85,25 @@ func Run(s *options.ServerRunOptions) error {
 		return err
 	}
 
-	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
-	genericConfig := genericapiserver.NewConfig(). // create the new config
-							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
-							ApplySecureServingOptions(s.SecureServing).
-							ApplyInsecureServingOptions(s.InsecureServing).
-							ApplyAuthenticationOptions(s.Authentication).
-							ApplyRBACSuperUser(s.Authorization.RBACSuperUser)
-
 	serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange)
 	if err != nil {
-		glog.Fatalf("Error determining service IP ranges: %v", err)
+		return fmt.Errorf("error determining service IP ranges: %v", err)
 	}
-	if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil {
+
-		glog.Fatalf("Failed to generate service certificate: %v", err)
+	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil {
+		return fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
+
+	genericConfig, err := genericapiserver.NewConfig(). // create the new config
+								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+								ApplyInsecureServingOptions(s.InsecureServing).
+								ApplyAuthenticationOptions(s.Authentication).
+								ApplyRBACSuperUser(s.Authorization.RBACSuperUser).
+								ApplySecureServingOptions(s.SecureServing)
+	if err != nil {
+		return fmt.Errorf("failed to configure https: %s", err)
 	}
 
 	capabilities.Initialize(capabilities.Capabilities{
@@ -120,7 +125,7 @@ func Run(s *options.ServerRunOptions) error {
 		var installSSH genericapiserver.InstallSSHKey
 		cloud, err := cloudprovider.InitCloudProvider(s.GenericServerRunOptions.CloudProvider, s.GenericServerRunOptions.CloudConfigFile)
 		if err != nil {
-			glog.Fatalf("Cloud provider could not be initialized: %v", err)
+			return fmt.Errorf("cloud provider could not be initialized: %v", err)
 		}
 		if cloud != nil {
 			if instances, supported := cloud.Instances(); supported {
@@ -128,7 +133,7 @@ func Run(s *options.ServerRunOptions) error {
 			}
 		}
 		if s.KubeletConfig.Port == 0 {
-			glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.")
+			return fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified")
 		}
 		// Set up the tunneler
 		// TODO(cjcullen): If we want this to handle per-kubelet ports or other
@@ -173,7 +178,7 @@ func Run(s *options.ServerRunOptions) error {
 
 	storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
 	if err != nil {
-		glog.Fatalf("error generating storage version map: %s", err)
+		return fmt.Errorf("error generating storage version map: %s", err)
 	}
 	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
 		s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
@@ -182,7 +187,7 @@ func Run(s *options.ServerRunOptions) error {
 		[]schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")},
 		master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig)
 	if err != nil {
-		glog.Fatalf("error in initializing storage factory: %s", err)
+		return fmt.Errorf("error in initializing storage factory: %s", err)
 	}
 	storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs"))
 	storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
@@ -221,20 +226,20 @@ func Run(s *options.ServerRunOptions) error {
 		// go directly to etcd to avoid recursive auth insanity
 		storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
 		if err != nil {
-			glog.Fatalf("Unable to get serviceaccounts storage: %v", err)
+			return fmt.Errorf("unable to get serviceaccounts storage: %v", err)
 		}
 		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
 	}
 
 	apiAuthenticator, securityDefinitions, err := authenticator.New(authenticatorConfig)
 	if err != nil {
-		glog.Fatalf("Invalid Authentication Config: %v", err)
+		return fmt.Errorf("invalid Authentication Config: %v", err)
 	}
 
 	privilegedLoopbackToken := uuid.NewRandom().String()
-	selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken)
+	selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
 	if err != nil {
-		glog.Fatalf("Failed to create clientset: %v", err)
+		return fmt.Errorf("failed to create clientset: %v", err)
 	}
 	client, err := internalclientset.NewForConfig(selfClientConfig)
 	if err != nil {
@@ -245,14 +250,14 @@ func Run(s *options.ServerRunOptions) error {
 	authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
 	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationConfig)
 	if err != nil {
-		glog.Fatalf("Invalid Authorization Config: %v", err)
+		return fmt.Errorf("invalid Authorization Config: %v", err)
 	}
 
 	admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
 	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
 	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
 	if err != nil {
-		glog.Fatalf("Failed to initialize plugins: %v", err)
+		return fmt.Errorf("failed to initialize plugins: %v", err)
 	}
 
 	proxyTransport := utilnet.SetTransportDefaults(&http.Transport{

examples/apiserver/apiserver.go

@@ -36,6 +36,8 @@ import (
 
 	// Install the testgroup API
 	_ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install"
+
+	"github.com/golang/glog"
 )
 
 const (
@@ -93,20 +95,21 @@ func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
 	if errs := serverOptions.InsecureServing.Validate("insecure-port"); len(errs) > 0 {
 		return utilerrors.NewAggregate(errs)
 	}
+	if err := serverOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(serverOptions.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
+		glog.Fatalf("Error creating self-signed certificates: %v", err)
+	}
 
-	config := genericapiserver.NewConfig().
+	config, err := genericapiserver.NewConfig().
 		ApplyOptions(serverOptions.GenericServerRunOptions).
-		ApplySecureServingOptions(serverOptions.SecureServing).
 		ApplyInsecureServingOptions(serverOptions.InsecureServing).
 		ApplyAuthenticationOptions(serverOptions.Authentication).
-		Complete()
+		ApplySecureServingOptions(serverOptions.SecureServing)
-	if err := config.MaybeGenerateServingCerts(); err != nil {
+	if err != nil {
-		// this wasn't treated as fatal for this process before
+		return fmt.Errorf("failed to configure https: %s", err)
-		fmt.Printf("Error creating cert: %v", err)
 	}
 
 	config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
-	s, err := config.New()
+	s, err := config.Complete().New()
 	if err != nil {
 		return fmt.Errorf("Error in bringing up the server: %v", err)
 	}

federation/cmd/federation-apiserver/app/BUILD

@@ -42,7 +42,6 @@ go_library(
         "//pkg/generated/openapi:go_default_library",
         "//pkg/genericapiserver:go_default_library",
         "//pkg/genericapiserver/authorizer:go_default_library",
-        "//pkg/genericapiserver/options:go_default_library",
         "//pkg/registry/cachesize:go_default_library",
         "//pkg/registry/core/configmap/etcd:go_default_library",
         "//pkg/registry/core/event/etcd:go_default_library",

federation/cmd/federation-apiserver/app/server.go

@@ -20,6 +20,7 @@ limitations under the License.
 package app
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
@@ -37,7 +38,6 @@ import (
 	"k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
-	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	"k8s.io/kubernetes/pkg/registry/cachesize"
 	"k8s.io/kubernetes/pkg/registry/generic"
 	genericregistry "k8s.io/kubernetes/pkg/registry/generic/registry"
@@ -73,16 +73,19 @@ func Run(s *options.ServerRunOptions) error {
 		return err
 	}
 
-	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
+	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
-	genericConfig := genericapiserver.NewConfig(). // create the new config
+		return fmt.Errorf("error creating self-signed certificates: %v", err)
-							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+	}
-							ApplySecureServingOptions(s.SecureServing).
-							ApplyInsecureServingOptions(s.InsecureServing).
-							ApplyAuthenticationOptions(s.Authentication).
-							ApplyRBACSuperUser(s.Authorization.RBACSuperUser)
 
-	if err := genericConfig.MaybeGenerateServingCerts(); err != nil {
+	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
-		glog.Fatalf("Failed to generate service certificate: %v", err)
+	genericConfig, err := genericapiserver.NewConfig(). // create the new config
+								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+								ApplyInsecureServingOptions(s.InsecureServing).
+								ApplyAuthenticationOptions(s.Authentication).
+								ApplyRBACSuperUser(s.Authorization.RBACSuperUser).
+								ApplySecureServingOptions(s.SecureServing)
+	if err != nil {
+		return fmt.Errorf("failed to configure https: %s", err)
 	}
 
 	// TODO: register cluster federation resources here.
@@ -130,7 +133,7 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	privilegedLoopbackToken := uuid.NewRandom().String()
-	selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken)
+	selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
 	if err != nil {
 		glog.Fatalf("Failed to create clientset: %v", err)
 	}

pkg/genericapiserver/BUILD

@@ -14,6 +14,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "config.go",
+        "config_selfclient.go",
         "default_storage_factory_builder.go",
         "discovery.go",
         "doc.go",
@@ -117,6 +118,7 @@ go_test(
         "//pkg/storage/storagebackend:go_default_library",
         "//pkg/util/cert:go_default_library",
         "//pkg/util/clock:go_default_library",
+        "//pkg/util/config:go_default_library",
         "//pkg/util/net:go_default_library",
         "//pkg/util/sets:go_default_library",
         "//pkg/version:go_default_library",

pkg/genericapiserver/config.go

@@ -17,12 +17,14 @@ limitations under the License.
 package genericapiserver
 
 import (
+	"crypto/tls"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"path"
 	"regexp"
 	goruntime "runtime"
 	"sort"
@@ -58,7 +60,6 @@ import (
 	"k8s.io/kubernetes/pkg/genericapiserver/routes"
 	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
 	"k8s.io/kubernetes/pkg/runtime"
-	certutil "k8s.io/kubernetes/pkg/util/cert"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/version"
 	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
@@ -169,35 +170,21 @@ type ServingInfo struct {
 type SecureServingInfo struct {
 	ServingInfo
 
-	// ServerCert is the TLS cert info for serving secure traffic
+	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
-	ServerCert GeneratableKeyCert
+	// allowed to be in SNICerts.
-	// SNICerts are named CertKeys for serving secure traffic with SNI support.
+	Cert *tls.Certificate
-	SNICerts []NamedCertKey
+
+	// CACert is an optional certificate authority used for the loopback connection of the Admission controllers.
+	// If this is nil, the certificate authority is extracted from Cert or a matching SNI certificate.
+	CACert *tls.Certificate
+
+	// SNICerts are the TLS certificates by name used for SNI.
+	SNICerts map[string]*tls.Certificate
+
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
 }
 
-type CertKey struct {
-	// CertFile is a file containing a PEM-encoded certificate
-	CertFile string
-	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
-	KeyFile string
-}
-
-type NamedCertKey struct {
-	CertKey
-
-	// Names is a list of domain patterns: fully qualified domain names, possibly prefixed with
-	// wildcard segments.
-	Names []string
-}
-
-type GeneratableKeyCert struct {
-	CertKey
-	// Generate indicates that the cert/key pair should be generated if its not present.
-	Generate bool
-}
-
 // NewConfig returns a Config struct with the default values
 func NewConfig() *Config {
 	longRunningRE := regexp.MustCompile(options.DefaultLongRunningRequestRE)
@@ -238,45 +225,69 @@ func NewConfig() *Config {
 	return config.ApplyOptions(defaultOptions)
 }
 
-func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) *Config {
+func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) (*Config, error) {
 	if secureServing == nil || secureServing.ServingOptions.BindPort <= 0 {
-		return c
+		return c, nil
 	}
 
 	secureServingInfo := &SecureServingInfo{
 		ServingInfo: ServingInfo{
 			BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)),
 		},
-		ServerCert: GeneratableKeyCert{
-			CertKey: CertKey{
-				CertFile: secureServing.ServerCert.CertKey.CertFile,
-				KeyFile:  secureServing.ServerCert.CertKey.KeyFile,
-			},
-		},
-		SNICerts: []NamedCertKey{},
 		ClientCA: secureServing.ClientCA,
 	}
-	if secureServing.ServerCert.CertKey.CertFile == "" && secureServing.ServerCert.CertKey.KeyFile == "" {
+
-		secureServingInfo.ServerCert.Generate = true
+	serverCertFile, serverKeyFile := secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile
-		secureServingInfo.ServerCert.CertFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".crt")
+
-		secureServingInfo.ServerCert.KeyFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".key")
+	// load main cert
+	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
+		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("unable to load server certificate: %v", err)
+		}
+		secureServingInfo.Cert = &tlsCert
 	}
 
-	secureServingInfo.SNICerts = nil
+	// optionally load CA cert
-	for _, nkc := range secureServing.SNICertKeys {
+	if len(secureServing.ServerCert.CACertFile) != 0 {
-		secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{
+		pemData, err := ioutil.ReadFile(secureServing.ServerCert.CACertFile)
-			CertKey: CertKey{
+		if err != nil {
-				KeyFile:  nkc.KeyFile,
+			return nil, fmt.Errorf("failed to read certificate authority from %q: %v", secureServing.ServerCert.CACertFile, err)
-				CertFile: nkc.CertFile,
+		}
-			},
+		block, pemData := pem.Decode(pemData)
-			Names: nkc.Names,
+		if block == nil {
+			return nil, fmt.Errorf("no certificate found in certificate authority file %q", secureServing.ServerCert.CACertFile)
+		}
+		if block.Type != "CERTIFICATE" {
+			return nil, fmt.Errorf("expected CERTIFICATE block in certiticate authority file %q, found: %s", secureServing.ServerCert.CACertFile, block.Type)
+		}
+		secureServingInfo.CACert = &tls.Certificate{
+			Certificate: [][]byte{block.Bytes},
+		}
+	}
+
+	// load SNI certs
+	namedTlsCerts := make([]namedTlsCert, 0, len(secureServing.SNICertKeys))
+	for _, nck := range secureServing.SNICertKeys {
+		tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile)
+		namedTlsCerts = append(namedTlsCerts, namedTlsCert{
+			tlsCert: tlsCert,
+			names:   nck.Names,
 		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to load SNI cert and key: %v", err)
+		}
+	}
+	var err error
+	secureServingInfo.SNICerts, err = getNamedCertificateMap(namedTlsCerts)
+	if err != nil {
+		return nil, err
 	}
 
 	c.SecureServingInfo = secureServingInfo
 	c.ReadWritePort = secureServing.ServingOptions.BindPort
 
-	return c
+	return c, nil
 }
 
 func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOptions) *Config {
@@ -456,38 +467,6 @@ func (c completedConfig) New() (*GenericAPIServer, error) {
 	return s, nil
 }
 
-// MaybeGenerateServingCerts generates serving certificates if requested and needed.
-func (c *Config) MaybeGenerateServingCerts(alternateIPs ...net.IP) error {
-	// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
-	// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
-	if c.SecureServingInfo != nil && c.SecureServingInfo.ServerCert.Generate {
-		canReadCertAndKey, err := certutil.CanReadCertAndKey(c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile)
-		if err != nil {
-			return err
-		}
-		if canReadCertAndKey {
-			return nil
-		}
-		// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
-		alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
-
-		if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil {
-			return fmt.Errorf("unable to generate self signed cert: %v", err)
-		} else {
-			if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil {
-				return err
-			}
-
-			if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil {
-				return err
-			}
-			glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile)
-		}
-	}
-
-	return nil
-}
-
 func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) (secure, insecure http.Handler) {
 	generic := func(handler http.Handler) http.Handler {
 		handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")

pkg/genericapiserver/config_selfclient.go

@@ -0,0 +1,138 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package genericapiserver
+
+import (
+	"bytes"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"net"
+
+	"k8s.io/kubernetes/pkg/client/restclient"
+
+	"github.com/golang/glog"
+)
+
+// NewSelfClientConfig returns a clientconfig which can be used to talk to this apiserver.
+func NewSelfClientConfig(secureServingInfo *SecureServingInfo, insecureServingInfo *ServingInfo, token string) (*restclient.Config, error) {
+	if cfg, err := secureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil {
+		if insecureServingInfo == nil {
+			// be fatal if insecure port is not available
+			return cfg, err
+		} else {
+			glog.Warningf("Failed to create secure local client, falling back to insecure local connection: %v", err)
+		}
+	}
+	if cfg, err := insecureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil {
+		return cfg, err
+	}
+
+	return nil, errors.New("Unable to set url for apiserver local client")
+}
+
+func (s *SecureServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) {
+	if s == nil || (s.Cert == nil && len(s.SNICerts) == 0) {
+		return nil, nil
+	}
+
+	host, port, err := net.SplitHostPort(s.ServingInfo.BindAddress)
+	if err != nil {
+		// should never happen
+		return nil, fmt.Errorf("invalid secure bind address: %q", s.ServingInfo.BindAddress)
+	}
+	if host == "0.0.0.0" {
+		// compare MaybeDefaultWithSelfSignedCerts which adds "localhost" to the cert as alternateDNS
+		host = "localhost"
+	}
+
+	clientConfig := &restclient.Config{
+		// Increase QPS limits. The client is currently passed to all admission plugins,
+		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
+		// for more details. Once #22422 is fixed, we may want to remove it.
+		QPS:         50,
+		Burst:       100,
+		Host:        "https://" + net.JoinHostPort(host, port),
+		BearerToken: token,
+	}
+
+	// find certificate for host: either explicitly given, from the server cert bundle or one of the SNI certs
+	var derCA []byte
+	if s.CACert != nil {
+		derCA = s.CACert.Certificate[0]
+	}
+	if derCA == nil && s.Cert != nil {
+		x509Cert, err := x509.ParseCertificate(s.Cert.Certificate[0])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse server certificate: %v", err)
+		}
+
+		if (net.ParseIP(host) != nil && certMatchesIP(x509Cert, host)) || certMatchesName(x509Cert, host) {
+			derCA = s.Cert.Certificate[0]
+		}
+	}
+	if derCA == nil && net.ParseIP(host) == nil {
+		if cert, found := s.SNICerts[host]; found {
+			derCA = cert.Certificate[0]
+		}
+	}
+	if derCA == nil {
+		return nil, fmt.Errorf("failed to find certificate which matches %q", host)
+	}
+	pemCA := bytes.Buffer{}
+	if err := pem.Encode(&pemCA, &pem.Block{Type: "CERTIFICATE", Bytes: derCA}); err != nil {
+		return nil, err
+	}
+	clientConfig.CAData = pemCA.Bytes()
+
+	return clientConfig, nil
+}
+
+func (s *ServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) {
+	if s == nil {
+		return nil, nil
+	}
+	return &restclient.Config{
+		Host: s.BindAddress,
+		// Increase QPS limits. The client is currently passed to all admission plugins,
+		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
+		// for more details. Once #22422 is fixed, we may want to remove it.
+		QPS:   50,
+		Burst: 100,
+	}, nil
+}
+
+func certMatchesName(cert *x509.Certificate, name string) bool {
+	for _, certName := range cert.DNSNames {
+		if certName == name {
+			return true
+		}
+	}
+
+	return false
+}
+
+func certMatchesIP(cert *x509.Certificate, ip string) bool {
+	for _, certIP := range cert.IPAddresses {
+		if certIP.String() == ip {
+			return true
+		}
+	}
+
+	return false
+}

pkg/genericapiserver/options/BUILD

@@ -28,14 +28,15 @@ go_library(
         "//pkg/apiserver/authenticator:go_default_library",
         "//pkg/client/clientset_generated/release_1_5/typed/authentication/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/release_1_5/typed/authorization/v1beta1:go_default_library",
-        "//pkg/client/restclient:go_default_library",
         "//pkg/client/unversioned/clientcmd:go_default_library",
         "//pkg/controller/informers:go_default_library",
         "//pkg/genericapiserver/authorizer:go_default_library",
         "//pkg/runtime/schema:go_default_library",
         "//pkg/storage/storagebackend:go_default_library",
+        "//pkg/util/cert:go_default_library",
         "//pkg/util/config:go_default_library",
         "//pkg/util/net:go_default_library",
+        "//vendor:github.com/golang/glog",
         "//vendor:github.com/spf13/pflag",
     ],
 )

pkg/genericapiserver/options/server_run_options.go

@@ -75,7 +75,6 @@ type ServerRunOptions struct {
 	// for testing). This is not actually exposed as a flag.
 	DefaultStorageVersions string
 	TargetRAMMB            int
-	TLSCAFile              string
 	WatchCacheSizes        []string
 }
 

pkg/genericapiserver/options/serving.go

@@ -17,14 +17,14 @@ limitations under the License.
 package options
 
 import (
-	"errors"
 	"fmt"
 	"net"
-	"strconv"
+	"path"
 
+	"github.com/golang/glog"
 	"github.com/spf13/pflag"
 
-	"k8s.io/kubernetes/pkg/client/restclient"
+	certutil "k8s.io/kubernetes/pkg/util/cert"
 	"k8s.io/kubernetes/pkg/util/config"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
 )
@@ -43,14 +43,10 @@ type SecureServingOptions struct {
 	SNICertKeys []config.NamedCertKey
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
-
-	// ServerCA is the certificate bundle for the signer of your serving certificate.  Used for building a loopback
-	// connection to the API server for admission.
-	ServerCA string
 }
 
 type CertKey struct {
-	// CertFile is a file containing a PEM-encoded certificate
+	// CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain
 	CertFile string
 	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
 	KeyFile string
@@ -59,6 +55,8 @@ type CertKey struct {
 type GeneratableKeyCert struct {
 	CertKey CertKey
 
+	// CACertFile is an optional file containing the certificate chain for CertKey.CertFile
+	CACertFile string
 	// CertDirectory is a directory that will contain the certificates.  If the cert and key aren't specifically set
 	// this will be used to derive a match with the "pair-name"
 	CertDirectory string
@@ -80,31 +78,6 @@ func NewSecureServingOptions() *SecureServingOptions {
 	}
 }
 
-func (s *SecureServingOptions) NewSelfClientConfig(token string) *restclient.Config {
-	if s == nil || s.ServingOptions.BindPort <= 0 || len(s.ServerCA) == 0 {
-		return nil
-	}
-
-	clientConfig := &restclient.Config{
-		// Increase QPS limits. The client is currently passed to all admission plugins,
-		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
-		// for more details. Once #22422 is fixed, we may want to remove it.
-		QPS:   50,
-		Burst: 100,
-	}
-
-	// Use secure port if the ServerCA is specified
-	host := s.ServingOptions.BindAddress.String()
-	if host == "0.0.0.0" {
-		host = "localhost"
-	}
-	clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.ServingOptions.BindPort))
-	clientConfig.CAFile = s.ServerCA
-	clientConfig.BearerToken = token
-
-	return clientConfig
-}
-
 func (s *SecureServingOptions) Validate() []error {
 	errors := []error{}
 	if s == nil {
@@ -138,6 +111,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
 		"File containing the default x509 private key matching --tls-cert-file.")
 
+	fs.StringVar(&s.ServerCert.CACertFile, "tls-ca-file", s.ServerCert.CACertFile, "If set, this "+
+		"certificate authority will used for secure access from Admission "+
+		"Controllers. This must be a valid PEM-encoded CA bundle. Altneratively, the certificate authority "+
+		"can be appended to the certificate provided by --tls-cert-file.")
+
 	fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
 		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
 		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
@@ -151,18 +129,12 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"If set, any request presenting a client certificate signed by one of "+
 		"the authorities in the client-ca-file is authenticated with an identity "+
 		"corresponding to the CommonName of the client certificate.")
-
-	fs.StringVar(&s.ServerCA, "tls-ca-file", s.ServerCA, "If set, this "+
-		"certificate authority will used for secure access from Admission "+
-		"Controllers. This must be a valid PEM-encoded CA bundle.")
-
 }
 
 func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
 	fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress,
 		"DEPRECATED: see --bind-address instead.")
 	fs.MarkDeprecated("public-address-override", "see --bind-address instead.")
-
 }
 
 func NewInsecureServingOptions() *ServingOptions {
@@ -182,23 +154,6 @@ func (s ServingOptions) Validate(portArg string) []error {
 	return errors
 }
 
-func (s *ServingOptions) NewSelfClientConfig(token string) *restclient.Config {
-	if s == nil || s.BindPort <= 0 {
-		return nil
-	}
-	clientConfig := &restclient.Config{
-		// Increase QPS limits. The client is currently passed to all admission plugins,
-		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
-		// for more details. Once #22422 is fixed, we may want to remove it.
-		QPS:   50,
-		Burst: 100,
-	}
-
-	clientConfig.Host = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
-
-	return clientConfig
-}
-
 func (s *ServingOptions) DefaultExternalAddress() (net.IP, error) {
 	return utilnet.ChooseBindAddress(s.BindAddress)
 }
@@ -224,14 +179,47 @@ func (s *ServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
 	fs.MarkDeprecated("port", "see --insecure-port instead.")
 }
 
-// Returns a clientconfig which can be used to talk to this apiserver.
+func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateIPs ...net.IP) error {
-func NewSelfClientConfig(secureServingOptions *SecureServingOptions, insecureServingOptions *ServingOptions, token string) (*restclient.Config, error) {
+	keyCert := &s.ServerCert.CertKey
-	if cfg := secureServingOptions.NewSelfClientConfig(token); cfg != nil {
+
-		return cfg, nil
+	if s == nil || len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
-	}
+		return nil
-	if cfg := insecureServingOptions.NewSelfClientConfig(token); cfg != nil {
-		return cfg, nil
 	}
 
-	return nil, errors.New("Unable to set url for apiserver local client")
+	keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt")
+	keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key")
+
+	canReadCertAndKey, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile)
+	if err != nil {
+		return err
+	}
+	if !canReadCertAndKey {
+		// TODO: It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
+		// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
+		// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
+		alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
+
+		// add either the bind address or localhost to the valid alternates
+		bindIP := s.ServingOptions.BindAddress.String()
+		if bindIP == "0.0.0.0" {
+			alternateDNS = append(alternateDNS, "localhost")
+		} else {
+			alternateIPs = append(alternateIPs, s.ServingOptions.BindAddress)
+		}
+
+		if cert, key, err := certutil.GenerateSelfSignedCertKey(publicAddress, alternateIPs, alternateDNS); err != nil {
+			return fmt.Errorf("unable to generate self signed cert: %v", err)
+		} else {
+			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {
+				return err
+			}
+
+			if err := certutil.WriteKey(keyCert.KeyFile, key); err != nil {
+				return err
+			}
+			glog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
+		}
+	}
+
+	return nil
 }

pkg/genericapiserver/serve.go

@@ -42,17 +42,12 @@ const (
 // be loaded or the initial listen call fails. The actual server loop (stoppable by closing
 // stopCh) runs in a go routine, i.e. serveSecurely does not block.
 func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
-	namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts)
-	if err != nil {
-		return fmt.Errorf("unable to load SNI certificates: %v", err)
-	}
-
 	secureServer := &http.Server{
 		Addr:           s.SecureServingInfo.BindAddress,
 		Handler:        s.Handler,
 		MaxHeaderBytes: 1 << 20,
 		TLSConfig: &tls.Config{
-			NameToCertificate: namedCerts,
+			NameToCertificate: s.SecureServingInfo.SNICerts,
 			// Can't use SSLv3 because of POODLE and BEAST
 			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
 			// Can't use TLSv1.1 because of RC4 cipher usage
@@ -62,19 +57,15 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
 		},
 	}
 
-	if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 {
+	if s.SecureServingInfo.Cert != nil {
-		secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
+		secureServer.TLSConfig.Certificates = []tls.Certificate{*s.SecureServingInfo.Cert}
-		secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile)
-		if err != nil {
-			return fmt.Errorf("unable to load server certificate: %v", err)
-		}
 	}
 
 	// append all named certs. Otherwise, the go tls stack will think no SNI processing
 	// is necessary because there is only one cert anyway.
 	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
 	// cert will become the default cert. That's what we expect anyway.
-	for _, c := range namedCerts {
+	for _, c := range s.SecureServingInfo.SNICerts {
 		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
 	}
 
@@ -91,6 +82,7 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
 	}
 
 	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
+	var err error
 	s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh)
 	return err
 }
@@ -186,48 +178,40 @@ func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int
 	return tcpAddr.Port, nil
 }
 
-// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in
+type namedTlsCert struct {
-// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded.
+	tlsCert tls.Certificate
-// Returns nil if len(namedCertKeys) == 0
-func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) {
-	if len(namedCertKeys) == 0 {
-		return nil, nil
-	}
 
-	// load keys
+	// names is a list of domain patterns: fully qualified domain names, possibly prefixed with
-	tlsCerts := make([]tls.Certificate, len(namedCertKeys))
+	// wildcard segments.
-	for i := range namedCertKeys {
+	names []string
-		var err error
+}
-		nkc := &namedCertKeys[i]
-		tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile)
-		if err != nil {
-			return nil, err
-		}
-	}
 
+// getNamedCertificateMap returns a map of *tls.Certificate by name. It's is
+// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
+// cannot be loaded. Returns nil if len(certs) == 0
+func getNamedCertificateMap(certs []namedTlsCert) (map[string]*tls.Certificate, error) {
 	// register certs with implicit names first, reverse order such that earlier trump over the later
-	tlsCertsByName := map[string]*tls.Certificate{}
+	byName := map[string]*tls.Certificate{}
-	for i := len(namedCertKeys) - 1; i >= 0; i-- {
+	for i := len(certs) - 1; i >= 0; i-- {
-		nkc := &namedCertKeys[i]
+		if len(certs[i].names) > 0 {
-		if len(nkc.Names) > 0 {
 			continue
 		}
-		cert := &tlsCerts[i]
+		cert := &certs[i].tlsCert
 
 		// read names from certificate common names and DNS names
 		if len(cert.Certificate) == 0 {
-			return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile)
+			return nil, fmt.Errorf("empty SNI certificate, skipping")
 		}
 		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
 		if err != nil {
-			return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err)
+			return nil, fmt.Errorf("parse error for SNI certificate: %v", err)
 		}
 		cn := x509Cert.Subject.CommonName
 		if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
-			tlsCertsByName[cn] = cert
+			byName[cn] = cert
 		}
 		for _, san := range x509Cert.DNSNames {
-			tlsCertsByName[san] = cert
+			byName[san] = cert
 		}
 		// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
 		// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
@@ -235,17 +219,14 @@ func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certi
 
 	// register certs with explicit names last, overwriting every of the implicit ones,
 	// again in reverse order.
-	for i := len(namedCertKeys) - 1; i >= 0; i-- {
+	for i := len(certs) - 1; i >= 0; i-- {
-		nkc := &namedCertKeys[i]
+		namedCert := &certs[i]
-		if len(nkc.Names) == 0 {
+		for _, name := range namedCert.names {
-			continue
+			byName[name] = &certs[i].tlsCert
-		}
-		for _, name := range nkc.Names {
-			tlsCertsByName[name] = &tlsCerts[i]
 		}
 	}
 
-	return tlsCertsByName, nil
+	return byName, nil
 }
 
 // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted

pkg/genericapiserver/serve_test.go

@@ -20,15 +20,18 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"os"
 	"testing"
 
-	utilcert "k8s.io/kubernetes/pkg/util/cert"
-
 	"github.com/stretchr/testify/assert"
+
+	"k8s.io/kubernetes/pkg/genericapiserver/options"
+	utilcert "k8s.io/kubernetes/pkg/util/cert"
+	"k8s.io/kubernetes/pkg/util/config"
 )
 
 type TestCertSpec struct {
@@ -41,47 +44,6 @@ type NamedTestCertSpec struct {
 	explicitNames []string // as --tls-sni-cert-key explicit names
 }
 
-func createTestCerts(spec TestCertSpec) (certFilePath, keyFilePath string, err error) {
-	var ips []net.IP
-	for _, ip := range spec.ips {
-		ips = append(ips, net.ParseIP(ip))
-	}
-
-	certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, ips, spec.names)
-	if err != nil {
-		return "", "", err
-	}
-
-	certFile, err := ioutil.TempFile(os.TempDir(), "cert")
-	if err != nil {
-		return "", "", err
-	}
-
-	keyFile, err := ioutil.TempFile(os.TempDir(), "key")
-	if err != nil {
-		os.Remove(certFile.Name())
-		return "", "", err
-	}
-
-	_, err = certFile.Write(certPem)
-	if err != nil {
-		os.Remove(certFile.Name())
-		os.Remove(keyFile.Name())
-		return "", "", err
-	}
-	certFile.Close()
-
-	_, err = keyFile.Write(keyPem)
-	if err != nil {
-		os.Remove(certFile.Name())
-		os.Remove(keyFile.Name())
-		return "", "", err
-	}
-	keyFile.Close()
-
-	return certFile.Name(), keyFile.Name(), nil
-}
-
 func TestGetNamedCertificateMap(t *testing.T) {
 	tests := []struct {
 		certs         []NamedTestCertSpec
@@ -225,26 +187,21 @@ func TestGetNamedCertificateMap(t *testing.T) {
 
 NextTest:
 	for i, test := range tests {
-		var namedCertKeys []NamedCertKey
+		var namedTLSCerts []namedTlsCert
 		bySignature := map[string]int{} // index in test.certs by cert signature
 		for j, c := range test.certs {
-			certFile, keyFile, err := createTestCerts(c.TestCertSpec)
+			cert, err := createTestTLSCerts(c.TestCertSpec)
 			if err != nil {
 				t.Errorf("%d - failed to create cert %d: %v", i, j, err)
 				continue NextTest
 			}
-			defer os.Remove(certFile)
-			defer os.Remove(keyFile)
 
-			namedCertKeys = append(namedCertKeys, NamedCertKey{
+			namedTLSCerts = append(namedTLSCerts, namedTlsCert{
-				CertKey: CertKey{
+				tlsCert: cert,
-					KeyFile:  keyFile,
+				names:   c.explicitNames,
-					CertFile: certFile,
-				},
-				Names: c.explicitNames,
 			})
 
-			sig, err := certFileSignature(certFile, keyFile)
+			sig, err := certSignature(cert)
 			if err != nil {
 				t.Errorf("%d - failed to get signature for %d: %v", i, j, err)
 				continue NextTest
@@ -252,7 +209,7 @@ NextTest:
 			bySignature[sig] = j
 		}
 
-		certMap, err := getNamedCertificateMap(namedCertKeys)
+		certMap, err := getNamedCertificateMap(namedTLSCerts)
 		if err == nil && len(test.errorString) != 0 {
 			t.Errorf("%d - expected no error, got: %v", i, err)
 		} else if err != nil && err.Error() != test.errorString {
@@ -363,19 +320,30 @@ func TestServerRunWithSNI(t *testing.T) {
 		},
 	}
 
+	tempDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir)
+
 NextTest:
 	for i, test := range tests {
 		// create server cert
-		serverCertFile, serverKeyFile, err := createTestCerts(test.Cert)
+		serverCertBundleFile, serverKeyFile, err := createTestCertFiles(tempDir, test.Cert)
 		if err != nil {
 			t.Errorf("%d - failed to create server cert: %v", i, err)
+			continue NextTest
 		}
-		defer os.Remove(serverCertFile)
+		ca, err := caCertFromBundle(serverCertBundleFile)
-		defer os.Remove(serverKeyFile)
+		if err != nil {
+			t.Errorf("%d - failed to extract ca cert from server cert bundle: %v", i, err)
+			continue NextTest
+		}
+		caCerts := []*x509.Certificate{ca}
 
 		// create SNI certs
-		var namedCertKeys []NamedCertKey
+		var namedCertKeys []config.NamedCertKey
-		serverSig, err := certFileSignature(serverCertFile, serverKeyFile)
+		serverSig, err := certFileSignature(serverCertBundleFile, serverKeyFile)
 		if err != nil {
 			t.Errorf("%d - failed to get server cert signature: %v", i, err)
 			continue NextTest
@@ -384,24 +352,27 @@ NextTest:
 			serverSig: -1,
 		}
 		for j, c := range test.SNICerts {
-			certFile, keyFile, err := createTestCerts(c.TestCertSpec)
+			certBundleFile, keyFile, err := createTestCertFiles(tempDir, c.TestCertSpec)
 			if err != nil {
 				t.Errorf("%d - failed to create SNI cert %d: %v", i, j, err)
 				continue NextTest
 			}
-			defer os.Remove(certFile)
-			defer os.Remove(keyFile)
 
-			namedCertKeys = append(namedCertKeys, NamedCertKey{
+			namedCertKeys = append(namedCertKeys, config.NamedCertKey{
-				CertKey: CertKey{
+				KeyFile:  keyFile,
-					KeyFile:  keyFile,
+				CertFile: certBundleFile,
-					CertFile: certFile,
+				Names:    c.explicitNames,
-				},
-				Names: c.explicitNames,
 			})
 
+			ca, err := caCertFromBundle(certBundleFile)
+			if err != nil {
+				t.Errorf("%d - failed to extract ca cert from SNI cert %d: %v", i, j, err)
+				continue NextTest
+			}
+			caCerts = append(caCerts, ca)
+
 			// store index in namedCertKeys with the signature as the key
-			sig, err := certFileSignature(certFile, keyFile)
+			sig, err := certFileSignature(certBundleFile, keyFile)
 			if err != nil {
 				t.Errorf("%d - failed get SNI cert %d signature: %v", i, j, err)
 				continue NextTest
@@ -416,17 +387,22 @@ NextTest:
 		defer etcdserver.Terminate(t)
 
 		config.EnableIndex = true
-		config.SecureServingInfo = &SecureServingInfo{
+		_, err = config.ApplySecureServingOptions(&options.SecureServingOptions{
-			ServingInfo: ServingInfo{
+			ServingOptions: options.ServingOptions{
-				BindAddress: "localhost:0",
+				BindAddress: net.ParseIP("127.0.0.1"),
+				BindPort:    6443,
 			},
-			ServerCert: GeneratableKeyCert{
+			ServerCert: options.GeneratableKeyCert{
-				CertKey: CertKey{
+				CertKey: options.CertKey{
-					CertFile: serverCertFile,
+					CertFile: serverCertBundleFile,
 					KeyFile:  serverKeyFile,
 				},
 			},
-			SNICerts: namedCertKeys,
+			SNICertKeys: namedCertKeys,
+		})
+		if err != nil {
+			t.Errorf("%d - failed applying the SecureServingOptions: %v", i, err)
+			continue NextTest
 		}
 		config.InsecureServingInfo = nil
 
@@ -436,27 +412,18 @@ NextTest:
 			continue NextTest
 		}
 
+		// patch in a 0-port to enable auto port allocation
+		s.SecureServingInfo.BindAddress = "127.0.0.1:0"
+
 		if err := s.serveSecurely(stopCh); err != nil {
 			t.Errorf("%d - failed running the server: %v", i, err)
 			continue NextTest
 		}
 
-		// load certificates into a pool
+		// load ca certificates into a pool
 		roots := x509.NewCertPool()
-		certFiles := []string{serverCertFile}
+		for _, caCert := range caCerts {
-		for _, c := range namedCertKeys {
+			roots.AddCert(caCert)
-			certFiles = append(certFiles, c.CertFile)
-		}
-		for _, certFile := range certFiles {
-			bs, err := ioutil.ReadFile(certFile)
-			if err != nil {
-				t.Errorf("%d - error reading %q: %v", i, certFile, err)
-				continue NextTest
-			}
-			if ok := roots.AppendCertsFromPEM(bs); !ok {
-				t.Errorf("%d - error adding cert %q to the pool", i, certFile)
-				continue NextTest
-			}
 		}
 
 		// try to dial
@@ -485,6 +452,77 @@ NextTest:
 	}
 }
 
+func parseIPList(ips []string) []net.IP {
+	var netIPs []net.IP
+	for _, ip := range ips {
+		netIPs = append(netIPs, net.ParseIP(ip))
+	}
+	return netIPs
+}
+
+func createTestTLSCerts(spec TestCertSpec) (tlsCert tls.Certificate, err error) {
+	certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names)
+	if err != nil {
+		return tlsCert, err
+	}
+
+	tlsCert, err = tls.X509KeyPair(certPem, keyPem)
+	return tlsCert, err
+}
+
+func createTestCertFiles(dir string, spec TestCertSpec) (certFilePath, keyFilePath string, err error) {
+	certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names)
+	if err != nil {
+		return "", "", err
+	}
+
+	certFile, err := ioutil.TempFile(dir, "cert")
+	if err != nil {
+		return "", "", err
+	}
+
+	keyFile, err := ioutil.TempFile(dir, "key")
+	if err != nil {
+		return "", "", err
+	}
+
+	_, err = certFile.Write(certPem)
+	if err != nil {
+		return "", "", err
+	}
+	certFile.Close()
+
+	_, err = keyFile.Write(keyPem)
+	if err != nil {
+		return "", "", err
+	}
+	keyFile.Close()
+
+	return certFile.Name(), keyFile.Name(), nil
+}
+
+func caCertFromBundle(bundlePath string) (*x509.Certificate, error) {
+	pemData, err := ioutil.ReadFile(bundlePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// fetch last block
+	var block *pem.Block
+	for {
+		var nextBlock *pem.Block
+		nextBlock, pemData = pem.Decode(pemData)
+		if nextBlock == nil {
+			if block == nil {
+				return nil, fmt.Errorf("no certificate found in %q", bundlePath)
+
+			}
+			return x509.ParseCertificate(block.Bytes)
+		}
+		block = nextBlock
+	}
+}
+
 func x509CertSignature(cert *x509.Certificate) string {
 	return base64.StdEncoding.EncodeToString(cert.Signature)
 }
@@ -494,13 +532,13 @@ func certFileSignature(certFile, keyFile string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	return certSignature(cert)
+}
 
+func certSignature(cert tls.Certificate) (string, error) {
 	x509Certs, err := x509.ParseCertificates(cert.Certificate[0])
 	if err != nil {
 		return "", err
 	}
-	if len(x509Certs) == 0 {
-		return "", fmt.Errorf("expected at least one cert after reparsing cert %q", certFile)
-	}
 	return x509CertSignature(x509Certs[0]), nil
 }

test/test_owners.csv

@@ -140,6 +140,7 @@ Federated Services DNS non-local federated service missing local service should
 Federated Services DNS non-local federated service should be able to discover a non-local federated service,jlowdermilk,1
 Federated Services DNS should be able to discover a federated service,derekwaynecarr,1
 Federated Services Service creation should create matching services in underlying clusters,jbeda,1
+Federated Services Service creation should not be deleted from underlying clusters when it is deleted,sttts,0
 Federated Services Service creation should succeed,rmmh,1
 Federated ingresses Federated Ingresses Ingress connectivity and DNS should be able to connect to a federated ingress via its load balancer,rmmh,1
 Federated ingresses Federated Ingresses should be created and deleted successfully,dchen1107,1
@@ -444,7 +445,7 @@ Services should be able to create a functioning NodePort service,bprashanth,0
 Services should be able to up and down services,bprashanth,0
 Services should check NodePort out-of-range,bprashanth,0
 Services should create endpoints for unready pods,maisem,0
-Serivces should only allow access from service loadbalancer source ranges,freehan,0
+Services should only allow access from service loadbalancer source ranges,sttts,0
 Services should preserve source pod IP for traffic thru service cluster IP,Random-Liu,1
 Services should prevent NodePort collisions,bprashanth,0
 Services should provide secure master service,bprashanth,0
@@ -546,7 +547,6 @@ k8s.io/kubernetes/pkg/api/errors,yifan-gu,1
 k8s.io/kubernetes/pkg/api/events,jlowdermilk,1
 k8s.io/kubernetes/pkg/api/install,timothysc,1
 k8s.io/kubernetes/pkg/api/meta,fabioy,1
-k8s.io/kubernetes/pkg/api/pod,piosz,1
 k8s.io/kubernetes/pkg/api/resource,smarterclayton,1
 k8s.io/kubernetes/pkg/api/service,spxtr,1
 k8s.io/kubernetes/pkg/api/testapi,caesarxuchao,1
@@ -554,6 +554,9 @@ k8s.io/kubernetes/pkg/api/unversioned,kevin-wangzefeng,1
 k8s.io/kubernetes/pkg/api/unversioned/validation,brendandburns,1
 k8s.io/kubernetes/pkg/api/util,ghodss,1
 k8s.io/kubernetes/pkg/api/v1,vulpecula,1
+k8s.io/kubernetes/pkg/api/v1/endpoints,sttts,0
+k8s.io/kubernetes/pkg/api/v1/pod,sttts,0
+k8s.io/kubernetes/pkg/api/v1/service,sttts,0
 k8s.io/kubernetes/pkg/api/validation,smarterclayton,1
 k8s.io/kubernetes/pkg/api/validation/path,luxas,1
 k8s.io/kubernetes/pkg/apimachinery,gmarek,1
@@ -787,6 +790,7 @@ k8s.io/kubernetes/pkg/registry/policy/poddisruptionbudget/etcd,xiang90,1
 k8s.io/kubernetes/pkg/registry/storage/storageclass,brendandburns,1
 k8s.io/kubernetes/pkg/registry/storage/storageclass/etcd,eparis,1
 k8s.io/kubernetes/pkg/runtime,wojtek-t,0
+k8s.io/kubernetes/pkg/runtime/schema,sttts,0
 k8s.io/kubernetes/pkg/runtime/serializer,wojtek-t,0
 k8s.io/kubernetes/pkg/runtime/serializer/json,wojtek-t,0
 k8s.io/kubernetes/pkg/runtime/serializer/protobuf,wojtek-t,0