Merge pull request #40947 from deads2k/apiserver-04-invert

Automatic merge from submit-queue (batch tested with PRs 41023, 41031, 40947)

apiserver command line options lead to config

Logically command line options lead to config, not the other way around.  We're clean enough now we can actually do the inversion.

WIP because I have some test cycles to fix, but this is all the meat.

@kubernetes/sig-api-machinery-misc

cmd/kube-aggregator/pkg/cmd/server/start.go

@@ -112,13 +112,14 @@ func (o AggregatorOptions) RunAggregator() error {
 
 	genericAPIServerConfig := genericapiserver.NewConfig().
 		WithSerializer(api.Codecs)
-	if _, err := genericAPIServerConfig.ApplySecureServingOptions(o.SecureServing); err != nil {
+
+	if err := o.SecureServing.ApplyTo(genericAPIServerConfig); err != nil {
+		return fmt.Errorf("failed to configure https: %s", err)
+	}
+	if err := o.Authentication.ApplyTo(genericAPIServerConfig); err != nil {
 		return err
 	}
-	if _, err := genericAPIServerConfig.ApplyDelegatingAuthenticationOptions(o.Authentication); err != nil {
+	if err := o.Authorization.ApplyTo(genericAPIServerConfig); err != nil {
-		return err
-	}
-	if _, err := genericAPIServerConfig.ApplyDelegatingAuthorizationOptions(o.Authorization); err != nil {
 		return err
 	}
 	genericAPIServerConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(

cmd/kube-apiserver/app/server.go

@@ -106,15 +106,19 @@ func Run(s *options.ServerRunOptions) error {
 
 	// create config from options
 	genericConfig := genericapiserver.NewConfig().
-		WithSerializer(api.Codecs).
+		WithSerializer(api.Codecs)
-		ApplyOptions(s.GenericServerRunOptions).
-		ApplyInsecureServingOptions(s.InsecureServing)
 
-	if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
+	if err := s.GenericServerRunOptions.ApplyTo(genericConfig); err != nil {
-		return fmt.Errorf("failed to configure https: %s", err)
+		return err
 	}
-	if err = s.Authentication.Apply(genericConfig); err != nil {
+	if err := s.InsecureServing.ApplyTo(genericConfig); err != nil {
-		return fmt.Errorf("failed to configure authentication: %s", err)
+		return err
+	}
+	if err := s.SecureServing.ApplyTo(genericConfig); err != nil {
+		return err
+	}
+	if err := s.Authentication.ApplyTo(genericConfig); err != nil {
+		return err
 	}
 
 	capabilities.Initialize(capabilities.Capabilities{

examples/apiserver/apiserver.go

@@ -104,14 +104,18 @@ func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
 
 	// create config from options
 	config := genericapiserver.NewConfig().
-		WithSerializer(api.Codecs).
+		WithSerializer(api.Codecs)
-		ApplyOptions(serverOptions.GenericServerRunOptions).
-		ApplyInsecureServingOptions(serverOptions.InsecureServing)
 
-	if _, err := config.ApplySecureServingOptions(serverOptions.SecureServing); err != nil {
+	if err := serverOptions.GenericServerRunOptions.ApplyTo(config); err != nil {
+		return err
+	}
+	if err := serverOptions.InsecureServing.ApplyTo(config); err != nil {
+		return err
+	}
+	if err := serverOptions.SecureServing.ApplyTo(config); err != nil {
 		return fmt.Errorf("failed to configure https: %s", err)
 	}
-	if err := serverOptions.Authentication.Apply(config); err != nil {
+	if err := serverOptions.Authentication.ApplyTo(config); err != nil {
 		return fmt.Errorf("failed to configure authentication: %s", err)
 	}
 

federation/cmd/federation-apiserver/app/server.go

@@ -89,15 +89,19 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	genericConfig := genericapiserver.NewConfig().
-		WithSerializer(api.Codecs).
+		WithSerializer(api.Codecs)
-		ApplyOptions(s.GenericServerRunOptions).
-		ApplyInsecureServingOptions(s.InsecureServing)
 
-	if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
+	if err := s.GenericServerRunOptions.ApplyTo(genericConfig); err != nil {
-		return fmt.Errorf("failed to configure https: %s", err)
+		return err
 	}
-	if err := s.Authentication.Apply(genericConfig); err != nil {
+	if err := s.InsecureServing.ApplyTo(genericConfig); err != nil {
-		return fmt.Errorf("failed to configure authentication: %s", err)
+		return err
+	}
+	if err := s.SecureServing.ApplyTo(genericConfig); err != nil {
+		return err
+	}
+	if err := s.Authentication.ApplyTo(genericConfig); err != nil {
+		return err
 	}
 
 	// TODO: register cluster federation resources here.

pkg/kubeapiserver/options/authentication.go

@@ -297,7 +297,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.Au
 	return ret
 }
 
-func (o *BuiltInAuthenticationOptions) Apply(c *genericapiserver.Config) error {
+func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {
 	if o == nil {
 		return nil
 	}

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -19,10 +19,8 @@ package server
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/pem"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	goruntime "runtime"
@@ -34,7 +32,6 @@ import (
 	"github.com/emicklei/go-restful/swagger"
 	"github.com/go-openapi/spec"
 	"github.com/pborman/uuid"
-	"gopkg.in/natefinch/lumberjack.v2"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	openapicommon "k8s.io/apimachinery/pkg/openapi"
@@ -56,7 +53,6 @@ import (
 	genericfilters "k8s.io/apiserver/pkg/server/filters"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/mux"
-	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/server/routes"
 	restclient "k8s.io/client-go/rest"
 	certutil "k8s.io/client-go/util/cert"
@@ -193,25 +189,23 @@ type SecureServingInfo struct {
 
 // NewConfig returns a Config struct with the default values
 func NewConfig() *Config {
-	config := &Config{
+	return &Config{
-		ReadWritePort:          6443,
+		ReadWritePort:               6443,
-		RequestContextMapper:   apirequest.NewRequestContextMapper(),
+		RequestContextMapper:        apirequest.NewRequestContextMapper(),
-		BuildHandlerChainsFunc: DefaultBuildHandlerChain,
+		BuildHandlerChainsFunc:      DefaultBuildHandlerChain,
-		LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix),
+		LegacyAPIGroupPrefixes:      sets.NewString(DefaultLegacyAPIPrefix),
-		HealthzChecks:          []healthz.HealthzChecker{healthz.PingHealthz},
+		HealthzChecks:               []healthz.HealthzChecker{healthz.PingHealthz},
-		EnableIndex:            true,
+		EnableIndex:                 true,
+		EnableGarbageCollection:     true,
+		EnableProfiling:             true,
+		MaxRequestsInFlight:         400,
+		MaxMutatingRequestsInFlight: 200,
+		MinRequestTimeout:           1800,
 
 		// Default to treating watch as a long-running operation
 		// Generic API servers have no inherent long-running subresources
 		LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),
 	}
-
-	// this keeps the defaults in sync
-	defaultOptions := options.NewServerRunOptions()
-	// unset fields that can be overridden to avoid setting values so that we won't end up with lingering values.
-	// TODO we probably want to run the defaults the other way.  A default here drives it in the CLI flags
-	defaultOptions.AuditLogPath = ""
-	return config.ApplyOptions(defaultOptions)
 }
 
 func (c *Config) WithSerializer(codecs serializer.CodecFactory) *Config {
@@ -257,82 +251,6 @@ func DefaultSwaggerConfig() *swagger.Config {
 	}
 }
 
-func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) (*Config, error) {
-	if secureServing == nil || secureServing.ServingOptions.BindPort <= 0 {
-		return c, nil
-	}
-
-	secureServingInfo := &SecureServingInfo{
-		ServingInfo: ServingInfo{
-			BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)),
-		},
-	}
-
-	serverCertFile, serverKeyFile := secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile
-
-	// load main cert
-	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
-		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
-		if err != nil {
-			return nil, fmt.Errorf("unable to load server certificate: %v", err)
-		}
-		secureServingInfo.Cert = &tlsCert
-	}
-
-	// optionally load CA cert
-	if len(secureServing.ServerCert.CACertFile) != 0 {
-		pemData, err := ioutil.ReadFile(secureServing.ServerCert.CACertFile)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read certificate authority from %q: %v", secureServing.ServerCert.CACertFile, err)
-		}
-		block, pemData := pem.Decode(pemData)
-		if block == nil {
-			return nil, fmt.Errorf("no certificate found in certificate authority file %q", secureServing.ServerCert.CACertFile)
-		}
-		if block.Type != "CERTIFICATE" {
-			return nil, fmt.Errorf("expected CERTIFICATE block in certiticate authority file %q, found: %s", secureServing.ServerCert.CACertFile, block.Type)
-		}
-		secureServingInfo.CACert = &tls.Certificate{
-			Certificate: [][]byte{block.Bytes},
-		}
-	}
-
-	// load SNI certs
-	namedTlsCerts := make([]namedTlsCert, 0, len(secureServing.SNICertKeys))
-	for _, nck := range secureServing.SNICertKeys {
-		tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile)
-		namedTlsCerts = append(namedTlsCerts, namedTlsCert{
-			tlsCert: tlsCert,
-			names:   nck.Names,
-		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to load SNI cert and key: %v", err)
-		}
-	}
-	var err error
-	secureServingInfo.SNICerts, err = getNamedCertificateMap(namedTlsCerts)
-	if err != nil {
-		return nil, err
-	}
-
-	c.SecureServingInfo = secureServingInfo
-	c.ReadWritePort = secureServing.ServingOptions.BindPort
-
-	return c, nil
-}
-
-func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOptions) *Config {
-	if insecureServing == nil || insecureServing.BindPort <= 0 {
-		return c
-	}
-
-	c.InsecureServingInfo = &ServingInfo{
-		BindAddress: net.JoinHostPort(insecureServing.BindAddress.String(), strconv.Itoa(insecureServing.BindPort)),
-	}
-
-	return c
-}
-
 func (c *Config) ApplyClientCert(clientCAFile string) (*Config, error) {
 	if c.SecureServingInfo != nil {
 		if len(clientCAFile) > 0 {
@@ -352,83 +270,6 @@ func (c *Config) ApplyClientCert(clientCAFile string) (*Config, error) {
 	return c, nil
 }
 
-func (c *Config) ApplyDelegatingAuthenticationOptions(o *options.DelegatingAuthenticationOptions) (*Config, error) {
-	if o == nil {
-		return c, nil
-	}
-
-	var err error
-	c, err = c.ApplyClientCert(o.ClientCert.ClientCA)
-	if err != nil {
-		return nil, fmt.Errorf("unable to load client CA file: %v", err)
-	}
-	c, err = c.ApplyClientCert(o.RequestHeader.ClientCAFile)
-	if err != nil {
-		return nil, fmt.Errorf("unable to load client CA file: %v", err)
-	}
-
-	cfg, err := o.ToAuthenticationConfig()
-	if err != nil {
-		return nil, err
-	}
-	authenticator, securityDefinitions, err := cfg.New()
-	if err != nil {
-		return nil, err
-	}
-
-	c.Authenticator = authenticator
-	if c.OpenAPIConfig != nil {
-		c.OpenAPIConfig.SecurityDefinitions = securityDefinitions
-	}
-	c.SupportsBasicAuth = false
-
-	return c, nil
-}
-
-func (c *Config) ApplyDelegatingAuthorizationOptions(o *options.DelegatingAuthorizationOptions) (*Config, error) {
-	if o == nil {
-		return c, nil
-	}
-
-	cfg, err := o.ToAuthorizationConfig()
-	if err != nil {
-		return nil, err
-	}
-	authorizer, err := cfg.New()
-	if err != nil {
-		return nil, err
-	}
-
-	c.Authorizer = authorizer
-
-	return c, nil
-}
-
-// ApplyOptions applies the run options to the method receiver and returns self
-func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
-	if len(options.AuditLogPath) != 0 {
-		c.AuditWriter = &lumberjack.Logger{
-			Filename:   options.AuditLogPath,
-			MaxAge:     options.AuditLogMaxAge,
-			MaxBackups: options.AuditLogMaxBackups,
-			MaxSize:    options.AuditLogMaxSize,
-		}
-	}
-
-	c.CorsAllowedOriginList = options.CorsAllowedOriginList
-	c.EnableGarbageCollection = options.EnableGarbageCollection
-	c.EnableProfiling = options.EnableProfiling
-	c.EnableContentionProfiling = options.EnableContentionProfiling
-	c.EnableSwaggerUI = options.EnableSwaggerUI
-	c.ExternalAddress = options.ExternalHost
-	c.MaxRequestsInFlight = options.MaxRequestsInFlight
-	c.MaxMutatingRequestsInFlight = options.MaxMutatingRequestsInFlight
-	c.MinRequestTimeout = options.MinRequestTimeout
-	c.PublicAddress = options.AdvertiseAddress
-
-	return c
-}
-
 type completedConfig struct {
 	*Config
 }

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -40,13 +40,13 @@ import (
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
-	apirequest "k8s.io/apiserver/pkg/endpoints/request"
-	"k8s.io/apiserver/pkg/server/healthz"
-	restclient "k8s.io/client-go/rest"
 	genericapi "k8s.io/apiserver/pkg/endpoints"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/apiserver/pkg/server/healthz"
 	genericmux "k8s.io/apiserver/pkg/server/mux"
 	"k8s.io/apiserver/pkg/server/routes"
+	restclient "k8s.io/client-go/rest"
 )
 
 // Info about an API group.
@@ -212,6 +212,11 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
 	<-stopCh
 }
 
+// EffectiveSecurePort returns the secure port we bound to.
+func (s *GenericAPIServer) EffectiveSecurePort() int {
+	return s.effectiveSecurePort
+}
+
 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource
 func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
 	for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {

staging/src/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -17,11 +17,13 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/spf13/pflag"
 
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/server"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -128,6 +130,35 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 	s.RequestHeader.AddFlags(fs)
 }
 
+func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.Config) error {
+	var err error
+	c, err = c.ApplyClientCert(s.ClientCert.ClientCA)
+	if err != nil {
+		return fmt.Errorf("unable to load client CA file: %v", err)
+	}
+	c, err = c.ApplyClientCert(s.RequestHeader.ClientCAFile)
+	if err != nil {
+		return fmt.Errorf("unable to load client CA file: %v", err)
+	}
+
+	cfg, err := s.ToAuthenticationConfig()
+	if err != nil {
+		return err
+	}
+	authenticator, securityDefinitions, err := cfg.New()
+	if err != nil {
+		return err
+	}
+
+	c.Authenticator = authenticator
+	if c.OpenAPIConfig != nil {
+		c.OpenAPIConfig.SecurityDefinitions = securityDefinitions
+	}
+	c.SupportsBasicAuth = false
+
+	return nil
+}
+
 func (s *DelegatingAuthenticationOptions) ToAuthenticationConfig() (authenticatorfactory.DelegatingAuthenticatorConfig, error) {
 	tokenClient, err := s.newTokenAccessReview()
 	if err != nil {

staging/src/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spf13/pflag"
 
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/server"
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1beta1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -69,6 +70,20 @@ func (s *DelegatingAuthorizationOptions) AddFlags(fs *pflag.FlagSet) {
 		"The duration to cache 'unauthorized' responses from the webhook authorizer.")
 }
 
+func (s *DelegatingAuthorizationOptions) ApplyTo(c *server.Config) error {
+	cfg, err := s.ToAuthorizationConfig()
+	if err != nil {
+		return err
+	}
+	authorizer, err := cfg.New()
+	if err != nil {
+		return err
+	}
+
+	c.Authorizer = authorizer
+	return nil
+}
+
 func (s *DelegatingAuthorizationOptions) ToAuthorizationConfig() (authorizerfactory.DelegatingAuthorizerConfig, error) {
 	sarClient, err := s.newSubjectAccessReview()
 	if err != nil {

staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -23,6 +23,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/server"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 
@@ -30,6 +31,7 @@ import (
 	_ "k8s.io/apiserver/pkg/features"
 
 	"github.com/spf13/pflag"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 // ServerRunOptions contains the options while running a generic api server.
@@ -62,21 +64,48 @@ type ServerRunOptions struct {
 }
 
 func NewServerRunOptions() *ServerRunOptions {
+	defaults := server.NewConfig()
+
 	return &ServerRunOptions{
 		AdmissionControl:            "AlwaysAdmit",
 		DefaultStorageMediaType:     "application/json",
 		DeleteCollectionWorkers:     1,
-		EnableGarbageCollection:     true,
+		EnableGarbageCollection:     defaults.EnableGarbageCollection,
-		EnableProfiling:             true,
+		EnableProfiling:             defaults.EnableProfiling,
 		EnableContentionProfiling:   false,
 		EnableWatchCache:            true,
-		MaxRequestsInFlight:         400,
+		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
-		MaxMutatingRequestsInFlight: 200,
+		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
-		MinRequestTimeout:           1800,
+		MinRequestTimeout:           defaults.MinRequestTimeout,
 		RuntimeConfig:               make(utilflag.ConfigurationMap),
 	}
 }
 
+// ApplyOptions applies the run options to the method receiver and returns self
+func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
+	if len(s.AuditLogPath) != 0 {
+		c.AuditWriter = &lumberjack.Logger{
+			Filename:   s.AuditLogPath,
+			MaxAge:     s.AuditLogMaxAge,
+			MaxBackups: s.AuditLogMaxBackups,
+			MaxSize:    s.AuditLogMaxSize,
+		}
+	}
+
+	c.CorsAllowedOriginList = s.CorsAllowedOriginList
+	c.EnableGarbageCollection = s.EnableGarbageCollection
+	c.EnableProfiling = s.EnableProfiling
+	c.EnableContentionProfiling = s.EnableContentionProfiling
+	c.EnableSwaggerUI = s.EnableSwaggerUI
+	c.ExternalAddress = s.ExternalHost
+	c.MaxRequestsInFlight = s.MaxRequestsInFlight
+	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
+	c.MinRequestTimeout = s.MinRequestTimeout
+	c.PublicAddress = s.AdvertiseAddress
+
+	return nil
+}
+
 // DefaultAdvertiseAddress sets the field AdvertiseAddress if
 // unset. The field will be set based on the SecureServingOptions. If
 // the SecureServingOptions is not present, DefaultExternalAddress

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -17,14 +17,19 @@ limitations under the License.
 package options
 
 import (
+	"crypto/tls"
+	"encoding/pem"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"path"
+	"strconv"
 
 	"github.com/golang/glog"
 	"github.com/spf13/pflag"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/server"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	certutil "k8s.io/client-go/util/cert"
 )
@@ -130,6 +135,70 @@ func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
 	fs.MarkDeprecated("public-address-override", "see --bind-address instead.")
 }
 
+func (s *SecureServingOptions) ApplyTo(c *server.Config) error {
+	if s.ServingOptions.BindPort <= 0 {
+		return nil
+	}
+
+	secureServingInfo := &server.SecureServingInfo{
+		ServingInfo: server.ServingInfo{
+			BindAddress: net.JoinHostPort(s.ServingOptions.BindAddress.String(), strconv.Itoa(s.ServingOptions.BindPort)),
+		},
+	}
+
+	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
+
+	// load main cert
+	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
+		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
+		if err != nil {
+			return fmt.Errorf("unable to load server certificate: %v", err)
+		}
+		secureServingInfo.Cert = &tlsCert
+	}
+
+	// optionally load CA cert
+	if len(s.ServerCert.CACertFile) != 0 {
+		pemData, err := ioutil.ReadFile(s.ServerCert.CACertFile)
+		if err != nil {
+			return fmt.Errorf("failed to read certificate authority from %q: %v", s.ServerCert.CACertFile, err)
+		}
+		block, pemData := pem.Decode(pemData)
+		if block == nil {
+			return fmt.Errorf("no certificate found in certificate authority file %q", s.ServerCert.CACertFile)
+		}
+		if block.Type != "CERTIFICATE" {
+			return fmt.Errorf("expected CERTIFICATE block in certiticate authority file %q, found: %s", s.ServerCert.CACertFile, block.Type)
+		}
+		secureServingInfo.CACert = &tls.Certificate{
+			Certificate: [][]byte{block.Bytes},
+		}
+	}
+
+	// load SNI certs
+	namedTLSCerts := make([]server.NamedTLSCert, 0, len(s.SNICertKeys))
+	for _, nck := range s.SNICertKeys {
+		tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile)
+		namedTLSCerts = append(namedTLSCerts, server.NamedTLSCert{
+			TLSCert: tlsCert,
+			Names:   nck.Names,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to load SNI cert and key: %v", err)
+		}
+	}
+	var err error
+	secureServingInfo.SNICerts, err = server.GetNamedCertificateMap(namedTLSCerts)
+	if err != nil {
+		return err
+	}
+
+	c.SecureServingInfo = secureServingInfo
+	c.ReadWritePort = s.ServingOptions.BindPort
+
+	return nil
+}
+
 func NewInsecureServingOptions() *ServingOptions {
 	return &ServingOptions{
 		BindAddress: net.ParseIP("127.0.0.1"),
@@ -172,6 +241,18 @@ func (s *ServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
 	fs.MarkDeprecated("port", "see --insecure-port instead.")
 }
 
+func (s *ServingOptions) ApplyTo(c *server.Config) error {
+	if s.BindPort <= 0 {
+		return nil
+	}
+
+	c.InsecureServingInfo = &server.ServingInfo{
+		BindAddress: net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)),
+	}
+
+	return nil
+}
+
 func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateIPs ...net.IP) error {
 	if s == nil {
 		return nil

staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package server
+package options
 
 import (
 	"crypto/tls"
@@ -26,17 +26,30 @@ import (
 	"net"
 	"os"
 	"reflect"
-	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/version"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	. "k8s.io/apiserver/pkg/server"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	utilcert "k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 )
 
+func setUp(t *testing.T) Config {
+	scheme := runtime.NewScheme()
+	codecs := serializer.NewCodecFactory(scheme)
+
+	config := NewConfig().WithSerializer(codecs)
+	config.RequestContextMapper = genericapirequest.NewRequestContextMapper()
+
+	return *config
+}
+
 type TestCertSpec struct {
 	host       string
 	names, ips []string // in certificate
@@ -190,7 +203,7 @@ func TestGetNamedCertificateMap(t *testing.T) {
 
 NextTest:
 	for i, test := range tests {
-		var namedTLSCerts []namedTlsCert
+		var namedTLSCerts []NamedTLSCert
 		bySignature := map[string]int{} // index in test.certs by cert signature
 		for j, c := range test.certs {
 			cert, err := createTestTLSCerts(c.TestCertSpec)
@@ -199,9 +212,9 @@ NextTest:
 				continue NextTest
 			}
 
-			namedTLSCerts = append(namedTLSCerts, namedTlsCert{
+			namedTLSCerts = append(namedTLSCerts, NamedTLSCert{
-				tlsCert: cert,
+				TLSCert: cert,
-				names:   c.explicitNames,
+				Names:   c.explicitNames,
 			})
 
 			sig, err := certSignature(cert)
@@ -212,7 +225,7 @@ NextTest:
 			bySignature[sig] = j
 		}
 
-		certMap, err := getNamedCertificateMap(namedTLSCerts)
+		certMap, err := GetNamedCertificateMap(namedTLSCerts)
 		if err == nil && len(test.errorString) != 0 {
 			t.Errorf("%d - expected no error, got: %v", i, err)
 		} else if err != nil && err.Error() != test.errorString {
@@ -461,27 +474,26 @@ NextTest:
 		stopCh := make(chan struct{})
 
 		// launch server
-		etcdserver, config, _ := setUp(t)
+		config := setUp(t)
-		defer etcdserver.Terminate(t)
 
 		v := fakeVersion()
 		config.Version = &v
 
 		config.EnableIndex = true
-		_, err = config.ApplySecureServingOptions(&options.SecureServingOptions{
+		secureOptions := &SecureServingOptions{
-			ServingOptions: options.ServingOptions{
+			ServingOptions: ServingOptions{
 				BindAddress: net.ParseIP("127.0.0.1"),
 				BindPort:    6443,
 			},
-			ServerCert: options.GeneratableKeyCert{
+			ServerCert: GeneratableKeyCert{
-				CertKey: options.CertKey{
+				CertKey: CertKey{
 					CertFile: serverCertBundleFile,
 					KeyFile:  serverKeyFile,
 				},
 			},
 			SNICertKeys: namedCertKeys,
-		})
+		}
-		if err != nil {
+		if err := secureOptions.ApplyTo(&config); err != nil {
 			t.Errorf("%q - failed applying the SecureServingOptions: %v", title, err)
 			continue NextTest
 		}
@@ -496,10 +508,14 @@ NextTest:
 		// patch in a 0-port to enable auto port allocation
 		s.SecureServingInfo.BindAddress = "127.0.0.1:0"
 
-		if err := s.serveSecurely(stopCh); err != nil {
+		// add poststart hook to know when the server is up.
-			t.Errorf("%q - failed running the server: %v", title, err)
+		startedCh := make(chan struct{})
-			continue NextTest
+		s.AddPostStartHook("test-notifier", func(context PostStartHookContext) error {
-		}
+			close(startedCh)
+			return nil
+		})
+		preparedServer := s.PrepareRun()
+		go preparedServer.Run(stopCh)
 
 		// load ca certificates into a pool
 		roots := x509.NewCertPool()
@@ -507,8 +523,11 @@ NextTest:
 			roots.AddCert(caCert)
 		}
 
+		<-startedCh
+
+		effectiveSecurePort := fmt.Sprintf("%d", preparedServer.EffectiveSecurePort())
 		// try to dial
-		addr := fmt.Sprintf("localhost:%d", s.effectiveSecurePort)
+		addr := fmt.Sprintf("localhost:%s", effectiveSecurePort)
 		t.Logf("Dialing %s as %q", addr, test.ServerName)
 		conn, err := tls.Dial("tcp", addr, &tls.Config{
 			RootCAs:    roots,
@@ -536,7 +555,7 @@ NextTest:
 		if len(test.SelfClientBindAddressOverride) != 0 {
 			host = test.SelfClientBindAddressOverride
 		}
-		config.SecureServingInfo.ServingInfo.BindAddress = net.JoinHostPort(host, strconv.Itoa(s.effectiveSecurePort))
+		config.SecureServingInfo.ServingInfo.BindAddress = net.JoinHostPort(host, effectiveSecurePort)
 		cfg, err := config.SecureServingInfo.NewSelfClientConfig("some-token")
 		if test.ExpectSelfClientError {
 			if err == nil {
@@ -654,3 +673,13 @@ func certSignature(cert tls.Certificate) (string, error) {
 	}
 	return x509CertSignature(x509Certs[0]), nil
 }
+
+func fakeVersion() version.Info {
+	return version.Info{
+		Major:        "42",
+		Minor:        "42",
+		GitVersion:   "42",
+		GitCommit:    "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
+		GitTreeState: "Dirty",
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/serve.go

@@ -173,25 +173,25 @@ func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int
 	return tcpAddr.Port, nil
 }
 
-type namedTlsCert struct {
+type NamedTLSCert struct {
-	tlsCert tls.Certificate
+	TLSCert tls.Certificate
 
 	// names is a list of domain patterns: fully qualified domain names, possibly prefixed with
 	// wildcard segments.
-	names []string
+	Names []string
 }
 
 // getNamedCertificateMap returns a map of *tls.Certificate by name. It's is
 // suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
 // cannot be loaded. Returns nil if len(certs) == 0
-func getNamedCertificateMap(certs []namedTlsCert) (map[string]*tls.Certificate, error) {
+func GetNamedCertificateMap(certs []NamedTLSCert) (map[string]*tls.Certificate, error) {
 	// register certs with implicit names first, reverse order such that earlier trump over the later
 	byName := map[string]*tls.Certificate{}
 	for i := len(certs) - 1; i >= 0; i-- {
-		if len(certs[i].names) > 0 {
+		if len(certs[i].Names) > 0 {
 			continue
 		}
-		cert := &certs[i].tlsCert
+		cert := &certs[i].TLSCert
 
 		// read names from certificate common names and DNS names
 		if len(cert.Certificate) == 0 {
@@ -216,8 +216,8 @@ func getNamedCertificateMap(certs []namedTlsCert) (map[string]*tls.Certificate,
 	// again in reverse order.
 	for i := len(certs) - 1; i >= 0; i-- {
 		namedCert := &certs[i]
-		for _, name := range namedCert.names {
+		for _, name := range namedCert.Names {
-			byName[name] = &certs[i].tlsCert
+			byName[name] = &certs[i].TLSCert
 		}
 	}
 

staging/src/k8s.io/apiserver/pkg/server/storage_factory_test.go

@@ -28,7 +28,6 @@ import (
 	"k8s.io/apiserver/pkg/apis/example"
 	exampleinstall "k8s.io/apiserver/pkg/apis/example/install"
 	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
-	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 )
 
@@ -128,7 +127,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
 	defaultEtcdLocation := []string{"http://127.0.0.1"}
 	for i, test := range testCases {
 		defaultConfig := storagebackend.Config{
-			Prefix:     options.DefaultEtcdPathPrefix,
+			Prefix:     "/registry",
 			ServerList: defaultEtcdLocation,
 			Copier:     scheme,
 		}

vendor/BUILD

@@ -9072,7 +9072,6 @@ go_library(
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/pborman/uuid",
         "//vendor:github.com/pkg/errors",
-        "//vendor:gopkg.in/natefinch/lumberjack.v2",
         "//vendor:k8s.io/apimachinery/pkg/apimachinery",
         "//vendor:k8s.io/apimachinery/pkg/apimachinery/registered",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
@@ -9103,7 +9102,6 @@ go_library(
         "//vendor:k8s.io/apiserver/pkg/server/filters",
         "//vendor:k8s.io/apiserver/pkg/server/healthz",
         "//vendor:k8s.io/apiserver/pkg/server/mux",
-        "//vendor:k8s.io/apiserver/pkg/server/options",
         "//vendor:k8s.io/apiserver/pkg/server/routes",
         "//vendor:k8s.io/apiserver/pkg/storage/storagebackend",
         "//vendor:k8s.io/client-go/rest",
@@ -14102,6 +14100,7 @@ go_library(
     deps = [
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/spf13/pflag",
+        "//vendor:gopkg.in/natefinch/lumberjack.v2",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
         "//vendor:k8s.io/apimachinery/pkg/runtime",
         "//vendor:k8s.io/apimachinery/pkg/util/net",
@@ -14109,6 +14108,7 @@ go_library(
         "//vendor:k8s.io/apiserver/pkg/authentication/authenticatorfactory",
         "//vendor:k8s.io/apiserver/pkg/authorization/authorizerfactory",
         "//vendor:k8s.io/apiserver/pkg/features",
+        "//vendor:k8s.io/apiserver/pkg/server",
         "//vendor:k8s.io/apiserver/pkg/storage/storagebackend",
         "//vendor:k8s.io/apiserver/pkg/util/feature",
         "//vendor:k8s.io/apiserver/pkg/util/flag",
@@ -14775,13 +14775,11 @@ go_test(
     srcs = [
         "k8s.io/apiserver/pkg/server/genericapiserver_test.go",
         "k8s.io/apiserver/pkg/server/resource_config_test.go",
-        "k8s.io/apiserver/pkg/server/serve_test.go",
         "k8s.io/apiserver/pkg/server/storage_factory_test.go",
     ],
     library = ":k8s.io/apiserver/pkg/server",
     tags = ["automanaged"],
     deps = [
-        "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/generated/openapi:go_default_library",
         "//vendor:github.com/go-openapi/spec",
         "//vendor:github.com/stretchr/testify/assert",
@@ -14803,12 +14801,9 @@ go_test(
         "//vendor:k8s.io/apiserver/pkg/authorization/authorizer",
         "//vendor:k8s.io/apiserver/pkg/endpoints/request",
         "//vendor:k8s.io/apiserver/pkg/registry/rest",
-        "//vendor:k8s.io/apiserver/pkg/server/options",
         "//vendor:k8s.io/apiserver/pkg/storage/etcd/testing",
         "//vendor:k8s.io/apiserver/pkg/storage/storagebackend",
-        "//vendor:k8s.io/apiserver/pkg/util/flag",
         "//vendor:k8s.io/client-go/pkg/api",
-        "//vendor:k8s.io/client-go/util/cert",
     ],
 )
 
@@ -15257,3 +15252,21 @@ go_library(
         "//vendor:k8s.io/apiserver/pkg/apis/apiserver",
     ],
 )
+
+go_test(
+    name = "k8s.io/apiserver/pkg/server/options_test",
+    srcs = ["k8s.io/apiserver/pkg/server/options/serving_test.go"],
+    library = ":k8s.io/apiserver/pkg/server/options",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//vendor:github.com/stretchr/testify/assert",
+        "//vendor:k8s.io/apimachinery/pkg/runtime",
+        "//vendor:k8s.io/apimachinery/pkg/runtime/serializer",
+        "//vendor:k8s.io/apimachinery/pkg/version",
+        "//vendor:k8s.io/apiserver/pkg/endpoints/request",
+        "//vendor:k8s.io/apiserver/pkg/server",
+        "//vendor:k8s.io/apiserver/pkg/util/flag",
+        "//vendor:k8s.io/client-go/util/cert",
+    ],
+)