diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index a7249a22f07..d22b706c8a8 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -29,6 +29,7 @@ go_library( "//pkg/auth/authorizer/union:go_default_library", "//pkg/auth/user:go_default_library", "//pkg/capabilities:go_default_library", + "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers:go_default_library", "//pkg/controller/informers:go_default_library", diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index cdfbffe540c..2dcabb0e480 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -31,7 +31,11 @@ import ( // ServerRunOptions runs a kubernetes api server. type ServerRunOptions struct { - GenericServerRunOptions *genericoptions.ServerRunOptions + GenericServerRunOptions *genericoptions.ServerRunOptions + Etcd *genericoptions.EtcdOptions + SecureServing *genericoptions.SecureServingOptions + InsecureServing *genericoptions.ServingOptions + AllowPrivileged bool EventTTL time.Duration KubeletConfig kubeletclient.KubeletClientConfig @@ -47,8 +51,12 @@ type ServerRunOptions struct { // NewServerRunOptions creates a new ServerRunOptions object with default parameters func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ - GenericServerRunOptions: genericoptions.NewServerRunOptions().WithEtcdOptions(), - EventTTL: 1 * time.Hour, + GenericServerRunOptions: genericoptions.NewServerRunOptions(), + Etcd: genericoptions.NewEtcdOptions(), + SecureServing: genericoptions.NewSecureServingOptions(), + InsecureServing: genericoptions.NewInsecureServingOptions(), + + EventTTL: 1 * time.Hour, KubeletConfig: kubeletclient.KubeletClientConfig{ Port: ports.KubeletPort, PreferredAddressTypes: []string{ @@ -69,8 +77,13 @@ func NewServerRunOptions() *ServerRunOptions { func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { // Add the generic flags. s.GenericServerRunOptions.AddUniversalFlags(fs) - //Add etcd specific flags. - s.GenericServerRunOptions.Etcd.AddEtcdStorageFlags(fs) + + s.Etcd.AddFlags(fs) + s.SecureServing.AddFlags(pflag.CommandLine) + s.SecureServing.AddDeprecatedFlags(pflag.CommandLine) + s.InsecureServing.AddFlags(pflag.CommandLine) + s.InsecureServing.AddDeprecatedFlags(pflag.CommandLine) + // Note: the weird ""+ in below lines seems to be the only way to get gofmt to // arrange these text blocks sensibly. Grrr. diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 75421efcac3..163d7c7f3f9 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -44,6 +44,7 @@ import ( authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union" "k8s.io/kubernetes/pkg/auth/user" "k8s.io/kubernetes/pkg/capabilities" + "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller/informers" serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" @@ -81,13 +82,19 @@ cluster's shared state through which all other components interact.`, // Run runs the specified APIServer. This should never exit. func Run(s *options.ServerRunOptions) error { - if errs := s.GenericServerRunOptions.Etcd.Validate(); len(errs) > 0 { + if errs := s.Etcd.Validate(); len(errs) > 0 { return utilerrors.NewAggregate(errs) } + if err := s.GenericServerRunOptions.DefaultExternalAddress(s.SecureServing, s.InsecureServing); err != nil { + return err + } + genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) genericConfig := genericapiserver.NewConfig(). // create the new config ApplyOptions(s.GenericServerRunOptions). // apply the options selected - Complete() // set default values based on the known values + ApplySecureServingOptions(s.SecureServing). + ApplyInsecureServingOptions(s.InsecureServing). + Complete() // set default values based on the known values serviceIPRange, apiServerServiceIP, err := genericapiserver.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange) if err != nil { @@ -145,7 +152,7 @@ func Run(s *options.ServerRunOptions) error { // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} - if s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize == 0 { + if s.Etcd.StorageConfig.DeserializationCacheSize == 0 { // When size of cache is not explicitly set, estimate its size based on // target memory usage. glog.V(2).Infof("Initalizing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB) @@ -161,9 +168,9 @@ func Run(s *options.ServerRunOptions) error { // size to compute its size. We may even go further and measure // collective sizes of the objects in the cache. clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60 - s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize = 25 * clusterSize - if s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize < 1000 { - s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize = 1000 + s.Etcd.StorageConfig.DeserializationCacheSize = 25 * clusterSize + if s.Etcd.StorageConfig.DeserializationCacheSize < 1000 { + s.Etcd.StorageConfig.DeserializationCacheSize = 1000 } } @@ -172,7 +179,7 @@ func Run(s *options.ServerRunOptions) error { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( - s.GenericServerRunOptions.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, + s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, // FIXME: this GroupVersionResource override should be configurable []schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")}, @@ -182,7 +189,7 @@ func Run(s *options.ServerRunOptions) error { } storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) - for _, override := range s.GenericServerRunOptions.Etcd.EtcdServersOverrides { + for _, override := range s.Etcd.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) @@ -203,9 +210,9 @@ func Run(s *options.ServerRunOptions) error { } // Default to the private server key for service account token signing - if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.SecureServing.ServerCert.CertKey.KeyFile != "" { - if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.SecureServing.ServerCert.CertKey.KeyFile) { - s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.SecureServing.ServerCert.CertKey.KeyFile} + if len(s.ServiceAccountKeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" { + if authenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) { + s.ServiceAccountKeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile} } else { glog.Warning("No TLS key provided, service account token authentication disabled") } @@ -226,7 +233,7 @@ func Run(s *options.ServerRunOptions) error { Anonymous: s.GenericServerRunOptions.AnonymousAuth, AnyToken: s.GenericServerRunOptions.EnableAnyToken, BasicAuthFile: s.GenericServerRunOptions.BasicAuthFile, - ClientCAFile: s.GenericServerRunOptions.SecureServing.ClientCA, + ClientCAFile: s.SecureServing.ClientCA, TokenAuthFile: s.GenericServerRunOptions.TokenAuthFile, OIDCIssuerURL: s.GenericServerRunOptions.OIDCIssuerURL, OIDCClientID: s.GenericServerRunOptions.OIDCClientID, @@ -248,11 +255,11 @@ func Run(s *options.ServerRunOptions) error { } privilegedLoopbackToken := uuid.NewRandom().String() - selfClientConfig, err := genericoptions.NewSelfClientConfig(s.GenericServerRunOptions.SecureServing, s.GenericServerRunOptions.InsecureServing, privilegedLoopbackToken) + selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } - client, err := s.GenericServerRunOptions.NewSelfClient(privilegedLoopbackToken) + client, err := internalclientset.NewForConfig(selfClientConfig) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index 71f12a06913..694676a6d94 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -55,22 +55,48 @@ func newStorageFactory() genericapiserver.StorageFactory { return storageFactory } -func NewServerRunOptions() *genericoptions.ServerRunOptions { - serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions().WithSecureServingOptions().WithInsecureServingOptions() - serverOptions.InsecureServing.BindPort = InsecurePort - return serverOptions +type ServerRunOptions struct { + GenericServerRunOptions *genericoptions.ServerRunOptions + Etcd *genericoptions.EtcdOptions + SecureServing *genericoptions.SecureServingOptions + InsecureServing *genericoptions.ServingOptions } -func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error { +func NewServerRunOptions() *ServerRunOptions { + s := ServerRunOptions{ + GenericServerRunOptions: genericoptions.NewServerRunOptions(), + Etcd: genericoptions.NewEtcdOptions(), + SecureServing: genericoptions.NewSecureServingOptions(), + InsecureServing: genericoptions.NewInsecureServingOptions(), + } + s.InsecureServing.BindPort = InsecurePort + s.SecureServing.ServingOptions.BindPort = SecurePort + + return &s +} + +func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error { // Set ServiceClusterIPRange _, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") - serverOptions.ServiceClusterIPRange = *serviceClusterIPRange + serverOptions.GenericServerRunOptions.ServiceClusterIPRange = *serviceClusterIPRange serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} - genericvalidation.ValidateRunOptions(serverOptions) + + genericvalidation.ValidateRunOptions(serverOptions.GenericServerRunOptions) if errs := serverOptions.Etcd.Validate(); len(errs) > 0 { return utilerrors.NewAggregate(errs) } - config := genericapiserver.NewConfig().ApplyOptions(serverOptions).Complete() + if errs := serverOptions.SecureServing.Validate(); len(errs) > 0 { + return utilerrors.NewAggregate(errs) + } + if errs := serverOptions.InsecureServing.Validate("insecure-port"); len(errs) > 0 { + return utilerrors.NewAggregate(errs) + } + + config := genericapiserver.NewConfig(). + ApplyOptions(serverOptions.GenericServerRunOptions). + ApplySecureServingOptions(serverOptions.SecureServing). + ApplyInsecureServingOptions(serverOptions.InsecureServing). + Complete() if err := config.MaybeGenerateServingCerts(); err != nil { // this wasn't treated as fatal for this process before fmt.Printf("Error creating cert: %v", err) diff --git a/examples/apiserver/server/main.go b/examples/apiserver/server/main.go index 5c69edf708f..5dff01318a7 100644 --- a/examples/apiserver/server/main.go +++ b/examples/apiserver/server/main.go @@ -30,10 +30,14 @@ func main() { // Parse command line flags. serverRunOptions.AddUniversalFlags(pflag.CommandLine) - serverRunOptions.AddEtcdStorageFlags(pflag.CommandLine) + serverRunOptions.Etcd.AddFlags(pflag.CommandLine) + serverRunOptions.SecureServing.AddFlags(pflag.CommandLine) + serverRunOptions.SecureServing.AddDeprecatedFlags(pflag.CommandLine) + serverRunOptions.InsecureServing.AddFlags(pflag.CommandLine) + serverRunOptions.InsecureServing.AddDeprecatedFlags(pflag.CommandLine) flag.InitFlags() - if err := apiserver.Run(serverRunOptions, wait.NeverStop); err != nil { + if err := serverRunOptions.Run(wait.NeverStop); err != nil { glog.Fatalf("Error in bringing up the server: %v", err) } } diff --git a/federation/cmd/federation-apiserver/app/BUILD b/federation/cmd/federation-apiserver/app/BUILD index 75c7fe1160e..3194543485b 100644 --- a/federation/cmd/federation-apiserver/app/BUILD +++ b/federation/cmd/federation-apiserver/app/BUILD @@ -38,6 +38,7 @@ go_library( "//pkg/apiserver/authenticator:go_default_library", "//pkg/auth/authorizer/union:go_default_library", "//pkg/auth/user:go_default_library", + "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/cloudprovider/providers:go_default_library", "//pkg/controller/informers:go_default_library", "//pkg/generated/openapi:go_default_library", @@ -58,6 +59,7 @@ go_library( "//pkg/registry/generic/registry:go_default_library", "//pkg/routes:go_default_library", "//pkg/runtime/schema:go_default_library", + "//pkg/util/errors:go_default_library", "//pkg/util/wait:go_default_library", "//pkg/version:go_default_library", "//plugin/pkg/admission/admit:go_default_library", diff --git a/federation/cmd/federation-apiserver/app/options/options.go b/federation/cmd/federation-apiserver/app/options/options.go index b1d8af4ac0a..49084fb30f6 100644 --- a/federation/cmd/federation-apiserver/app/options/options.go +++ b/federation/cmd/federation-apiserver/app/options/options.go @@ -28,14 +28,22 @@ import ( // Runtime options for the federation-apiserver. type ServerRunOptions struct { GenericServerRunOptions *genericoptions.ServerRunOptions - EventTTL time.Duration + Etcd *genericoptions.EtcdOptions + SecureServing *genericoptions.SecureServingOptions + InsecureServing *genericoptions.ServingOptions + + EventTTL time.Duration } // NewServerRunOptions creates a new ServerRunOptions object with default values. func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ - GenericServerRunOptions: genericoptions.NewServerRunOptions().WithEtcdOptions(), - EventTTL: 1 * time.Hour, + GenericServerRunOptions: genericoptions.NewServerRunOptions(), + Etcd: genericoptions.NewEtcdOptions(), + SecureServing: genericoptions.NewSecureServingOptions(), + InsecureServing: genericoptions.NewInsecureServingOptions(), + + EventTTL: 1 * time.Hour, } return &s } @@ -44,8 +52,9 @@ func NewServerRunOptions() *ServerRunOptions { func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { // Add the generic flags. s.GenericServerRunOptions.AddUniversalFlags(fs) - //Add etcd specific flags. - s.GenericServerRunOptions.Etcd.AddEtcdStorageFlags(fs) + s.Etcd.AddFlags(fs) + s.SecureServing.AddFlags(fs) + s.InsecureServing.AddFlags(fs) fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, "Amount of time to retain events. Default is 1h.") diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index 21b58340b08..89abe7bab72 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/apiserver/authenticator" authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union" "k8s.io/kubernetes/pkg/auth/user" + "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/controller/informers" "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" @@ -44,6 +45,7 @@ import ( "k8s.io/kubernetes/pkg/registry/generic/registry" "k8s.io/kubernetes/pkg/routes" "k8s.io/kubernetes/pkg/runtime/schema" + utilerrors "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/version" authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union" @@ -67,13 +69,19 @@ cluster's shared state through which all other components interact.`, // Run runs the specified APIServer. This should never exit. func Run(s *options.ServerRunOptions) error { - if errs := s.GenericServerRunOptions.Etcd.Validate(); len(errs) > 0 { - glog.Fatal(errs) + if errs := s.Etcd.Validate(); len(errs) > 0 { + utilerrors.NewAggregate(errs) } + if err := s.GenericServerRunOptions.DefaultExternalAddress(s.SecureServing, s.InsecureServing); err != nil { + return err + } + genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) genericConfig := genericapiserver.NewConfig(). // create the new config ApplyOptions(s.GenericServerRunOptions). // apply the options selected - Complete() // set default values based on the known values + ApplySecureServingOptions(s.SecureServing). + ApplyInsecureServingOptions(s.InsecureServing). + Complete() // set default values based on the known values if err := genericConfig.MaybeGenerateServingCerts(); err != nil { glog.Fatalf("Failed to generate service certificate: %v", err) @@ -82,23 +90,23 @@ func Run(s *options.ServerRunOptions) error { // TODO: register cluster federation resources here. resourceConfig := genericapiserver.NewResourceConfig() - if s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize == 0 { + if s.Etcd.StorageConfig.DeserializationCacheSize == 0 { // When size of cache is not explicitly set, set it to 50000 - s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize = 50000 + s.Etcd.StorageConfig.DeserializationCacheSize = 50000 } storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( - s.GenericServerRunOptions.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, + s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, []schema.GroupVersionResource{}, resourceConfig, s.GenericServerRunOptions.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } - for _, override := range s.GenericServerRunOptions.Etcd.EtcdServersOverrides { + for _, override := range s.Etcd.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) @@ -122,7 +130,7 @@ func Run(s *options.ServerRunOptions) error { Anonymous: s.GenericServerRunOptions.AnonymousAuth, AnyToken: s.GenericServerRunOptions.EnableAnyToken, BasicAuthFile: s.GenericServerRunOptions.BasicAuthFile, - ClientCAFile: s.GenericServerRunOptions.SecureServing.ClientCA, + ClientCAFile: s.SecureServing.ClientCA, TokenAuthFile: s.GenericServerRunOptions.TokenAuthFile, OIDCIssuerURL: s.GenericServerRunOptions.OIDCIssuerURL, OIDCClientID: s.GenericServerRunOptions.OIDCClientID, @@ -137,11 +145,11 @@ func Run(s *options.ServerRunOptions) error { } privilegedLoopbackToken := uuid.NewRandom().String() - selfClientConfig, err := genericoptions.NewSelfClientConfig(s.GenericServerRunOptions.SecureServing, s.GenericServerRunOptions.InsecureServing, privilegedLoopbackToken) + selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken) if err != nil { glog.Fatalf("Failed to create clientset: %v", err) } - client, err := s.GenericServerRunOptions.NewSelfClient(privilegedLoopbackToken) + client, err := internalclientset.NewForConfig(selfClientConfig) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index e7a1958cd90..957d87b48a1 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -54,7 +54,6 @@ import ( genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" "k8s.io/kubernetes/pkg/runtime" certutil "k8s.io/kubernetes/pkg/util/cert" - utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/version" ) @@ -230,6 +229,59 @@ func NewConfig() *Config { return config.ApplyOptions(defaultOptions) } +func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) *Config { + if secureServing == nil || secureServing.ServingOptions.BindPort <= 0 { + return c + } + + secureServingInfo := &SecureServingInfo{ + ServingInfo: ServingInfo{ + BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)), + }, + ServerCert: GeneratableKeyCert{ + CertKey: CertKey{ + CertFile: secureServing.ServerCert.CertKey.CertFile, + KeyFile: secureServing.ServerCert.CertKey.KeyFile, + }, + }, + SNICerts: []NamedCertKey{}, + ClientCA: secureServing.ClientCA, + } + if secureServing.ServerCert.CertKey.CertFile == "" && secureServing.ServerCert.CertKey.KeyFile == "" { + secureServingInfo.ServerCert.Generate = true + secureServingInfo.ServerCert.CertFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".crt") + secureServingInfo.ServerCert.KeyFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".key") + } + + secureServingInfo.SNICerts = nil + for _, nkc := range secureServing.SNICertKeys { + secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{ + CertKey: CertKey{ + KeyFile: nkc.KeyFile, + CertFile: nkc.CertFile, + }, + Names: nkc.Names, + }) + } + + c.SecureServingInfo = secureServingInfo + c.ReadWritePort = secureServing.ServingOptions.BindPort + + return c +} + +func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOptions) *Config { + if insecureServing == nil || insecureServing.BindPort <= 0 { + return c + } + + c.InsecureServingInfo = &ServingInfo{ + BindAddress: net.JoinHostPort(insecureServing.BindAddress.String(), strconv.Itoa(insecureServing.BindPort)), + } + + return c +} + // ApplyOptions applies the run options to the method receiver and returns self func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { if len(options.AuditLogPath) != 0 { @@ -241,48 +293,6 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { } } - if options.SecureServing != nil && options.SecureServing.ServingOptions.BindPort > 0 { - secureServingInfo := &SecureServingInfo{ - ServingInfo: ServingInfo{ - BindAddress: net.JoinHostPort(options.SecureServing.ServingOptions.BindAddress.String(), strconv.Itoa(options.SecureServing.ServingOptions.BindPort)), - }, - ServerCert: GeneratableKeyCert{ - CertKey: CertKey{ - CertFile: options.SecureServing.ServerCert.CertKey.CertFile, - KeyFile: options.SecureServing.ServerCert.CertKey.KeyFile, - }, - }, - SNICerts: []NamedCertKey{}, - ClientCA: options.SecureServing.ClientCA, - } - if options.SecureServing.ServerCert.CertKey.CertFile == "" && options.SecureServing.ServerCert.CertKey.KeyFile == "" { - secureServingInfo.ServerCert.Generate = true - secureServingInfo.ServerCert.CertFile = path.Join(options.SecureServing.ServerCert.CertDirectory, options.SecureServing.ServerCert.PairName+".crt") - secureServingInfo.ServerCert.KeyFile = path.Join(options.SecureServing.ServerCert.CertDirectory, options.SecureServing.ServerCert.PairName+".key") - } - - secureServingInfo.SNICerts = nil - for _, nkc := range options.SecureServing.SNICertKeys { - secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{ - CertKey: CertKey{ - KeyFile: nkc.KeyFile, - CertFile: nkc.CertFile, - }, - Names: nkc.Names, - }) - } - - c.SecureServingInfo = secureServingInfo - c.ReadWritePort = options.SecureServing.ServingOptions.BindPort - } - - if options.InsecureServing != nil && options.InsecureServing.BindPort > 0 { - insecureServingInfo := &ServingInfo{ - BindAddress: net.JoinHostPort(options.InsecureServing.BindAddress.String(), strconv.Itoa(options.InsecureServing.BindPort)), - } - c.InsecureServingInfo = insecureServingInfo - } - c.AuthorizerRBACSuperUser = options.AuthorizationRBACSuperUser c.CorsAllowedOriginList = options.CorsAllowedOriginList c.EnableGarbageCollection = options.EnableGarbageCollection @@ -483,17 +493,6 @@ func (s *GenericAPIServer) installAPI(c *Config) { func DefaultAndValidateRunOptions(options *options.ServerRunOptions) { genericvalidation.ValidateRunOptions(options) - // If advertise-address is not specified, use bind-address. If bind-address - // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default - // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) - if options.SecureServing != nil && (options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified()) { - hostIP, err := utilnet.ChooseBindAddress(options.SecureServing.ServingOptions.BindAddress) - if err != nil { - glog.Fatalf("Unable to find suitable network address.error='%v' . "+ - "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) - } - options.AdvertiseAddress = hostIP - } glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress) // Set default value for ExternalAddress if not specified. diff --git a/pkg/genericapiserver/options/BUILD b/pkg/genericapiserver/options/BUILD index 28f47608f97..8b6f319af02 100644 --- a/pkg/genericapiserver/options/BUILD +++ b/pkg/genericapiserver/options/BUILD @@ -15,9 +15,9 @@ go_library( srcs = [ "authenticator.go", "doc.go", - "etcd_options.go", + "etcd.go", "server_run_options.go", - "serving_options.go", + "serving.go", ], tags = ["automanaged"], deps = [ @@ -25,7 +25,6 @@ go_library( "//pkg/api:go_default_library", "//pkg/apimachinery/registered:go_default_library", "//pkg/apiserver/authenticator:go_default_library", - "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/restclient:go_default_library", "//pkg/runtime/schema:go_default_library", "//pkg/storage/storagebackend:go_default_library", diff --git a/pkg/genericapiserver/options/etcd_options.go b/pkg/genericapiserver/options/etcd.go similarity index 96% rename from pkg/genericapiserver/options/etcd_options.go rename to pkg/genericapiserver/options/etcd.go index 53ab03baa89..4d1b3124505 100644 --- a/pkg/genericapiserver/options/etcd_options.go +++ b/pkg/genericapiserver/options/etcd.go @@ -34,7 +34,7 @@ type EtcdOptions struct { EtcdServersOverrides []string } -func NewDefaultEtcdOptions() *EtcdOptions { +func NewEtcdOptions() *EtcdOptions { return &EtcdOptions{ StorageConfig: storagebackend.Config{ Prefix: DefaultEtcdPathPrefix, @@ -55,7 +55,7 @@ func (s *EtcdOptions) Validate() []error { } // AddEtcdFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet -func (s *EtcdOptions) AddEtcdStorageFlags(fs *pflag.FlagSet) { +func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+ "Per-resource etcd servers overrides, comma separated. The individual override "+ "format: group/resource#servers, where servers are http://ip:port, semicolon separated.") diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index 620b5af8df4..e7220868cf6 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -17,6 +17,7 @@ limitations under the License. package options import ( + "fmt" "net" "strings" "time" @@ -24,7 +25,6 @@ import ( "k8s.io/kubernetes/pkg/admission" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apimachinery/registered" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/util/config" utilnet "k8s.io/kubernetes/pkg/util/net" @@ -51,10 +51,6 @@ var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABA // ServerRunOptions contains the options while running a generic api server. type ServerRunOptions struct { - Etcd *EtcdOptions - SecureServing *SecureServingOptions - InsecureServing *ServingOptions - AdmissionControl string AdmissionControlConfigFile string AdvertiseAddress net.IP @@ -140,19 +136,28 @@ func NewServerRunOptions() *ServerRunOptions { } } -func (o *ServerRunOptions) WithEtcdOptions() *ServerRunOptions { - o.Etcd = NewDefaultEtcdOptions() - return o -} +func (s *ServerRunOptions) DefaultExternalAddress(secure *SecureServingOptions, insecure *ServingOptions) error { + if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { + switch { + case secure != nil: + hostIP, err := secure.ServingOptions.DefaultExternalAddress() + if err != nil { + return fmt.Errorf("Unable to find suitable network address.error='%v'. "+ + "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) + } + s.AdvertiseAddress = hostIP -func (o *ServerRunOptions) WithSecureServingOptions() *ServerRunOptions { - o.SecureServing = NewDefaultSecureServingOptions() - return o -} + case insecure != nil: + hostIP, err := insecure.DefaultExternalAddress() + if err != nil { + return fmt.Errorf("Unable to find suitable network address.error='%v'. "+ + "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) + } + s.AdvertiseAddress = hostIP + } + } -func (o *ServerRunOptions) WithInsecureServingOptions() *ServerRunOptions { - o.InsecureServing = NewDefaultInsecureServingOptions() - return o + return nil } // StorageGroupsToEncodingVersion returns a map from group name to group version, @@ -201,15 +206,6 @@ func mergeGroupVersionIntoMap(gvList string, dest map[string]schema.GroupVersion return nil } -// Returns a clientset which can be used to talk to this apiserver. -func (s *ServerRunOptions) NewSelfClient(token string) (clientset.Interface, error) { - clientConfig, err := NewSelfClientConfig(s.SecureServing, s.InsecureServing, token) - if err != nil { - return nil, err - } - return clientset.NewForConfig(clientConfig) -} - // AddFlags adds flags for a specific APIServer to the specified FlagSet func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { // Note: the weird ""+ in below lines seems to be the only way to get gofmt to diff --git a/pkg/genericapiserver/options/serving_options.go b/pkg/genericapiserver/options/serving.go similarity index 93% rename from pkg/genericapiserver/options/serving_options.go rename to pkg/genericapiserver/options/serving.go index 1bf66efcb29..8487b9e35af 100644 --- a/pkg/genericapiserver/options/serving_options.go +++ b/pkg/genericapiserver/options/serving.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/client/restclient" "k8s.io/kubernetes/pkg/util/config" + utilnet "k8s.io/kubernetes/pkg/util/net" ) type ServingOptions struct { @@ -66,7 +67,7 @@ type GeneratableKeyCert struct { PairName string } -func NewDefaultSecureServingOptions() *SecureServingOptions { +func NewSecureServingOptions() *SecureServingOptions { return &SecureServingOptions{ ServingOptions: ServingOptions{ BindAddress: net.ParseIP("0.0.0.0"), @@ -80,9 +81,10 @@ func NewDefaultSecureServingOptions() *SecureServingOptions { } func (s *SecureServingOptions) NewSelfClientConfig(token string) *restclient.Config { - if s == nil || s.ServingOptions.BindPort <= 0 && len(s.ServerCA) == 0 { + if s == nil || s.ServingOptions.BindPort <= 0 || len(s.ServerCA) == 0 { return nil } + clientConfig := &restclient.Config{ // Increase QPS limits. The client is currently passed to all admission plugins, // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 @@ -113,7 +115,7 @@ func (s *SecureServingOptions) Validate() []error { return errors } -func (s *SecureServingOptions) AddSecureServingFlags(fs *pflag.FlagSet) { +func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { fs.IPVar(&s.ServingOptions.BindAddress, "bind-address", s.ServingOptions.BindAddress, ""+ "The IP address on which to listen for the --secure-port port. The "+ "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ @@ -156,14 +158,14 @@ func (s *SecureServingOptions) AddSecureServingFlags(fs *pflag.FlagSet) { } -func (s *SecureServingOptions) AddDeprecatedSecureServingFlags(fs *pflag.FlagSet) { +func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress, "DEPRECATED: see --bind-address instead.") fs.MarkDeprecated("public-address-override", "see --bind-address instead.") } -func NewDefaultInsecureServingOptions() *ServingOptions { +func NewInsecureServingOptions() *ServingOptions { return &ServingOptions{ BindAddress: net.ParseIP("127.0.0.1"), BindPort: 8080, @@ -197,7 +199,11 @@ func (s *ServingOptions) NewSelfClientConfig(token string) *restclient.Config { return clientConfig } -func (s *ServingOptions) AddInsecureServingFlags(fs *pflag.FlagSet) { +func (s *ServingOptions) DefaultExternalAddress() (net.IP, error) { + return utilnet.ChooseBindAddress(s.BindAddress) +} + +func (s *ServingOptions) AddFlags(fs *pflag.FlagSet) { fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+ "The IP address on which to serve the --insecure-port (set to 0.0.0.0 for all interfaces). "+ "Defaults to localhost.") @@ -209,7 +215,7 @@ func (s *ServingOptions) AddInsecureServingFlags(fs *pflag.FlagSet) { "port. This is performed by nginx in the default setup.") } -func (s *ServingOptions) AddDeprecatedInsecureServingFlags(fs *pflag.FlagSet) { +func (s *ServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.IPVar(&s.BindAddress, "address", s.BindAddress, "DEPRECATED: see --insecure-bind-address instead.") fs.MarkDeprecated("address", "see --insecure-bind-address instead.") diff --git a/pkg/genericapiserver/validation/universal_validation.go b/pkg/genericapiserver/validation/universal_validation.go index 088fe50422d..72e683acfc3 100644 --- a/pkg/genericapiserver/validation/universal_validation.go +++ b/pkg/genericapiserver/validation/universal_validation.go @@ -49,23 +49,6 @@ func verifyServiceNodePort(options *options.ServerRunOptions) []error { return errors } -func verifySecureAndInsecurePort(options *options.ServerRunOptions) []error { - errors := []error{} - errors = append(errors, options.SecureServing.Validate()...) - errors = append(errors, options.InsecureServing.Validate("insecure-port")...) - - if (options.SecureServing == nil || options.SecureServing.ServingOptions.BindPort == 0) && - (options.InsecureServing == nil || options.InsecureServing.BindPort == 0) { - glog.Fatalf("--secure-port and --insecure-port cannot be turned off at the same time.") - } - - if options.SecureServing != nil && options.InsecureServing != nil && - options.SecureServing.ServingOptions.BindPort == options.InsecureServing.BindPort { - errors = append(errors, fmt.Errorf("--secure-port and --insecure-port cannot use the same port.")) - } - return errors -} - func ValidateRunOptions(options *options.ServerRunOptions) { errors := []error{} if errs := verifyClusterIPFlags(options); len(errs) > 0 { @@ -74,9 +57,6 @@ func ValidateRunOptions(options *options.ServerRunOptions) { if errs := verifyServiceNodePort(options); len(errs) > 0 { errors = append(errors, errs...) } - if errs := verifySecureAndInsecurePort(options); len(errs) > 0 { - errors = append(errors, errs...) - } if err := utilerrors.NewAggregate(errors); err != nil { glog.Fatalf("Validate server run options failed: %v", err) } diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 27a4adc5027..7e7eeee4982 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -41,7 +41,7 @@ func NewAPIServer() *APIServer { // Start starts the apiserver, returns when apiserver is ready. func (a *APIServer) Start() error { config := options.NewServerRunOptions() - config.GenericServerRunOptions.Etcd.StorageConfig.ServerList = []string{getEtcdClientURL()} + config.Etcd.StorageConfig.ServerList = []string{getEtcdClientURL()} _, ipnet, err := net.ParseCIDR(clusterIPRange) if err != nil { return err diff --git a/test/integration/discoverysummarizer/discoverysummarizer_test.go b/test/integration/discoverysummarizer/discoverysummarizer_test.go index 353cd8733f1..a392f8033d5 100644 --- a/test/integration/discoverysummarizer/discoverysummarizer_test.go +++ b/test/integration/discoverysummarizer/discoverysummarizer_test.go @@ -68,15 +68,15 @@ func runDiscoverySummarizer(t *testing.T) string { func runAPIServer(t *testing.T, stopCh <-chan struct{}) string { serverRunOptions := apiserver.NewServerRunOptions() // Change the ports, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel. - serverRunOptions.SecurePort = 6443 + 3 - serverRunOptions.InsecurePort = 8080 + 3 + serverRunOptions.SecureServing.ServingOptions.BindPort = 6443 + 3 + serverRunOptions.InsecureServing.BindPort = 8080 + 3 go func() { - if err := apiserver.Run(serverRunOptions, stopCh); err != nil { + if err := serverRunOptions.Run(stopCh); err != nil { t.Fatalf("Error in bringing up the example apiserver: %v", err) } }() - serverURL := fmt.Sprintf("http://localhost:%d", serverRunOptions.InsecurePort) + serverURL := fmt.Sprintf("http://localhost:%d", serverRunOptions.InsecureServing.BindPort) if err := waitForServerUp(serverURL); err != nil { t.Fatalf("%v", err) } diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 21ebffa89f8..4ffce261bcd 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -44,7 +44,7 @@ func TestRunServer(t *testing.T) { serverIP := fmt.Sprintf("http://localhost:%d", apiserver.InsecurePort) stopCh := make(chan struct{}) go func() { - if err := apiserver.Run(apiserver.NewServerRunOptions(), stopCh); err != nil { + if err := apiserver.NewServerRunOptions().Run(stopCh); err != nil { t.Fatalf("Error in bringing up the server: %v", err) } }() @@ -65,7 +65,7 @@ func TestRunSecureServer(t *testing.T) { options := apiserver.NewServerRunOptions() options.InsecureServing.BindPort = 0 options.SecureServing.ServingOptions.BindPort = apiserver.SecurePort - if err := apiserver.Run(options, stopCh); err != nil { + if err := options.Run(stopCh); err != nil { t.Fatalf("Error in bringing up the server: %v", err) } }() diff --git a/test/integration/federation/server_test.go b/test/integration/federation/server_test.go index 8747939bd00..d1bfa722587 100644 --- a/test/integration/federation/server_test.go +++ b/test/integration/federation/server_test.go @@ -88,11 +88,11 @@ var groupVersions = []schema.GroupVersion{ func TestRun(t *testing.T) { s := options.NewServerRunOptions() - s.GenericServerRunOptions.SecurePort = securePort - s.GenericServerRunOptions.InsecurePort = insecurePort + s.SecureServing.ServingOptions.BindPort = securePort + s.InsecureServing.BindPort = insecurePort _, ipNet, _ := net.ParseCIDR("10.10.10.0/24") s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet - s.GenericServerRunOptions.StorageConfig.ServerList = []string{"http://localhost:2379"} + s.Etcd.StorageConfig.ServerList = []string{"http://localhost:2379"} go func() { if err := app.Run(s); err != nil { t.Fatalf("Error in bringing up the server: %v", err)