diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index a3936e17689..78ae941b768 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -202,9 +202,9 @@ func Run(s *options.ServerRunOptions) error { } // Default to the private server key for service account token signing - if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.TLSPrivateKeyFile != "" { - if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.TLSPrivateKeyFile) { - s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.TLSPrivateKeyFile} + if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.SecureServingOptions.ServerCert.CertKey.KeyFile != "" { + if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.SecureServingOptions.ServerCert.CertKey.KeyFile) { + s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.SecureServingOptions.ServerCert.CertKey.KeyFile} } else { glog.Warning("No TLS key provided, service account token authentication disabled") } @@ -225,7 +225,7 @@ func Run(s *options.ServerRunOptions) error { Anonymous: s.GenericServerRunOptions.AnonymousAuth, AnyToken: s.GenericServerRunOptions.EnableAnyToken, BasicAuthFile: s.GenericServerRunOptions.BasicAuthFile, - ClientCAFile: s.GenericServerRunOptions.ClientCAFile, + ClientCAFile: s.GenericServerRunOptions.SecureServingOptions.ClientCA, TokenAuthFile: s.GenericServerRunOptions.TokenAuthFile, OIDCIssuerURL: s.GenericServerRunOptions.OIDCIssuerURL, OIDCClientID: s.GenericServerRunOptions.OIDCClientID, diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index 316f2744af2..8580fe520a3 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -56,7 +56,7 @@ func newStorageFactory() genericapiserver.StorageFactory { } func NewServerRunOptions() *genericoptions.ServerRunOptions { - serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions() + serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions().WithSecureServingOptions() serverOptions.InsecurePort = InsecurePort return serverOptions } diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index f18a9d6e2ac..fcdac6f7ef0 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -121,7 +121,7 @@ func Run(s *options.ServerRunOptions) error { Anonymous: s.GenericServerRunOptions.AnonymousAuth, AnyToken: s.GenericServerRunOptions.EnableAnyToken, BasicAuthFile: s.GenericServerRunOptions.BasicAuthFile, - ClientCAFile: s.GenericServerRunOptions.ClientCAFile, + ClientCAFile: s.GenericServerRunOptions.SecureServingOptions.ClientCA, TokenAuthFile: s.GenericServerRunOptions.TokenAuthFile, OIDCIssuerURL: s.GenericServerRunOptions.OIDCIssuerURL, OIDCClientID: s.GenericServerRunOptions.OIDCClientID, diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index 3f4eb7ef652..c661030d62c 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -226,7 +226,6 @@ func NewConfig() *Config { defaultOptions := options.NewServerRunOptions() // unset fields that can be overridden to avoid setting values so that we won't end up with lingering values. // TODO we probably want to run the defaults the other way. A default here drives it in the CLI flags - defaultOptions.SecurePort = 0 defaultOptions.InsecurePort = 0 defaultOptions.AuditLogPath = "" return config.ApplyOptions(defaultOptions) @@ -243,28 +242,28 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { } } - if options.SecurePort > 0 { + if options.SecureServingOptions != nil && options.SecureServingOptions.ServingOptions.BindPort > 0 { secureServingInfo := &SecureServingInfo{ ServingInfo: ServingInfo{ - BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)), + BindAddress: net.JoinHostPort(options.SecureServingOptions.ServingOptions.BindAddress.String(), strconv.Itoa(options.SecureServingOptions.ServingOptions.BindPort)), }, ServerCert: GeneratableKeyCert{ CertKey: CertKey{ - CertFile: options.TLSCertFile, - KeyFile: options.TLSPrivateKeyFile, + CertFile: options.SecureServingOptions.ServerCert.CertKey.CertFile, + KeyFile: options.SecureServingOptions.ServerCert.CertKey.KeyFile, }, }, SNICerts: []NamedCertKey{}, - ClientCA: options.ClientCAFile, + ClientCA: options.SecureServingOptions.ClientCA, } - if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" { + if options.SecureServingOptions.ServerCert.CertKey.CertFile == "" && options.SecureServingOptions.ServerCert.CertKey.KeyFile == "" { secureServingInfo.ServerCert.Generate = true - secureServingInfo.ServerCert.CertFile = path.Join(options.CertDirectory, "apiserver.crt") - secureServingInfo.ServerCert.KeyFile = path.Join(options.CertDirectory, "apiserver.key") + secureServingInfo.ServerCert.CertFile = path.Join(options.SecureServingOptions.ServerCert.CertDirectory, options.SecureServingOptions.ServerCert.PairName+".crt") + secureServingInfo.ServerCert.KeyFile = path.Join(options.SecureServingOptions.ServerCert.CertDirectory, options.SecureServingOptions.ServerCert.PairName+".key") } secureServingInfo.SNICerts = nil - for _, nkc := range options.SNICertKeys { + for _, nkc := range options.SecureServingOptions.SNICertKeys { secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{ CertKey: CertKey{ KeyFile: nkc.KeyFile, @@ -275,7 +274,7 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { } c.SecureServingInfo = secureServingInfo - c.ReadWritePort = options.SecurePort + c.ReadWritePort = options.SecureServingOptions.ServingOptions.BindPort } if options.InsecurePort > 0 { @@ -488,8 +487,8 @@ func DefaultAndValidateRunOptions(options *options.ServerRunOptions) { // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) - if options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified() { - hostIP, err := utilnet.ChooseBindAddress(options.BindAddress) + if options.SecureServingOptions != nil && (options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified()) { + hostIP, err := utilnet.ChooseBindAddress(options.SecureServingOptions.ServingOptions.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index e021107e3fd..b67d6d73daa 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -54,7 +54,8 @@ var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABA // ServerRunOptions contains the options while running a generic api server. type ServerRunOptions struct { - Etcd *EtcdOptions + Etcd *EtcdOptions + SecureServingOptions *SecureServingOptions AdmissionControl string AdmissionControlConfigFile string @@ -70,9 +71,6 @@ type ServerRunOptions struct { AnonymousAuth bool BasicAuthFile string - BindAddress net.IP - CertDirectory string - ClientCAFile string CloudConfigFile string CloudProvider string CorsAllowedOriginList []string @@ -107,7 +105,6 @@ type ServerRunOptions struct { RequestHeaderClientCAFile string RequestHeaderAllowedNames []string RuntimeConfig config.ConfigurationMap - SecurePort int ServiceClusterIPRange net.IPNet // TODO: make this a list ServiceNodePortRange utilnet.PortRange StorageVersions string @@ -117,9 +114,6 @@ type ServerRunOptions struct { DefaultStorageVersions string TargetRAMMB int TLSCAFile string - TLSCertFile string - TLSPrivateKeyFile string - SNICertKeys []config.NamedCertKey TokenAuthFile string EnableAnyToken bool WatchCacheSizes []string @@ -132,8 +126,6 @@ func NewServerRunOptions() *ServerRunOptions { AuthorizationMode: "AlwaysAllow", AuthorizationWebhookCacheAuthorizedTTL: 5 * time.Minute, AuthorizationWebhookCacheUnauthorizedTTL: 30 * time.Second, - BindAddress: net.ParseIP("0.0.0.0"), - CertDirectory: "/var/run/kubernetes", DefaultStorageMediaType: "application/json", DefaultStorageVersions: registered.AllPreferredGroupVersions(), DeleteCollectionWorkers: 1, @@ -149,7 +141,6 @@ func NewServerRunOptions() *ServerRunOptions { MaxRequestsInFlight: 400, MinRequestTimeout: 1800, RuntimeConfig: make(config.ConfigurationMap), - SecurePort: 6443, ServiceNodePortRange: DefaultServiceNodePortRange, StorageVersions: registered.AllPreferredGroupVersions(), } @@ -159,6 +150,10 @@ func (o *ServerRunOptions) WithEtcdOptions() *ServerRunOptions { o.Etcd = NewDefaultEtcdOptions() return o } +func (o *ServerRunOptions) WithSecureServingOptions() *ServerRunOptions { + o.SecureServingOptions = NewDefaultSecureServingOptions() + return o +} // StorageGroupsToEncodingVersion returns a map from group name to group version, // computed from s.StorageVersions flag. @@ -225,15 +220,16 @@ func (s *ServerRunOptions) NewSelfClientConfig(token string) (*restclient.Config Burst: 100, } - // Use secure port if the TLSCAFile is specified - if s.SecurePort > 0 && len(s.TLSCAFile) > 0 { - host := s.BindAddress.String() + // Use secure port if the ServerCA is specified + if s.SecureServingOptions != nil && s.SecureServingOptions.ServingOptions.BindPort > 0 && len(s.SecureServingOptions.ServerCA) > 0 { + host := s.SecureServingOptions.ServingOptions.BindAddress.String() if host == "0.0.0.0" { host = "localhost" } - clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.SecurePort)) - clientConfig.CAFile = s.TLSCAFile + clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.SecureServingOptions.ServingOptions.BindPort)) + clientConfig.CAFile = s.SecureServingOptions.ServerCA clientConfig.BearerToken = token + } else if s.InsecurePort > 0 { clientConfig.Host = net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)) } else { @@ -293,24 +289,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "If set, the file that will be used to admit requests to the secure port of the API server "+ "via http basic authentication.") - fs.IPVar(&s.BindAddress, "public-address-override", s.BindAddress, - "DEPRECATED: see --bind-address instead.") - fs.MarkDeprecated("public-address-override", "see --bind-address instead.") - - fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+ - "The IP address on which to listen for the --secure-port port. The "+ - "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ - "clients. If blank, all interfaces will be used (0.0.0.0).") - - fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, ""+ - "The directory where the TLS certs are located (by default /var/run/kubernetes). "+ - "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") - - fs.StringVar(&s.ClientCAFile, "client-ca-file", s.ClientCAFile, ""+ - "If set, any request presenting a client certificate signed by one of "+ - "the authorities in the client-ca-file is authenticated with an identity "+ - "corresponding to the CommonName of the client certificate.") - fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") @@ -448,10 +426,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "apis// can be used to turn on/off specific resources. api/all and "+ "api/legacy are special keys to control all and legacy api versions respectively.") - fs.IntVar(&s.SecurePort, "secure-port", s.SecurePort, ""+ - "The port on which to serve HTTPS with authentication and authorization. If 0, "+ - "don't serve HTTPS at all.") - fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+ "A CIDR notation IP range from which to assign service cluster IPs. This must not "+ "overlap with any IP ranges assigned to nodes for pods.") @@ -481,28 +455,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "It defaults to a list of preferred versions of all registered groups, "+ "which is derived from the KUBE_API_VERSIONS environment variable.") - fs.StringVar(&s.TLSCAFile, "tls-ca-file", s.TLSCAFile, "If set, this "+ - "certificate authority will used for secure access from Admission "+ - "Controllers. This must be a valid PEM-encoded CA bundle.") - - fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+ - "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ - "after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+ - "--tls-private-key-file are not provided, a self-signed certificate and key "+ - "are generated for the public address and saved to /var/run/kubernetes.") - - fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile, - "File containing the default x509 private key matching --tls-cert-file.") - - fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+ - "A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+ - "domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+ - "segments. If no domain patterns are provided, the names of the certificate are "+ - "extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+ - "trump over extracted names. For multiple key/certificate pairs, use the "+ - "--tls-sni-cert-key multiple times. "+ - "Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".") - fs.StringVar(&s.TokenAuthFile, "token-auth-file", s.TokenAuthFile, ""+ "If set, the file that will be used to secure the secure port of the API server "+ "via token authentication.") diff --git a/pkg/genericapiserver/options/serving_options.go b/pkg/genericapiserver/options/serving_options.go new file mode 100644 index 00000000000..1bfb60ab064 --- /dev/null +++ b/pkg/genericapiserver/options/serving_options.go @@ -0,0 +1,147 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "net" + + "github.com/spf13/pflag" + + "k8s.io/kubernetes/pkg/util/config" +) + +type ServingOptions struct { + BindAddress net.IP + BindPort int +} + +type SecureServingOptions struct { + ServingOptions ServingOptions + + // ServerCert is the TLS cert info for serving secure traffic + ServerCert GeneratableKeyCert + // SNICertKeys are named CertKeys for serving secure traffic with SNI support. + SNICertKeys []config.NamedCertKey + // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates + ClientCA string + + // ServerCA is the certificate bundle for the signer of your serving certificate. Used for building a loopback + // connection to the API server for admission. + ServerCA string +} + +type CertKey struct { + // CertFile is a file containing a PEM-encoded certificate + CertFile string + // KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile + KeyFile string +} + +type GeneratableKeyCert struct { + CertKey CertKey + + // CertDirectory is a directory that will contain the certificates. If the cert and key aren't specifically set + // this will be used to derive a match with the "pair-name" + CertDirectory string + // PairName is the name which will be used with CertDirectory to make a cert and key names + // It becomes CertDirector/PairName.crt and CertDirector/PairName.key + PairName string +} + +func NewDefaultSecureServingOptions() *SecureServingOptions { + return &SecureServingOptions{ + ServingOptions: ServingOptions{ + BindAddress: net.ParseIP("0.0.0.0"), + BindPort: 6443, + }, + ServerCert: GeneratableKeyCert{ + PairName: "apiserver", + CertDirectory: "/var/run/kubernetes", + }, + } +} + +func (s *SecureServingOptions) Validate() []error { + errors := []error{} + if s == nil { + return errors + } + + errors = append(errors, s.ServingOptions.Validate("secure-port")...) + return errors +} + +func (s ServingOptions) Validate(portArg string) []error { + errors := []error{} + + if s.BindPort < 0 || s.BindPort > 65535 { + errors = append(errors, fmt.Errorf("--%v %v must be between 0 and 65535, inclusive. 0 for turning off secure port.", portArg, s.BindPort)) + } + + return errors +} + +func (s *SecureServingOptions) AddSecureServingFlags(fs *pflag.FlagSet) { + fs.IPVar(&s.ServingOptions.BindAddress, "bind-address", s.ServingOptions.BindAddress, ""+ + "The IP address on which to listen for the --secure-port port. The "+ + "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ + "clients. If blank, all interfaces will be used (0.0.0.0).") + + fs.IntVar(&s.ServingOptions.BindPort, "secure-port", s.ServingOptions.BindPort, ""+ + "The port on which to serve HTTPS with authentication and authorization. If 0, "+ + "don't serve HTTPS at all.") + + fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+ + "The directory where the TLS certs are located (by default /var/run/kubernetes). "+ + "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") + + fs.StringVar(&s.ServerCert.CertKey.CertFile, "tls-cert-file", s.ServerCert.CertKey.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+ + "--tls-private-key-file are not provided, a self-signed certificate and key "+ + "are generated for the public address and saved to /var/run/kubernetes.") + + fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile, + "File containing the default x509 private key matching --tls-cert-file.") + + fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+ + "A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+ + "domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+ + "segments. If no domain patterns are provided, the names of the certificate are "+ + "extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+ + "trump over extracted names. For multiple key/certificate pairs, use the "+ + "--tls-sni-cert-key multiple times. "+ + "Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".") + + fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+ + "If set, any request presenting a client certificate signed by one of "+ + "the authorities in the client-ca-file is authenticated with an identity "+ + "corresponding to the CommonName of the client certificate.") + + fs.StringVar(&s.ServerCA, "tls-ca-file", s.ServerCA, "If set, this "+ + "certificate authority will used for secure access from Admission "+ + "Controllers. This must be a valid PEM-encoded CA bundle.") + +} + +func (s *SecureServingOptions) AddDeprecatedSecureServingFlags(fs *pflag.FlagSet) { + fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress, + "DEPRECATED: see --bind-address instead.") + fs.MarkDeprecated("public-address-override", "see --bind-address instead.") + +} diff --git a/pkg/genericapiserver/validation/universal_validation.go b/pkg/genericapiserver/validation/universal_validation.go index 6577500dd8e..4821db9bc38 100644 --- a/pkg/genericapiserver/validation/universal_validation.go +++ b/pkg/genericapiserver/validation/universal_validation.go @@ -51,19 +51,17 @@ func verifyServiceNodePort(options *options.ServerRunOptions) []error { func verifySecureAndInsecurePort(options *options.ServerRunOptions) []error { errors := []error{} - if options.SecurePort < 0 || options.SecurePort > 65535 { - errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port.", options.SecurePort)) - } + errors = append(errors, options.SecureServingOptions.Validate()...) if options.InsecurePort < 0 || options.InsecurePort > 65535 { errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 65535, inclusive. 0 for turning off insecure port.", options.InsecurePort)) } - if options.SecurePort == 0 && options.InsecurePort == 0 { + if (options.SecureServingOptions == nil || options.SecureServingOptions.ServingOptions.BindPort == 0) && options.InsecurePort == 0 { glog.Fatalf("--secure-port and --insecure-port cannot be turned off at the same time.") } - if options.SecurePort == options.InsecurePort { + if options.SecureServingOptions != nil && options.SecureServingOptions.ServingOptions.BindPort == options.InsecurePort { errors = append(errors, fmt.Errorf("--secure-port and --insecure-port cannot use the same port.")) } return errors diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 858d2ee390e..4b9a33c9bd2 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -64,7 +64,7 @@ func TestRunSecureServer(t *testing.T) { go func() { options := apiserver.NewServerRunOptions() options.InsecurePort = 0 - options.SecurePort = apiserver.SecurePort + options.SecureServingOptions.ServingOptions.BindPort = apiserver.SecurePort if err := apiserver.Run(options, stopCh); err != nil { t.Fatalf("Error in bringing up the server: %v", err) }