diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 8f9e3a92174..4e0ca207b1c 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -166,6 +166,7 @@ func TestAddFlags(t *testing.T) { PairName: "apiserver", }, HTTP2MaxStreamsPerConnection: 42, + Required: true, }), InsecureServing: &kubeoptions.InsecureServingOptions{ BindAddress: net.ParseIP("127.0.0.1"), diff --git a/pkg/kubeapiserver/options/serving.go b/pkg/kubeapiserver/options/serving.go index 85b102c9751..c2bad3e892d 100644 --- a/pkg/kubeapiserver/options/serving.go +++ b/pkg/kubeapiserver/options/serving.go @@ -36,6 +36,7 @@ func NewSecureServingOptions() *genericoptions.SecureServingOptionsWithLoopback return genericoptions.WithLoopback(&genericoptions.SecureServingOptions{ BindAddress: net.ParseIP("0.0.0.0"), BindPort: 6443, + Required: true, ServerCert: genericoptions.GeneratableKeyCert{ PairName: "apiserver", CertDirectory: "/var/run/kubernetes", diff --git a/pkg/master/controller.go b/pkg/master/controller.go index ac5a86db1b6..466209195f8 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -79,6 +79,11 @@ type Controller struct { // NewBootstrapController returns a controller for watching the core capabilities of the master func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTStorage, serviceClient coreclient.ServicesGetter, nsClient coreclient.NamespacesGetter, eventClient coreclient.EventsGetter) *Controller { + _, publicServicePort, err := c.GenericConfig.SecureServing.HostPort() + if err != nil { + glog.Fatalf("failed to get listener address: %v", err) + } + return &Controller{ ServiceClient: serviceClient, NamespaceClient: nsClient, @@ -104,7 +109,7 @@ func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.Lega ServicePort: c.ExtraConfig.APIServerServicePort, ExtraServicePorts: c.ExtraConfig.ExtraServicePorts, ExtraEndpointPorts: c.ExtraConfig.ExtraEndpointPorts, - PublicServicePort: c.GenericConfig.ReadWritePort, + PublicServicePort: publicServicePort, KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort, } } diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index eb70321696a..360d11931ff 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -113,6 +113,9 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, informers.SharedI TLSClientConfig: &tls.Config{}, }) + // set fake SecureServingInfo because the listener port is needed for the kubernetes service + config.GenericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}} + clientset, err := kubernetes.NewForConfig(config.GenericConfig.LoopbackClientConfig) if err != nil { t.Fatalf("unable to create client set due to %v", err) @@ -122,6 +125,23 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, informers.SharedI return server, *config, sharedInformers, assert.New(t) } +type fakeLocalhost443Listener struct{} + +func (fakeLocalhost443Listener) Accept() (net.Conn, error) { + return nil, nil +} + +func (fakeLocalhost443Listener) Close() error { + return nil +} + +func (fakeLocalhost443Listener) Addr() net.Addr { + return &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: 443, + } +} + // TestLegacyRestStorageStrategies ensures that all Storage objects which are using the generic registry Store have // their various strategies properly wired up. This surfaced as a bug where strategies defined Export functions, but // they were never used outside of unit tests because the export strategies were not assigned inside the Store. diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 5097a98b62b..43fb57cd468 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -181,9 +181,6 @@ type Config struct { // values below here are targets for removal //=========================================================================== - // The port on PublicAddress where a read-write server will be installed. - // Defaults to 6443 if not set. - ReadWritePort int // PublicAddress is the IP address where members of the cluster (kubelet, // kube-proxy, services, etc.) can reach the GenericAPIServer. // If nil or 0.0.0.0, the host's default interface will be used. @@ -250,7 +247,6 @@ type AuthorizationInfo struct { func NewConfig(codecs serializer.CodecFactory) *Config { return &Config{ Serializer: codecs, - ReadWritePort: 443, BuildHandlerChainFunc: DefaultBuildHandlerChain, HandlerChainWaitGroup: new(utilwaitgroup.SafeWaitGroup), LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix), @@ -354,16 +350,21 @@ type CompletedConfig struct { // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { - host := c.ExternalAddress - if host == "" && c.PublicAddress != nil { - host = c.PublicAddress.String() + if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { + c.ExternalAddress = c.PublicAddress.String() } - // if there is no port, and we have a ReadWritePort, use that - if _, _, err := net.SplitHostPort(host); err != nil && c.ReadWritePort != 0 { - host = net.JoinHostPort(host, strconv.Itoa(c.ReadWritePort)) + // if there is no port, and we listen on one securely, use that one + if _, _, err := net.SplitHostPort(c.ExternalAddress); err != nil { + if c.SecureServing == nil { + glog.Fatalf("cannot derive external address port without listening on a secure port.") + } + _, port, err := c.SecureServing.HostPort() + if err != nil { + glog.Fatalf("cannot derive external address from the secure port: %v", err) + } + c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - c.ExternalAddress = host if c.OpenAPIConfig != nil && c.OpenAPIConfig.SecurityDefinitions != nil { // Setup OpenAPI security: all APIs will have the same authentication for now. @@ -615,3 +616,19 @@ func NewRequestInfoResolver(c *Config) *apirequest.RequestInfoFactory { GrouplessAPIPrefixes: legacyAPIPrefixes, } } + +func (s *SecureServingInfo) HostPort() (string, int, error) { + if s == nil || s.Listener == nil { + return "", 0, fmt.Errorf("no listener found") + } + addr := s.Listener.Addr().String() + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return "", 0, fmt.Errorf("failed to get port from listener address %q: %v", addr, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid non-numeric port %q", portStr) + } + return host, port, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index 13f0c7409e1..07fece4c7ae 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -34,6 +34,7 @@ import ( func TestNewWithDelegate(t *testing.T) { delegateConfig := NewConfig(codecs) + delegateConfig.ExternalAddress = "192.168.10.4:443" delegateConfig.PublicAddress = net.ParseIP("192.168.10.4") delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") delegateConfig.LoopbackClientConfig = &rest.Config{} @@ -64,6 +65,7 @@ func TestNewWithDelegate(t *testing.T) { delegateServer.PrepareRun() wrappingConfig := NewConfig(codecs) + wrappingConfig.ExternalAddress = "192.168.10.4:443" wrappingConfig.PublicAddress = net.ParseIP("192.168.10.4") wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") wrappingConfig.LoopbackClientConfig = &rest.Config{} diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index bde27d4833d..a3dda1a44ff 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -125,6 +125,7 @@ func testGetOpenAPIDefinitions(_ kubeopenapi.ReferenceCallback) map[string]kubeo // setUp is a convience function for setting up for (most) tests. func setUp(t *testing.T) (Config, *assert.Assertions) { config := NewConfig(codecs) + config.ExternalAddress = "192.168.10.4:443" config.PublicAddress = net.ParseIP("192.168.10.4") config.LegacyAPIGroupPrefixes = sets.NewString("/api") config.LoopbackClientConfig = &restclient.Config{} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index ee9cebacd04..7a2f570fc7c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -40,6 +40,8 @@ type SecureServingOptions struct { // BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp", // "tcp4", and "tcp6". BindNetwork string + // Required set to true means that BindPort cannot be zero. + Required bool // Listener is the secure server network listener. // either Listener or BindAddress/BindPort/BindNetwork is set, @@ -102,7 +104,9 @@ func (s *SecureServingOptions) Validate() []error { errors := []error{} - if s.BindPort < 0 || s.BindPort > 65535 { + if s.Required && s.BindPort < 1 || s.BindPort > 65535 { + errors = append(errors, fmt.Errorf("--secure-port %v must be between 1 and 65535, inclusive. It cannot turned off with 0", s.BindPort)) + } else if s.BindPort < 0 || s.BindPort > 65535 { errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port", s.BindPort)) } @@ -118,9 +122,14 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { "The IP address on which to listen for the --secure-port port. The "+ "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ "clients. If blank, all interfaces will be used (0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces).") - fs.IntVar(&s.BindPort, "secure-port", s.BindPort, ""+ - "The port on which to serve HTTPS with authentication and authorization. If 0, "+ - "don't serve HTTPS at all.") + + desc := "The port on which to serve HTTPS with authentication and authorization." + if s.Required { + desc += "It cannot switched off with 0." + } else { + desc += "If 0, don't serve HTTPS at all." + } + fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc) fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+ "The directory where the TLS certs are located. "+ diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go index 8d249cb54b4..dd6e0e1a7f5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go @@ -48,8 +48,6 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(c *server.Config) error { return nil } - c.ReadWritePort = s.BindPort - // create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and // let the server return it when the loopback client connects. certPem, keyPem, err := certutil.GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil) diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 070c802657a..d3e6179979e 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -225,6 +225,10 @@ func NewIntegrationTestMasterConfig() *master.Config { masterConfig := NewMasterConfig() masterConfig.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4") masterConfig.ExtraConfig.APIResourceConfigSource = master.DefaultAPIResourceConfigSource() + + // TODO: get rid of these tests or port them to secure serving + masterConfig.GenericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}} + return masterConfig } @@ -291,6 +295,9 @@ func NewMasterConfig() *master.Config { genericConfig.Version = &kubeVersion genericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + // TODO: get rid of these tests or port them to secure serving + genericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}} + err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, genericConfig) if err != nil { panic(err) @@ -329,3 +336,20 @@ func SharedEtcd() *storagebackend.Config { cfg.ServerList = []string{GetEtcdURL()} return cfg } + +type fakeLocalhost443Listener struct{} + +func (fakeLocalhost443Listener) Accept() (net.Conn, error) { + return nil, nil +} + +func (fakeLocalhost443Listener) Close() error { + return nil +} + +func (fakeLocalhost443Listener) Addr() net.Addr { + return &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: 443, + } +}