From d82f98c9b3dbbdf7daed2a17fb44ecc3ac7d23ad Mon Sep 17 00:00:00 2001 From: deads2k Date: Mon, 31 Oct 2016 15:51:52 -0400 Subject: [PATCH] remove non-generic options from genericapiserver.Config --- cmd/kube-apiserver/app/server.go | 15 +++- pkg/genericapiserver/BUILD | 4 +- pkg/genericapiserver/config.go | 88 +++---------------- pkg/genericapiserver/genericapiserver.go | 5 -- pkg/genericapiserver/genericapiserver_test.go | 17 ++-- pkg/genericapiserver/services.go | 54 ++++++++++++ pkg/master/BUILD | 2 +- pkg/master/controller.go | 18 ++-- pkg/master/master.go | 77 ++++++++++++++-- pkg/master/master_test.go | 14 +-- pkg/registry/core/rest/storage_core.go | 14 +-- test/integration/framework/master_utils.go | 2 + test/integration/master/master_test.go | 18 ++-- 13 files changed, 191 insertions(+), 137 deletions(-) create mode 100644 pkg/genericapiserver/services.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 917b0acdabb..8dd8a754fb6 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -86,7 +86,11 @@ func Run(s *options.ServerRunOptions) error { ApplyOptions(s.GenericServerRunOptions). // apply the options selected Complete() // set default values based on the known values - if err := genericConfig.MaybeGenerateServingCerts(); err != nil { + serviceIPRange, apiServerServiceIP, err := genericapiserver.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange) + if err != nil { + glog.Fatalf("Error determining service IP ranges: %v", err) + } + if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil { glog.Fatalf("Failed to generate service certificate: %v", err) } @@ -322,6 +326,15 @@ func Run(s *options.ServerRunOptions) error { ProxyTransport: proxyTransport, Tunneler: tunneler, + + ServiceIPRange: serviceIPRange, + APIServerServiceIP: apiServerServiceIP, + APIServerServicePort: 443, + + ServiceNodePortRange: s.GenericServerRunOptions.ServiceNodePortRange, + KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort, + + MasterCount: s.GenericServerRunOptions.MasterCount, } if s.GenericServerRunOptions.EnableWatchCache { diff --git a/pkg/genericapiserver/BUILD b/pkg/genericapiserver/BUILD index c2864e1ee08..985ab477c41 100644 --- a/pkg/genericapiserver/BUILD +++ b/pkg/genericapiserver/BUILD @@ -15,6 +15,7 @@ go_library( srcs = [ "config.go", "default_storage_factory_builder.go", + "discovery.go", "doc.go", "genericapiserver.go", "healthz.go", @@ -23,6 +24,7 @@ go_library( "resource_encoding_config.go", "reststorage_interfaces.go", "serve.go", + "services.go", "storage_factory.go", "tunneler.go", ], @@ -102,11 +104,11 @@ go_test( "//pkg/auth/user:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/genericapiserver/options:go_default_library", - "//pkg/registry/core/service/ipallocator:go_default_library", "//pkg/storage/etcd/testing:go_default_library", "//pkg/storage/storagebackend:go_default_library", "//pkg/util/cert:go_default_library", "//pkg/util/clock:go_default_library", + "//pkg/util/net:go_default_library", "//pkg/util/sets:go_default_library", "//pkg/version:go_default_library", "//vendor:github.com/go-openapi/spec", diff --git a/pkg/genericapiserver/config.go b/pkg/genericapiserver/config.go index 62d34eee7bd..9fbca9acbc7 100644 --- a/pkg/genericapiserver/config.go +++ b/pkg/genericapiserver/config.go @@ -50,7 +50,6 @@ import ( "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/genericapiserver/routes" genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" - ipallocator "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/runtime" certutil "k8s.io/kubernetes/pkg/util/cert" utilnet "k8s.io/kubernetes/pkg/util/net" @@ -107,13 +106,13 @@ type Config struct { // Note that it is up to the request handlers to ignore or honor this timeout. In seconds. MinRequestTimeout int - // Number of masters running; all masters must be started with the - // same value for this field. (Numbers > 1 currently untested.) - MasterCount int - SecureServingInfo *SecureServingInfo InsecureServingInfo *ServingInfo + // DiscoveryAddresses is used to build the IPs pass to discovery. If nil, the ExternalAddress is + // always reported + DiscoveryAddresses DiscoveryAddresses + // The port on PublicAddress where a read-write server will be installed. // Defaults to 6443 if not set. ReadWritePort int @@ -126,34 +125,6 @@ type Config struct { // If nil or 0.0.0.0, the host's default interface will be used. PublicAddress net.IP - // The range of IPs to be assigned to services with type=ClusterIP or greater - ServiceClusterIPRange *net.IPNet - - // The IP address for the GenericAPIServer service (must be inside ServiceClusterIPRange) - ServiceReadWriteIP net.IP - - // Port for the apiserver service. - ServiceReadWritePort int - - // The range of ports to be assigned to services with type=NodePort or greater - ServiceNodePortRange utilnet.PortRange - - // Additional ports to be exposed on the GenericAPIServer service - // extraServicePorts is injectable in the event that more ports - // (other than the default 443/tcp) are exposed on the GenericAPIServer - // and those ports need to be load balanced by the GenericAPIServer - // service because this pkg is linked by out-of-tree projects - // like openshift which want to use the GenericAPIServer but also do - // more stuff. - ExtraServicePorts []api.ServicePort - // Additional ports to be exposed on the GenericAPIServer endpoints - // Port names should align with ports defined in ExtraServicePorts - ExtraEndpointPorts []api.EndpointPort - - // If non-zero, the "kubernetes" services uses this port as NodePort. - // TODO(sttts): move into master - KubernetesServiceNodePort int - // EnableOpenAPISupport enables OpenAPI support. Allow downstream customers to disable OpenAPI spec. EnableOpenAPISupport bool @@ -221,9 +192,7 @@ func NewConfig() *Config { config := &Config{ Serializer: api.Codecs, - MasterCount: 1, ReadWritePort: 6443, - ServiceReadWritePort: 443, RequestContextMapper: api.NewRequestContextMapper(), BuildHandlerChainsFunc: DefaultBuildHandlerChain, LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix), @@ -318,13 +287,9 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config { c.EnableProfiling = options.EnableProfiling c.EnableSwaggerUI = options.EnableSwaggerUI c.ExternalAddress = options.ExternalHost - c.KubernetesServiceNodePort = options.KubernetesServiceNodePort - c.MasterCount = options.MasterCount c.MaxRequestsInFlight = options.MaxRequestsInFlight c.MinRequestTimeout = options.MinRequestTimeout c.PublicAddress = options.AdvertiseAddress - c.ServiceClusterIPRange = &options.ServiceClusterIPRange - c.ServiceNodePortRange = options.ServiceNodePortRange c.SupportsBasicAuth = len(options.BasicAuthFile) > 0 return c @@ -337,35 +302,6 @@ type completedConfig struct { // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. func (c *Config) Complete() completedConfig { - if c.ServiceClusterIPRange == nil || c.ServiceClusterIPRange.IP == nil { - defaultNet := "10.0.0.0/24" - glog.Warningf("Network range for service cluster IPs is unspecified. Defaulting to %v.", defaultNet) - _, serviceClusterIPRange, err := net.ParseCIDR(defaultNet) - if err != nil { - glog.Fatalf("Unable to parse CIDR: %v", err) - } - if size := ipallocator.RangeSize(serviceClusterIPRange); size < 8 { - glog.Fatalf("The service cluster IP range must be at least %d IP addresses", 8) - } - c.ServiceClusterIPRange = serviceClusterIPRange - } - if c.ServiceReadWriteIP == nil { - // Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP. - serviceReadWriteIP, err := ipallocator.GetIndexedIP(c.ServiceClusterIPRange, 1) - if err != nil { - glog.Fatalf("Failed to generate service read-write IP for GenericAPIServer service: %v", err) - } - glog.V(4).Infof("Setting GenericAPIServer service IP to %q (read-write).", serviceReadWriteIP) - c.ServiceReadWriteIP = serviceReadWriteIP - } - if c.ServiceNodePortRange.Size == 0 { - // TODO: Currently no way to specify an empty range (do we need to allow this?) - // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) - // but then that breaks the strict nestedness of ServiceType. - // Review post-v1 - c.ServiceNodePortRange = options.DefaultServiceNodePortRange - glog.Infof("Node port range unspecified. Defaulting to %v.", c.ServiceNodePortRange) - } if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { hostAndPort := c.PublicAddress.String() if c.ReadWritePort != 0 { @@ -395,6 +331,10 @@ func (c *Config) Complete() completedConfig { } } } + if c.DiscoveryAddresses == nil { + c.DiscoveryAddresses = DefaultDiscoveryAddresses{DefaultAddress: c.ExternalAddress} + } + return completedConfig{c} } @@ -430,14 +370,8 @@ func (c completedConfig) New() (*GenericAPIServer, error) { return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil") } - discoveryAddresses := DefaultDiscoveryAddresses{DefaultAddress: c.ExternalAddress} - if c.ServiceClusterIPRange != nil { - discoveryAddresses.DiscoveryCIDRRules = append(discoveryAddresses.DiscoveryCIDRRules, - DiscoveryCIDRRule{IPRange: *c.ServiceClusterIPRange, Address: net.JoinHostPort(c.ServiceReadWriteIP.String(), strconv.Itoa(c.ServiceReadWritePort))}) - } - s := &GenericAPIServer{ - discoveryAddresses: discoveryAddresses, + discoveryAddresses: c.DiscoveryAddresses, LoopbackClientConfig: c.LoopbackClientConfig, legacyAPIGroupPrefixes: c.LegacyAPIGroupPrefixes, admissionControl: c.AdmissionControl, @@ -447,7 +381,6 @@ func (c completedConfig) New() (*GenericAPIServer, error) { minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second, enableSwaggerSupport: c.EnableSwaggerSupport, - MasterCount: c.MasterCount, SecureServingInfo: c.SecureServingInfo, InsecureServingInfo: c.InsecureServingInfo, ExternalAddress: c.ExternalAddress, @@ -470,12 +403,11 @@ func (c completedConfig) New() (*GenericAPIServer, error) { } // MaybeGenerateServingCerts generates serving certificates if requested and needed. -func (c completedConfig) MaybeGenerateServingCerts() error { +func (c completedConfig) MaybeGenerateServingCerts(alternateIPs ...net.IP) error { // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") if c.SecureServingInfo != nil && c.SecureServingInfo.ServerCert.Generate && !certutil.CanReadCertOrKey(c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile) { // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? - alternateIPs := []net.IP{c.ServiceReadWriteIP} alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil { diff --git a/pkg/genericapiserver/genericapiserver.go b/pkg/genericapiserver/genericapiserver.go index 3ed71cb715e..79f21d8902d 100644 --- a/pkg/genericapiserver/genericapiserver.go +++ b/pkg/genericapiserver/genericapiserver.go @@ -19,7 +19,6 @@ package genericapiserver import ( "fmt" "mime" - "net" "net/http" "sort" "strings" @@ -147,10 +146,6 @@ type GenericAPIServer struct { healthzLock sync.Mutex healthzChecks []healthz.HealthzChecker healthzCreated bool - - // See Config.$name for documentation of these flags: - - MasterCount int } func init() { diff --git a/pkg/genericapiserver/genericapiserver_test.go b/pkg/genericapiserver/genericapiserver_test.go index 3e105fee44e..c1920b0a466 100644 --- a/pkg/genericapiserver/genericapiserver_test.go +++ b/pkg/genericapiserver/genericapiserver_test.go @@ -25,7 +25,6 @@ import ( "net/http" "net/http/httptest" "reflect" - "strconv" "testing" "k8s.io/kubernetes/pkg/api" @@ -376,20 +375,17 @@ func TestDiscoveryAtAPIS(t *testing.T) { } func TestGetServerAddressByClientCIDRs(t *testing.T) { - s, etcdserver, config, _ := newMaster(t) - defer etcdserver.Terminate(t) - publicAddressCIDRMap := []unversioned.ServerAddressByClientCIDR{ { ClientCIDR: "0.0.0.0/0", - ServerAddress: config.ExternalAddress, + ServerAddress: "ExternalAddress", }, } internalAddressCIDRMap := []unversioned.ServerAddressByClientCIDR{ publicAddressCIDRMap[0], { - ClientCIDR: config.ServiceClusterIPRange.String(), - ServerAddress: net.JoinHostPort(config.ServiceReadWriteIP.String(), strconv.Itoa(config.ServiceReadWritePort)), + ClientCIDR: "10.0.0.0/24", + ServerAddress: "serviceIP", }, } internalIP := "10.0.0.1" @@ -455,8 +451,13 @@ func TestGetServerAddressByClientCIDRs(t *testing.T) { }, } + _, ipRange, _ := net.ParseCIDR("10.0.0.0/24") + discoveryAddresses := DefaultDiscoveryAddresses{DefaultAddress: "ExternalAddress"} + discoveryAddresses.DiscoveryCIDRRules = append(discoveryAddresses.DiscoveryCIDRRules, + DiscoveryCIDRRule{IPRange: *ipRange, Address: "serviceIP"}) + for i, test := range testCases { - if a, e := s.discoveryAddresses.ServerAddressByClientCIDRs(utilnet.GetClientIP(&test.Request)), test.ExpectedMap; reflect.DeepEqual(e, a) != true { + if a, e := discoveryAddresses.ServerAddressByClientCIDRs(utilnet.GetClientIP(&test.Request)), test.ExpectedMap; reflect.DeepEqual(e, a) != true { t.Fatalf("test case %d failed. expected: %v, actual: %v", i+1, e, a) } } diff --git a/pkg/genericapiserver/services.go b/pkg/genericapiserver/services.go new file mode 100644 index 00000000000..d6f71207a3a --- /dev/null +++ b/pkg/genericapiserver/services.go @@ -0,0 +1,54 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "fmt" + "net" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" +) + +// DefaultServiceIPRange takes a the serviceIPRange flag and returns the defaulted service ip range (if needed), +// api server service IP, and an error +// TODO move this out of the genericapiserver package +func DefaultServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) { + serviceClusterIPRange := passedServiceClusterIPRange + if passedServiceClusterIPRange.IP == nil { + defaultNet := "10.0.0.0/24" + glog.Infof("Network range for service cluster IPs is unspecified. Defaulting to %v.", defaultNet) + _, defaultServiceClusterIPRange, err := net.ParseCIDR(defaultNet) + if err != nil { + return net.IPNet{}, net.IP{}, err + } + serviceClusterIPRange = *defaultServiceClusterIPRange + } + if size := ipallocator.RangeSize(&serviceClusterIPRange); size < 8 { + return net.IPNet{}, net.IP{}, fmt.Errorf("The service cluster IP range must be at least %d IP addresses", 8) + } + + // Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP. + apiServerServiceIP, err := ipallocator.GetIndexedIP(&serviceClusterIPRange, 1) + if err != nil { + return net.IPNet{}, net.IP{}, err + } + glog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP) + + return serviceClusterIPRange, apiServerServiceIP, nil +} diff --git a/pkg/master/BUILD b/pkg/master/BUILD index bde0e4f0bc2..a504b4ef046 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -52,6 +52,7 @@ go_library( "//pkg/apis/storage/v1beta1:go_default_library", "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", "//pkg/genericapiserver:go_default_library", + "//pkg/genericapiserver/options:go_default_library", "//pkg/healthz:go_default_library", "//pkg/kubelet/client:go_default_library", "//pkg/master/thirdparty:go_default_library", @@ -116,7 +117,6 @@ go_test( "//pkg/generated/openapi:go_default_library", "//pkg/genericapiserver:go_default_library", "//pkg/kubelet/client:go_default_library", - "//pkg/registry/core/service/ipallocator:go_default_library", "//pkg/registry/registrytest:go_default_library", "//pkg/runtime:go_default_library", "//pkg/storage/etcd/testing:go_default_library", diff --git a/pkg/master/controller.go b/pkg/master/controller.go index 880d1fdfa6f..01ef1f6f0a6 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -50,7 +50,7 @@ type Controller struct { ServiceClusterIPRegistry rangeallocation.RangeRegistry ServiceClusterIPInterval time.Duration - ServiceClusterIPRange *net.IPNet + ServiceClusterIPRange net.IPNet ServiceNodePortRegistry rangeallocation.RangeRegistry ServiceNodePortInterval time.Duration @@ -87,21 +87,21 @@ func (c *Config) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTSto SystemNamespacesInterval: 1 * time.Minute, ServiceClusterIPRegistry: legacyRESTStorage.ServiceClusterIPAllocator, - ServiceClusterIPRange: c.GenericConfig.ServiceClusterIPRange, + ServiceClusterIPRange: c.ServiceIPRange, ServiceClusterIPInterval: 3 * time.Minute, ServiceNodePortRegistry: legacyRESTStorage.ServiceNodePortAllocator, - ServiceNodePortRange: c.GenericConfig.ServiceNodePortRange, + ServiceNodePortRange: c.ServiceNodePortRange, ServiceNodePortInterval: 3 * time.Minute, PublicIP: c.GenericConfig.PublicAddress, - ServiceIP: c.GenericConfig.ServiceReadWriteIP, - ServicePort: c.GenericConfig.ServiceReadWritePort, - ExtraServicePorts: c.GenericConfig.ExtraServicePorts, - ExtraEndpointPorts: c.GenericConfig.ExtraEndpointPorts, + ServiceIP: c.APIServerServiceIP, + ServicePort: c.APIServerServicePort, + ExtraServicePorts: c.ExtraServicePorts, + ExtraEndpointPorts: c.ExtraEndpointPorts, PublicServicePort: c.GenericConfig.ReadWritePort, - KubernetesServiceNodePort: c.GenericConfig.KubernetesServiceNodePort, + KubernetesServiceNodePort: c.KubernetesServiceNodePort, } } @@ -117,7 +117,7 @@ func (c *Controller) Start() { return } - repairClusterIPs := servicecontroller.NewRepair(c.ServiceClusterIPInterval, c.ServiceRegistry, c.ServiceClusterIPRange, c.ServiceClusterIPRegistry) + repairClusterIPs := servicecontroller.NewRepair(c.ServiceClusterIPInterval, c.ServiceRegistry, &c.ServiceClusterIPRange, c.ServiceClusterIPRegistry) repairNodePorts := portallocatorcontroller.NewRepair(c.ServiceNodePortInterval, c.ServiceRegistry, c.ServiceNodePortRange, c.ServiceNodePortRegistry) // run all of the controllers once prior to returning from Start. diff --git a/pkg/master/master.go b/pkg/master/master.go index 1d415864b7f..b7828c0d82a 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -18,8 +18,10 @@ package master import ( "fmt" + "net" "net/http" "reflect" + "strconv" "time" "k8s.io/kubernetes/pkg/api" @@ -37,9 +39,11 @@ import ( storageapiv1beta1 "k8s.io/kubernetes/pkg/apis/storage/v1beta1" coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/genericapiserver" + "k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/healthz" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/thirdparty" + utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/kubernetes/pkg/registry/generic" "k8s.io/kubernetes/pkg/registry/generic/registry" @@ -84,6 +88,38 @@ type Config struct { EnableUISupport bool EnableLogsSupport bool ProxyTransport http.RoundTripper + + // Values to build the IP addresses used by discovery + // The range of IPs to be assigned to services with type=ClusterIP or greater + ServiceIPRange net.IPNet + // The IP address for the GenericAPIServer service (must be inside ServiceIPRange) + APIServerServiceIP net.IP + // Port for the apiserver service. + APIServerServicePort int + + // TODO, we can probably group service related items into a substruct to make it easier to configure + // the API server items and `Extra*` fields likely fit nicely together. + + // The range of ports to be assigned to services with type=NodePort or greater + ServiceNodePortRange utilnet.PortRange + // Additional ports to be exposed on the GenericAPIServer service + // extraServicePorts is injectable in the event that more ports + // (other than the default 443/tcp) are exposed on the GenericAPIServer + // and those ports need to be load balanced by the GenericAPIServer + // service because this pkg is linked by out-of-tree projects + // like openshift which want to use the GenericAPIServer but also do + // more stuff. + ExtraServicePorts []api.ServicePort + // Additional ports to be exposed on the GenericAPIServer endpoints + // Port names should align with ports defined in ExtraServicePorts + ExtraEndpointPorts []api.EndpointPort + // If non-zero, the "kubernetes" services uses this port as NodePort. + // TODO(sttts): move into master + KubernetesServiceNodePort int + + // Number of masters running; all masters must be started with the + // same value for this field. (Numbers > 1 currently untested.) + MasterCount int } // EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be @@ -106,6 +142,31 @@ type completedConfig struct { func (c *Config) Complete() completedConfig { c.GenericConfig.Complete() + serviceIPRange, apiServerServiceIP, err := genericapiserver.DefaultServiceIPRange(c.ServiceIPRange) + if err != nil { + glog.Fatalf("Error determining service IP ranges: %v", err) + } + if c.ServiceIPRange.IP == nil { + c.ServiceIPRange = serviceIPRange + } + if c.APIServerServiceIP == nil { + c.APIServerServiceIP = apiServerServiceIP + } + + discoveryAddresses := genericapiserver.DefaultDiscoveryAddresses{DefaultAddress: c.GenericConfig.ExternalAddress} + discoveryAddresses.DiscoveryCIDRRules = append(discoveryAddresses.DiscoveryCIDRRules, + genericapiserver.DiscoveryCIDRRule{IPRange: c.ServiceIPRange, Address: net.JoinHostPort(c.APIServerServiceIP.String(), strconv.Itoa(c.APIServerServicePort))}) + c.GenericConfig.DiscoveryAddresses = discoveryAddresses + + if c.ServiceNodePortRange.Size == 0 { + // TODO: Currently no way to specify an empty range (do we need to allow this?) + // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) + // but then that breaks the strict nestedness of ServiceType. + // Review post-v1 + c.ServiceNodePortRange = options.DefaultServiceNodePortRange + glog.Infof("Node port range unspecified. Defaulting to %v.", c.ServiceNodePortRange) + } + // enable swagger UI only if general UI support is on c.GenericConfig.EnableSwaggerUI = c.GenericConfig.EnableSwaggerUI && c.EnableUISupport @@ -116,7 +177,7 @@ func (c *Config) Complete() completedConfig { if c.EndpointReconcilerConfig.Reconciler == nil { // use a default endpoint reconciler if nothing is set endpointClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) - c.EndpointReconcilerConfig.Reconciler = NewMasterCountEndpointReconciler(c.GenericConfig.MasterCount, endpointClient) + c.EndpointReconcilerConfig.Reconciler = NewMasterCountEndpointReconciler(c.MasterCount, endpointClient) } // this has always been hardcoded true in the past @@ -170,13 +231,13 @@ func (c completedConfig) New() (*Master, error) { // install legacy rest storage if c.GenericConfig.APIResourceConfigSource.AnyResourcesForVersionEnabled(apiv1.SchemeGroupVersion) { legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{ - StorageFactory: c.StorageFactory, - ProxyTransport: c.ProxyTransport, - KubeletClientConfig: c.KubeletClientConfig, - EventTTL: c.EventTTL, - ServiceClusterIPRange: c.GenericConfig.ServiceClusterIPRange, - ServiceNodePortRange: c.GenericConfig.ServiceNodePortRange, - LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, + StorageFactory: c.StorageFactory, + ProxyTransport: c.ProxyTransport, + KubeletClientConfig: c.KubeletClientConfig, + EventTTL: c.EventTTL, + ServiceIPRange: c.ServiceIPRange, + ServiceNodePortRange: c.ServiceNodePortRange, + LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, } m.InstallLegacyAPI(c.Config, restOptionsFactory.NewFor, legacyRESTStorageProvider) } diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index 80e1108871e..3d14b39167c 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -66,7 +66,9 @@ func setUp(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert. server, storageConfig := etcdtesting.NewUnsecuredEtcd3TestClientServer(t) config := &Config{ - GenericConfig: genericapiserver.NewConfig(), + GenericConfig: genericapiserver.NewConfig(), + APIServerServicePort: 443, + MasterCount: 1, } resourceEncoding := genericapiserver.NewDefaultResourceEncodingConfig() @@ -142,16 +144,6 @@ func newLimitedMaster(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Confi return master, etcdserver, config, assert } -// TestNew verifies that the New function returns a Master -// using the configuration properly. -func TestNew(t *testing.T) { - master, etcdserver, _, assert := newMaster(t) - defer etcdserver.Terminate(t) - - // these values get defaulted - assert.Equal(master.GenericAPIServer.MasterCount, 1) -} - // TestVersion tests /version func TestVersion(t *testing.T) { s, etcdserver, _, _ := newMaster(t) diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index c919879e5cd..bcfca24cbea 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -75,9 +75,9 @@ type LegacyRESTStorageProvider struct { KubeletClientConfig kubeletclient.KubeletClientConfig EventTTL time.Duration - // ServiceClusterIPRange is used to build cluster IPs for discovery. - ServiceClusterIPRange *net.IPNet - ServiceNodePortRange utilnet.PortRange + // ServiceIPRange is used to build cluster IPs for discovery. + ServiceIPRange net.IPNet + ServiceNodePortRange utilnet.PortRange LoopbackClientConfig *restclient.Config } @@ -154,9 +154,9 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi restStorage.ServiceRegistry = service.NewRegistry(serviceRESTStorage) var serviceClusterIPRegistry rangeallocation.RangeRegistry - serviceClusterIPRange := c.ServiceClusterIPRange - if serviceClusterIPRange == nil { - return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, fmt.Errorf("service clusterIPRange is nil") + serviceClusterIPRange := c.ServiceIPRange + if serviceClusterIPRange.IP == nil { + return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, fmt.Errorf("service clusterIPRange is missing") } serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services")) @@ -164,7 +164,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err } - ServiceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface { + ServiceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(&serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface { mem := allocator.NewAllocationMap(max, rangeSpec) // TODO etcdallocator package to return a storage interface via the storageFactory etcd := etcdallocator.NewEtcd(mem, "/ranges/serviceips", api.Resource("serviceipallocations"), serviceStorageConfig) diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 4403b918693..ad1f6e3fc8d 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -359,6 +359,8 @@ func NewMasterConfig() *master.Config { EnableCoreControllers: true, EnableWatchCache: true, KubeletClientConfig: kubeletclient.KubeletClientConfig{Port: 10250}, + APIServerServicePort: 443, + MasterCount: 1, } } diff --git a/test/integration/master/master_test.go b/test/integration/master/master_test.go index 264826c28e4..356d0da5c54 100644 --- a/test/integration/master/master_test.go +++ b/test/integration/master/master_test.go @@ -425,11 +425,11 @@ func TestMasterService(t *testing.T) { func TestServiceAlloc(t *testing.T) { cfg := framework.NewIntegrationTestMasterConfig() - _, cidr, err := net.ParseCIDR("192.168.0.0/30") + _, cidr, err := net.ParseCIDR("192.168.0.0/29") if err != nil { t.Fatalf("bad cidr: %v", err) } - cfg.GenericConfig.ServiceClusterIPRange = cidr + cfg.ServiceIPRange = *cidr _, s := framework.RunAMaster(cfg) defer s.Close() @@ -460,13 +460,15 @@ func TestServiceAlloc(t *testing.T) { t.Fatalf("creating kubernetes service timed out") } - // Make a service. - if _, err := client.Core().Services(api.NamespaceDefault).Create(svc(1)); err != nil { - t.Fatalf("got unexpected error: %v", err) + // make 5 more services to take up all IPs + for i := 0; i < 5; i++ { + if _, err := client.Core().Services(api.NamespaceDefault).Create(svc(i)); err != nil { + t.Error(err) + } } - // Make a second service. It will fail because we're out of cluster IPs - if _, err := client.Core().Services(api.NamespaceDefault).Create(svc(2)); err != nil { + // Make another service. It will fail because we're out of cluster IPs + if _, err := client.Core().Services(api.NamespaceDefault).Create(svc(8)); err != nil { if !strings.Contains(err.Error(), "range is full") { t.Errorf("unexpected error text: %v", err) } @@ -488,7 +490,7 @@ func TestServiceAlloc(t *testing.T) { } // This time creating the second service should work. - if _, err := client.Core().Services(api.NamespaceDefault).Create(svc(2)); err != nil { + if _, err := client.Core().Services(api.NamespaceDefault).Create(svc(8)); err != nil { t.Fatalf("got unexpected error: %v", err) } }