From 7d09148ad70d2b65202106f6b354af5671160647 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 6 Sep 2017 17:46:05 +0200 Subject: [PATCH 1/8] apiserver: separate apiserver specific configs into ExtraConfig --- cmd/kube-apiserver/app/aggregator.go | 12 ++-- cmd/kube-apiserver/app/apiextensions.go | 5 +- cmd/kube-apiserver/app/server.go | 61 ++++++++--------- pkg/master/controller.go | 18 ++--- pkg/master/master.go | 67 ++++++++++--------- pkg/master/master_test.go | 34 +++++----- .../pkg/apiserver/apiserver.go | 8 ++- .../pkg/cmd/server/start.go | 6 +- .../test/integration/registration_test.go | 2 +- .../test/integration/testserver/start.go | 6 +- .../pkg/apiserver/apiserver.go | 25 ++++--- .../kube-aggregator/pkg/cmd/server/start.go | 12 ++-- .../pkg/apiserver/apiserver.go | 5 ++ .../sample-apiserver/pkg/cmd/server/start.go | 1 + test/integration/apiserver/apiserver_test.go | 2 +- test/integration/auth/rbac_test.go | 2 +- .../etcd/etcd_storage_path_test.go | 2 +- test/integration/framework/master_utils.go | 20 +++--- test/integration/master/master_test.go | 2 +- test/integration/openshift/openshift_test.go | 8 ++- 20 files changed, 165 insertions(+), 133 deletions(-) diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index 2fc6944957f..1c31d537e9e 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -77,11 +77,13 @@ func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, command aggregatorConfig := &aggregatorapiserver.Config{ GenericConfig: &genericConfig, - CoreKubeInformers: externalInformers, - ProxyClientCert: certBytes, - ProxyClientKey: keyBytes, - ServiceResolver: serviceResolver, - ProxyTransport: proxyTransport, + ExtraConfig: aggregatorapiserver.ExtraConfig{ + CoreKubeInformers: externalInformers, + ProxyClientCert: certBytes, + ProxyClientKey: keyBytes, + ServiceResolver: serviceResolver, + ProxyTransport: proxyTransport, + }, } return aggregatorConfig, nil diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 5c0b19f1ece..1d2697cab66 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -41,11 +41,12 @@ func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, comm apiextensionsConfig := &apiextensionsapiserver.Config{ GenericConfig: &genericConfig, - CRDRESTOptionsGetter: apiextensionscmd.NewCRDRESTOptionsGetter(etcdOptions), + ExtraConfig: apiextensionsapiserver.ExtraConfig{ + CRDRESTOptionsGetter: apiextensionscmd.NewCRDRESTOptionsGetter(etcdOptions), + }, } return apiextensionsConfig, nil - } func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) { diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index fae0ef9f80e..1923512000c 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -172,8 +172,8 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc if err != nil { return nil, err } - aggregatorConfig.ProxyTransport = proxyTransport - aggregatorConfig.ServiceResolver = serviceResolver + aggregatorConfig.ExtraConfig.ProxyTransport = proxyTransport + aggregatorConfig.ExtraConfig.ServiceResolver = serviceResolver aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers) if err != nil { // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines @@ -312,40 +312,41 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele config := &master.Config{ GenericConfig: genericConfig, + ExtraConfig: master.ExtraConfig{ + ClientCARegistrationHook: master.ClientCARegistrationHook{ + ClientCA: clientCA, + RequestHeaderUsernameHeaders: s.Authentication.RequestHeader.UsernameHeaders, + RequestHeaderGroupHeaders: s.Authentication.RequestHeader.GroupHeaders, + RequestHeaderExtraHeaderPrefixes: s.Authentication.RequestHeader.ExtraHeaderPrefixes, + RequestHeaderCA: requestHeaderProxyCA, + RequestHeaderAllowedNames: s.Authentication.RequestHeader.AllowedNames, + }, - ClientCARegistrationHook: master.ClientCARegistrationHook{ - ClientCA: clientCA, - RequestHeaderUsernameHeaders: s.Authentication.RequestHeader.UsernameHeaders, - RequestHeaderGroupHeaders: s.Authentication.RequestHeader.GroupHeaders, - RequestHeaderExtraHeaderPrefixes: s.Authentication.RequestHeader.ExtraHeaderPrefixes, - RequestHeaderCA: requestHeaderProxyCA, - RequestHeaderAllowedNames: s.Authentication.RequestHeader.AllowedNames, + APIResourceConfigSource: storageFactory.APIResourceConfigSource, + StorageFactory: storageFactory, + EnableCoreControllers: true, + EventTTL: s.EventTTL, + KubeletClientConfig: s.KubeletConfig, + EnableUISupport: true, + EnableLogsSupport: s.EnableLogsHandler, + ProxyTransport: proxyTransport, + + Tunneler: nodeTunneler, + + ServiceIPRange: serviceIPRange, + APIServerServiceIP: apiServerServiceIP, + APIServerServicePort: 443, + + ServiceNodePortRange: s.ServiceNodePortRange, + KubernetesServiceNodePort: s.KubernetesServiceNodePort, + + MasterCount: s.MasterCount, }, - - APIResourceConfigSource: storageFactory.APIResourceConfigSource, - StorageFactory: storageFactory, - EnableCoreControllers: true, - EventTTL: s.EventTTL, - KubeletClientConfig: s.KubeletConfig, - EnableUISupport: true, - EnableLogsSupport: s.EnableLogsHandler, - ProxyTransport: proxyTransport, - - Tunneler: nodeTunneler, - - ServiceIPRange: serviceIPRange, - APIServerServiceIP: apiServerServiceIP, - APIServerServicePort: 443, - - ServiceNodePortRange: s.ServiceNodePortRange, - KubernetesServiceNodePort: s.KubernetesServiceNodePort, - - MasterCount: s.MasterCount, } if nodeTunneler != nil { // Use the nodeTunneler's dialer to connect to the kubelet - config.KubeletClientConfig.Dial = nodeTunneler.Dial + config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial } return config, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil diff --git a/pkg/master/controller.go b/pkg/master/controller.go index 0e922e17b82..085687b13ca 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -82,28 +82,28 @@ func (c *Config) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTSto ServiceClient: serviceClient, NamespaceClient: nsClient, - EndpointReconciler: c.EndpointReconcilerConfig.Reconciler, - EndpointInterval: c.EndpointReconcilerConfig.Interval, + EndpointReconciler: c.ExtraConfig.EndpointReconcilerConfig.Reconciler, + EndpointInterval: c.ExtraConfig.EndpointReconcilerConfig.Interval, SystemNamespaces: []string{metav1.NamespaceSystem, metav1.NamespacePublic}, SystemNamespacesInterval: 1 * time.Minute, ServiceClusterIPRegistry: legacyRESTStorage.ServiceClusterIPAllocator, - ServiceClusterIPRange: c.ServiceIPRange, + ServiceClusterIPRange: c.ExtraConfig.ServiceIPRange, ServiceClusterIPInterval: 3 * time.Minute, ServiceNodePortRegistry: legacyRESTStorage.ServiceNodePortAllocator, - ServiceNodePortRange: c.ServiceNodePortRange, + ServiceNodePortRange: c.ExtraConfig.ServiceNodePortRange, ServiceNodePortInterval: 3 * time.Minute, PublicIP: c.GenericConfig.PublicAddress, - ServiceIP: c.APIServerServiceIP, - ServicePort: c.APIServerServicePort, - ExtraServicePorts: c.ExtraServicePorts, - ExtraEndpointPorts: c.ExtraEndpointPorts, + ServiceIP: c.ExtraConfig.APIServerServiceIP, + ServicePort: c.ExtraConfig.APIServerServicePort, + ExtraServicePorts: c.ExtraConfig.ExtraServicePorts, + ExtraEndpointPorts: c.ExtraConfig.ExtraEndpointPorts, PublicServicePort: c.GenericConfig.ReadWritePort, - KubernetesServiceNodePort: c.KubernetesServiceNodePort, + KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort, } } diff --git a/pkg/master/master.go b/pkg/master/master.go index e5d5420fd59..ba3365de675 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -85,9 +85,7 @@ const ( DefaultEndpointReconcilerInterval = 10 * time.Second ) -type Config struct { - GenericConfig *genericapiserver.Config - +type ExtraConfig struct { ClientCARegistrationHook ClientCARegistrationHook APIResourceConfigSource serverstorage.APIResourceConfigSource @@ -135,6 +133,11 @@ type Config struct { MasterCount int } +type Config struct { + GenericConfig *genericapiserver.Config + ExtraConfig ExtraConfig +} + // EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be // used by the master. type EndpointReconcilerConfig struct { @@ -157,42 +160,42 @@ type completedConfig struct { func (c *Config) Complete() completedConfig { c.GenericConfig.Complete() - serviceIPRange, apiServerServiceIP, err := DefaultServiceIPRange(c.ServiceIPRange) + serviceIPRange, apiServerServiceIP, err := DefaultServiceIPRange(c.ExtraConfig.ServiceIPRange) if err != nil { glog.Fatalf("Error determining service IP ranges: %v", err) } - if c.ServiceIPRange.IP == nil { - c.ServiceIPRange = serviceIPRange + if c.ExtraConfig.ServiceIPRange.IP == nil { + c.ExtraConfig.ServiceIPRange = serviceIPRange } - if c.APIServerServiceIP == nil { - c.APIServerServiceIP = apiServerServiceIP + if c.ExtraConfig.APIServerServiceIP == nil { + c.ExtraConfig.APIServerServiceIP = apiServerServiceIP } discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: c.GenericConfig.ExternalAddress} discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules, - discovery.CIDRRule{IPRange: c.ServiceIPRange, Address: net.JoinHostPort(c.APIServerServiceIP.String(), strconv.Itoa(c.APIServerServicePort))}) + discovery.CIDRRule{IPRange: c.ExtraConfig.ServiceIPRange, Address: net.JoinHostPort(c.ExtraConfig.APIServerServiceIP.String(), strconv.Itoa(c.ExtraConfig.APIServerServicePort))}) c.GenericConfig.DiscoveryAddresses = discoveryAddresses - if c.ServiceNodePortRange.Size == 0 { + if c.ExtraConfig.ServiceNodePortRange.Size == 0 { // TODO: Currently no way to specify an empty range (do we need to allow this?) // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) // but then that breaks the strict nestedness of ServiceType. // Review post-v1 - c.ServiceNodePortRange = options.DefaultServiceNodePortRange - glog.Infof("Node port range unspecified. Defaulting to %v.", c.ServiceNodePortRange) + c.ExtraConfig.ServiceNodePortRange = options.DefaultServiceNodePortRange + glog.Infof("Node port range unspecified. Defaulting to %v.", c.ExtraConfig.ServiceNodePortRange) } // enable swagger UI only if general UI support is on - c.GenericConfig.EnableSwaggerUI = c.GenericConfig.EnableSwaggerUI && c.EnableUISupport + c.GenericConfig.EnableSwaggerUI = c.GenericConfig.EnableSwaggerUI && c.ExtraConfig.EnableUISupport - if c.EndpointReconcilerConfig.Interval == 0 { - c.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval + if c.ExtraConfig.EndpointReconcilerConfig.Interval == 0 { + c.ExtraConfig.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval } - if c.EndpointReconcilerConfig.Reconciler == nil { + if c.ExtraConfig.EndpointReconcilerConfig.Reconciler == nil { // use a default endpoint reconciler if nothing is set endpointClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) - c.EndpointReconcilerConfig.Reconciler = NewMasterCountEndpointReconciler(c.MasterCount, endpointClient) + c.ExtraConfig.EndpointReconcilerConfig.Reconciler = NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointClient) } // this has always been hardcoded true in the past @@ -211,7 +214,7 @@ func (c *Config) SkipComplete() completedConfig { // Certain config fields must be specified, including: // KubeletClientConfig func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Master, error) { - if reflect.DeepEqual(c.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) { + if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) { return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig") } @@ -220,10 +223,10 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) return nil, err } - if c.EnableUISupport { + if c.ExtraConfig.EnableUISupport { routes.UIRedirect{}.Install(s.Handler.NonGoRestfulMux) } - if c.EnableLogsSupport { + if c.ExtraConfig.EnableLogsSupport { routes.Logs{}.Install(s.Handler.GoRestfulContainer) } @@ -232,14 +235,14 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } // install legacy rest storage - if c.APIResourceConfigSource.AnyResourcesForVersionEnabled(apiv1.SchemeGroupVersion) { + if c.ExtraConfig.APIResourceConfigSource.AnyResourcesForVersionEnabled(apiv1.SchemeGroupVersion) { legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{ - StorageFactory: c.StorageFactory, - ProxyTransport: c.ProxyTransport, - KubeletClientConfig: c.KubeletClientConfig, - EventTTL: c.EventTTL, - ServiceIPRange: c.ServiceIPRange, - ServiceNodePortRange: c.ServiceNodePortRange, + StorageFactory: c.ExtraConfig.StorageFactory, + ProxyTransport: c.ExtraConfig.ProxyTransport, + KubeletClientConfig: c.ExtraConfig.KubeletClientConfig, + EventTTL: c.ExtraConfig.EventTTL, + ServiceIPRange: c.ExtraConfig.ServiceIPRange, + ServiceNodePortRange: c.ExtraConfig.ServiceNodePortRange, LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, } m.InstallLegacyAPI(c.Config, c.Config.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider) @@ -270,13 +273,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) appsrest.RESTStorageProvider{}, admissionregistrationrest.RESTStorageProvider{}, } - m.InstallAPIs(c.Config.APIResourceConfigSource, c.Config.GenericConfig.RESTOptionsGetter, restStorageProviders...) + m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...) - if c.Tunneler != nil { - m.installTunneler(c.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes()) + if c.ExtraConfig.Tunneler != nil { + m.installTunneler(c.ExtraConfig.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes()) } - m.GenericAPIServer.AddPostStartHookOrDie("ca-registration", c.ClientCARegistrationHook.PostStartHook) + m.GenericAPIServer.AddPostStartHookOrDie("ca-registration", c.ExtraConfig.ClientCARegistrationHook.PostStartHook) return m, nil } @@ -287,7 +290,7 @@ func (m *Master) InstallLegacyAPI(c *Config, restOptionsGetter generic.RESTOptio glog.Fatalf("Error building core storage: %v", err) } - if c.EnableCoreControllers { + if c.ExtraConfig.EnableCoreControllers { coreClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient) m.GenericAPIServer.AddPostStartHookOrDie("bootstrap-controller", bootstrapController.PostStartHook) diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index 8c3bc7ca89f..89f2b2ae22b 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -71,9 +71,11 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion config := &Config{ GenericConfig: genericapiserver.NewConfig(api.Codecs), - APIResourceConfigSource: DefaultAPIResourceConfigSource(), - APIServerServicePort: 443, - MasterCount: 1, + ExtraConfig: ExtraConfig{ + APIResourceConfigSource: DefaultAPIResourceConfigSource(), + APIServerServicePort: 443, + MasterCount: 1, + }, } resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(api.Registry) @@ -95,16 +97,16 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion kubeVersion := kubeversion.Get() config.GenericConfig.Version = &kubeVersion - config.StorageFactory = storageFactory + config.ExtraConfig.StorageFactory = storageFactory config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}} config.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4") config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") config.GenericConfig.RequestContextMapper = genericapirequest.NewRequestContextMapper() config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}} config.GenericConfig.EnableMetrics = true - config.EnableCoreControllers = false - config.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} - config.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ + config.ExtraConfig.EnableCoreControllers = false + config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} + config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ Dial: func(network, addr string) (net.Conn, error) { return nil, nil }, TLSClientConfig: &tls.Config{}, }) @@ -126,12 +128,12 @@ func TestLegacyRestStorageStrategies(t *testing.T) { defer etcdserver.Terminate(t) storageProvider := corerest.LegacyRESTStorageProvider{ - StorageFactory: masterCfg.StorageFactory, - ProxyTransport: masterCfg.ProxyTransport, - KubeletClientConfig: masterCfg.KubeletClientConfig, - EventTTL: masterCfg.EventTTL, - ServiceIPRange: masterCfg.ServiceIPRange, - ServiceNodePortRange: masterCfg.ServiceNodePortRange, + StorageFactory: masterCfg.ExtraConfig.StorageFactory, + ProxyTransport: masterCfg.ExtraConfig.ProxyTransport, + KubeletClientConfig: masterCfg.ExtraConfig.KubeletClientConfig, + EventTTL: masterCfg.ExtraConfig.EventTTL, + ServiceIPRange: masterCfg.ExtraConfig.ServiceIPRange, + ServiceNodePortRange: masterCfg.ExtraConfig.ServiceNodePortRange, LoopbackClientConfig: masterCfg.GenericConfig.LoopbackClientConfig, } @@ -162,7 +164,7 @@ func TestCertificatesRestStorageStrategies(t *testing.T) { defer etcdserver.Terminate(t) certStorageProvider := certificatesrest.RESTStorageProvider{} - apiGroupInfo, _ := certStorageProvider.NewRESTStorage(masterCfg.APIResourceConfigSource, masterCfg.GenericConfig.RESTOptionsGetter) + apiGroupInfo, _ := certStorageProvider.NewRESTStorage(masterCfg.ExtraConfig.APIResourceConfigSource, masterCfg.GenericConfig.RESTOptionsGetter) exceptions := registrytest.StrategyExceptions{ HasExportStrategy: []string{ @@ -205,8 +207,8 @@ func limitedAPIResourceConfigSource() *serverstorage.ResourceConfig { // newLimitedMaster only enables the core group, the extensions group, the batch group, and the autoscaling group. func newLimitedMaster(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { etcdserver, config, assert := setUp(t) - config.APIResourceConfigSource = limitedAPIResourceConfigSource() - master, err := config.Complete().New(genericapiserver.EmptyDelegate) + config.ExtraConfig.APIResourceConfigSource = limitedAPIResourceConfigSource() + master, err := config.Complete(sharedInformers).New(genericapiserver.EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the master: %v", err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 1d1409d9a4d..b5b659a643b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -78,10 +78,14 @@ func init() { Scheme.AddUnversionedTypes(unversionedVersion, unversionedTypes...) } +type ExtraConfig struct { + CRDRESTOptionsGetter genericregistry.RESTOptionsGetter +} type Config struct { GenericConfig *genericapiserver.Config + ExtraConfig ExtraConfig +} - CRDRESTOptionsGetter genericregistry.RESTOptionsGetter } type CustomResourceDefinitions struct { @@ -173,7 +177,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions().Lister(), s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(), delegateHandler, - c.CRDRESTOptionsGetter, + c.ExtraConfig.CRDRESTOptionsGetter, c.GenericConfig.AdmissionControl, ) s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go index 38c266e7073..a2dccd5a066 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go @@ -100,8 +100,10 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err } config := &apiserver.Config{ - GenericConfig: serverConfig, - CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd), + GenericConfig: serverConfig, + ExtraConfig: apiserver.ExtraConfig{ + CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd), + }, } return config, nil } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/registration_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/registration_test.go index a9bab6aec2a..a72ce2dcf9e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/registration_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/registration_test.go @@ -457,7 +457,7 @@ func TestEtcdStorage(t *testing.T) { } func getPrefixFromConfig(t *testing.T, config *extensionsapiserver.Config) string { - extensionsOptionsGetter, ok := config.CRDRESTOptionsGetter.(extensionsapiserver.CRDRESTOptionsGetter) + extensionsOptionsGetter, ok := config.ExtraConfig.CRDRESTOptionsGetter.(extensionsapiserver.CRDRESTOptionsGetter) if !ok { t.Fatal("can't obtain etcd prefix: unable to cast config.CRDRESTOptionsGetter to extensionsapiserver.CRDRESTOptionsGetter") } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go index de23379c068..2c83a9335c1 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go @@ -88,8 +88,10 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) { customResourceDefinitionRESTOptionsGetter.StorageConfig.Copier = extensionsapiserver.UnstructuredCopier{} config := &extensionsapiserver.Config{ - GenericConfig: genericConfig, - CRDRESTOptionsGetter: customResourceDefinitionRESTOptionsGetter, + GenericConfig: genericConfig, + ExtraConfig: extensionsapiserver.ExtraConfig{ + CRDRESTOptionsGetter: customResourceDefinitionRESTOptionsGetter, + }, } return config, nil diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index 65873c88e0c..bdf55a9688f 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -70,9 +70,7 @@ func init() { // legacyAPIServiceName is the fixed name of the only non-groupified API version const legacyAPIServiceName = "v1." -type Config struct { - GenericConfig *genericapiserver.Config - +type ExtraConfig struct { // CoreKubeInformers is used to watch kube resources CoreKubeInformers kubeinformers.SharedInformerFactory @@ -89,6 +87,11 @@ type Config struct { ServiceResolver ServiceResolver } +type Config struct { + GenericConfig *genericapiserver.RecommendedConfig + ExtraConfig ExtraConfig +} + // APIAggregator contains state for a Kubernetes cluster master/api server. type APIAggregator struct { GenericAPIServer *genericapiserver.GenericAPIServer @@ -168,14 +171,14 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg GenericAPIServer: genericServer, delegateHandler: delegationTarget.UnprotectedHandler(), contextMapper: c.GenericConfig.RequestContextMapper, - proxyClientCert: c.ProxyClientCert, - proxyClientKey: c.ProxyClientKey, - proxyTransport: c.ProxyTransport, + proxyClientCert: c.ExtraConfig.ProxyClientCert, + proxyClientKey: c.ExtraConfig.ProxyClientKey, + proxyTransport: c.ExtraConfig.ProxyTransport, proxyHandlers: map[string]*proxyHandler{}, handledGroups: sets.String{}, lister: informerFactory.Apiregistration().InternalVersion().APIServices().Lister(), APIRegistrationInformers: informerFactory, - serviceResolver: c.ServiceResolver, + serviceResolver: c.ExtraConfig.ServiceResolver, } apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiregistration.GroupName, registry, Scheme, metav1.ParameterCodec, Codecs) @@ -198,17 +201,17 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler) s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler) - apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().InternalVersion().APIServices(), c.CoreKubeInformers.Core().V1().Services(), s) + apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().InternalVersion().APIServices(), c.ExtraConfig.CoreKubeInformers.Core().V1().Services(), s) availableController := statuscontrollers.NewAvailableConditionController( informerFactory.Apiregistration().InternalVersion().APIServices(), - c.CoreKubeInformers.Core().V1().Services(), - c.CoreKubeInformers.Core().V1().Endpoints(), + c.ExtraConfig.CoreKubeInformers.Core().V1().Services(), + c.ExtraConfig.CoreKubeInformers.Core().V1().Endpoints(), apiregistrationClient.Apiregistration(), ) s.GenericAPIServer.AddPostStartHook("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error { informerFactory.Start(context.StopCh) - c.CoreKubeInformers.Start(context.StopCh) + c.ExtraConfig.CoreKubeInformers.Start(context.StopCh) return nil }) s.GenericAPIServer.AddPostStartHook("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error { diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 1db47144b33..d5214d7a6ea 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -151,16 +151,18 @@ func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error { serviceResolver := apiserver.NewClusterIPServiceResolver(kubeInformers.Core().V1().Services().Lister()) config := apiserver.Config{ - GenericConfig: serverConfig, - CoreKubeInformers: kubeInformers, - ServiceResolver: serviceResolver, + GenericConfig: serverConfig, + ExtraConfig: apiserver.ExtraConfig{ + CoreKubeInformers: kubeInformers, + ServiceResolver: serviceResolver, + }, } - config.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile) + config.ExtraConfig.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile) if err != nil { return err } - config.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile) + config.ExtraConfig.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile) if err != nil { return err } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 33ebf143c6a..32b3acfa2c1 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -61,10 +61,15 @@ func init() { ) } +type ExtraConfig struct { + // Place you custom config here. +} + type Config struct { GenericConfig *genericapiserver.Config // SharedInformerFactory provides shared informers for resources SharedInformerFactory informers.SharedInformerFactory + ExtraConfig ExtraConfig } // WardleServer contains state for a Kubernetes cluster master/api server. diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 916ac18f4de..69b2aad9871 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -126,6 +126,7 @@ func (o WardleServerOptions) Config() (*apiserver.Config, error) { config := &apiserver.Config{ GenericConfig: serverConfig, SharedInformerFactory: informerFactory, + ExtraConfig: apiserver.ExtraConfig{}, } return config, nil } diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 33d0cb1a994..5fc84d0ab7a 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -44,7 +44,7 @@ import ( func setup(t *testing.T) (*httptest.Server, clientset.Interface, framework.CloseFunc) { masterConfig := framework.NewIntegrationTestMasterConfig() - masterConfig.EnableCoreControllers = false + masterConfig.ExtraConfig.EnableCoreControllers = false _, s, closeFn := framework.RunAMaster(masterConfig) clientSet, err := clientset.NewForConfig(&restclient.Config{Host: s.URL}) diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index 58093392b0b..f1bf44c3470 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -75,7 +75,7 @@ type testRESTOptionsGetter struct { } func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := getter.config.StorageFactory.NewConfig(resource) + storageConfig, err := getter.config.ExtraConfig.StorageFactory.NewConfig(resource) if err != nil { return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err) } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 96b85aec5f2..41d96532168 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -690,7 +690,7 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV t.Fatal(err) } - kubeAPIServerConfig.APIResourceConfigSource = &allResourceSource{} // force enable all resources + kubeAPIServerConfig.ExtraConfig.APIResourceConfigSource = &allResourceSource{} // force enable all resources kubeAPIServer, err := app.CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.EmptyDelegate, sharedInformers) if err != nil { diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 9577cdce550..d279812c03c 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -357,22 +357,24 @@ func NewMasterConfig() *master.Config { } return &master.Config{ - GenericConfig: genericConfig, - APIResourceConfigSource: master.DefaultAPIResourceConfigSource(), - StorageFactory: storageFactory, - EnableCoreControllers: true, - KubeletClientConfig: kubeletclient.KubeletClientConfig{Port: 10250}, - APIServerServicePort: 443, - MasterCount: 1, + GenericConfig: genericConfig, + ExtraConfig: master.ExtraConfig{ + APIResourceConfigSource: master.DefaultAPIResourceConfigSource(), + StorageFactory: storageFactory, + EnableCoreControllers: true, + KubeletClientConfig: kubeletclient.KubeletClientConfig{Port: 10250}, + APIServerServicePort: 443, + MasterCount: 1, + }, } } // Returns the master config appropriate for most integration tests. func NewIntegrationTestMasterConfig() *master.Config { masterConfig := NewMasterConfig() - masterConfig.EnableCoreControllers = true + masterConfig.ExtraConfig.EnableCoreControllers = true masterConfig.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4") - masterConfig.APIResourceConfigSource = master.DefaultAPIResourceConfigSource() + masterConfig.ExtraConfig.APIResourceConfigSource = master.DefaultAPIResourceConfigSource() return masterConfig } diff --git a/test/integration/master/master_test.go b/test/integration/master/master_test.go index 22dda4b4b23..3355192e69b 100644 --- a/test/integration/master/master_test.go +++ b/test/integration/master/master_test.go @@ -548,7 +548,7 @@ func TestServiceAlloc(t *testing.T) { if err != nil { t.Fatalf("bad cidr: %v", err) } - cfg.ServiceIPRange = *cidr + cfg.ExtraConfig.ServiceIPRange = *cidr _, s, closeFn := framework.RunAMaster(cfg) defer closeFn() diff --git a/test/integration/openshift/openshift_test.go b/test/integration/openshift/openshift_test.go index a29b1a55b16..40b62455b72 100644 --- a/test/integration/openshift/openshift_test.go +++ b/test/integration/openshift/openshift_test.go @@ -30,9 +30,11 @@ func TestMasterExportsSymbols(t *testing.T) { GenericConfig: &genericapiserver.Config{ EnableMetrics: true, }, - EnableCoreControllers: false, - EnableUISupport: false, - EnableLogsSupport: false, + ExtraConfig: master.ExtraConfig{ + EnableCoreControllers: false, + EnableUISupport: false, + EnableLogsSupport: false, + }, } _ = &master.Master{ GenericAPIServer: &genericapiserver.GenericAPIServer{}, From 1bcea54104cb7f53e58924dd5413cf4ba7ceb587 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 6 Sep 2017 18:06:18 +0200 Subject: [PATCH 2/8] apiserver: make config completion structural recursion --- pkg/master/controller.go | 2 +- pkg/master/master.go | 34 +++++++++------- .../pkg/apiserver/apiserver.go | 31 +++++++------- .../src/k8s.io/apiserver/pkg/server/config.go | 15 +++---- .../apiserver/pkg/server/config_test.go | 2 +- .../pkg/server/genericapiserver_test.go | 6 +-- .../pkg/apiserver/apiserver.go | 40 ++++++++++--------- .../pkg/apiserver/apiserver.go | 26 ++++++------ .../sample-apiserver/pkg/cmd/server/start.go | 5 +-- 9 files changed, 87 insertions(+), 74 deletions(-) diff --git a/pkg/master/controller.go b/pkg/master/controller.go index 085687b13ca..feb347237a9 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -77,7 +77,7 @@ type Controller struct { } // NewBootstrapController returns a controller for watching the core capabilities of the master -func (c *Config) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTStorage, serviceClient coreclient.ServicesGetter, nsClient coreclient.NamespacesGetter) *Controller { +func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTStorage, serviceClient coreclient.ServicesGetter, nsClient coreclient.NamespacesGetter) *Controller { return &Controller{ ServiceClient: serviceClient, NamespaceClient: nsClient, diff --git a/pkg/master/master.go b/pkg/master/master.go index ba3365de675..184d10a282b 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -138,6 +138,16 @@ type Config struct { ExtraConfig ExtraConfig } +type completedConfig struct { + GenericConfig genericapiserver.CompletedConfig + ExtraConfig *ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig +} + // EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be // used by the master. type EndpointReconcilerConfig struct { @@ -152,13 +162,12 @@ type Master struct { ClientCARegistrationHook ClientCARegistrationHook } -type completedConfig struct { - *Config -} - // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() completedConfig { - c.GenericConfig.Complete() +func (cfg *Config) Complete() CompletedConfig { + c := completedConfig{ + cfg.GenericConfig.Complete(), + &cfg.ExtraConfig, + } serviceIPRange, apiServerServiceIP, err := DefaultServiceIPRange(c.ExtraConfig.ServiceIPRange) if err != nil { @@ -201,12 +210,7 @@ func (c *Config) Complete() completedConfig { // this has always been hardcoded true in the past c.GenericConfig.EnableMetrics = true - return completedConfig{c} -} - -// SkipComplete provides a way to construct a server instance without config completion. -func (c *Config) SkipComplete() completedConfig { - return completedConfig{c} + return CompletedConfig{&c} } // New returns a new instance of Master from the given config. @@ -218,7 +222,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig") } - s, err := c.Config.GenericConfig.SkipComplete().New("kube-apiserver", delegationTarget) // completion is done in Complete, no need for a second time + s, err := c.GenericConfig.New("kube-apiserver", delegationTarget) if err != nil { return nil, err } @@ -245,7 +249,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) ServiceNodePortRange: c.ExtraConfig.ServiceNodePortRange, LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, } - m.InstallLegacyAPI(c.Config, c.Config.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider) + m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider) } // The order here is preserved in discovery. @@ -284,7 +288,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) return m, nil } -func (m *Master) InstallLegacyAPI(c *Config, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) { +func (m *Master) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) { legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter) if err != nil { glog.Fatalf("Error building core storage: %v", err) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index b5b659a643b..e98f20d1c97 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -81,11 +81,20 @@ func init() { type ExtraConfig struct { CRDRESTOptionsGetter genericregistry.RESTOptionsGetter } + type Config struct { GenericConfig *genericapiserver.Config ExtraConfig ExtraConfig } +type completedConfig struct { + GenericConfig genericapiserver.CompletedConfig + ExtraConfig *ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig } type CustomResourceDefinitions struct { @@ -95,31 +104,25 @@ type CustomResourceDefinitions struct { Informers internalinformers.SharedInformerFactory } -type completedConfig struct { - *Config -} - // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() completedConfig { - c.GenericConfig.EnableDiscovery = false - c.GenericConfig.Complete() +func (cfg *Config) Complete() CompletedConfig { + c := completedConfig{ + cfg.GenericConfig.Complete(), + &cfg.ExtraConfig, + } + c.GenericConfig.EnableDiscovery = false c.GenericConfig.Version = &version.Info{ Major: "0", Minor: "1", } - return completedConfig{c} -} - -// SkipComplete provides a way to construct a server instance without config completion. -func (c *Config) SkipComplete() completedConfig { - return completedConfig{c} + return CompletedConfig{&c} } // New returns a new instance of CustomResourceDefinitions from the given config. func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) { - genericServer, err := c.Config.GenericConfig.SkipComplete().New("apiextensions-apiserver", delegationTarget) // completion is done in Complete, no need for a second time + genericServer, err := c.GenericConfig.New("apiextensions-apiserver", delegationTarget) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index f3b37518238..062c5b3359d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -62,6 +62,7 @@ import ( certutil "k8s.io/client-go/util/cert" openapicommon "k8s.io/kube-openapi/pkg/common" + // install apis _ "k8s.io/apiserver/pkg/apis/apiserver/install" ) @@ -302,9 +303,14 @@ type completedConfig struct { *Config } +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig +} + // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. -func (c *Config) Complete() completedConfig { +func (c *Config) Complete() CompletedConfig { if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { hostAndPort := c.PublicAddress.String() if c.ReadWritePort != 0 { @@ -379,12 +385,7 @@ func (c *Config) Complete() completedConfig { c.RequestInfoResolver = NewRequestInfoResolver(c) } - return completedConfig{c} -} - -// SkipComplete provides a way to construct a server instance without config completion. -func (c *Config) SkipComplete() completedConfig { - return completedConfig{c} + return CompletedConfig{&completedConfig{c}} } // New creates a new server which logically combines the handling chain with the passed server. diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index c94046d0b8e..8fd48f1fc68 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -52,7 +52,7 @@ func TestNewWithDelegate(t *testing.T) { return fmt.Errorf("delegate failed healthcheck") })) - delegateServer, err := delegateConfig.SkipComplete().New("test", EmptyDelegate) + delegateServer, err := delegateConfig.Complete().New("test", EmptyDelegate) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index 215960a678e..e74fbf10fb8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -147,7 +147,7 @@ func TestInstallAPIGroups(t *testing.T) { config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"} - s, err := config.SkipComplete().New("test", EmptyDelegate) + s, err := config.Complete().New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } @@ -351,7 +351,7 @@ func TestCustomHandlerChain(t *testing.T) { called = true }) - s, err := config.SkipComplete().New("test", EmptyDelegate) + s, err := config.Complete().New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } @@ -406,7 +406,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { kubeVersion := fakeVersion() config.Version = &kubeVersion - s, err := config.SkipComplete().New("test", EmptyDelegate) + s, err := config.Complete().New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index bdf55a9688f..1fff050c193 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -88,10 +88,20 @@ type ExtraConfig struct { } type Config struct { - GenericConfig *genericapiserver.RecommendedConfig + GenericConfig *genericapiserver.Config ExtraConfig ExtraConfig } +type completedConfig struct { + GenericConfig genericapiserver.CompletedConfig + ExtraConfig *ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig +} + // APIAggregator contains state for a Kubernetes cluster master/api server. type APIAggregator struct { GenericAPIServer *genericapiserver.GenericAPIServer @@ -124,41 +134,35 @@ type APIAggregator struct { openAPIAggregationController *openapicontroller.AggregationController } -type completedConfig struct { - *Config -} - // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() completedConfig { +func (cfg *Config) Complete() CompletedConfig { + c := completedConfig{ + cfg.GenericConfig.Complete(), + &cfg.ExtraConfig, + } + // the kube aggregator wires its own discovery mechanism // TODO eventually collapse this by extracting all of the discovery out c.GenericConfig.EnableDiscovery = false - c.GenericConfig.Complete() - version := version.Get() c.GenericConfig.Version = &version - return completedConfig{c} -} - -// SkipComplete provides a way to construct a server instance without config completion. -func (c *Config) SkipComplete() completedConfig { - return completedConfig{c} + return CompletedConfig{&c} } // New returns a new instance of APIAggregator from the given config. func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.DelegationTarget) (*APIAggregator, error) { // Prevent generic API server to install OpenAPI handler. Aggregator server // has its own customized OpenAPI handler. - openApiConfig := c.Config.GenericConfig.OpenAPIConfig - c.Config.GenericConfig.OpenAPIConfig = nil + openApiConfig := c.GenericConfig.OpenAPIConfig + c.GenericConfig.OpenAPIConfig = nil - genericServer, err := c.Config.GenericConfig.SkipComplete().New("kube-aggregator", delegationTarget) // completion is done in Complete, no need for a second time + genericServer, err := c.GenericConfig.New("kube-aggregator", delegationTarget) if err != nil { return nil, err } - apiregistrationClient, err := internalclientset.NewForConfig(c.Config.GenericConfig.LoopbackClientConfig) + apiregistrationClient, err := internalclientset.NewForConfig(c.GenericConfig.LoopbackClientConfig) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 32b3acfa2c1..c6c0f36ccf4 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -67,8 +67,6 @@ type ExtraConfig struct { type Config struct { GenericConfig *genericapiserver.Config - // SharedInformerFactory provides shared informers for resources - SharedInformerFactory informers.SharedInformerFactory ExtraConfig ExtraConfig } @@ -78,29 +76,33 @@ type WardleServer struct { } type completedConfig struct { - *Config + GenericConfig genericapiserver.CompletedConfig + ExtraConfig *ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig } // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() completedConfig { - c.GenericConfig.Complete() +func (cfg *Config) Complete() CompletedConfig { + c := completedConfig{ + cfg.GenericConfig.Complete(), + &cfg.ExtraConfig, + } c.GenericConfig.Version = &version.Info{ Major: "1", Minor: "0", } - return completedConfig{c} -} - -// SkipComplete provides a way to construct a server instance without config completion. -func (c *Config) SkipComplete() completedConfig { - return completedConfig{c} + return CompletedConfig{&c} } // New returns a new instance of WardleServer from the given config. func (c completedConfig) New() (*WardleServer, error) { - genericServer, err := c.Config.GenericConfig.SkipComplete().New("sample-apiserver", genericapiserver.EmptyDelegate) // completion is done in Complete, no need for a second time + genericServer, err := c.GenericConfig.New("sample-apiserver", genericapiserver.EmptyDelegate) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 69b2aad9871..7ebc41e187e 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -124,8 +124,7 @@ func (o WardleServerOptions) Config() (*apiserver.Config, error) { } config := &apiserver.Config{ - GenericConfig: serverConfig, - SharedInformerFactory: informerFactory, + GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{}, } return config, nil @@ -143,7 +142,7 @@ func (o WardleServerOptions) RunWardleServer(stopCh <-chan struct{}) error { } server.GenericAPIServer.AddPostStartHook("start-sample-server-informers", func(context genericapiserver.PostStartHookContext) error { - config.SharedInformerFactory.Start(context.StopCh) + config.GenericConfig.SharedInformerFactory.Start(context.StopCh) return nil }) From dffe50f8bd820295f7f1fbc56a6269b6b8c6966b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 6 Sep 2017 18:15:43 +0200 Subject: [PATCH 3/8] apiserver: allow disabling authz/n via options --- .../test/integration/testserver/start.go | 20 +++---------------- .../pkg/server/options/authentication.go | 5 +++++ .../pkg/server/options/authorization.go | 5 +++++ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go index 2c83a9335c1..c5c593c8c43 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go @@ -30,7 +30,6 @@ import ( "k8s.io/apiextensions-apiserver/pkg/cmd/server" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/authorization/authorizerfactory" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/client-go/dynamic" ) @@ -44,7 +43,8 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) { options := server.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr) options.RecommendedOptions.Audit.LogOptions.Path = "-" options.RecommendedOptions.SecureServing.BindPort = port - options.RecommendedOptions.Authentication.SkipInClusterLookup = true + options.RecommendedOptions.Authentication = nil // disable + options.RecommendedOptions.Authorization = nil // disable options.RecommendedOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1") etcdURL, ok := os.LookupEnv("KUBE_INTEGRATION_ETCD_URL") if !ok { @@ -53,26 +53,12 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) { options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{etcdURL} options.RecommendedOptions.Etcd.StorageConfig.Prefix = uuid.New() - // TODO stop copying this - // because there isn't currently a way to disable authentication or authorization from options - // explode options.Config here genericConfig := genericapiserver.NewConfig(extensionsapiserver.Codecs) - genericConfig.Authenticator = nil - genericConfig.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() if err := options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } - if err := options.RecommendedOptions.Etcd.ApplyTo(genericConfig); err != nil { - return nil, err - } - if err := options.RecommendedOptions.SecureServing.ApplyTo(genericConfig); err != nil { - return nil, err - } - if err := options.RecommendedOptions.Audit.ApplyTo(genericConfig); err != nil { - return nil, err - } - if err := options.RecommendedOptions.Features.ApplyTo(genericConfig); err != nil { + if err := options.RecommendedOptions.ApplyTo(genericConfig); err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go index 1a5c728c5b2..624343fc479 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go @@ -143,6 +143,11 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { } func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.Config) error { + if s == nil { + c.Authenticator = nil + return nil + } + clientCA, err := s.getClientCA() if err != nil { return err diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go index ddc32fc9092..3d356958f5d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go @@ -71,6 +71,11 @@ func (s *DelegatingAuthorizationOptions) AddFlags(fs *pflag.FlagSet) { } func (s *DelegatingAuthorizationOptions) ApplyTo(c *server.Config) error { + if s == nil { + c.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + return nil + } + cfg, err := s.ToAuthorizationConfig() if err != nil { return err From ca3f7453464f6866a3bf467c8b9d8e132484cfb4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 6 Sep 2017 18:22:23 +0200 Subject: [PATCH 4/8] apiserver: stratify versioned informer construction --- cmd/kube-apiserver/app/aggregator.go | 5 ++- cmd/kube-apiserver/app/apiextensions.go | 8 +++-- cmd/kube-apiserver/app/server.go | 9 ++--- .../cmd/federation-apiserver/app/server.go | 11 ++++++- pkg/master/master.go | 5 +-- pkg/master/master_openapi_test.go | 4 +-- pkg/master/master_test.go | 14 ++++---- .../pkg/apiserver/apiserver.go | 2 +- .../pkg/cmd/server/start.go | 2 +- .../test/integration/testserver/start.go | 2 +- .../src/k8s.io/apiserver/pkg/server/config.go | 33 ++++++++++++++++--- .../apiserver/pkg/server/config_test.go | 8 ++--- .../pkg/server/genericapiserver_test.go | 12 +++---- .../apiserver/pkg/server/options/admission.go | 9 ++--- .../pkg/server/options/recommended.go | 26 +++++++++++---- .../apiserver/pkg/server/options/serving.go | 24 -------------- .../pkg/server/options/serving_test.go | 2 +- .../pkg/apiserver/apiserver.go | 2 +- .../kube-aggregator/pkg/cmd/server/start.go | 2 +- .../pkg/apiserver/apiserver.go | 3 +- .../sample-apiserver/pkg/cmd/server/start.go | 4 +-- .../etcd/etcd_storage_path_test.go | 4 +-- test/integration/examples/apiserver_test.go | 4 +-- test/integration/framework/master_utils.go | 4 +-- 24 files changed, 115 insertions(+), 84 deletions(-) diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index 1c31d537e9e..aab55da9c28 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -76,7 +76,10 @@ func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, command } aggregatorConfig := &aggregatorapiserver.Config{ - GenericConfig: &genericConfig, + GenericConfig: &genericapiserver.RecommendedConfig{ + Config: genericConfig, + SharedInformerFactory: externalInformers, + }, ExtraConfig: aggregatorapiserver.ExtraConfig{ CoreKubeInformers: externalInformers, ProxyClientCert: certBytes, diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 1d2697cab66..aa9c19fe3db 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -25,10 +25,11 @@ import ( apiextensionscmd "k8s.io/apiextensions-apiserver/pkg/cmd/server" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + kubeexternalinformers "k8s.io/client-go/informers" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) -func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, commandOptions *options.ServerRunOptions) (*apiextensionsapiserver.Config, error) { +func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, externalInformers kubeexternalinformers.SharedInformerFactory, commandOptions *options.ServerRunOptions) (*apiextensionsapiserver.Config, error) { // make a shallow copy to let us twiddle a few things // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions genericConfig := kubeAPIServerConfig @@ -40,7 +41,10 @@ func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, comm genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} apiextensionsConfig := &apiextensionsapiserver.Config{ - GenericConfig: &genericConfig, + GenericConfig: &genericapiserver.RecommendedConfig{ + Config: genericConfig, + SharedInformerFactory: externalInformers, + }, ExtraConfig: apiextensionsapiserver.ExtraConfig{ CRDRESTOptionsGetter: apiextensionscmd.NewCRDRESTOptionsGetter(etcdOptions), }, diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 1923512000c..df6fd7cb5c2 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -133,7 +133,7 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc // TPRs are enabled and not yet beta, since this these are the successor, they fall under the same enablement rule // If additional API servers are added, they should be gated. - apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, runOptions) + apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, runOptions) if err != nil { return nil, err } @@ -142,7 +142,7 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc return nil, err } - kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers) + kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers, versionedInformers) if err != nil { return nil, err } @@ -191,8 +191,8 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc } // CreateKubeAPIServer creates and wires a workable kube-apiserver -func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory) (*master.Master, error) { - kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer) +func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory) (*master.Master, error) { + kubeAPIServer, err := kubeAPIServerConfig.Complete(versionedInformers).New(delegateAPIServer) if err != nil { return nil, err } @@ -466,6 +466,7 @@ func BuildGenericConfig(s *options.ServerRunOptions) (*genericapiserver.Config, err = s.Admission.ApplyTo( genericConfig, + versionedInformers, pluginInitializer) if err != nil { return nil, nil, nil, nil, nil, fmt.Errorf("failed to initialize admission: %v", err) diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index 47f3054cc55..78d50ea2abe 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -39,6 +39,8 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" serverstorage "k8s.io/apiserver/pkg/server/storage" + clientgoinformers "k8s.io/client-go/informers" + clientgoclientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes" openapicommon "k8s.io/kube-openapi/pkg/common" federationv1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1" @@ -189,6 +191,12 @@ func NonBlockingRun(s *options.ServerRunOptions, stopCh <-chan struct{}) error { } sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute) + clientgoExternalClient, err := clientgoclientset.NewForConfig(genericConfig.LoopbackClientConfig) + if err != nil { + return fmt.Errorf("failed to create real external clientset: %v", err) + } + versionedInformers := clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) + authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers) apiAuthorizer, _, err := authorizationConfig.New() if err != nil { @@ -210,6 +218,7 @@ func NonBlockingRun(s *options.ServerRunOptions, stopCh <-chan struct{}) error { err = s.Admission.ApplyTo( genericConfig, + versionedInformers, pluginInitializer, ) if err != nil { @@ -235,7 +244,7 @@ func NonBlockingRun(s *options.ServerRunOptions, stopCh <-chan struct{}) error { cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes) } - m, err := genericConfig.Complete().New("federation", genericapiserver.EmptyDelegate) + m, err := genericConfig.Complete(versionedInformers).New("federation", genericapiserver.EmptyDelegate) if err != nil { return err } diff --git a/pkg/master/master.go b/pkg/master/master.go index 184d10a282b..b2117e00f2b 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -62,6 +62,7 @@ import ( "github.com/prometheus/client_golang/prometheus" // RESTStorage installers + "k8s.io/client-go/informers" admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest" appsrest "k8s.io/kubernetes/pkg/registry/apps/rest" authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest" @@ -163,9 +164,9 @@ type Master struct { } // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (cfg *Config) Complete() CompletedConfig { +func (cfg *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { c := completedConfig{ - cfg.GenericConfig.Complete(), + cfg.GenericConfig.Complete(informers), &cfg.ExtraConfig, } diff --git a/pkg/master/master_openapi_test.go b/pkg/master/master_openapi_test.go index 4629e4159eb..92501ca40f5 100644 --- a/pkg/master/master_openapi_test.go +++ b/pkg/master/master_openapi_test.go @@ -41,7 +41,7 @@ import ( // TestValidOpenAPISpec verifies that the open api is added // at the proper endpoint and the spec is valid. func TestValidOpenAPISpec(t *testing.T) { - etcdserver, config, assert := setUp(t) + etcdserver, config, sharedInformers, assert := setUp(t) defer etcdserver.Terminate(t) config.GenericConfig.EnableIndex = true @@ -54,7 +54,7 @@ func TestValidOpenAPISpec(t *testing.T) { } config.GenericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig() - master, err := config.Complete().New(genericapiserver.EmptyDelegate) + master, err := config.Complete(sharedInformers).New(genericapiserver.EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the master: %v", err) } diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index 89f2b2ae22b..11ec1e0a01b 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -66,11 +66,11 @@ import ( ) // setUp is a convience function for setting up for (most) tests. -func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertions) { +func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, informers.SharedInformerFactory, *assert.Assertions) { server, storageConfig := etcdtesting.NewUnsecuredEtcd3TestClientServer(t, api.Scheme) config := &Config{ - GenericConfig: genericapiserver.NewConfig(api.Codecs), + GenericConfig: genericapiserver.NewConfig(api.Codecs), ExtraConfig: ExtraConfig{ APIResourceConfigSource: DefaultAPIResourceConfigSource(), APIServerServicePort: 443, @@ -115,9 +115,9 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion if err != nil { t.Fatalf("unable to create client set due to %v", err) } - config.GenericConfig.SharedInformerFactory = informers.NewSharedInformerFactory(clientset, config.GenericConfig.LoopbackClientConfig.Timeout) + sharedInformers := informers.NewSharedInformerFactory(clientset, config.GenericConfig.LoopbackClientConfig.Timeout) - return server, *config, assert.New(t) + return server, *config, sharedInformers, assert.New(t) } // TestLegacyRestStorageStrategies ensures that all Storage objects which are using the generic registry Store have @@ -180,9 +180,9 @@ func TestCertificatesRestStorageStrategies(t *testing.T) { } func newMaster(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { - etcdserver, config, assert := setUp(t) + etcdserver, config, sharedInformers, assert := setUp(t) - master, err := config.Complete().New(genericapiserver.EmptyDelegate) + master, err := config.Complete(sharedInformers).New(genericapiserver.EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the master: %v", err) } @@ -206,7 +206,7 @@ func limitedAPIResourceConfigSource() *serverstorage.ResourceConfig { // newLimitedMaster only enables the core group, the extensions group, the batch group, and the autoscaling group. func newLimitedMaster(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { - etcdserver, config, assert := setUp(t) + etcdserver, config, sharedInformers, assert := setUp(t) config.ExtraConfig.APIResourceConfigSource = limitedAPIResourceConfigSource() master, err := config.Complete(sharedInformers).New(genericapiserver.EmptyDelegate) if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index e98f20d1c97..de90cef9068 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -83,7 +83,7 @@ type ExtraConfig struct { } type Config struct { - GenericConfig *genericapiserver.Config + GenericConfig *genericapiserver.RecommendedConfig ExtraConfig ExtraConfig } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go index a2dccd5a066..b38d564b8a4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go @@ -94,7 +94,7 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } - serverConfig := genericapiserver.NewConfig(apiserver.Codecs) + serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go index c5c593c8c43..5c8efe7cf71 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go @@ -53,7 +53,7 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) { options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{etcdURL} options.RecommendedOptions.Etcd.StorageConfig.Prefix = uuid.New() - genericConfig := genericapiserver.NewConfig(extensionsapiserver.Codecs) + genericConfig := genericapiserver.NewRecommendedConfig(extensionsapiserver.Codecs) if err := options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 062c5b3359d..2c459d9b680 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -122,8 +122,6 @@ type Config struct { // Will default to a value based on secure serving info and available ipv4 IPs. ExternalAddress string - // SharedInformerFactory provides shared informers for resources - SharedInformerFactory informers.SharedInformerFactory //=========================================================================== // Fields you probably don't care about changing //=========================================================================== @@ -187,6 +185,13 @@ type Config struct { PublicAddress net.IP } +type RecommendedConfig struct { + Config + + // SharedInformerFactory provides shared informers for resources + SharedInformerFactory informers.SharedInformerFactory +} + type SecureServingInfo struct { // BindAddress is the ip:port to serve on BindAddress string @@ -242,6 +247,13 @@ func NewConfig(codecs serializer.CodecFactory) *Config { } } +// NewRecommendedConfig returns a RecommendedConfig struct with the default values +func NewRecommendedConfig(codecs serializer.CodecFactory) *RecommendedConfig { + return &RecommendedConfig{ + Config: *NewConfig(codecs), + } +} + func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, scheme *runtime.Scheme) *openapicommon.Config { defNamer := apiopenapi.NewDefinitionNamer(scheme) return &openapicommon.Config{ @@ -301,6 +313,13 @@ func (c *Config) ApplyClientCert(clientCAFile string) (*Config, error) { type completedConfig struct { *Config + + //=========================================================================== + // values below here are filled in during completion + //=========================================================================== + + // SharedInformerFactory provides shared informers for resources + SharedInformerFactory informers.SharedInformerFactory } type CompletedConfig struct { @@ -310,7 +329,7 @@ type CompletedConfig struct { // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. -func (c *Config) Complete() CompletedConfig { +func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { hostAndPort := c.PublicAddress.String() if c.ReadWritePort != 0 { @@ -385,7 +404,13 @@ func (c *Config) Complete() CompletedConfig { c.RequestInfoResolver = NewRequestInfoResolver(c) } - return CompletedConfig{&completedConfig{c}} + return CompletedConfig{&completedConfig{c, informers}} +} + +// Complete fills in any fields not set that are required to have valid data and can be derived +// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. +func (c *RecommendedConfig) Complete() CompletedConfig { + return c.Config.Complete(c.SharedInformerFactory) } // New creates a new server which logically combines the handling chain with the passed server. diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index 8fd48f1fc68..9f0ac45c5d1 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -44,7 +44,6 @@ func TestNewWithDelegate(t *testing.T) { if clientset == nil { t.Fatal("unable to create fake client set") } - delegateConfig.SharedInformerFactory = informers.NewSharedInformerFactory(clientset, delegateConfig.LoopbackClientConfig.Timeout) delegateHealthzCalled := false delegateConfig.HealthzChecks = append(delegateConfig.HealthzChecks, healthz.NamedCheck("delegate-health", func(r *http.Request) error { @@ -52,7 +51,8 @@ func TestNewWithDelegate(t *testing.T) { return fmt.Errorf("delegate failed healthcheck") })) - delegateServer, err := delegateConfig.Complete().New("test", EmptyDelegate) + sharedInformers := informers.NewSharedInformerFactory(clientset, delegateConfig.LoopbackClientConfig.Timeout) + delegateServer, err := delegateConfig.Complete(sharedInformers).New("test", EmptyDelegate) if err != nil { t.Fatal(err) } @@ -73,7 +73,6 @@ func TestNewWithDelegate(t *testing.T) { wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") wrappingConfig.LoopbackClientConfig = &rest.Config{} wrappingConfig.SwaggerConfig = DefaultSwaggerConfig() - wrappingConfig.SharedInformerFactory = informers.NewSharedInformerFactory(clientset, wrappingConfig.LoopbackClientConfig.Timeout) wrappingHealthzCalled := false wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error { @@ -81,7 +80,8 @@ func TestNewWithDelegate(t *testing.T) { return fmt.Errorf("wrapping failed healthcheck") })) - wrappingServer, err := wrappingConfig.Complete().New("test", delegateServer) + sharedInformers = informers.NewSharedInformerFactory(clientset, wrappingConfig.LoopbackClientConfig.Timeout) + wrappingServer, err := wrappingConfig.Complete(sharedInformers).New("test", delegateServer) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index e74fbf10fb8..54a6540fdaf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -96,7 +96,6 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion if clientset == nil { t.Fatal("unable to create fake client set") } - config.SharedInformerFactory = informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout) // TODO restore this test, but right now, eliminate our cycle // config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, runtime.NewScheme()) @@ -107,7 +106,8 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion // }, // } config.SwaggerConfig = DefaultSwaggerConfig() - config.Complete() + sharedInformers := informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout) + config.Complete(sharedInformers) return etcdServer, *config, assert.New(t) } @@ -115,7 +115,7 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion func newMaster(t *testing.T) (*GenericAPIServer, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { etcdserver, config, assert := setUp(t) - s, err := config.Complete().New("test", EmptyDelegate) + s, err := config.Complete(nil).New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } @@ -147,7 +147,7 @@ func TestInstallAPIGroups(t *testing.T) { config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"} - s, err := config.Complete().New("test", EmptyDelegate) + s, err := config.Complete(nil).New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } @@ -351,7 +351,7 @@ func TestCustomHandlerChain(t *testing.T) { called = true }) - s, err := config.Complete().New("test", EmptyDelegate) + s, err := config.Complete(nil).New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } @@ -406,7 +406,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { kubeVersion := fakeVersion() config.Version = &kubeVersion - s, err := config.Complete().New("test", EmptyDelegate) + s, err := config.Complete(nil).New("test", EmptyDelegate) if err != nil { t.Fatalf("Error in bringing up the server: %v", err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/admission.go b/staging/src/k8s.io/apiserver/pkg/server/options/admission.go index 5604681825f..40c34ce3d3a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/admission.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/admission.go @@ -24,6 +24,7 @@ import ( "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/server" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" ) @@ -63,17 +64,17 @@ func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) { // genericconfig.LoopbackClientConfig // genericconfig.SharedInformerFactory // genericconfig.Authorizer -func (a *AdmissionOptions) ApplyTo(serverCfg *server.Config, pluginInitializers ...admission.PluginInitializer) error { +func (a *AdmissionOptions) ApplyTo(c *server.Config, informers informers.SharedInformerFactory, pluginInitializers ...admission.PluginInitializer) error { pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(a.PluginNames, a.ConfigFile) if err != nil { return fmt.Errorf("failed to read plugin config: %v", err) } - clientset, err := kubernetes.NewForConfig(serverCfg.LoopbackClientConfig) + clientset, err := kubernetes.NewForConfig(c.LoopbackClientConfig) if err != nil { return err } - genericInitializer, err := initializer.New(clientset, serverCfg.SharedInformerFactory, serverCfg.Authorizer) + genericInitializer, err := initializer.New(clientset, informers, c.Authorizer) if err != nil { return err } @@ -86,7 +87,7 @@ func (a *AdmissionOptions) ApplyTo(serverCfg *server.Config, pluginInitializers return err } - serverCfg.AdmissionControl = admissionChain + c.AdmissionControl = admissionChain return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index a0efcb49a52..2cf8f919701 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -17,11 +17,16 @@ limitations under the License. package options import ( + "fmt" + "time" + "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" + clientgoinformers "k8s.io/client-go/informers" + clientgoclientset "k8s.io/client-go/kubernetes" ) // RecommendedOptions contains the recommended options for running an API server @@ -55,26 +60,33 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) { o.Features.AddFlags(fs) } -func (o *RecommendedOptions) ApplyTo(config *server.Config) error { - if err := o.Etcd.ApplyTo(config); err != nil { +func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { + if err := o.Etcd.ApplyTo(&config.Config); err != nil { return err } - if err := o.SecureServing.ApplyTo(config); err != nil { + if err := o.SecureServing.ApplyTo(&config.Config); err != nil { return err } - if err := o.Authentication.ApplyTo(config); err != nil { + if err := o.Authentication.ApplyTo(&config.Config); err != nil { return err } - if err := o.Authorization.ApplyTo(config); err != nil { + if err := o.Authorization.ApplyTo(&config.Config); err != nil { return err } - if err := o.Audit.ApplyTo(config); err != nil { + if err := o.Audit.ApplyTo(&config.Config); err != nil { return err } - if err := o.Features.ApplyTo(config); err != nil { + if err := o.Features.ApplyTo(&config.Config); err != nil { return err } + // do convenience work for RecommendedOptions users + clientgoExternalClient, err := clientgoclientset.NewForConfig(config.LoopbackClientConfig) + if err != nil { + return fmt.Errorf("failed to create real external clientset: %v", err) + } + config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) + return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index b21f0002c34..8cd6bfc693f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -24,7 +24,6 @@ import ( "net" "path" "strconv" - "time" "github.com/golang/glog" "github.com/pborman/uuid" @@ -33,9 +32,6 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/server" utilflag "k8s.io/apiserver/pkg/util/flag" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" certutil "k8s.io/client-go/util/cert" ) @@ -175,26 +171,6 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error { c.SecureServingInfo.SNICerts[server.LoopbackClientServerNameOverride] = &tlsCert } - // create shared informers, if not explicitly set use in cluster config. - // do not fail on an error, this allows an external API server to startup - // outside of a kube cluster. - var clientCfg *rest.Config - err = nil - if s.useLoopbackCfg { - clientCfg = c.LoopbackClientConfig - } else { - clientCfg, err = rest.InClusterConfig() - } - if err != nil { - glog.Errorf("Couldn't create in cluster config due to %v. SharedInformerFactory will not be set.", err) - return nil - } - clientset, err := kubernetes.NewForConfig(clientCfg) - if err != nil { - glog.Errorf("Couldn't create clientset due to %v. SharedInformerFactory will not be set.", err) - return nil - } - c.SharedInformerFactory = informers.NewSharedInformerFactory(clientset, 10*time.Minute) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go index c70a2ccb4e6..6d41262573a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go @@ -487,7 +487,7 @@ NextTest: return } - s, err := config.Complete().New("test", server.EmptyDelegate) + s, err := config.Complete(nil).New("test", server.EmptyDelegate) if err != nil { t.Errorf("%q - failed creating the server: %v", title, err) return diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index 1fff050c193..a85794e166d 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -88,7 +88,7 @@ type ExtraConfig struct { } type Config struct { - GenericConfig *genericapiserver.Config + GenericConfig *genericapiserver.RecommendedConfig ExtraConfig ExtraConfig } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index d5214d7a6ea..b5cc1e20826 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -118,7 +118,7 @@ func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error { return fmt.Errorf("error creating self-signed certificates: %v", err) } - serverConfig := genericapiserver.NewConfig(apiserver.Codecs) + serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return err diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index c6c0f36ccf4..e36dc1360ae 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -30,7 +30,6 @@ import ( "k8s.io/sample-apiserver/pkg/apis/wardle" "k8s.io/sample-apiserver/pkg/apis/wardle/install" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" - informers "k8s.io/sample-apiserver/pkg/client/informers_generated/internalversion" wardleregistry "k8s.io/sample-apiserver/pkg/registry" fischerstorage "k8s.io/sample-apiserver/pkg/registry/wardle/fischer" flunderstorage "k8s.io/sample-apiserver/pkg/registry/wardle/flunder" @@ -66,7 +65,7 @@ type ExtraConfig struct { } type Config struct { - GenericConfig *genericapiserver.Config + GenericConfig *genericapiserver.RecommendedConfig ExtraConfig ExtraConfig } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 7ebc41e187e..8bdeb19e1b5 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -104,7 +104,7 @@ func (o WardleServerOptions) Config() (*apiserver.Config, error) { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } - serverConfig := genericapiserver.NewConfig(apiserver.Codecs) + serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } @@ -119,7 +119,7 @@ func (o WardleServerOptions) Config() (*apiserver.Config, error) { return nil, err } - if err := o.Admission.ApplyTo(serverConfig, admissionInitializer); err != nil { + if err := o.Admission.ApplyTo(&serverConfig.Config, serverConfig.SharedInformerFactory, admissionInitializer); err != nil { return nil, err } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 41d96532168..b631514ab3d 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -685,14 +685,14 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV if err != nil { t.Fatal(err) } - kubeAPIServerConfig, sharedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) + kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) if err != nil { t.Fatal(err) } kubeAPIServerConfig.ExtraConfig.APIResourceConfigSource = &allResourceSource{} // force enable all resources - kubeAPIServer, err := app.CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.EmptyDelegate, sharedInformers) + kubeAPIServer, err := app.CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.EmptyDelegate, sharedInformers, versionedInformers) if err != nil { t.Fatal(err) } diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 671f4cc84a2..a36b30c0ac7 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -116,13 +116,13 @@ func TestAggregatedAPIServer(t *testing.T) { if err != nil { t.Fatal(err) } - kubeAPIServerConfig, sharedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) + kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) if err != nil { t.Fatal(err) } kubeClientConfigValue.Store(kubeAPIServerConfig.GenericConfig.LoopbackClientConfig) - kubeAPIServer, err := app.CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.EmptyDelegate, sharedInformers) + kubeAPIServer, err := app.CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.EmptyDelegate, sharedInformers, versionedInformers) if err != nil { t.Fatal(err) } diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index d279812c03c..235d692a302 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -248,9 +248,9 @@ func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Serv if err != nil { glog.Fatal(err) } - masterConfig.GenericConfig.SharedInformerFactory = extinformers.NewSharedInformerFactory(clientset, masterConfig.GenericConfig.LoopbackClientConfig.Timeout) - m, err = masterConfig.Complete().New(genericapiserver.EmptyDelegate) + sharedInformers := extinformers.NewSharedInformerFactory(clientset, masterConfig.GenericConfig.LoopbackClientConfig.Timeout) + m, err = masterConfig.Complete(sharedInformers).New(genericapiserver.EmptyDelegate) if err != nil { closeFn() glog.Fatalf("error in bringing up the master: %v", err) From 2b64d3a0fd2ccdad4b2f21acb484a36e04381856 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 7 Sep 2017 09:39:31 +0200 Subject: [PATCH 5/8] apiserver: split core API creation from secure serving --- cmd/kube-apiserver/app/server.go | 2 - .../cmd/federation-apiserver/app/server.go | 1 - .../pkg/cmd/server/start.go | 3 + .../src/k8s.io/apiserver/pkg/server/config.go | 4 +- .../apiserver/pkg/server/options/coreapi.go | 79 +++++++++++++++++++ .../pkg/server/options/recommended.go | 23 +++--- .../apiserver/pkg/server/options/serving.go | 13 --- .../kube-aggregator/pkg/cmd/server/start.go | 4 + test/integration/examples/apiserver_test.go | 1 + 9 files changed, 99 insertions(+), 31 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index df6fd7cb5c2..b38b9d8c172 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -618,8 +618,6 @@ func defaultOptions(s *options.ServerRunOptions) error { if err != nil { return fmt.Errorf("error determining service IP ranges: %v", err) } - s.SecureServing.ForceLoopbackConfigUsage() - if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { return fmt.Errorf("error creating self-signed certificates: %v", err) } diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index 78d50ea2abe..1bd94b7c47f 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -105,7 +105,6 @@ func NonBlockingRun(s *options.ServerRunOptions, stopCh <-chan struct{}) error { if err := s.CloudProvider.DefaultExternalHost(s.GenericServerRunOptions); err != nil { return fmt.Errorf("error setting the external host value: %v", err) } - s.SecureServing.ForceLoopbackConfigUsage() s.Authentication.ApplyAuthorization(s.Authorization) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go index b38d564b8a4..ccf52cfeb7a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go @@ -49,6 +49,9 @@ func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomRes StdErr: errOut, } + // the shared informer is not needed for kube-aggregator. Disable the kubeconfig flag and the client creation. + o.RecommendedOptions.CoreAPI = nil + return o } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 2c459d9b680..69c0e4497eb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -188,7 +188,9 @@ type Config struct { type RecommendedConfig struct { Config - // SharedInformerFactory provides shared informers for resources + // SharedInformerFactory provides shared informers for Kubernetes resources. This value is set by + // RecommendedOptions.CoreAPI.ApplyTo called by RecommendedOptions.ApplyTo. It uses an in-cluster client config + // by default, or the kubeconfig given with kubeconfig command line flag. SharedInformerFactory informers.SharedInformerFactory } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go b/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go new file mode 100644 index 00000000000..01d489ba04d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go @@ -0,0 +1,79 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "time" + + "github.com/spf13/pflag" + "k8s.io/apiserver/pkg/server" + clientgoinformers "k8s.io/client-go/informers" + clientgoclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// CoreAPIOptions contains options to configure the connection to a core API Kubernetes apiserver. +type CoreAPIOptions struct { + // CoreAPIKubeconfigPath is a filename for a kubeconfig file to contact the core API server with. + // If it is not set, the in cluster config is used. + CoreAPIKubeconfigPath string +} + +func NewCoreAPIOptions() *CoreAPIOptions { + return &CoreAPIOptions{} +} + +func (o *CoreAPIOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.CoreAPIKubeconfigPath, "kubeconfig", o.CoreAPIKubeconfigPath, + "kubeconfig file pointing at the 'core' kubernetes server.") +} + +func (o *CoreAPIOptions) ApplyTo(config *server.RecommendedConfig) error { + if o == nil { + return nil + } + + // create shared informer for Kubernetes APIs + var kubeconfig *rest.Config + var err error + if len(o.CoreAPIKubeconfigPath) > 0 { + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfigPath} + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + kubeconfig, err = loader.ClientConfig() + if err != nil { + return fmt.Errorf("failed to load kubeconfig at %q: %v", o.CoreAPIKubeconfigPath, err) + } + } else { + kubeconfig, err = rest.InClusterConfig() + if err != nil { + return err + } + } + clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeconfig) + if err != nil { + return fmt.Errorf("failed to create Kubernetes clientset: %v", err) + } + config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) + + return nil +} + +func (o *CoreAPIOptions) Validate() []error { + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index 2cf8f919701..97460b87792 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -17,20 +17,16 @@ limitations under the License. package options import ( - "fmt" - "time" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" - clientgoinformers "k8s.io/client-go/informers" - clientgoclientset "k8s.io/client-go/kubernetes" ) -// RecommendedOptions contains the recommended options for running an API server -// If you add something to this list, it should be in a logical grouping +// RecommendedOptions contains the recommended options for running an API server. +// If you add something to this list, it should be in a logical grouping. +// Each of them can be nil to leave the feature unconfigured on ApplyTo. type RecommendedOptions struct { Etcd *EtcdOptions SecureServing *SecureServingOptions @@ -38,6 +34,7 @@ type RecommendedOptions struct { Authorization *DelegatingAuthorizationOptions Audit *AuditOptions Features *FeatureOptions + CoreAPI *CoreAPIOptions } func NewRecommendedOptions(prefix string, copier runtime.ObjectCopier, codec runtime.Codec) *RecommendedOptions { @@ -48,6 +45,7 @@ func NewRecommendedOptions(prefix string, copier runtime.ObjectCopier, codec run Authorization: NewDelegatingAuthorizationOptions(), Audit: NewAuditOptions(), Features: NewFeatureOptions(), + CoreAPI: NewCoreAPIOptions(), } } @@ -58,6 +56,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) { o.Authorization.AddFlags(fs) o.Audit.AddFlags(fs) o.Features.AddFlags(fs) + o.CoreAPI.AddFlags(fs) } func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { @@ -79,14 +78,9 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { if err := o.Features.ApplyTo(&config.Config); err != nil { return err } - - // do convenience work for RecommendedOptions users - clientgoExternalClient, err := clientgoclientset.NewForConfig(config.LoopbackClientConfig) - if err != nil { - return fmt.Errorf("failed to create real external clientset: %v", err) + if err := o.CoreAPI.ApplyTo(config); err != nil { + return err } - config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - return nil } @@ -98,6 +92,7 @@ func (o *RecommendedOptions) Validate() []error { errors = append(errors, o.Authorization.Validate()...) errors = append(errors, o.Audit.Validate()...) errors = append(errors, o.Features.Validate()...) + errors = append(errors, o.CoreAPI.Validate()...) return errors } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index 8cd6bfc693f..38c07a41a54 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -43,9 +43,6 @@ type SecureServingOptions struct { ServerCert GeneratableKeyCert // SNICertKeys are named CertKeys for serving secure traffic with SNI support. SNICertKeys []utilflag.NamedCertKey - - // when set determines whether to use loopback configuration to create shared informers. - useLoopbackCfg bool } type CertKey struct { @@ -174,16 +171,6 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error { return nil } -// ForceLoopbackConfigUsage forces the usage of the loopback configuration -// to create SharedInformerFactory. The primary client of this method -// is kube API server, no other API server is the source of truth for kube APIs. -// -// Note: -// this method MUST be called prior to ApplyTo to take an effect. -func (s *SecureServingOptions) ForceLoopbackConfigUsage() { - s.useLoopbackCfg = true -} - func (s *SecureServingOptions) applyServingInfoTo(c *server.Config) error { if s.BindPort <= 0 { return nil diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index b5cc1e20826..c47bb0654db 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -99,6 +99,10 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { StdOut: out, StdErr: err, } + + // the shared informer is not needed for kube-aggregator. Disable the kubeconfig flag and the client creation. + o.RecommendedOptions.CoreAPI = nil + return o } diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index a36b30c0ac7..512dce8f944 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -192,6 +192,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--authorization-kubeconfig", kubeconfigFile.Name(), "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, + "--kubeconfig", kubeconfigFile.Name(), }) if err := wardleCmd.Execute(); err != nil { t.Log(err) From b153268da79d2acf14e042945959801c3dba8221 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 7 Sep 2017 09:40:09 +0200 Subject: [PATCH 6/8] apiserver: avoid panics on nil sub-option structs --- .../src/k8s.io/apiserver/pkg/server/options/audit.go | 12 ++++++++++++ .../apiserver/pkg/server/options/authentication.go | 4 ++++ .../apiserver/pkg/server/options/authorization.go | 4 ++++ .../k8s.io/apiserver/pkg/server/options/coreapi.go | 4 ++++ .../src/k8s.io/apiserver/pkg/server/options/etcd.go | 12 ++++++++++++ .../k8s.io/apiserver/pkg/server/options/feature.go | 12 ++++++++++++ .../k8s.io/apiserver/pkg/server/options/serving.go | 12 ++++++++++++ 7 files changed, 60 insertions(+) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go index 3ccb275428b..b75772f069f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go @@ -87,6 +87,10 @@ func NewAuditOptions() *AuditOptions { // Validate checks invalid config combination func (o *AuditOptions) Validate() []error { + if o == nil { + return nil + } + allErrors := []error{} if !advancedAuditingEnabled() { @@ -137,6 +141,10 @@ func (o *AuditOptions) Validate() []error { } func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + fs.StringVar(&o.PolicyFile, "audit-policy-file", o.PolicyFile, "Path to the file that defines the audit policy configuration. Requires the 'AdvancedAuditing' feature gate."+ " With AdvancedAuditing, a profile is required to enable auditing.") @@ -146,6 +154,10 @@ func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) { } func (o *AuditOptions) ApplyTo(c *server.Config) error { + if o == nil { + return nil + } + // Apply legacy audit options if advanced audit is not enabled. if !advancedAuditingEnabled() { return o.LogOptions.legacyApplyTo(c) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go index 624343fc479..28c933bea54 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go @@ -43,6 +43,10 @@ type RequestHeaderAuthenticationOptions struct { } func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { + if s == nil { + return + } + fs.StringSliceVar(&s.UsernameHeaders, "requestheader-username-headers", s.UsernameHeaders, ""+ "List of request headers to inspect for usernames. X-Remote-User is common.") diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go index 3d356958f5d..9a452d11e6b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go @@ -57,6 +57,10 @@ func (s *DelegatingAuthorizationOptions) Validate() []error { } func (s *DelegatingAuthorizationOptions) AddFlags(fs *pflag.FlagSet) { + if s == nil { + return + } + fs.StringVar(&s.RemoteKubeConfigFile, "authorization-kubeconfig", s.RemoteKubeConfigFile, ""+ "kubeconfig file pointing at the 'core' kubernetes server with enough rights to create "+ " subjectaccessreviews.authorization.k8s.io.") diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go b/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go index 01d489ba04d..edcf3a7b058 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go @@ -40,6 +40,10 @@ func NewCoreAPIOptions() *CoreAPIOptions { } func (o *CoreAPIOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + fs.StringVar(&o.CoreAPIKubeconfigPath, "kubeconfig", o.CoreAPIKubeconfigPath, "kubeconfig file pointing at the 'core' kubernetes server.") } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 0a222ac79bc..95b20fce75c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -71,6 +71,10 @@ func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions { } func (s *EtcdOptions) Validate() []error { + if s == nil { + return nil + } + allErrors := []error{} if len(s.StorageConfig.ServerList) == 0 { allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified")) @@ -85,6 +89,10 @@ func (s *EtcdOptions) Validate() []error { // AddEtcdFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { + if s == nil { + return + } + fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+ "Per-resource etcd servers overrides, comma separated. The individual override "+ "format: group/resource#servers, where servers are http://ip:port, semicolon separated.") @@ -132,6 +140,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { } func (s *EtcdOptions) ApplyTo(c *server.Config) error { + if s == nil { + return nil + } + s.addEtcdHealthEndpoint(c) c.RESTOptionsGetter = &SimpleRestOptionsFactory{Options: *s} return nil diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/feature.go b/staging/src/k8s.io/apiserver/pkg/server/options/feature.go index cd62c7c67f7..7e02a183b7a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/feature.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/feature.go @@ -40,6 +40,10 @@ func NewFeatureOptions() *FeatureOptions { } func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/") fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", o.EnableContentionProfiling, @@ -49,6 +53,10 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) { } func (o *FeatureOptions) ApplyTo(c *server.Config) error { + if o == nil { + return nil + } + c.EnableProfiling = o.EnableProfiling c.EnableContentionProfiling = o.EnableContentionProfiling c.EnableSwaggerUI = o.EnableSwaggerUI @@ -57,6 +65,10 @@ func (o *FeatureOptions) ApplyTo(c *server.Config) error { } func (o *FeatureOptions) Validate() []error { + if o == nil { + return nil + } + errs := []error{} return errs } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index 38c07a41a54..775476c3d9f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -81,6 +81,10 @@ func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) { } func (s *SecureServingOptions) Validate() []error { + if s == nil { + return nil + } + errors := []error{} if s.BindPort < 0 || s.BindPort > 65535 { @@ -91,6 +95,10 @@ func (s *SecureServingOptions) Validate() []error { } func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { + if s == nil { + return + } + fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+ "The IP address on which to listen for the --secure-port port. The "+ "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ @@ -136,6 +144,10 @@ func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { // ApplyTo fills up serving information in the server configuration. func (s *SecureServingOptions) ApplyTo(c *server.Config) error { + if s == nil { + return nil + } + if s.BindPort <= 0 { return nil } From d99c7df360b88433a05599fb0d761b881af779f4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 8 Sep 2017 14:36:38 +0200 Subject: [PATCH 7/8] kube-aggregator: use shared informers from RecommendedConfig --- cmd/kube-apiserver/app/aggregator.go | 9 ++-- .../hostpath-pods/insecure-etcd-pod.yaml | 2 +- .../pkg/apiserver/apiserver.go | 12 ++---- .../kube-aggregator/pkg/cmd/server/start.go | 41 ++----------------- test/integration/examples/apiserver_test.go | 2 +- 5 files changed, 13 insertions(+), 53 deletions(-) diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index aab55da9c28..28bb181b71e 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -81,11 +81,10 @@ func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, command SharedInformerFactory: externalInformers, }, ExtraConfig: aggregatorapiserver.ExtraConfig{ - CoreKubeInformers: externalInformers, - ProxyClientCert: certBytes, - ProxyClientKey: keyBytes, - ServiceResolver: serviceResolver, - ProxyTransport: proxyTransport, + ProxyClientCert: certBytes, + ProxyClientKey: keyBytes, + ServiceResolver: serviceResolver, + ProxyTransport: proxyTransport, }, } diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml index af7f2969ea3..4cc60730324 100644 --- a/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml +++ b/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml @@ -12,7 +12,7 @@ spec: args: - "/usr/local/bin/kube-aggregator" - "--secure-port=9443" - - "--core-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" + - "--kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" - "--authentication-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" - "--authorization-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" - "--proxy-client-cert-file=/var/run/auth-proxy-client/client-auth-proxy.crt" diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index a85794e166d..3e70b7a5e22 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -30,7 +30,6 @@ import ( genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" - kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/pkg/version" "k8s.io/kube-aggregator/pkg/apis/apiregistration" @@ -71,9 +70,6 @@ func init() { const legacyAPIServiceName = "v1." type ExtraConfig struct { - // CoreKubeInformers is used to watch kube resources - CoreKubeInformers kubeinformers.SharedInformerFactory - // ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use // this to confirm the proxy's identity ProxyClientCert []byte @@ -205,17 +201,17 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler) s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler) - apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().InternalVersion().APIServices(), c.ExtraConfig.CoreKubeInformers.Core().V1().Services(), s) + apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().InternalVersion().APIServices(), c.GenericConfig.SharedInformerFactory.Core().V1().Services(), s) availableController := statuscontrollers.NewAvailableConditionController( informerFactory.Apiregistration().InternalVersion().APIServices(), - c.ExtraConfig.CoreKubeInformers.Core().V1().Services(), - c.ExtraConfig.CoreKubeInformers.Core().V1().Endpoints(), + c.GenericConfig.SharedInformerFactory.Core().V1().Services(), + c.GenericConfig.SharedInformerFactory.Core().V1().Endpoints(), apiregistrationClient.Apiregistration(), ) s.GenericAPIServer.AddPostStartHook("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error { informerFactory.Start(context.StopCh) - c.ExtraConfig.CoreKubeInformers.Start(context.StopCh) + c.GenericConfig.SharedInformerFactory.Start(context.StopCh) return nil }) s.GenericAPIServer.AddPostStartHook("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error { diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index c47bb0654db..66b1ac5f905 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "io/ioutil" - "time" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -30,10 +29,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" - kubeinformers "k8s.io/client-go/informers" - kubeclientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" ) @@ -48,10 +43,6 @@ type AggregatorOptions struct { ProxyClientCertFile string ProxyClientKeyFile string - // CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server with - // If it is not set, the in cluster config is used - CoreAPIKubeconfig string - StdOut io.Writer StdErr io.Writer } @@ -86,9 +77,6 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) { o.RecommendedOptions.AddFlags(fs) fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server") fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server") - fs.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+ - "kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+ - " services,endpoints. If not set, the in-cluster config is used") } // NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible @@ -100,9 +88,6 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { StdErr: err, } - // the shared informer is not needed for kube-aggregator. Disable the kubeconfig flag and the client creation. - o.RecommendedOptions.CoreAPI = nil - return o } @@ -132,36 +117,16 @@ func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error { sets.NewString("attach", "exec", "proxy", "log", "portforward"), ) - var kubeconfig *rest.Config - var err error - if len(o.CoreAPIKubeconfig) > 0 { - loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig} - loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) - - kubeconfig, err = loader.ClientConfig() - - } else { - kubeconfig, err = rest.InClusterConfig() - } - if err != nil { - return err - } - - coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig) - if err != nil { - return err - } - kubeInformers := kubeinformers.NewSharedInformerFactory(coreAPIServerClient, 5*time.Minute) - serviceResolver := apiserver.NewClusterIPServiceResolver(kubeInformers.Core().V1().Services().Lister()) + serviceResolver := apiserver.NewClusterIPServiceResolver(serverConfig.SharedInformerFactory.Core().V1().Services().Lister()) config := apiserver.Config{ GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{ - CoreKubeInformers: kubeInformers, - ServiceResolver: serviceResolver, + ServiceResolver: serviceResolver, }, } + var err error config.ExtraConfig.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile) if err != nil { return err diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 512dce8f944..48b052f9ab0 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -264,7 +264,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--requestheader-username-headers", "", "--proxy-client-cert-file", proxyClientCertFile.Name(), "--proxy-client-key-file", proxyClientKeyFile.Name(), - "--core-kubeconfig", kubeconfigFile.Name(), + "--kubeconfig", kubeconfigFile.Name(), "--authentication-kubeconfig", kubeconfigFile.Name(), "--authorization-kubeconfig", kubeconfigFile.Name(), "--etcd-servers", framework.GetEtcdURL(), From fbd310dbc7312fcae4267dd64326a1e7b4a0a8ae Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 6 Sep 2017 18:18:12 +0200 Subject: [PATCH 8/8] Update bazel --- federation/cmd/federation-apiserver/app/BUILD | 1 + pkg/master/BUILD | 1 + .../apiextensions-apiserver/test/integration/testserver/BUILD | 1 - staging/src/k8s.io/apiserver/pkg/server/options/BUILD | 1 + staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD | 1 - staging/src/k8s.io/kube-aggregator/pkg/cmd/server/BUILD | 4 ---- staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD | 1 - 7 files changed, 3 insertions(+), 7 deletions(-) diff --git a/federation/cmd/federation-apiserver/app/BUILD b/federation/cmd/federation-apiserver/app/BUILD index 1b43b5d4833..da5179e3451 100644 --- a/federation/cmd/federation-apiserver/app/BUILD +++ b/federation/cmd/federation-apiserver/app/BUILD @@ -80,6 +80,7 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/filters:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", + "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", ], diff --git a/pkg/master/BUILD b/pkg/master/BUILD index 89bbc5c4a36..31d17e7057c 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -94,6 +94,7 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", + "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD index 484ddb50607..87c46151ae1 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD @@ -24,7 +24,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", - "//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/names:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 525e5b20ba6..c7d7c2d5afb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -31,6 +31,7 @@ go_library( "audit.go", "authentication.go", "authorization.go", + "coreapi.go", "doc.go", "etcd.go", "feature.go", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD index 2aa58ebed09..48ff6a7cea6 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD @@ -62,7 +62,6 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/proxy:go_default_library", - "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/informers/core/v1:go_default_library", "//vendor/k8s.io/client-go/listers/core/v1:go_default_library", "//vendor/k8s.io/client-go/pkg/version:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/BUILD index b366c51c89b..f16dc39a852 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/BUILD @@ -16,10 +16,6 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/filters:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", - "//vendor/k8s.io/client-go/informers:go_default_library", - "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/client-go/rest:go_default_library", - "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apiserver:go_default_library", ], diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD index 79ed70f8257..4247f22cf04 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD @@ -29,7 +29,6 @@ go_library( "//vendor/k8s.io/sample-apiserver/pkg/apis/wardle:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/apis/wardle/install:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1:go_default_library", - "//vendor/k8s.io/sample-apiserver/pkg/client/informers_generated/internalversion:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/registry:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/registry/wardle/fischer:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/registry/wardle/flunder:go_default_library",