From d631f9b7e9e9bec131d171a7a859455498fdeb49 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 10 Jul 2019 18:37:24 -0400 Subject: [PATCH] Use metadata informers instead of dynamic informers in controller manager All controllers in controller-manager that deal with objects generically work with those objects without needing the full object. Update the GC and quota controller to use PartialObjectMetadata input objects which is faster and more efficient. --- cmd/kube-controller-manager/app/BUILD | 2 +- .../app/controllermanager.go | 42 ++++++++++--------- cmd/kube-controller-manager/app/core.go | 5 +-- cmd/kube-controller-manager/app/core_test.go | 8 ++-- pkg/controller/BUILD | 2 +- pkg/controller/informer_factory.go | 18 ++++---- test/integration/garbagecollector/BUILD | 2 +- .../garbage_collector_test.go | 6 +-- vendor/modules.txt | 2 - 9 files changed, 43 insertions(+), 44 deletions(-) diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index b1f12174587..1fb0323e8bd 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -121,10 +121,10 @@ go_library( "//staging/src/k8s.io/client-go/discovery/cached:go_default_library", "//staging/src/k8s.io/client-go/discovery/cached/memory:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/dynamic/dynamicinformer:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/metadata:go_default_library", + "//staging/src/k8s.io/client-go/metadata/metadatainformer:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/restmapper:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 9978079f41b..a26b3e84aa4 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -31,7 +31,7 @@ import ( "github.com/spf13/cobra" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -43,10 +43,10 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/term" cacheddiscovery "k8s.io/client-go/discovery/cached" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" + "k8s.io/client-go/metadata/metadatainformer" restclient "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/leaderelection" @@ -239,7 +239,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { } controllerContext.InformerFactory.Start(controllerContext.Stop) - controllerContext.GenericInformerFactory.Start(controllerContext.Stop) + controllerContext.ObjectOrMetadataInformerFactory.Start(controllerContext.Stop) close(controllerContext.InformersStarted) select {} @@ -295,9 +295,11 @@ type ControllerContext struct { // InformerFactory gives access to informers for the controller. InformerFactory informers.SharedInformerFactory - // GenericInformerFactory gives access to informers for typed resources - // and dynamic resources. - GenericInformerFactory controller.InformerFactory + // ObjectOrMetadataInformerFactory gives access to informers for typed resources + // and dynamic resources by their metadata. All generic controllers currently use + // object metadata - if a future controller needs access to the full object this + // would become GenericInformerFactory and take a dynamic client. + ObjectOrMetadataInformerFactory controller.InformerFactory // ComponentConfig provides access to init options for a given controller ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration @@ -448,8 +450,8 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien versionedClient := rootClientBuilder.ClientOrDie("shared-informers") sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)()) - dynamicClient := dynamic.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("dynamic-informers")) - dynamicInformers := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, ResyncPeriod(s)()) + metadataClient := metadata.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("metadata-informers")) + metadataInformers := metadatainformer.NewSharedInformerFactory(metadataClient, ResyncPeriod(s)()) // If apiserver is not running we should wait for some time and fail only then. This is particularly // important when we start apiserver and controller manager at the same time. @@ -477,17 +479,17 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien } ctx := ControllerContext{ - ClientBuilder: clientBuilder, - InformerFactory: sharedInformers, - GenericInformerFactory: controller.NewInformerFactory(sharedInformers, dynamicInformers), - ComponentConfig: s.ComponentConfig, - RESTMapper: restMapper, - AvailableResources: availableResources, - Cloud: cloud, - LoopMode: loopMode, - Stop: stop, - InformersStarted: make(chan struct{}), - ResyncPeriod: ResyncPeriod(s), + ClientBuilder: clientBuilder, + InformerFactory: sharedInformers, + ObjectOrMetadataInformerFactory: controller.NewInformerFactory(sharedInformers, metadataInformers), + ComponentConfig: s.ComponentConfig, + RESTMapper: restMapper, + AvailableResources: availableResources, + Cloud: cloud, + LoopMode: loopMode, + Stop: stop, + InformersStarted: make(chan struct{}), + ResyncPeriod: ResyncPeriod(s), } return ctx, nil } diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 293e1e56b63..64528678e8d 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -33,7 +33,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilfeature "k8s.io/apiserver/pkg/util/feature" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" - "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/metadata" restclient "k8s.io/client-go/rest" @@ -334,7 +333,7 @@ func startResourceQuotaController(ctx ControllerContext) (http.Handler, bool, er QuotaClient: resourceQuotaControllerClient.CoreV1(), ResourceQuotaInformer: ctx.InformerFactory.Core().V1().ResourceQuotas(), ResyncPeriod: controller.StaticResyncPeriodFunc(ctx.ComponentConfig.ResourceQuotaController.ResourceQuotaSyncPeriod.Duration), - InformerFactory: ctx.GenericInformerFactory, + InformerFactory: ctx.ObjectOrMetadataInformerFactory, ReplenishmentResyncPeriod: ctx.ResyncPeriod, DiscoveryFunc: discoveryFunc, IgnoredResourcesFunc: quotaConfiguration.IgnoredResources, @@ -439,7 +438,7 @@ func startGarbageCollectorController(ctx ControllerContext) (http.Handler, bool, ctx.RESTMapper, deletableResources, ignoredResources, - ctx.GenericInformerFactory, + ctx.ObjectOrMetadataInformerFactory, ctx.InformersStarted, ) if err != nil { diff --git a/cmd/kube-controller-manager/app/core_test.go b/cmd/kube-controller-manager/app/core_test.go index 38aef728c7b..1ca43f6ce7e 100644 --- a/cmd/kube-controller-manager/app/core_test.go +++ b/cmd/kube-controller-manager/app/core_test.go @@ -123,10 +123,10 @@ func TestController_DiscoveryError(t *testing.T) { testClientBuilder := TestClientBuilder{clientset: testClientset} testInformerFactory := informers.NewSharedInformerFactoryWithOptions(testClientset, time.Duration(1)) ctx := ControllerContext{ - ClientBuilder: testClientBuilder, - InformerFactory: testInformerFactory, - GenericInformerFactory: testInformerFactory, - InformersStarted: make(chan struct{}), + ClientBuilder: testClientBuilder, + InformerFactory: testInformerFactory, + ObjectOrMetadataInformerFactory: testInformerFactory, + InformersStarted: make(chan struct{}), } for funcName, controllerInit := range controllerInitFuncMap { _, _, err := controllerInit(ctx) diff --git a/pkg/controller/BUILD b/pkg/controller/BUILD index 09ecdb16032..c4a87570d6c 100644 --- a/pkg/controller/BUILD +++ b/pkg/controller/BUILD @@ -80,11 +80,11 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", - "//staging/src/k8s.io/client-go/dynamic/dynamicinformer:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/metadata/metadatainformer:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", diff --git a/pkg/controller/informer_factory.go b/pkg/controller/informer_factory.go index f6fb65288d9..11f3272e707 100644 --- a/pkg/controller/informer_factory.go +++ b/pkg/controller/informer_factory.go @@ -18,8 +18,8 @@ package controller import ( "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" + "k8s.io/client-go/metadata/metadatainformer" ) // InformerFactory creates informers for each group version resource. @@ -29,28 +29,28 @@ type InformerFactory interface { } type informerFactory struct { - typedInformerFactory informers.SharedInformerFactory - dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory + typedInformerFactory informers.SharedInformerFactory + metadataInformerFactory metadatainformer.SharedInformerFactory } func (i *informerFactory) ForResource(resource schema.GroupVersionResource) (informers.GenericInformer, error) { informer, err := i.typedInformerFactory.ForResource(resource) if err != nil { - return i.dynamicInformerFactory.ForResource(resource), nil + return i.metadataInformerFactory.ForResource(resource), nil } return informer, nil } func (i *informerFactory) Start(stopCh <-chan struct{}) { i.typedInformerFactory.Start(stopCh) - i.dynamicInformerFactory.Start(stopCh) + i.metadataInformerFactory.Start(stopCh) } // NewInformerFactory creates a new InformerFactory which works with both typed -// resources and dynamic resources -func NewInformerFactory(typedInformerFactory informers.SharedInformerFactory, dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory) InformerFactory { +// resources and metadata-only resources +func NewInformerFactory(typedInformerFactory informers.SharedInformerFactory, metadataInformerFactory metadatainformer.SharedInformerFactory) InformerFactory { return &informerFactory{ - typedInformerFactory: typedInformerFactory, - dynamicInformerFactory: dynamicInformerFactory, + typedInformerFactory: typedInformerFactory, + metadataInformerFactory: metadataInformerFactory, } } diff --git a/test/integration/garbagecollector/BUILD b/test/integration/garbagecollector/BUILD index 4d7e234027b..9e1b132c6d4 100644 --- a/test/integration/garbagecollector/BUILD +++ b/test/integration/garbagecollector/BUILD @@ -29,10 +29,10 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/client-go/discovery/cached/memory:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/dynamic/dynamicinformer:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/metadata:go_default_library", + "//staging/src/k8s.io/client-go/metadata/metadatainformer:go_default_library", "//staging/src/k8s.io/client-go/restmapper:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//test/integration:go_default_library", diff --git a/test/integration/garbagecollector/garbage_collector_test.go b/test/integration/garbagecollector/garbage_collector_test.go index 6f2a0dca4b7..dee8be7478e 100644 --- a/test/integration/garbagecollector/garbage_collector_test.go +++ b/test/integration/garbagecollector/garbage_collector_test.go @@ -38,10 +38,10 @@ import ( "k8s.io/apiserver/pkg/storage/names" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" - "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/metadata" + "k8s.io/client-go/metadata/metadatainformer" "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/cache" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" @@ -242,7 +242,7 @@ func setupWithServer(t *testing.T, result *kubeapiservertesting.TestServer, work t.Fatalf("failed to create dynamicClient: %v", err) } sharedInformers := informers.NewSharedInformerFactory(clientSet, 0) - dynamicInformers := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) + metadataInformers := metadatainformer.NewSharedInformerFactory(metadataClient, 0) alwaysStarted := make(chan struct{}) close(alwaysStarted) gc, err := garbagecollector.NewGarbageCollector( @@ -250,7 +250,7 @@ func setupWithServer(t *testing.T, result *kubeapiservertesting.TestServer, work restMapper, deletableResources, garbagecollector.DefaultIgnoredResources(), - controller.NewInformerFactory(sharedInformers, dynamicInformers), + controller.NewInformerFactory(sharedInformers, metadataInformers), alwaysStarted, ) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index ba7454d9c36..b1a9adf775f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1315,8 +1315,6 @@ k8s.io/client-go/discovery/cached/disk k8s.io/client-go/discovery/cached/memory k8s.io/client-go/discovery/fake k8s.io/client-go/dynamic -k8s.io/client-go/dynamic/dynamicinformer -k8s.io/client-go/dynamic/dynamiclister k8s.io/client-go/dynamic/fake k8s.io/client-go/informers k8s.io/client-go/informers/admissionregistration