From 007d7a802f2f634bcb21c86c81bd91bf18165499 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 11 Nov 2016 14:10:10 +1100 Subject: [PATCH] Add option to disable federation ingress controller --- .../federation-controller-manager/app/BUILD | 17 ++++ .../app/controllermanager.go | 65 ++++++++++++-- .../app/controllermanager_test.go | 88 +++++++++++++++++++ .../app/options/BUILD | 1 + .../app/options/options.go | 8 ++ .../pkg/federation-controller/ingress/BUILD | 1 + .../ingress/ingress_controller.go | 12 +++ .../ingress/ingress_controller_test.go | 10 +-- test/test_owners.csv | 1 + 9 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 federation/cmd/federation-controller-manager/app/controllermanager_test.go diff --git a/federation/cmd/federation-controller-manager/app/BUILD b/federation/cmd/federation-controller-manager/app/BUILD index 86e57a07853..ea2311606f9 100644 --- a/federation/cmd/federation-controller-manager/app/BUILD +++ b/federation/cmd/federation-controller-manager/app/BUILD @@ -33,9 +33,13 @@ go_library( "//federation/pkg/federation-controller/secret:go_default_library", "//federation/pkg/federation-controller/service:go_default_library", "//federation/pkg/federation-controller/util:go_default_library", + "//pkg/apis/meta/v1:go_default_library", "//pkg/client/restclient:go_default_library", + "//pkg/client/typed/discovery:go_default_library", "//pkg/client/unversioned/clientcmd:go_default_library", "//pkg/healthz:go_default_library", + "//pkg/runtime/schema:go_default_library", + "//pkg/util/config:go_default_library", "//pkg/util/configz:go_default_library", "//pkg/util/wait:go_default_library", "//pkg/version:go_default_library", @@ -45,3 +49,16 @@ go_library( "//vendor:github.com/spf13/pflag", ], ) + +go_test( + name = "go_default_test", + srcs = ["controllermanager_test.go"], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//federation/pkg/federation-controller/ingress:go_default_library", + "//pkg/apis/meta/v1:go_default_library", + "//pkg/runtime/schema:go_default_library", + "//pkg/util/config:go_default_library", + ], +) diff --git a/federation/cmd/federation-controller-manager/app/controllermanager.go b/federation/cmd/federation-controller-manager/app/controllermanager.go index b2f0781a3a2..02431bbb6e4 100644 --- a/federation/cmd/federation-controller-manager/app/controllermanager.go +++ b/federation/cmd/federation-controller-manager/app/controllermanager.go @@ -39,6 +39,7 @@ import ( secretcontroller "k8s.io/kubernetes/federation/pkg/federation-controller/secret" servicecontroller "k8s.io/kubernetes/federation/pkg/federation-controller/service" "k8s.io/kubernetes/federation/pkg/federation-controller/util" + metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/client/restclient" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" "k8s.io/kubernetes/pkg/healthz" @@ -50,6 +51,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/kubernetes/pkg/client/typed/discovery" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/util/config" ) const ( @@ -155,6 +159,12 @@ func StartControllers(s *options.CMServer, restClientCfg *restclient.Config) err glog.Fatalf("Cloud provider could not be initialized: %v", err) } + discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(restClientCfg) + serverResources, err := discoveryClient.ServerResources() + if err != nil { + glog.Fatalf("Could not find resources from API Server: %v", err) + } + glog.Infof("Loading client config for namespace controller %q", "namespace-controller") nsClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "namespace-controller")) namespaceController := namespacecontroller.NewNamespaceController(nsClientset) @@ -182,11 +192,13 @@ func StartControllers(s *options.CMServer, restClientCfg *restclient.Config) err // TODO: rename s.ConcurentReplicaSetSyncs go deploymentController.Run(s.ConcurrentReplicaSetSyncs, wait.NeverStop) - glog.Infof("Loading client config for ingress controller %q", "ingress-controller") - ingClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "ingress-controller")) - ingressController := ingresscontroller.NewIngressController(ingClientset) - glog.Infof("Running ingress controller") - ingressController.Run(wait.NeverStop) + if controllerEnabled(s.Controllers, serverResources, ingresscontroller.ControllerName, ingresscontroller.RequiredResources, true) { + glog.Infof("Loading client config for ingress controller %q", "ingress-controller") + ingClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "ingress-controller")) + ingressController := ingresscontroller.NewIngressController(ingClientset) + glog.Infof("Running ingress controller") + ingressController.Run(wait.NeverStop) + } glog.Infof("Loading client config for service controller %q", servicecontroller.UserAgentName) scClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, servicecontroller.UserAgentName)) @@ -209,3 +221,46 @@ func restClientConfigFromSecret(master string) (*restclient.Config, error) { } return restClientCfg, nil } + +func controllerEnabled(controllers config.ConfigurationMap, serverResources []*metav1.APIResourceList, controller string, requiredResources []schema.GroupVersionResource, defaultValue bool) bool { + controllerConfig, ok := controllers[controller] + if ok { + if controllerConfig == "false" { + glog.Infof("%s controller disabled by config", controller) + return false + } + if controllerConfig == "true" { + if !hasRequiredResources(serverResources, requiredResources) { + glog.Fatalf("%s controller enabled explicitly but API Server does not have required resources", controller) + panic("unreachable") + } + return true + } + } else if defaultValue { + if !hasRequiredResources(serverResources, requiredResources) { + glog.Warningf("%s controller disabled because API Server does not have required resources", controller) + return false + } + } + return defaultValue +} + +func hasRequiredResources(serverResources []*metav1.APIResourceList, requiredResources []schema.GroupVersionResource) bool { + for _, resource := range requiredResources { + found := false + for _, serverResource := range serverResources { + if serverResource.GroupVersion == resource.GroupVersion().String() { + for _, apiResource := range serverResource.APIResources { + if apiResource.Name == resource.Resource { + found = true + break + } + } + } + } + if !found { + return false + } + } + return true +} diff --git a/federation/cmd/federation-controller-manager/app/controllermanager_test.go b/federation/cmd/federation-controller-manager/app/controllermanager_test.go new file mode 100644 index 00000000000..e229a430b2c --- /dev/null +++ b/federation/cmd/federation-controller-manager/app/controllermanager_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + ingresscontroller "k8s.io/kubernetes/federation/pkg/federation-controller/ingress" + metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/util/config" + "testing" +) + +func TestControllerEnabled(t *testing.T) { + + testCases := []struct { + controllersConfig config.ConfigurationMap + serverResources []*metav1.APIResourceList + controller string + requiredResources []schema.GroupVersionResource + defaultValue bool + expectedResult bool + }{ + // no override, API server has Ingress enabled + { + controllersConfig: config.ConfigurationMap{}, + serverResources: []*metav1.APIResourceList{ + { + GroupVersion: "extensions/v1beta1", + APIResources: []metav1.APIResource{ + {Name: "ingresses", Namespaced: true, Kind: "Ingress"}, + }, + }, + }, + controller: ingresscontroller.ControllerName, + requiredResources: ingresscontroller.RequiredResources, + defaultValue: true, + expectedResult: true, + }, + // no override, API server has Ingress disabled + { + controllersConfig: config.ConfigurationMap{}, + serverResources: []*metav1.APIResourceList{}, + controller: ingresscontroller.ControllerName, + requiredResources: ingresscontroller.RequiredResources, + defaultValue: true, + expectedResult: false, + }, + // API server has Ingress enabled, override config to disable Ingress controller + { + controllersConfig: config.ConfigurationMap{ + ingresscontroller.ControllerName: "false", + }, + serverResources: []*metav1.APIResourceList{ + { + GroupVersion: "extensions/v1beta1", + APIResources: []metav1.APIResource{ + {Name: "ingresses", Namespaced: true, Kind: "Ingress"}, + }, + }, + }, + controller: ingresscontroller.ControllerName, + requiredResources: ingresscontroller.RequiredResources, + defaultValue: true, + expectedResult: false, + }, + } + + for _, test := range testCases { + actualEnabled := controllerEnabled(test.controllersConfig, test.serverResources, test.controller, test.requiredResources, test.defaultValue) + if actualEnabled != test.expectedResult { + t.Errorf("%s controller: expected %v, got %v", test.controller, test.expectedResult, actualEnabled) + } + } +} diff --git a/federation/cmd/federation-controller-manager/app/options/BUILD b/federation/cmd/federation-controller-manager/app/options/BUILD index a7a9347c479..9470a81e036 100644 --- a/federation/cmd/federation-controller-manager/app/options/BUILD +++ b/federation/cmd/federation-controller-manager/app/options/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/apis/componentconfig:go_default_library", "//pkg/apis/meta/v1:go_default_library", "//pkg/client/leaderelection:go_default_library", + "//pkg/util/config:go_default_library", "//vendor:github.com/spf13/pflag", ], ) diff --git a/federation/cmd/federation-controller-manager/app/options/options.go b/federation/cmd/federation-controller-manager/app/options/options.go index 77612bf9390..eb7882452b8 100644 --- a/federation/cmd/federation-controller-manager/app/options/options.go +++ b/federation/cmd/federation-controller-manager/app/options/options.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/client/leaderelection" + "k8s.io/kubernetes/pkg/util/config" ) type ControllerManagerConfiguration struct { @@ -67,6 +68,8 @@ type ControllerManagerConfiguration struct { LeaderElection componentconfig.LeaderElectionConfiguration `json:"leaderElection"` // contentType is contentType of requests sent to apiserver. ContentType string `json:"contentType"` + // ConfigurationMap determining which controllers should be enabled or disabled + Controllers config.ConfigurationMap `json:"controllers"` } // CMServer is the main context object for the controller manager. @@ -94,6 +97,7 @@ func NewCMServer() *CMServer { APIServerQPS: 20.0, APIServerBurst: 30, LeaderElection: leaderelection.DefaultLeaderElectionConfiguration(), + Controllers: make(config.ConfigurationMap), }, } return &s @@ -118,5 +122,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&s.APIServerBurst, "federated-api-burst", s.APIServerBurst, "Burst to use while talking with federation apiserver") fs.StringVar(&s.DnsProvider, "dns-provider", s.DnsProvider, "DNS provider. Valid values are: "+fmt.Sprintf("%q", dnsprovider.RegisteredDnsProviders())) fs.StringVar(&s.DnsConfigFile, "dns-provider-config", s.DnsConfigFile, "Path to config file for configuring DNS provider.") + fs.Var(&s.Controllers, "controllers", ""+ + "A set of key=value pairs that describe controller configuration that may be passed "+ + "to controller manager to enable/disable specific controllers. Valid options are: \n"+ + "ingress=true|false (default=true)") leaderelection.BindFlags(&s.LeaderElection, fs) } diff --git a/federation/pkg/federation-controller/ingress/BUILD b/federation/pkg/federation-controller/ingress/BUILD index 3b63433a4bd..5334078fa2f 100644 --- a/federation/pkg/federation-controller/ingress/BUILD +++ b/federation/pkg/federation-controller/ingress/BUILD @@ -30,6 +30,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/conversion:go_default_library", "//pkg/runtime:go_default_library", + "//pkg/runtime/schema:go_default_library", "//pkg/types:go_default_library", "//pkg/util/flowcontrol:go_default_library", "//pkg/watch:go_default_library", diff --git a/federation/pkg/federation-controller/ingress/ingress_controller.go b/federation/pkg/federation-controller/ingress/ingress_controller.go index 03e143f3e3c..ee4a90df94c 100644 --- a/federation/pkg/federation-controller/ingress/ingress_controller.go +++ b/federation/pkg/federation-controller/ingress/ingress_controller.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/conversion" pkgruntime "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util/flowcontrol" "k8s.io/kubernetes/pkg/watch" @@ -58,6 +59,11 @@ const ( // We wait for ingress to be created in this cluster before creating it any // other cluster. firstClusterAnnotation = "ingress.federation.kubernetes.io/first-cluster" + ControllerName = "ingress" +) + +var ( + RequiredResources = []schema.GroupVersionResource{extensionsv1beta1.SchemeGroupVersion.WithResource("ingresses")} ) type IngressController struct { @@ -494,6 +500,12 @@ func (ic *IngressController) reconcileConfigMapForCluster(clusterName string) { return } + ingressList := ic.ingressInformerStore.List() + if len(ingressList) <= 0 { + glog.V(4).Infof("No federated ingresses, ignore reconcile config map.") + return + } + if clusterName == allClustersKey { clusters, err := ic.configMapFederatedInformer.GetReadyClusters() if err != nil { diff --git a/federation/pkg/federation-controller/ingress/ingress_controller_test.go b/federation/pkg/federation-controller/ingress/ingress_controller_test.go index 5a3d987334d..aee5ccc6d34 100644 --- a/federation/pkg/federation-controller/ingress/ingress_controller_test.go +++ b/federation/pkg/federation-controller/ingress/ingress_controller_test.go @@ -129,15 +129,15 @@ func TestIngressController(t *testing.T) { t.Log("Adding Ingress UID ConfigMap to cluster 1") cluster1ConfigMapWatch.Add(cfg1) - t.Log("Checking that UID annotation on Cluster 1 annotation was correctly updated") - cluster := GetClusterFromChan(fedClusterUpdateChan) - assert.NotNil(t, cluster) - assert.Equal(t, cluster.ObjectMeta.Annotations[uidAnnotationKey], cfg1.Data[uidKey]) - // Test add federated ingress. t.Log("Adding Federated Ingress") fedIngressWatch.Add(&fedIngress) + t.Log("Checking that UID annotation on Cluster 1 annotation was correctly updated after adding Federated Ingress") + cluster := GetClusterFromChan(fedClusterUpdateChan) + assert.NotNil(t, cluster) + assert.Equal(t, cluster.ObjectMeta.Annotations[uidAnnotationKey], cfg1.Data[uidKey]) + t.Logf("Checking that approproate finalizers are added") // There should be 2 updates to add both the finalizers. updatedIngress := GetIngressFromChan(t, fedIngressUpdateChan) diff --git a/test/test_owners.csv b/test/test_owners.csv index 751b4754b1e..ef6d898de1a 100644 --- a/test/test_owners.csv +++ b/test/test_owners.csv @@ -519,6 +519,7 @@ k8s.io/kubernetes/cmd/mungedocs,mwielgus,1 k8s.io/kubernetes/examples,Random-Liu,0 k8s.io/kubernetes/federation/apis/federation/install,nikhiljindal,0 k8s.io/kubernetes/federation/apis/federation/validation,nikhiljindal,0 +k8s.io/kubernetes/federation/cmd/federation-controller-manager/app,kzwang,0 k8s.io/kubernetes/federation/pkg/dnsprovider,sttts,1 k8s.io/kubernetes/federation/pkg/dnsprovider/providers/aws/route53,cjcullen,1 k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns,jsafrane,1