From d9b75ed82bf5b80716d9d1c4682a725b165314b3 Mon Sep 17 00:00:00 2001 From: deads2k Date: Tue, 10 Jan 2017 15:11:25 -0500 Subject: [PATCH] add --controllers to controller manager --- cmd/hyperkube/kube-controller-manager.go | 2 +- cmd/kube-controller-manager/app/BUILD | 10 +++ .../app/controller_manager_test.go | 81 +++++++++++++++++++ .../app/controllermanager.go | 47 ++++++++++- cmd/kube-controller-manager/app/options/BUILD | 2 + .../app/options/options.go | 32 +++++++- .../controller-manager.go | 2 +- pkg/apis/componentconfig/types.go | 7 ++ .../componentconfig/zz_generated.deepcopy.go | 5 ++ 9 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 cmd/kube-controller-manager/app/controller_manager_test.go diff --git a/cmd/hyperkube/kube-controller-manager.go b/cmd/hyperkube/kube-controller-manager.go index b102a4023a0..4be4381605b 100644 --- a/cmd/hyperkube/kube-controller-manager.go +++ b/cmd/hyperkube/kube-controller-manager.go @@ -33,6 +33,6 @@ func NewKubeControllerManager() *Server { return app.Run(s) }, } - s.AddFlags(hks.Flags()) + s.AddFlags(hks.Flags(), app.KnownControllers(), app.ControllersDisabledByDefault.List()) return &hks } diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index d2023b59266..d94fd8ccbe7 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -5,6 +5,7 @@ licenses(["notice"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -98,6 +99,7 @@ go_library( "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", "//vendor:k8s.io/apimachinery/pkg/runtime/schema", "//vendor:k8s.io/apimachinery/pkg/runtime/serializer", + "//vendor:k8s.io/apimachinery/pkg/util/sets", "//vendor:k8s.io/apimachinery/pkg/util/wait", "//vendor:k8s.io/apiserver/pkg/healthz", ], @@ -118,3 +120,11 @@ filegroup( ], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["controller_manager_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = ["//vendor:k8s.io/apimachinery/pkg/util/sets"], +) diff --git a/cmd/kube-controller-manager/app/controller_manager_test.go b/cmd/kube-controller-manager/app/controller_manager_test.go new file mode 100644 index 00000000000..030bfce400c --- /dev/null +++ b/cmd/kube-controller-manager/app/controller_manager_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app implements a server that runs a set of active +// components. This includes replication controllers, service endpoints and +// nodes. +// +package app + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestIsControllerEnabled(t *testing.T) { + tcs := []struct { + name string + controllerName string + controllers []string + disabledByDefaultControllers []string + expected bool + }{ + { + name: "on by name", + controllerName: "bravo", + controllers: []string{"alpha", "bravo", "-charlie"}, + disabledByDefaultControllers: []string{"delta", "echo"}, + expected: true, + }, + { + name: "off by name", + controllerName: "charlie", + controllers: []string{"alpha", "bravo", "-charlie"}, + disabledByDefaultControllers: []string{"delta", "echo"}, + expected: false, + }, + { + name: "on by default", + controllerName: "alpha", + controllers: []string{"*"}, + disabledByDefaultControllers: []string{"delta", "echo"}, + expected: true, + }, + { + name: "off by default", + controllerName: "delta", + controllers: []string{"*"}, + disabledByDefaultControllers: []string{"delta", "echo"}, + expected: false, + }, + { + name: "off by default implicit, no star", + controllerName: "foxtrot", + controllers: []string{"alpha", "bravo", "-charlie"}, + disabledByDefaultControllers: []string{"delta", "echo"}, + expected: false, + }, + } + + for _, tc := range tcs { + actual := IsControllerEnabled(tc.controllerName, sets.NewString(tc.disabledByDefaultControllers...), tc.controllers...) + if actual != tc.expected { + t.Errorf("%v: expected %v, got %v", tc.name, tc.expected, actual) + } + } + +} diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 119cadeb5dc..347c8db29bb 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -32,6 +32,7 @@ import ( "time" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/healthz" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" @@ -71,7 +72,7 @@ const ( // NewControllerManagerCommand creates a *cobra.Command object with default parameters func NewControllerManagerCommand() *cobra.Command { s := options.NewCMServer() - s.AddFlags(pflag.CommandLine) + s.AddFlags(pflag.CommandLine, KnownControllers(), ControllersDisabledByDefault.List()) cmd := &cobra.Command{ Use: "kube-controller-manager", Long: `The Kubernetes controller manager is a daemon that embeds @@ -98,6 +99,10 @@ func ResyncPeriod(s *options.CMServer) func() time.Duration { // Run runs the CMServer. This should never exit. func Run(s *options.CMServer) error { + if err := s.Validate(KnownControllers(), ControllersDisabledByDefault.List()); err != nil { + return err + } + if c, err := configz.New("componentconfig"); err == nil { c.Set(s.KubeControllerManagerConfiguration) } else { @@ -216,11 +221,46 @@ type ControllerContext struct { Stop <-chan struct{} } +func (c ControllerContext) IsControllerEnabled(name string) bool { + return IsControllerEnabled(name, ControllersDisabledByDefault, c.Options.Controllers...) +} + +func IsControllerEnabled(name string, disabledByDefaultControllers sets.String, controllers ...string) bool { + hasStar := false + for _, controller := range controllers { + if controller == name { + return true + } + if controller == "-"+name { + return false + } + if controller == "*" { + hasStar = true + } + } + // if we get here, there was no explicit choice + if !hasStar { + // nothing on by default + return false + } + if disabledByDefaultControllers.Has(name) { + return false + } + + return true +} + // InitFunc is used to launch a particular controller. It may run additional "should I activate checks". // Any error returned will cause the controller process to `Fatal` // The bool indicates whether the controller was enabled. type InitFunc func(ctx ControllerContext) (bool, error) +func KnownControllers() []string { + return sets.StringKeySet(newControllerInitializers()).List() +} + +var ControllersDisabledByDefault = sets.NewString() + func newControllerInitializers() map[string]InitFunc { controllers := map[string]InitFunc{} controllers["endpoint"] = startEndpointController @@ -330,6 +370,11 @@ func StartControllers(controllers map[string]InitFunc, s *options.CMServer, root } for controllerName, initFn := range controllers { + if !ctx.IsControllerEnabled(controllerName) { + glog.Warningf("%q is disabled", controllerName) + continue + } + time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter)) glog.V(1).Infof("Starting %q", controllerName) diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index 15416dab713..d61571e7101 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -16,6 +16,8 @@ go_library( "//pkg/client/leaderelection:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/config:go_default_library", + "//pkg/util/errors:go_default_library", + "//pkg/util/sets:go_default_library", "//vendor:github.com/spf13/pflag", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", ], diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 7cb0535ee59..88111f373e0 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -19,6 +19,8 @@ limitations under the License. package options import ( + "fmt" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,6 +28,8 @@ import ( "k8s.io/kubernetes/pkg/client/leaderelection" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/util/config" + utilerrors "k8s.io/kubernetes/pkg/util/errors" + "k8s.io/kubernetes/pkg/util/sets" "github.com/spf13/pflag" ) @@ -42,6 +46,7 @@ type CMServer struct { func NewCMServer() *CMServer { s := CMServer{ KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ + Controllers: []string{"*"}, Port: ports.ControllerManagerPort, Address: "0.0.0.0", ConcurrentEndpointSyncs: 5, @@ -103,7 +108,11 @@ func NewCMServer() *CMServer { } // AddFlags adds flags for a specific CMServer to the specified FlagSet -func (s *CMServer) AddFlags(fs *pflag.FlagSet) { +func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) { + fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+ + "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ + "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", + strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", "))) fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on") fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)") fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.") @@ -188,3 +197,24 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { leaderelection.BindFlags(&s.LeaderElection, fs) config.DefaultFeatureGate.AddFlag(fs) } + +// Validate is used to validate the options and config before launching the controller manager +func (s *CMServer) Validate(allControllers []string, disabledByDefaultControllers []string) error { + var errs []error + + allControllersSet := sets.NewString(allControllers...) + for _, controller := range s.Controllers { + if controller == "*" { + continue + } + if strings.HasPrefix(controller, "-") { + controller = controller[1:] + } + + if !allControllersSet.Has(controller) { + errs = append(errs, fmt.Errorf("%q is not in the list of known controllers", controller)) + } + } + + return utilerrors.NewAggregate(errs) +} diff --git a/cmd/kube-controller-manager/controller-manager.go b/cmd/kube-controller-manager/controller-manager.go index 8487bae44dd..3f5c83b4fff 100644 --- a/cmd/kube-controller-manager/controller-manager.go +++ b/cmd/kube-controller-manager/controller-manager.go @@ -43,7 +43,7 @@ func init() { func main() { s := options.NewCMServer() - s.AddFlags(pflag.CommandLine) + s.AddFlags(pflag.CommandLine, app.KnownControllers(), app.ControllersDisabledByDefault.List()) flag.InitFlags() logs.InitLogs() diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 27512380c1a..51b9bb31edb 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -601,6 +601,13 @@ type LeaderElectionConfiguration struct { type KubeControllerManagerConfiguration struct { metav1.TypeMeta + // Controllers is the list of controllers to enable or disable + // '*' means "all enabled by default controllers" + // 'foo' means "enable 'foo'" + // '-foo' means "disable 'foo'" + // first item for a particular name wins + Controllers []string + // port is the port that the controller-manager's http service runs on. Port int32 // address is the IP address to serve on (set to 0.0.0.0 for all interfaces). diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index fd0cb23d286..b5253c32e14 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -73,6 +73,11 @@ func DeepCopy_componentconfig_KubeControllerManagerConfiguration(in interface{}, in := in.(*KubeControllerManagerConfiguration) out := out.(*KubeControllerManagerConfiguration) *out = *in + if in.Controllers != nil { + in, out := &in.Controllers, &out.Controllers + *out = make([]string, len(*in)) + copy(*out, *in) + } return nil } }