diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 3f7909af909..4981e2d3f9d 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -20,12 +20,14 @@ import ( "context" "fmt" "net" + "net/http" "os" "strings" "time" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" @@ -72,7 +74,7 @@ the cloud specific control loops shipped with Kubernetes.`, verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) - c, err := s.Config() + c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) @@ -87,7 +89,7 @@ the cloud specific control loops shipped with Kubernetes.`, } fs := cmd.Flags() - namedFlagSets := s.Flags() + namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List()) verflag.AddFlags(namedFlagSets.FlagSet("global")) globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) cmoptions.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) @@ -297,3 +299,29 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru select {} } + +// initFunc is used to launch a particular controller. It may run additional "should I activate checks". +// Any error returned will cause the controller process to `Fatal` +// The bool indicates whether the controller was enabled. +type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error) + +// KnownControllers indicate the default controller we are known. +func KnownControllers() []string { + ret := sets.StringKeySet(newControllerInitializers()) + return ret.List() +} + +// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers. +var ControllersDisabledByDefault = sets.NewString() + +// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func) +// paired to their initFunc. This allows for structured downstream composition and subdivision. +func newControllerInitializers() map[string]initFunc { + controllers := map[string]initFunc{} + controllers["cloud-node"] = startCloudNodeController + controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController + controllers["persistentvolume-binder"] = startPersistentVolumeLabelController + controllers["service"] = startServiceController + controllers["route"] = startRouteController + return controllers +} diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index 5e2a5979493..c3eea94ec46 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -22,8 +22,6 @@ import ( "net" "time" - "k8s.io/klog" - "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -36,6 +34,7 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" + "k8s.io/klog" ccmconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config" ccmconfigscheme "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme" ccmconfigv1alpha1 "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1" @@ -125,11 +124,9 @@ func NewDefaultComponentConfig(insecurePort int32) (*ccmconfig.CloudControllerMa } // Flags returns flags for a specific APIServer by section name -func (o *CloudControllerManagerOptions) Flags() apiserverflag.NamedFlagSets { +func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers []string) apiserverflag.NamedFlagSets { fss := apiserverflag.NamedFlagSets{} - o.Generic.AddFlags(&fss, []string{}, []string{}) - // TODO: Implement the --controllers flag fully for the ccm - fss.FlagSet("generic").MarkHidden("controllers") + o.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers) o.KubeCloudShared.AddFlags(fss.FlagSet("generic")) o.ServiceController.AddFlags(fss.FlagSet("service controller")) @@ -219,10 +216,10 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, } // Validate is used to validate config before launching the cloud controller manager -func (o *CloudControllerManagerOptions) Validate() error { +func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers []string) error { errors := []error{} - errors = append(errors, o.Generic.Validate(nil, nil)...) + errors = append(errors, o.Generic.Validate(allControllers, disabledByDefaultControllers)...) errors = append(errors, o.KubeCloudShared.Validate()...) errors = append(errors, o.ServiceController.Validate()...) errors = append(errors, o.SecureServing.Validate()...) @@ -246,8 +243,8 @@ func resyncPeriod(c *cloudcontrollerconfig.Config) func() time.Duration { } // Config return a cloud controller manager config objective -func (o *CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, error) { - if err := o.Validate(); err != nil { +func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers []string) (*cloudcontrollerconfig.Config, error) { + if err := o.Validate(allControllers, disabledByDefaultControllers); err != nil { return nil, err } diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index fce64bb9590..3f78c261731 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -115,7 +115,7 @@ func TestDefaultFlags(t *testing.T) { func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) s, _ := NewCloudControllerManagerOptions() - for _, f := range s.Flags().FlagSets { + for _, f := range s.Flags([]string{""}, []string{""}).FlagSets { fs.AddFlagSet(f) } @@ -131,6 +131,7 @@ func TestAddFlags(t *testing.T) { "--configure-cloud-routes=false", "--contention-profiling=true", "--controller-start-interval=2m", + "--controllers=foo,bar", "--http2-max-streams-per-connection=47", "--kube-api-burst=100", "--kube-api-content-type=application/vnd.kubernetes.protobuf", @@ -173,7 +174,7 @@ func TestAddFlags(t *testing.T) { Debugging: &cmoptions.DebuggingOptions{ EnableContentionProfiling: true, }, - Controllers: []string{"*"}, + Controllers: []string{"foo", "bar"}, }, KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ CloudProvider: &cmoptions.CloudProviderOptions{ diff --git a/cmd/cloud-controller-manager/app/testing/testserver.go b/cmd/cloud-controller-manager/app/testing/testserver.go index e5a3e9e5bad..638d9f3d6ff 100644 --- a/cmd/cloud-controller-manager/app/testing/testserver.go +++ b/cmd/cloud-controller-manager/app/testing/testserver.go @@ -83,7 +83,8 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err if err != nil { return TestServer{}, err } - namedFlagSets := s.Flags() + all, disabled := app.KnownControllers(), app.ControllersDisabledByDefault.List() + namedFlagSets := s.Flags(all, disabled) for _, f := range namedFlagSets.FlagSets { fs.AddFlagSet(f) } @@ -108,7 +109,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) } - config, err := s.Config() + config, err := s.Config(all, disabled) if err != nil { return result, fmt.Errorf("failed to create config from options: %v", err) } diff --git a/cmd/controller-manager/app/options/generic.go b/cmd/controller-manager/app/options/generic.go index 2089c7179f2..8d0dfc34f18 100644 --- a/cmd/controller-manager/app/options/generic.go +++ b/cmd/controller-manager/app/options/generic.go @@ -71,7 +71,6 @@ func (o *GenericControllerManagerConfigurationOptions) AddFlags(fss *apiserverfl genericfs.Float32Var(&o.ClientConnection.QPS, "kube-api-qps", o.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver.") genericfs.Int32Var(&o.ClientConnection.Burst, "kube-api-burst", o.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver.") genericfs.DurationVar(&o.ControllerStartInterval.Duration, "controller-start-interval", o.ControllerStartInterval.Duration, "Interval between starting controller managers.") - // TODO: complete the work of the cloud-controller-manager (and possibly other consumers of this code) respecting the --controllers flag genericfs.StringSliceVar(&o.Controllers, "controllers", o.Controllers, fmt.Sprintf(""+ "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 737b7d4e6cd..b840523db41 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -387,7 +387,7 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc if loopMode == IncludeCloudLoops { controllers["service"] = startServiceController controllers["route"] = startRouteController - controllers["cloudnodelifecycle"] = startCloudNodeLifecycleController + controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController // TODO: volume controller into the IncludeCloudLoops only set. } controllers["persistentvolume-binder"] = startPersistentVolumeBinderController