diff --git a/cmd/cloud-controller-manager/main.go b/cmd/cloud-controller-manager/main.go index b5e20b6543c..7c452b74800 100644 --- a/cmd/cloud-controller-manager/main.go +++ b/cmd/cloud-controller-manager/main.go @@ -32,7 +32,7 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cloud-provider" + cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" @@ -68,7 +68,15 @@ func main() { nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration fss := cliflag.NamedFlagSets{} nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller")) - controllerInitializers["nodeipam"] = nodeIpamController.startNodeIpamControllerWrapper + + controllerInitializers["nodeipam"] = app.ControllerInitFuncConstructor{ + // "node-controller" is the shared identity of all node controllers, including node, node lifecycle, and node ipam. + // See https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context. + InitContext: app.ControllerInitContext{ + ClientName: "node-controller", + }, + Constructor: nodeIpamController.StartNodeIpamControllerWrapper, + } command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop) diff --git a/cmd/cloud-controller-manager/nodeipamcontroller.go b/cmd/cloud-controller-manager/nodeipamcontroller.go index 76c302f8b8f..c90a632c6f8 100644 --- a/cmd/cloud-controller-manager/nodeipamcontroller.go +++ b/cmd/cloud-controller-manager/nodeipamcontroller.go @@ -52,7 +52,7 @@ type nodeIPAMController struct { nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions } -func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { +func (nodeIpamController *nodeIPAMController) StartNodeIpamControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { allErrors := nodeIpamController.nodeIPAMControllerOptions.Validate() if len(allErrors) > 0 { klog.Fatal("NodeIPAM controller values are not properly set.") @@ -60,11 +60,11 @@ func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(com nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration) return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startNodeIpamController(completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud) + return startNodeIpamController(initContext, completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud) } } -func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) { +func startNodeIpamController(initContext app.ControllerInitContext, ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) { var serviceCIDR *net.IPNet var secondaryServiceCIDR *net.IPNet @@ -147,7 +147,7 @@ func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, n nodeIpamController, err := nodeipamcontroller.NewNodeIpamController( ctx.InformerFactory.Core().V1().Nodes(), cloud, - ctx.ClientBuilder.ClientOrDie("node-controller"), + ctx.ClientBuilder.ClientOrDie(initContext.ClientName), clusterCIDRs, serviceCIDR, secondaryServiceCIDR, diff --git a/staging/src/k8s.io/cloud-provider/app/controllermanager.go b/staging/src/k8s.io/cloud-provider/app/controllermanager.go index 945ba2bf1d5..9730f1179ba 100644 --- a/staging/src/k8s.io/cloud-provider/app/controllermanager.go +++ b/staging/src/k8s.io/cloud-provider/app/controllermanager.go @@ -66,7 +66,7 @@ const ( // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters // initFuncConstructor is a map of named controller groups (you can start more than one in an init func) paired to their InitFuncConstructor. // additionalFlags provides controller specific flags to be included in the complete set of controller manager flags -func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command { +func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, controllerInitFuncConstructors map[string]ControllerInitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command { cmd := &cobra.Command{ Use: "cloud-controller-manager", Long: `The Cloud controller manager is a daemon that embeds @@ -75,7 +75,7 @@ the cloud specific control loops shipped with Kubernetes.`, verflag.PrintAndExitIfRequested() cliflag.PrintFlags(cmd.Flags()) - c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List()) + c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List()) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return err @@ -83,7 +83,7 @@ the cloud specific control loops shipped with Kubernetes.`, completedConfig := c.Complete() cloud := cloudInitializer(completedConfig) - controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud) + controllerInitializers := ConstructControllerInitializers(controllerInitFuncConstructors, completedConfig, cloud) if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) @@ -102,7 +102,7 @@ the cloud specific control loops shipped with Kubernetes.`, } fs := cmd.Flags() - namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List()) + namedFlagSets := s.Flags(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List()) verflag.AddFlags(namedFlagSets.FlagSet("global")) globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) @@ -308,11 +308,11 @@ type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprov type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error) // InitFuncConstructor is used to construct InitFunc -type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc +type InitFuncConstructor func(initcontext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc // ControllerNames indicate the default controller we are known. -func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string { - ret := sets.StringKeySet(initFuncConstructors) +func ControllerNames(controllerInitFuncConstructors map[string]ControllerInitFuncConstructor) []string { + ret := sets.StringKeySet(controllerInitFuncConstructors) return ret.List() } @@ -321,48 +321,80 @@ var ControllersDisabledByDefault = sets.NewString() // ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func) // paired to their InitFunc. This allows for structured downstream composition and subdivision. -func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc { +func ConstructControllerInitializers(controllerInitFuncConstructors map[string]ControllerInitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc { controllers := map[string]InitFunc{} - for name, constructor := range initFuncConstructors { - controllers[name] = constructor(completedConfig, cloud) + for name, constructor := range controllerInitFuncConstructors { + controllers[name] = constructor.Constructor(constructor.InitContext, completedConfig, cloud) } return controllers } +type ControllerInitFuncConstructor struct { + InitContext ControllerInitContext + Constructor InitFuncConstructor +} + +type ControllerInitContext struct { + ClientName string +} + // StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller -func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { +func StartCloudNodeControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startCloudNodeController(completedConfig, cloud, ctx.Stop) + return startCloudNodeController(initContext, completedConfig, cloud, ctx.Stop) } } -// startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller -func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { +// StartCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller +func StartCloudNodeLifecycleControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop) + return startCloudNodeLifecycleController(initContext, completedConfig, cloud, ctx.Stop) } } -// startServiceControllerWrapper is used to take cloud cofig as input and start service controller -func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { +// StartServiceControllerWrapper is used to take cloud cofig as input and start service controller +func StartServiceControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startServiceController(completedConfig, cloud, ctx.Stop) + return startServiceController(initContext, completedConfig, cloud, ctx.Stop) } } -// startRouteControllerWrapper is used to take cloud cofig as input and start route controller -func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { +// StartRouteControllerWrapper is used to take cloud cofig as input and start route controller +func StartRouteControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startRouteController(completedConfig, cloud, ctx.Stop) + return startRouteController(initContext, completedConfig, cloud, ctx.Stop) } } // DefaultInitFuncConstructors is a map of default named controller groups paired with InitFuncConstructor -var DefaultInitFuncConstructors = map[string]InitFuncConstructor{ - "cloud-node": StartCloudNodeControllerWrapper, - "cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper, - "service": startServiceControllerWrapper, - "route": startRouteControllerWrapper, +var DefaultInitFuncConstructors = map[string]ControllerInitFuncConstructor{ + // The cloud-node controller shares the "node-controller" identity with the cloud-node-lifecycle + // controller for historical reasons. See + // https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context. + "cloud-node": { + InitContext: ControllerInitContext{ + ClientName: "node-controller", + }, + Constructor: StartCloudNodeControllerWrapper, + }, + "cloud-node-lifecycle": { + InitContext: ControllerInitContext{ + ClientName: "node-controller", + }, + Constructor: StartCloudNodeLifecycleControllerWrapper, + }, + "service": { + InitContext: ControllerInitContext{ + ClientName: "service-controller", + }, + Constructor: StartServiceControllerWrapper, + }, + "route": { + InitContext: ControllerInitContext{ + ClientName: "route-controller", + }, + Constructor: StartRouteControllerWrapper, + }, } // CreateControllerContext creates a context struct containing references to resources needed by the diff --git a/staging/src/k8s.io/cloud-provider/app/core.go b/staging/src/k8s.io/cloud-provider/app/core.go index 0b939eb2b1f..c18b64d0af9 100644 --- a/staging/src/k8s.io/cloud-provider/app/core.go +++ b/staging/src/k8s.io/cloud-provider/app/core.go @@ -39,12 +39,12 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" ) -func startCloudNodeController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { +func startCloudNodeController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { // Start the CloudNodeController nodeController, err := cloudnodecontroller.NewCloudNodeController( ctx.SharedInformers.Core().V1().Nodes(), // cloud node controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), + ctx.ClientBuilder.ClientOrDie(initContext.ClientName), cloud, ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration, ) @@ -58,12 +58,12 @@ func startCloudNodeController(ctx *config.CompletedConfig, cloud cloudprovider.I return nil, true, nil } -func startCloudNodeLifecycleController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { +func startCloudNodeLifecycleController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { // Start the cloudNodeLifecycleController cloudNodeLifecycleController, err := cloudnodelifecyclecontroller.NewCloudNodeLifecycleController( ctx.SharedInformers.Core().V1().Nodes(), // cloud node lifecycle controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), + ctx.ClientBuilder.ClientOrDie(initContext.ClientName), cloud, ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, ) @@ -77,11 +77,11 @@ func startCloudNodeLifecycleController(ctx *config.CompletedConfig, cloud cloudp return nil, true, nil } -func startServiceController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { +func startServiceController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { // Start the service controller serviceController, err := servicecontroller.New( cloud, - ctx.ClientBuilder.ClientOrDie("service-controller"), + ctx.ClientBuilder.ClientOrDie(initContext.ClientName), ctx.SharedInformers.Core().V1().Services(), ctx.SharedInformers.Core().V1().Nodes(), ctx.ComponentConfig.KubeCloudShared.ClusterName, @@ -98,7 +98,7 @@ func startServiceController(ctx *config.CompletedConfig, cloud cloudprovider.Int return nil, true, nil } -func startRouteController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { +func startRouteController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { if !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { klog.Infof("Will not configure cloud provider routes, --configure-cloud-routes: %v", ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) return nil, false, nil @@ -134,7 +134,7 @@ func startRouteController(ctx *config.CompletedConfig, cloud cloudprovider.Inter routeController := routecontroller.New( routes, - ctx.ClientBuilder.ClientOrDie("route-controller"), + ctx.ClientBuilder.ClientOrDie(initContext.ClientName), ctx.SharedInformers.Core().V1().Nodes(), ctx.ComponentConfig.KubeCloudShared.ClusterName, clusterCIDRs, diff --git a/staging/src/k8s.io/cloud-provider/sample/basic_main.go b/staging/src/k8s.io/cloud-provider/sample/basic_main.go index 9f6386c5430..c3821156932 100644 --- a/staging/src/k8s.io/cloud-provider/sample/basic_main.go +++ b/staging/src/k8s.io/cloud-provider/sample/basic_main.go @@ -27,7 +27,7 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cloud-provider" + cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" @@ -49,7 +49,7 @@ func main() { } fss := cliflag.NamedFlagSets{} - command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop) + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers(), fss, wait.NeverStop) // TODO: once we switch everything over to Cobra commands, we can go back to calling // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the @@ -65,6 +65,30 @@ func main() { } } +// If custom ClientNames are used, as below, then the controller will not use +// the API server bootstrapped RBAC, and instead will require it to be installed +// separately. +func controllerInitializers() map[string]app.ControllerInitFuncConstructor { + controllerInitializers := app.DefaultInitFuncConstructors + if constructor, ok := controllerInitializers["cloud-node"]; ok { + constructor.InitContext.ClientName = "mycloud-external-cloud-node-controller" + controllerInitializers["cloud-node"] = constructor + } + if constructor, ok := controllerInitializers["cloud-node-lifecycle"]; ok { + constructor.InitContext.ClientName = "mycloud-external-cloud-node-lifecycle-controller" + controllerInitializers["cloud-node-lifecycle"] = constructor + } + if constructor, ok := controllerInitializers["service"]; ok { + constructor.InitContext.ClientName = "mycloud-external-service-controller" + controllerInitializers["service"] = constructor + } + if constructor, ok := controllerInitializers["route"]; ok { + constructor.InitContext.ClientName = "mycloud-external-route-controller" + controllerInitializers["route"] = constructor + } + return controllerInitializers +} + func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface { cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider